Good morning,
I have a problem with a slow throughput on CRS504-4XQ. I connected two esxi hosts (both using Mellanox® ConnectX®-6 Lx cards) with the switch. Both cards are rated for 25 Gb/s connection, but I when tried using iperf3, I got around 542 Mb/s transfer. When I tried direct connection, the speed is around 23.8 Gb/s
Any ideas?
What firmware version? Show configuration.
export file=somefile.txt hide-sensitive
Firmware is 7.13.2
2024-01-25 20:04:53 by RouterOS 7.13.2
software id = FW0D-G796
model = CRS504-4XQ
serial number = HD8082X97B6
/interface bridge
add name=bridge1 port-cost-mode=short
add admin-mac=18:FD:74:BF:F7:A4 auto-mac=no comment=defconf name=mgmt_bridge
port-cost-mode=short
/interface ethernet
set [ find default-name=ether1 ] l2mtu=1500
set [ find default-name=qsfp28-1-1 ] l2mtu=9000 mtu=9000
set [ find default-name=qsfp28-1-2 ] l2mtu=9000 mtu=9000
set [ find default-name=qsfp28-1-3 ] l2mtu=9000 mtu=9000
set [ find default-name=qsfp28-1-4 ] l2mtu=9000 mtu=9000
set [ find default-name=qsfp28-2-1 ] l2mtu=9000 mtu=9000
set [ find default-name=qsfp28-2-2 ] l2mtu=9000 mtu=9000
set [ find default-name=qsfp28-2-3 ] l2mtu=9000 mtu=9000
set [ find default-name=qsfp28-2-4 ] l2mtu=9000 mtu=9000
set [ find default-name=qsfp28-3-1 ] l2mtu=9000 mtu=9000
set [ find default-name=qsfp28-3-2 ] l2mtu=9000 mtu=9000
set [ find default-name=qsfp28-3-3 ] l2mtu=9000 mtu=9000
set [ find default-name=qsfp28-3-4 ] auto-negotiation=no l2mtu=9000 mtu=9000
set [ find default-name=qsfp28-4-1 ] auto-negotiation=no l2mtu=9000 mtu=9000
speed=10G-baseCR
set [ find default-name=qsfp28-4-2 ] l2mtu=9000 mtu=9000
set [ find default-name=qsfp28-4-3 ] l2mtu=9000 mtu=9000
set [ find default-name=qsfp28-4-4 ] l2mtu=9000 mtu=9000
/interface ethernet switch
set 0 l3-hw-offloading=yes
/interface list
add name=WAN
add name=LAN
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip hotspot profile
set [ find default=yes ] html-directory=hotspot
/port
set 0 name=serial0
/interface bridge port
add bridge=mgmt_bridge comment=defconf interface=ether1 internal-path-cost=10
path-cost=10
add bridge=bridge1 comment=defconf interface=qsfp28-1-1 internal-path-cost=10
path-cost=10
add bridge=bridge1 comment=defconf interface=qsfp28-1-3 internal-path-cost=10
path-cost=10
add bridge=bridge1 comment=defconf interface=qsfp28-2-1 internal-path-cost=10
path-cost=10
add bridge=mgmt_bridge comment=defconf ingress-filtering=no interface=
qsfp28-2-2 internal-path-cost=10 path-cost=10
add bridge=bridge1 comment=defconf interface=qsfp28-3-1 internal-path-cost=10
path-cost=10
add bridge=*1F comment=defconf interface=qsfp28-3-2 internal-path-cost=10
path-cost=10
add bridge=bridge1 comment=defconf interface=qsfp28-3-3 internal-path-cost=10
path-cost=10
add bridge=bridge1 comment=defconf interface=qsfp28-3-4 internal-path-cost=10
path-cost=10
add bridge=bridge1 comment=defconf interface=qsfp28-4-1 internal-path-cost=10
path-cost=10
add bridge=bridge1 comment=defconf interface=qsfp28-4-2 internal-path-cost=10
path-cost=10
add bridge=bridge1 comment=defconf interface=qsfp28-4-3 internal-path-cost=10
path-cost=10
add bridge=bridge1 comment=defconf ingress-filtering=no interface=qsfp28-4-4
internal-path-cost=10 path-cost=10
add bridge=mgmt_bridge ingress-filtering=no interface=qsfp28-1-2
internal-path-cost=10 path-cost=10
add bridge=mgmt_bridge ingress-filtering=no interface=qsfp28-1-4
internal-path-cost=10 path-cost=10
add bridge=bridge1 interface=qsfp28-2-4 internal-path-cost=10 path-cost=10
add bridge=bridge1 interface=qsfp28-2-3 internal-path-cost=10 path-cost=10
/ip neighbor discovery-settings
set discover-interface-list=all
/interface list member
add interface=ether1 list=WAN
add interface=qsfp28-1-1 list=LAN
add interface=qsfp28-1-2 list=LAN
add interface=qsfp28-1-3 list=LAN
add interface=qsfp28-1-4 list=LAN
add interface=qsfp28-2-1 list=LAN
add interface=qsfp28-2-2 list=LAN
add interface=qsfp28-2-3 list=LAN
add interface=qsfp28-2-4 list=LAN
add interface=qsfp28-3-1 list=LAN
add interface=qsfp28-3-2 list=LAN
add interface=qsfp28-3-3 list=LAN
add interface=qsfp28-3-4 list=LAN
add interface=qsfp28-4-1 list=LAN
add interface=qsfp28-4-2 list=LAN
add interface=qsfp28-4-3 list=LAN
add interface=qsfp28-4-4 list=LAN
/ip address
add address=192.168.1.232/24 interface=mgmt_bridge network=192.168.1.0
add address=192.168.3.253/24 interface=bridge1 network=192.168.3.0
/ip dhcp-server network
add address=192.168.3.0/24 gateway=192.168.3.253
/ip dns
set allow-remote-requests=yes servers=8.8.8.8
/ip route
add disabled=no dst-address=0.0.0.0/0 gateway=192.168.1.253 routing-table=
main suppress-hw-offload=no
/ip traffic-flow
set interfaces=LAN
/snmp
set enabled=yes
/system clock
set time-zone-name=Europe/Warsaw
/system identity
set name=Mikrotik_switch_3
/system note
set show-at-login=no
/system ntp client
set enabled=yes
/system ntp client servers
add address=pool.ntp.org
/system routerboard settings
set boot-os=router-os
Create only one bridge, and put all ports as members of this bridge. Probably one of the bridges is processed by the CPU. By the way this is switch not a router. To separate ports use vlan filtering. Read the docs! https://help.mikrotik.com/docs/display/ROS/Bridging+and+Switching#BridgingandSwitching-BridgeHardwareOffloading
This switch has two independent switch chips, one for the management interface and one for the rest. That is why I thought it can reach full speed with two independent bridges.
Looks like that messing with the assignment of the interfaces helped. I can reach wire speeds with two bridges, one only for the management interface, and the second one for the rest of the ports.
You should look on internal block diagram: https://i.mt.lv/cdn/product_files/CRS504-4XQ-IN_220412.png
Please consult how to make one hardware offloaded bridge?
I delete all bridge create one bridge, add to this bridge all port, but speed as before slow