I'm having a really odd issue that no amount of config resets seems to address. It's a very simple setup, with two 504's with MLAG do the bulk of the L2 switching work (in the main server room) and another two without MLAG in two other buildings respectively - from the MLAG stack to the other switches is a 100G link each, and between the latter is a LAG of 2x25Gbit. RSTP is set up to prioritise the 100G links over the slower LAG.
I understand that I can't do L3HW offload when using MLAG, so I've tried to set up inter-VLAN routing on one of the other switches - just 3 VLANs, I've set up VLAN interfaces and added them to the bridge as ports with the correct admission setting (accept only untagged + priority tagged), and assigned IP addresses to each VLAN interface. This is a simple static inter-VLAN routing setup. I had it working fine, but after a reboot /ip/routes shows no H flag on any route and inter-VLAN performance tanks. I had some Switch Rules, disabling all of them made no difference.
Now here's the really odd part - if I go in the bridge settings and either enable or disable Fast Forward (doesn't matter which), the routes immediately get L3 HW offload and perform perfectly (ie ~23Gbps iperf between two hosts on the main stack, separate VLANs/subnets). If I reboot the switch doing the routing, L3HW is gone and I'm down to ~250Mbps routing speeds until I turn Fast Forward on or off - again it doesn't matter. I have cleared all config and restored, I've even pasted the configs in manually and only enabled L3HW at the very end, in that case I still have to turn Bridge FF on or off to activate L3HW routing.
I would be very grateful if anyone has any ideas on this or had seen it before so we can raise an issue? It certainly feels like a bug to me.
Thanks
Alex
Config below:
Code: Select all
# jan/09/2023 13:26:14 by RouterOS 7.7rc4
# software id = LUSW-JDXH
#
# model = CRS504-4XQ
# serial number = HCE08C0Q3JH
/interface bridge
add fast-forward=no name=bridge pvid=999 vlan-filtering=yes
/interface ethernet
set [ find default-name=ether1 ] l2mtu=1584
set [ find default-name=qsfp28-1-1 ] l2mtu=9084 mtu=9000
set [ find default-name=qsfp28-1-2 ] l2mtu=9084 mtu=9000
set [ find default-name=qsfp28-1-3 ] l2mtu=9084 mtu=9000
set [ find default-name=qsfp28-1-4 ] l2mtu=9084 mtu=9000
set [ find default-name=qsfp28-2-1 ] auto-negotiation=no l2mtu=9084 mtu=9000 \
speed=25Gbps
set [ find default-name=qsfp28-2-2 ] auto-negotiation=no l2mtu=9084 mtu=9000 \
speed=25Gbps
set [ find default-name=qsfp28-2-3 ] auto-negotiation=no l2mtu=9084 mtu=9000 \
speed=25Gbps
set [ find default-name=qsfp28-2-4 ] auto-negotiation=no l2mtu=9084 mtu=9000 \
speed=25Gbps
set [ find default-name=qsfp28-3-1 ] auto-negotiation=no l2mtu=9084 mtu=9000 \
speed=10Gbps
set [ find default-name=qsfp28-3-2 ] auto-negotiation=no l2mtu=9084 mtu=9000 \
speed=10Gbps
set [ find default-name=qsfp28-3-3 ] auto-negotiation=no l2mtu=9084 mtu=9000 \
speed=10Gbps
set [ find default-name=qsfp28-3-4 ] auto-negotiation=no l2mtu=9084 mtu=9000 \
speed=10Gbps
set [ find default-name=qsfp28-4-1 ] auto-negotiation=no l2mtu=9084 mtu=9000 \
speed=10Gbps
set [ find default-name=qsfp28-4-2 ] auto-negotiation=no l2mtu=9084 mtu=9000 \
speed=10Gbps
set [ find default-name=qsfp28-4-3 ] auto-negotiation=no l2mtu=9084 mtu=9000 \
speed=10Gbps
set [ find default-name=qsfp28-4-4 ] auto-negotiation=no l2mtu=9084 mtu=9000 \
speed=10Gbps
/interface vlan
add interface=bridge name=IT vlan-id=9
add interface=bridge name=main vlan-id=1
add interface=bridge name=management vlan-id=7
/interface bonding
add mode=802.3ad mtu=9000 name=b3tob2 slaves=qsfp28-2-2,qsfp28-2-1 \
transmit-hash-policy=layer-3-and-4
add mode=802.3ad mtu=9000 name=downlink slaves="qsfp28-3-1,qsfp28-3-2,qsfp28-3\
-3,qsfp28-3-4,qsfp28-4-1,qsfp28-4-2,qsfp28-4-3,qsfp28-4-4" \
transmit-hash-policy=layer-3-and-4
/interface ethernet switch
set 0 l3-hw-offloading=yes
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip hotspot profile
set [ find default=yes ] html-directory=hotspot
/port
set 0 name=serial0
/interface bridge port
add bridge=bridge interface=qsfp28-1-1 path-cost=7 priority=0x50 pvid=999
add bridge=bridge interface=downlink pvid=999
add bridge=bridge frame-types=admit-only-untagged-and-priority-tagged \
interface=main
add bridge=bridge frame-types=admit-only-untagged-and-priority-tagged \
interface=management pvid=7
add bridge=bridge frame-types=admit-only-untagged-and-priority-tagged \
interface=IT pvid=9
add bridge=bridge interface=b3tob2 pvid=999
/interface ethernet switch l3hw-settings
set ipv6-hw=yes
/interface bridge vlan
# port with pvid added to untagged group which might cause problems, consider adding a seperate VLAN entry
add bridge=bridge tagged=bridge,downlink,qsfp28-1-1,b3tob2 vlan-ids=\
1-18,50,100-103
/ip address
add address=10.20.9.241/24 interface=IT network=10.20.9.0
add address=10.20.7.241/24 interface=management network=10.20.7.0
add address=10.20.19.241/24 interface=IT network=10.20.19.0
add address=172.22.254.241/16 interface=main network=172.22.0.0
/ip dns
set servers=10.20.12.32
/ip route
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway=10.20.19.254 \
pref-src="" routing-table=main scope=30 suppress-hw-offload=no \
target-scope=10
add disabled=no distance=1 dst-address=10.20.7.26/32 gateway=10.20.19.254 \
pref-src="" routing-table=main scope=30 suppress-hw-offload=no \
target-scope=10
add disabled=no distance=1 dst-address=10.20.7.30/32 gateway=10.20.19.254 \
pref-src="" routing-table=main scope=30 suppress-hw-offload=no \
target-scope=10
add disabled=no distance=1 dst-address=10.20.9.20/32 gateway=10.20.19.254 \
pref-src="" routing-table=main scope=30 suppress-hw-offload=no \
target-scope=10
add disabled=no distance=1 dst-address=10.20.9.22/32 gateway=10.20.19.254 \
pref-src="" routing-table=main scope=30 suppress-hw-offload=no \
target-scope=10
/system clock
set time-zone-name=Europe/London
/system identity
set name=100G-Barn3
/system routerboard settings
set boot-os=router-os
#error exporting /system/swos