Mlag breaks access to switch *half Solved*

Hardware MikroTik hardware general
1

I have two crs328-24p-4s+rm running 7.5 I enabled mlag over a bonded peer link, I am only using L2 features no l3 features on the switch and no acl. Once I enabled mlag I lost access to the switch via ip address but can still connect via winbox either wired or wirelessly. All traffic is passing and mlag is working, there are two other switches in play but all traffic is flowing. Can’t think of what could have changed at the address level to suddenly loss access.

[admin@core233] > export
# dec/11/2022 14:38:05 by RouterOS 7.5
# software id = MDNX-ZUMT
#
# model = CRS328-24P-4S+
# serial number = D41D0B26F339
/interface bridge
add admin-mac=C4:AD:34:F6:8E:37 auto-mac=no comment=defconf dhcp-snooping=yes igmp-snooping=yes name=bridge vlan-filtering=yes
/interface ethernet
set [ find default-name=ether15 ] name=Bedroom
set [ find default-name=ether16 ] name=KitchenAP
set [ find default-name=ether9 ] name=backyad
set [ find default-name=ether10 ] name=basement
set [ find default-name=sfp-sfpplus3 ] l2mtu=9000 mtu=9000 name=core96_sfp6
set [ find default-name=sfp-sfpplus2 ] advertise=10M-half,10M-full,100M-half,100M-full,1000M-half,1000M-full,10000M-full,2500M-full,5000M-full l2mtu=9000 mtu=9000 name=core98_sfp4
set [ find default-name=sfp-sfpplus1 ] l2mtu=9000 mtu=9000 name=core234-sfp2
set [ find default-name=sfp-sfpplus4 ] l2mtu=9000 mtu=9000 name=core234-sfp4
set [ find default-name=ether7 ] name=dead poe-out=off
set [ find default-name=ether13 ] poe-out=off
set [ find default-name=ether24 ] poe-out=off
set [ find default-name=ether3 ] name=ethport poe-out=off
set [ find default-name=ether14 ] name=frontdoor
set [ find default-name=ether5 ] name=garage
set [ find default-name=ether12 ] name=hall
set [ find default-name=ether4 ] name=left
set [ find default-name=ether8 ] name=leftf
set [ find default-name=ether18 ] name=plate
set [ find default-name=ether19 ] name=plate2
set [ find default-name=ether20 ] name=pool
set [ find default-name=ether17 ] name=rear
set [ find default-name=ether6 ] name=right
set [ find default-name=ether2 ] name=rightF
set [ find default-name=ether11 ] name=roof poe-out=off
/interface vlan
add interface=bridge name=MHQ vlan-id=103
/interface bonding
add mode=802.3ad mtu=9000 name=234-b slaves=core234-sfp2,core234-sfp4
add mlag-id=10 mode=802.3ad name=fw01-bond slaves=ether21
add mlag-id=20 mode=802.3ad name=fw02-bond slaves=ether22
/interface list
add name=WAN
add name=LAN
add name=cam
/interface lte apn
set [ find default=yes ] ip-type=ipv4 use-network-apn=no
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/port
set 0 name=serial0
/routing bgp template
set default disabled=no output.network=bgp-networks
/routing ospf instance
add disabled=no name=default-v2
/routing ospf area
add disabled=yes instance=default-v2 name=backbone-v2
/system logging action
set 3 remote=172.16.1.54 remote-port=7004
/interface bridge mlag
set bridge=bridge peer-port=234-b
/interface bridge port
add bridge=bridge comment=defconf frame-types=admit-only-untagged-and-priority-tagged ingress-filtering=no interface=ether1 pvid=103
add bridge=bridge comment=defconf frame-types=admit-only-untagged-and-priority-tagged ingress-filtering=no interface=rightF pvid=222
add bridge=bridge comment=defconf frame-types=admit-only-untagged-and-priority-tagged ingress-filtering=no interface=ethport pvid=106
add bridge=bridge comment=defconf frame-types=admit-only-untagged-and-priority-tagged ingress-filtering=no interface=left pvid=222
add bridge=bridge comment=defconf frame-types=admit-only-untagged-and-priority-tagged ingress-filtering=no interface=garage pvid=222
add bridge=bridge comment=defconf frame-types=admit-only-untagged-and-priority-tagged ingress-filtering=no interface=right pvid=222
add bridge=bridge comment=defconf frame-types=admit-only-untagged-and-priority-tagged ingress-filtering=no interface=dead
add bridge=bridge comment=defconf frame-types=admit-only-untagged-and-priority-tagged ingress-filtering=no interface=leftf pvid=222
add bridge=bridge comment=defconf frame-types=admit-only-untagged-and-priority-tagged ingress-filtering=no interface=backyad pvid=222
add bridge=bridge comment=defconf frame-types=admit-only-vlan-tagged interface=basement trusted=yes
add bridge=bridge comment=defconf frame-types=admit-only-vlan-tagged interface=roof
add bridge=bridge comment=defconf frame-types=admit-only-vlan-tagged interface=hall trusted=yes
add bridge=bridge comment=defconf frame-types=admit-only-untagged-and-priority-tagged ingress-filtering=no interface=ether13 pvid=103
add bridge=bridge comment=defconf frame-types=admit-only-untagged-and-priority-tagged ingress-filtering=no interface=frontdoor pvid=222
add bridge=bridge comment=defconf frame-types=admit-only-vlan-tagged interface=Bedroom trusted=yes
add bridge=bridge comment=defconf frame-types=admit-only-vlan-tagged interface=KitchenAP trusted=yes
add bridge=bridge comment=defconf frame-types=admit-only-untagged-and-priority-tagged ingress-filtering=no interface=rear pvid=222
add bridge=bridge comment=defconf frame-types=admit-only-untagged-and-priority-tagged ingress-filtering=no interface=plate pvid=222
add bridge=bridge comment=defconf frame-types=admit-only-untagged-and-priority-tagged ingress-filtering=no interface=plate2 pvid=222
add bridge=bridge comment=defconf frame-types=admit-only-untagged-and-priority-tagged interface=pool pvid=222
add bridge=bridge comment=defconf frame-types=admit-only-untagged-and-priority-tagged ingress-filtering=no interface=ether23 pvid=100
add bridge=bridge comment=defconf frame-types=admit-only-vlan-tagged interface=ether24 trusted=yes
add bridge=bridge comment=defconf frame-types=admit-only-vlan-tagged interface=core98_sfp4 trusted=yes
add bridge=bridge comment=defconf frame-types=admit-only-vlan-tagged interface=core96_sfp6 trusted=yes
add bridge=bridge interface=234-b pvid=99 trusted=yes
add bridge=bridge frame-types=admit-only-vlan-tagged hw=no interface=fw01-bond
add bridge=bridge frame-types=admit-only-vlan-tagged hw=no interface=fw02-bond
/ip neighbor discovery-settings
set discover-interface-list=all lldp-med-net-policy-vlan=1
/ip settings
set max-neighbor-entries=8192
/ipv6 settings
set disable-ipv6=yes
/interface bridge vlan
add bridge=bridge tagged=core98_sfp4,core234-sfp4,core234-sfp2,core96_sfp6,234-b untagged=rightF,left,garage,right,leftf,backyad,pool,frontdoor,rear,plate vlan-ids=222
add bridge=bridge tagged=core98_sfp4,core234-sfp2,core234-sfp4,core96_sfp6,234-b,fw01-bond untagged=ethport vlan-ids=106
add bridge=bridge tagged=core98_sfp4,core234-sfp2,core234-sfp4,core96_sfp6,234-b vlan-ids=105
add bridge=bridge tagged=basement,core98_sfp4,hall,core234-sfp2,core96_sfp6,KitchenAP,Bedroom,core234-sfp4,234-b vlan-ids=104
add bridge=bridge tagged=basement,core98_sfp4,hall,core234-sfp2,core96_sfp6,KitchenAP,Bedroom,core234-sfp4,234-b vlan-ids=102
add bridge=bridge tagged=basement,core98_sfp4,hall,core234-sfp2,core96_sfp6,KitchenAP,Bedroom,core234-sfp4,234-b vlan-ids=101
add bridge=bridge tagged=core98_sfp4,core96_sfp6,core234-sfp2,core234-sfp4,234-b vlan-ids=100
add bridge=bridge tagged=basement,core98_sfp4,hall,core234-sfp2,KitchenAP,core96_sfp6,Bedroom,core234-sfp4,bridge,234-b untagged=ether13,ether1 vlan-ids=103
add bridge=bridge tagged=core96_sfp6,core234-sfp2,core234-sfp4,234-b untagged=core98_sfp4 vlan-ids=420
/interface list member
add interface=ether1 list=WAN
add interface=rightF list=LAN
add interface=ethport list=LAN
add interface=left list=LAN
add interface=garage list=LAN
add interface=right list=LAN
add interface=dead list=LAN
add interface=leftf list=LAN
add interface=backyad list=LAN
add interface=basement list=LAN
add interface=roof list=LAN
add interface=hall list=LAN
add interface=ether13 list=LAN
add interface=frontdoor list=LAN
add interface=Bedroom list=LAN
add interface=KitchenAP list=LAN
add interface=rear list=LAN
add interface=plate list=LAN
add interface=plate2 list=LAN
add interface=pool list=LAN
add interface=ether21 list=LAN
add interface=ether22 list=LAN
add interface=ether23 list=LAN
add interface=ether24 list=LAN
add interface=core234-sfp2 list=LAN
add interface=core98_sfp4 list=LAN
add interface=core96_sfp6 list=LAN
add interface=core234-sfp4 list=LAN
add interface=backyad list=cam
add interface=ether13 list=cam
add interface=rear list=cam
add interface=right list=cam
add interface=rightF list=cam
add interface=left list=cam
add interface=leftf list=cam
add interface=garage list=cam
add interface=frontdoor list=cam
add interface=plate list=cam
add interface=plate2 list=cam
/interface ovpn-server server
set auth=sha1,md5
/ip address
add address=192.168.103.233/24 interface=MHQ network=192.168.103.0
/ip dhcp-client
add dhcp-options=clientid,clientid_duid,hostname interface=bridge
/ip dns
set servers=172.16.1.154
/ip route
add disabled=no dst-address=0.0.0.0/0 gateway=192.168.103.1
/snmp
set contact=netadmin enabled=yes location=MHQ trap-target=192.168.105.248
/system clock
set time-zone-name=America/Chicago
/system identity
set name=core233
/system logging
set 0 action=remote
set 1 action=remote
set 2 action=remote
set 3 action=remote
add action=remote topics=interface
add action=remote topics=bridge
/system ntp client
set enabled=yes
/system ntp client servers
add address=172.16.1.4
/system package update
set channel=testing
/system routerboard settings
set boot-os=router-os boot-protocol=dhcp
/system swos
set allow-from-ports=p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23,p24,p25,p26,p27,p28 identity=core233 static-ip-address=192.168.103.233
/tool sniffer
set file-name=temp2.pkt filter-interface=bridge
[admin@core233] >
v7.17beta [testing] is released!
2

Suggestion: do not use user admin.

3

Thank you that is solid advice, not sure it solves the problem, but it does solve a problem so thank you for that reminder to not be lazy.

4

This is half solved, seems to be an issue with host database when mlag is used. Rebooting one switch allowed it to be accessed via IP FROM THE SAME L2 ONLY, then Rebooting the other one did the same thing they both became available by ip in a browser but only if your on the same l2 routing to them fails. Also I noticed devices on ports not in a bond started dying I was seeing multiple Eternal and internal hits on the host table one last reboot of the switches seemed to clear out all the odd arp and now all devices are online again.

However I still can’t access via IP from another network like used to. So maybe 7.7 will fix this who knows.

Man I really appreciate that mod that gave me the suggestion I went back and looked at some of his other amazing contributions to this community. I can say as an SE at one house hold name network company I only wish our support staff showed so much care to our customer. You are beyond words good job sir.

5

Very thanks!!!

6

I’m having this problem, too. I see OP commenting that the mod gave solid advise, but the only comment I see from mod is the suggestion to not use username admin.

IS OP saying that somehow the username has something to do with this issue, or is he simply saying thanks for giving an off topic suggestion while his problem remains?

7

did you make any progress on this, I have similar / same issue on 2 CRS317 switches. Once I add a VLAN tagged on an interface it stops access between the switches, everything else seemingly working as expected but this behaviour makes me think its broken / implemented incorrectly.

8

It’s really broken, has been causing me weeks of problems - and now I found this thread. Not good. It will be dozens of hours of work to remove MLAG, and will reduce my level of redundancy. But is does explain the horrible struggle we’ve had since we’ve put a number of 'Tik MLAG pairs into production.

9

Version 7.8 didn’t solve the problem

10

Same for RouterOS 7.9. Still the same problem.

11

Same Problem. Connection is only intermittent when using MLAG

12

Seems you all are being affected from this known bug: http://forum.mikrotik.com/t/mlag-issue-mlag-functionality-flaps-lacp-system-id-of-secondary-when-primary-reboots/157426/1

13

For me 7.10 solved the problem.
True, at first, when the Mlag was turned on, access to both switches was lost. Restarting both helped.

Restarting the master does not affect the available secondary as it did before.
I will carefully watch how it will work in production.

MLAG Bridge not work in ROS 7.7 - 7.8 - 7.9, OK in ROS 7.6
14

There is some kind of problem anyway.
If you select a rate of 30 seconds (which is the default) on the bond 802.3ad mlag interface, the switches become unavailable by ip, current mac address. Reboot doesn’t help.
But if you choose 1 second, everything works.
Does anyone else work too?

/interface bonding
add lacp-rate=1sec mode=802.3ad name=bonding.mlag slaves=qsfpplus1-1.mlag,qsfpplus2-1.mlag
/interface bridge mlag
set bridge=bridge peer-port=bonding.mlag
15

I tried this, but it did not solve my issue. I still can’t access the switches on l3

16

after a certain number of reboots of both switches they become available
but I encountered a load of 100% even without traffic
here is the answer from technical support:

Thank you for the report!
We have managed to reproduce the issue locally in our labs and look forward to fixing it on upcoming RouterOS versions, unfortunately, I cannot provide a release date now.
As a workaround, I can only suggest disabling the MLAG.

17

Upgraded my two CRS317 switches running MLAG to v7.15.1 and continue to experience this problem.

swFloor → LAG → MLAG ( swFabric1 ← LACP Bond Peer Link → swFabric2)

Now the critical bit: To manage swFabric1 & swFabric2 they both have

/interface vlan
add interface=br-trunk name=vl-Switches vlan-id=254
/ip address
add address=172.17.254.(1 or 2)/24 interface=vl-Switches network=172.17.254.0

Remainder of setup is normal:
br-trunk has RSTP enabled
Peer Links are in LACP bond between swFabric1 & swFabric2
VLAN filtering is enabled on br-trunk, and all vlans tagged under /interface/bridge/vlan, except the peer link ports which are untagged with otherwise unused pvid (3999).

So when it works, it is fine, can access both swFabric1 & swFabric2 & any other switch on the vl-Switches VLAN.

But when I upgrade a switch, I lose access to some of the devices on vl-Switches.
e.g. Upgrade swFabric1, as it reboots all is well can access swFabric2 etc. Once swFabric1 comes back on line, suddenly I lose access to swFabric2.

Now, the interesting bit: To get it to work again I used a console cable on swFabric2 and DISABLED vl-Switches (under /interface vlan), and then ENABLED vl-Switches again. Suddenly all is good, I can access both fabric switches, and also any other devices that were on the same vlan as vl-switches but had ‘dropped off’.

So this appears to be problem with how Mikrotik are offloading the VLAN to the CPU with the MLAG setup. Maybe (a guess) the VLAN comes up BEFORE the MLAG and the Mikrotik logic handles it differently (Assumes VLAN not coming in via MLAG), but once system is up and ‘stable’ disabling and re-enabling the /interface/vlan xxx seems to fix it (So I assume Mikrotik logic sees that is is coming in via MLAG and handles it differently now).

I hope this helps anyone else fire-fighting this problem.

18

Crazy to think its been 2 years, I forgot all about opening this, giving up and removing mlag. Tried again this year thought it was working only to see nope still not working well, lost so many hours in rstp nightmare.

While I can’t afford to tear out all mikrotik switch access, I can see that if I want any hope at all of redundant switch connections to my firewall all have to replace at least on pair of MT switches.

I am on 7.14.2 thought about going to 7.16.1 but thought you know what why hasn’t worked correctly in years, might as well just give up.