Hey all!
I’m kind of lost for a solution with this new setup we have, hopefully you’ll be able to guide me!
The main goal of this setup : VRRP redundancy between dual CCR2216s into a pair of CRS520 switches in an MLAG configuration.
We also have an LACP link from our ISP which goes untagged into each MLAG switch, which each CCR reaches through a dedicated VLAN.
Issue : When R1 or R2 “dies”, all routing stop. Hosts in the same vlan can no longer ping their gateway (even if I try to ping .2 or .3). The WAN link dies and everything comes to a halt.. But if I plug back in R2, R1 starts routing & responding again to traffic. Can MLAG cause this behavior?
Here’s a brief overview of the configuration:
RouterOS Version : 7.21.3
WAN IP : /29 shared across R1 & R2 with the VRRP wan interface being a /32 shared ip (IPs below aren’t real)
LAN IPs : /24 on LAN VLANs with VRRP interface for each VLAN
CCR2216 connections : LACP of 2x100gbps. Each link going into each CRS520. (Example : R1 qsfp28-1-1 > SW0.1 qsfp-1–1. R2 qsfp28-1-1 > SW0.1 qsfp-2–1 and then the same for the second interface of each router)
CRS520 connections : linked together over MLAG pair of a single LACP bond using ports 3&4.
Export from R1:
/interface bridge
add frame-types=admit-only-vlan-tagged name=BR0 protocol-mode=none
vlan-filtering=yes
/interface ethernet
set [ find default-name=qsfp28-1-1 ] comment=AE1-1
set [ find default-name=qsfp28-1-3 ] advertise="10M-baseT-half,10M-baseT-full,
100M-baseT-half,100M-baseT-full,1G-baseT-half,1G-baseT-full,1G-baseX,2.5G-
baseT,2.5G-baseX,5G-baseT,10G-baseT,10G-baseSR-LR,10G-baseCR,40G-baseSR4-L
R4,40G-baseCR4,25G-baseSR-LR,25G-baseCR,50G-baseSR2-LR2,50G-baseCR2"
set [ find default-name=qsfp28-2-1 ] comment=AE1-2
set [ find default-name=qsfp28-2-3 ] advertise="10M-baseT-half,10M-baseT-full,
100M-baseT-half,100M-baseT-full,1G-baseT-half,1G-baseT-full,1G-baseX,2.5G-
baseT,2.5G-baseX,5G-baseT,10G-baseT,10G-baseSR-LR,10G-baseCR,40G-baseSR4-L
R4,40G-baseCR4,25G-baseSR-LR,25G-baseCR,50G-baseSR2-LR2,50G-baseCR2"
set [ find default-name=sfp28-2 ] auto-negotiation=no speed=1G-baseX
/interface vlan
add interface=BR0 l3-hw-offloading=no name=VL5 vlan-id=
5
add interface=BR0 l3-hw-offloading=no name=VL10
vlan-id=10
add interface=BR0 l3-hw-offloading=no name=VL20
vlan-id=20
add interface=BR0 l3-hw-offloading=no name=VL21
vlan-id=21
add interface=BR0 l3-hw-offloading=no name=VL25
vlan-id=25
add interface=BR0 l3-hw-offloading=no name=VL30
vlan-id=30
add interface=BR0 l3-hw-offloading=no name=VL31
vlan-id=31
add interface=BR0 l3-hw-offloading=no name=VL50
vlan-id=50
add interface=BR0 l3-hw-offloading=no name=VL60 vlan-id=
60
add interface=BR0 l3-hw-offloading=no name=VL61
vlan-id=61
add interface=BR0 l3-hw-offloading=no name=VL62 vlan-id=
62
add interface=BR0 l3-hw-offloading=no name=VL70
vlan-id=70
add interface=BR0 l3-hw-offloading=no name=VL80 vlan-id=80
add interface=BR0 l3-hw-offloading=no name=
VL90 vlan-id=90
add comment="ISP Trunk" interface=BR0 l3-hw-offloading=no name=VL102 vlan-id=
102
/interface bonding
add comment="Trunk to Core" lacp-rate=1sec mode=802.3ad name=AE1 slaves=
qsfp28-1-1,qsfp28-2-1 transmit-hash-policy=layer-3-and-4
/interface vrrp
add interface=VL102 name=vrrp-WAN on-backup=
"/ip firewall nat disable [find where comment=\"VRRP-WAN-SNAT\"]""
on-master=
"/ip firewall nat enable [find where comment=\"VRRP-WAN-SNAT\"]""
priority=254 sync-connection-tracking=yes vrid=102
add group-authority=self interface=VL10 name=vrrp10 priority=150
sync-connection-tracking=yes vrid=10
add group-authority=vrrp10 interface=VL20 name=vrrp20 priority=150 vrid=20
add group-authority=vrrp10 interface=VL21 name=vrrp21 priority=150 vrid=21
add group-authority=vrrp10 interface=VL25 name=vrrp25 priority=150 vrid=25
add group-authority=vrrp10 interface=VL30 name=vrrp30 priority=150 vrid=30
add group-authority=vrrp10 interface=VL31 name=vrrp31 priority=150 vrid=31
add group-authority=vrrp10 interface=VL50 name=vrrp50 priority=150 vrid=50
add group-authority=vrrp10 interface=VL60 name=vrrp60 priority=150 vrid=60
add group-authority=vrrp10 interface=VL61 name=vrrp61 priority=150 vrid=61
add group-authority=vrrp10 interface=VL62 name=vrrp62 priority=150 vrid=62
add group-authority=vrrp10 interface=VL70 name=vrrp70 priority=150 vrid=70
add group-authority=vrrp10 interface=VL80 name=vrrp80 priority=150 vrid=80
add group-authority=vrrp10 interface=VL90 name=vrrp90 priority=150 vrid=90
add group-authority=vrrp10 interface=VL5 name=vrrp5 priority=150 vrid=5
/interface bridge port
add bridge=BR0 interface=AE1
add bridge=BR0 interface=ether1 pvid=50
/ip firewall connection tracking
set enabled=yes
/ip neighbor discovery-settings
set discover-interface-list=!dynamic
/interface bridge vlan
add bridge=BR0 tagged=AE1,BR0 vlan-ids=5
add bridge=BR0 tagged=AE1,BR0 vlan-ids=10
add bridge=BR0 tagged=AE1,BR0 vlan-ids=20
add bridge=BR0 tagged=AE1,BR0 vlan-ids=25
add bridge=BR0 tagged=AE1,BR0 vlan-ids=30
add bridge=BR0 tagged=AE1,BR0 untagged=ether1
vlan-ids=50
add bridge=BR0 tagged=AE1,BR0 vlan-ids=60
add bridge=BR0 tagged=AE1,BR0 vlan-ids=61
add bridge=BR0 tagged=AE1,BR0 vlan-ids=70
add bridge=BR0 tagged=AE1,BR0 vlan-ids=90
add bridge=BR0 tagged=AE1,BR0 vlan-ids=62
add bridge=BR0 tagged=AE1,BR0 vlan-ids=31
add bridge=BR0 tagged=BR0,AE1 vlan-ids=21
add bridge=BR0 tagged=BR0,AE1 vlan-ids=80
add bridge=BR0 tagged=BR0,AE1 vlan-ids=102
/ip address
add address=10.0.5.2/24 interface=VL5 network=10.0.5.0
add address=10.0.1.2/24 interface=VL10 network=10.0.1.0
add address=10.0.20.2/24 interface=VL20 network=10.0.20.0
add address=10.0.25.2/24 interface=VL25 network=10.0.25.0
add address=10.0.30.2/24 interface=VL30 network=10.0.30.0
add address=10.0.31.2/24 interface=VL31 network=10.0.31.0
add address=10.0.50.2/24 interface=VL50 network=10.0.50.0
add address=10.0.60.2/24 interface=VL60 network=10.0.60.0
add address=10.0.61.2/24 interface=VL61 network=10.0.61.0
add address=10.0.62.2/24 interface=VL62 network=10.0.62.0
add address=10.0.70.2/24 interface=VL70 network=10.0.70.0
add address=10.0.90.2/24 interface=VL90 network=10.0.90.0
add address=10.0.5.1 interface=vrrp5 network=10.0.5.1
add address=10.0.1.1 interface=vrrp10 network=10.0.1.1
add address=10.0.20.1 interface=vrrp20 network=10.0.20.1
add address=10.0.25.1 interface=vrrp25 network=10.0.25.1
add address=10.0.30.1 interface=vrrp30 network=10.0.30.1
add address=10.0.31.1 interface=vrrp31 network=10.0.31.1
add address=10.0.60.1 interface=vrrp60 network=10.0.60.1
add address=10.0.61.1 interface=vrrp61 network=10.0.61.1
add address=10.0.62.1 interface=vrrp62 network=10.0.62.1
add address=10.0.70.1 interface=vrrp70 network=10.0.70.1
add address=10.0.90.1 interface=vrrp90 network=10.0.90.1
add address=10.0.80.2/24 interface=VL80 network=10.0.80.0
add address=10.0.21.2/24 interface=VL21 network=10.0.21.0
add address=10.0.80.1 interface=vrrp80 network=10.0.80.1
add address=10.0.21.1 interface=vrrp21 network=10.0.21.1
add address=192.168.112.98/29 comment="WAN real IP (R1)" interface=VL102
network=192.168.112.96
add address=192.168.112.99 comment="WAN VRRP VIP" interface=vrrp-WAN network=
192.168.112.99
add address=10.0.50.1 interface=vrrp50 network=10.0.50.1
/ip route
add comment="Default route via ISP" disabled=no distance=1 dst-address=\
0.0.0.0/0 gateway=192.168.112.97 routing-table=main scope=30 \
suppress-hw-offload=no target-scope=10
R2 configuration:
/interface bridge
add frame-types=admit-only-vlan-tagged name=BR0 protocol-mode=none
vlan-filtering=yes
/interface ethernet
set [ find default-name=qsfp28-1-3 ] advertise="10M-baseT-half,10M-baseT-full,
100M-baseT-half,100M-baseT-full,1G-baseT-half,1G-baseT-full,1G-baseX,2.5G-
baseT,2.5G-baseX,5G-baseT,10G-baseT,10G-baseSR-LR,10G-baseCR,40G-baseSR4-L
R4,40G-baseCR4,25G-baseSR-LR,25G-baseCR,50G-baseSR2-LR2,50G-baseCR2"
set [ find default-name=qsfp28-2-3 ] advertise="10M-baseT-half,10M-baseT-full,
100M-baseT-half,100M-baseT-full,1G-baseT-half,1G-baseT-full,1G-baseX,2.5G-
baseT,2.5G-baseX,5G-baseT,10G-baseT,10G-baseSR-LR,10G-baseCR,40G-baseSR4-L
R4,40G-baseCR4,25G-baseSR-LR,25G-baseCR,50G-baseSR2-LR2,50G-baseCR2"
/interface vlan
add interface=BR0 l3-hw-offloading=no name=VL5 vlan-id=
5
add interface=BR0 l3-hw-offloading=no name=VL10
vlan-id=10
add interface=BR0 l3-hw-offloading=no name=VL20
vlan-id=20
add interface=BR0 l3-hw-offloading=no name=VL21
vlan-id=21
add interface=BR0 l3-hw-offloading=no name=VL25
vlan-id=25
add interface=BR0 l3-hw-offloading=no name=VL30
vlan-id=30
add interface=BR0 l3-hw-offloading=no name=VL31
vlan-id=31
add interface=BR0 l3-hw-offloading=no name=VL50
vlan-id=50
add interface=BR0 l3-hw-offloading=no name=VL60 vlan-id=
60
add interface=BR0 l3-hw-offloading=no name=VL61
vlan-id=61
add interface=BR0 l3-hw-offloading=no name=VL62 vlan-id=
62
add interface=BR0 l3-hw-offloading=no name=VL70
vlan-id=70
add interface=BR0 l3-hw-offloading=no name=VL80 vlan-id=80
add interface=BR0 l3-hw-offloading=no name=
VL90 vlan-id=90
add interface=BR0 l3-hw-offloading=no name=VL102 vlan-id=
102
/interface bonding
add comment="Trunk to Core" lacp-rate=1sec mode=802.3ad name=AE1 slaves=
qsfp28-1-1,qsfp28-2-1 transmit-hash-policy=layer-3-and-4
/interface vrrp
add interface=VL102 name=vrrp-Wan on-backup=
"/ip firewall nat disable [find where comment="VRRP-WAN-SNAT"]"
on-master=
"/ip firewall nat enable [find where comment="VRRP-WAN-SNAT"]"
sync-connection-tracking=yes vrid=102
add group-authority=self interface=VL10 name=vrrp10 sync-connection-tracking=
yes vrid=10
add group-authority=vrrp10 interface=VL20 name=vrrp20 vrid=20
add group-authority=vrrp10 interface=VL21 name=vrrp21 vrid=21
add group-authority=vrrp10 interface=VL25 name=vrrp25 vrid=25
add group-authority=vrrp10 interface=VL30 name=vrrp30 vrid=30
add group-authority=vrrp10 interface=VL31 name=vrrp31 vrid=31
add group-authority=vrrp10 interface=VL50 name=vrrp50 vrid=50
add group-authority=vrrp10 interface=VL60 name=vrrp60 vrid=60
add group-authority=vrrp10 interface=VL61 name=vrrp61 vrid=61
add group-authority=vrrp10 interface=VL62 name=vrrp62 vrid=62
add group-authority=vrrp10 interface=VL70 name=vrrp70 vrid=70
add group-authority=vrrp10 interface=VL80 name=vrrp80 vrid=80
add group-authority=vrrp10 interface=VL90 name=vrrp90 vrid=90
add group-authority=vrrp10 interface=VL5 name=vrrp5 vrid=5
/interface bridge port
add bridge=BR0 interface=AE1
add bridge=BR0 interface=ether1 pvid=50
/ip firewall connection tracking
set enabled=yes
/ip neighbor discovery-settings
set discover-interface-list=!dynamic
/interface bridge vlan
add bridge=BR0 tagged=AE1,BR0 vlan-ids=5
add bridge=BR0 tagged=AE1,BR0 vlan-ids=10
add bridge=BR0 tagged=AE1,BR0 vlan-ids=20
add bridge=BR0 tagged=AE1,BR0 vlan-ids=25
add bridge=BR0 tagged=AE1,BR0 vlan-ids=30
add bridge=BR0 tagged=AE1,BR0 untagged=ether1
vlan-ids=50
add bridge=BR0 tagged=AE1,BR0 vlan-ids=60
add bridge=BR0 tagged=AE1,BR0 vlan-ids=61
add bridge=BR0 tagged=AE1,BR0 vlan-ids=70
add bridge=BR0 tagged=AE1,BR0 vlan-ids=90
add bridge=BR0 tagged=AE1,BR0 vlan-ids=62
add bridge=BR0 tagged=AE1,BR0 vlan-ids=31
add bridge=BR0 tagged=BR0,AE1 vlan-ids=21
add bridge=BR0 tagged=BR0,AE1 vlan-ids=80
add bridge=BR0 tagged=BR0,AE1 vlan-ids=102
/ip address
add address=192.168.112.100/29 comment="WAN real IP (R2)" interface=VL102
network=192.168.112.96
add address=192.168.112.99 comment="WAN VRRP VIP" interface=vrrp-Wan network=
192.168.112.99
add address=10.0.5.3/24 interface=VL5 network=10.0.5.0
add address=10.0.1.3/24 interface=VL10 network=10.0.1.0
add address=10.0.20.3/24 interface=VL20 network=10.0.20.0
add address=10.0.25.3/24 interface=VL25 network=10.0.25.0
add address=10.0.30.3/24 interface=VL30 network=10.0.30.0
add address=10.0.31.3/24 interface=VL31 network=10.0.31.0
add address=10.0.50.3/24 interface=VL50 network=10.0.50.0
add address=10.0.60.3/24 interface=VL60 network=10.0.60.0
add address=10.0.61.3/24 interface=VL61 network=10.0.61.0
add address=10.0.62.3/24 interface=VL62 network=10.0.62.0
add address=10.0.70.3/24 interface=VL70 network=10.0.70.0
add address=10.0.90.3/24 interface=VL90 network=10.0.90.0
add address=10.0.5.1 interface=vrrp5 network=10.0.5.1
add address=10.0.1.1 interface=vrrp10 network=10.0.1.1
add address=10.0.20.1 interface=vrrp20 network=10.0.20.1
add address=10.0.25.1 interface=vrrp25 network=10.0.25.1
add address=10.0.30.1 interface=vrrp30 network=10.0.30.1
add address=10.0.31.1 interface=vrrp31 network=10.0.31.1
add address=10.0.60.1 interface=vrrp60 network=10.0.60.1
add address=10.0.61.1 interface=vrrp61 network=10.0.61.1
add address=10.0.62.1 interface=vrrp62 network=10.0.62.1
add address=10.0.70.1 interface=vrrp70 network=10.0.70.1
add address=10.0.90.1 interface=vrrp90 network=10.0.90.1
add address=10.0.80.3/24 interface=VL80 network=10.0.80.0
add address=10.0.21.3/24 interface=VL21 network=10.0.21.0
add address=10.0.80.1 interface=vrrp80 network=10.0.80.1
add address=10.0.21.1 interface=vrrp21 network=10.0.21.1
add address=10.0.50.1 interface=vrrp50 network=10.0.50.1
/ip route
add comment="Default route via ISP" disabled=no distance=2 dst-address=
0.0.0.0/0 gateway=192.168.112.97 routing-table=main scope=30
suppress-hw-offload=no target-scope=10
SW0.1 & SW0.2 (identical configuration for both):
/interface bridge
add frame-types=admit-only-vlan-tagged name=BR0 vlan-filtering=yes
/interface vlan
add interface=BR0 name=VL50 vlan-id=50
/interface bonding
add comment=R1 lacp-rate=1sec mlag-id=100 mode=802.3ad name=AE1 slaves=
qsfp28-1-1 transmit-hash-policy=layer-3-and-4
add comment=R2 lacp-rate=1sec mlag-id=200 mode=802.3ad name=AE2 slaves=
qsfp28-2-1 transmit-hash-policy=layer-3-and-4
add comment=MLAG lacp-rate=1sec mode=802.3ad name=AE3 slaves=
qsfp28-3-1,qsfp28-4-1 transmit-hash-policy=layer-3-and-4
add comment="PVE1 Ceph Cluster Network" lacp-rate=1sec mlag-id=4 mode=802.3ad
name=AE4 slaves=qsfp28-5-1 transmit-hash-policy=layer-3-and-4
add comment="PVE1 Ceph Public Network" lacp-rate=1sec mlag-id=5 mode=802.3ad
name=AE5 slaves=qsfp28-6-1 transmit-hash-policy=layer-3-and-4
add comment="PVE2 Ceph Cluster Network" lacp-rate=1sec mlag-id=6 mode=802.3ad
name=AE6 slaves=qsfp28-7-1 transmit-hash-policy=layer-3-and-4
add comment="PVE2 Ceph Public Network" lacp-rate=1sec mlag-id=7 mode=802.3ad
name=AE7 slaves=qsfp28-8-1 transmit-hash-policy=layer-3-and-4
add comment="PVE3 Ceph Cluster Network" lacp-rate=1sec mlag-id=8 mode=802.3ad
name=AE8 slaves=qsfp28-9-1 transmit-hash-policy=layer-3-and-4
add comment="PVE3 Ceph Public Network" lacp-rate=1sec mlag-id=9 mode=802.3ad
name=AE9 slaves=qsfp28-10-1 transmit-hash-policy=layer-3-and-4
add comment="Trunk to SW0.3" lacp-rate=1sec mlag-id=10 mode=802.3ad name=AE10
slaves=qsfp28-11-1 transmit-hash-policy=layer-3-and-4
add comment="Trunk to ISP" lacp-rate=1sec mlag-id=11 mode=802.3ad name=AE11
slaves=sfp28-4 transmit-hash-policy=layer-2-and-3
add comment="Trunk to SW05" lacp-rate=1sec mlag-id=12 mode=802.3ad name=AE12
slaves=sfp28-3 transmit-hash-policy=layer-3-and-4
/interface bridge vlan
add bridge=BR0 tagged=
AE5,AE7,AE9,AE10,sfp28-2,sfp28-1,AE3,AE1,AE2 vlan-ids=5
add bridge=BR0 tagged=
AE5,AE7,AE9,AE10,AE12,sfp28-2,sfp28-1,AE3,AE2,AE1 untagged=qsfp28-14-1
vlan-ids=10
add bridge=BR0 tagged=
AE5,AE7,AE9,AE10,sfp28-2,sfp28-1,AE3,AE2,AE1 vlan-ids=20
add bridge=BR0 tagged=
AE5,AE7,AE9,AE10,sfp28-2,sfp28-1,AE3,AE2,AE1 vlan-ids=25
add bridge=BR0 tagged=
AE5,AE7,AE9,AE10,sfp28-2,sfp28-1,AE3,AE2,AE1 vlan-ids=30
add bridge=BR0 tagged=
AE5,AE7,AE9,AE10,AE12,sfp28-2,sfp28-1,BR0,AE3,AE2,AE1 untagged=ether1
vlan-ids=50
add bridge=BR0 tagged=
AE5,AE7,AE9,AE10,sfp28-2,sfp28-1,AE3,AE2,AE1 vlan-ids=60
add bridge=BR0 tagged=
AE5,AE7,AE9,AE10,sfp28-2,sfp28-1,AE3,AE2,AE1 vlan-ids=61
add bridge=BR0 tagged=AE5,AE7,AE9,AE10,AE3,AE2,AE1
vlan-ids=70
add bridge=BR0 tagged=
AE5,AE7,AE9,AE10,AE3,AE2,AE1 vlan-ids=90
add bridge=BR0 tagged=AE10,AE5,AE7,AE9,AE3,AE2,AE1
vlan-ids=21
add bridge=BR0 tagged=AE10,sfp28-2,AE3,AE2,AE1 vlan-ids=
62
add bridge=BR0 tagged=AE5,AE7,AE9,sfp28-2,AE3,AE2,AE1
vlan-ids=31
add bridge=BR0 tagged=AE3,AE1,AE2 untagged=AE11 vlan-ids=102
add bridge=BR0 tagged=AE1,AE2,AE3,sfp28-2 vlan-ids=80
Please let me know if there’s any alternatives to VRRP which would allow the equivalent of EVPN-MH without tons of scripts. If my configuration is garbage, please let me know too lol! (if it looks a little wonky i’ve removed lots of comments from the configurations") I’m looking for ideas and feedback or ideas to what can be causing these inconsistencies. Thanks!