VRRP over VLAN over BRIDGE

shaoranrch · March 7, 2016, 11:46pm

Hello,

Currently I’m trying to implement the following topology:

Currently I’m creating VLANs over a bridge on each router, and on top of each VLAN I’m creating VRRP interfaces. So far so good, except for some erratic behavior:

1.- Sometimes (not always, but around 20% of the time I think) when I force a failover situation or shutdown both routers just to get them back online, at least 1 won’t behave as it’s supposed, the VRRP interface will stay in INVALID status (I) to fix this I have to shutdown and reenable manually the interface.
2.- I can see around 5 - 15% packets lost on this configuration, even though it’s supposedly stable.
3.- Sometimes I get a 100% packet lost, and I can see the router getting frames on interfaces where they shouldn’t be (because the path is blocked due to RSTP on cisco switches)

Here are my configs, If you’d like to take a look. I’ve disabled RSTP and STP, because the routers are connected to Cisco switches using PVST+, I’ve set them to load balance between the vlans on a 50/50 ratio (setting root priority and port priority), I can see the ciscos disabling the corresponding interfaces as intended if I issue a “show spanning-tree”. So no broadcast storm here.

R01 config

/interface bridge
add comment="Interfaz para administracion de dispositivo" name=Loopback01
add admin-mac=00:00:5E:80:00:01 ageing-time=10s auto-mac=no comment="Bridge de conexion con los switches de distribucion" name=TO-DIST-Bridge protocol-mode=none
/interface ethernet
set [ find default-name=ether1 ] comment="Conexion con firewall de borde -pfsense-"
set [ find default-name=ether2 ] comment="Conexion con concentrador enlaces entre sedes"
set [ find default-name=ether3 ] comment="Conexion con SaaS.Mcy.R02"
set [ find default-name=ether4 ] comment="Conexion con SG200-SW01"
set [ find default-name=ether5 ] comment="Conexion con SG200-SW02"
/interface vrrp
add interface=TO-DIST-Bridge name=Gateway-Vlan1.mgmt priority=110
/ip neighbor discovery
set ether1 comment="Conexion con firewall de borde -pfsense-"
set ether2 comment="Conexion con concentrador enlaces entre sedes"
set ether3 comment="Conexion con SaaS.Mcy.R02"
set ether4 comment="Conexion con SG200-SW01"
set ether5 comment="Conexion con SG200-SW02"
set Loopback01 comment="Interfaz para administracion de dispositivo"
set TO-DIST-Bridge comment="Bridge de conexion con los switches de distribucion"
/interface vlan
add interface=TO-DIST-Bridge name=100-Usuarios-Administracion vlan-id=100
add interface=TO-DIST-Bridge name=110-Servidores vlan-id=110
add interface=TO-DIST-Bridge name=120-Telefonos-IP vlan-id=120
add interface=TO-DIST-Bridge name=130-WiFi-Administracion vlan-id=130
add interface=TO-DIST-Bridge name=140-WiFi-Inventario vlan-id=140
add interface=TO-DIST-Bridge name=150-WiFi-Invitados vlan-id=150
add interface=TO-DIST-Bridge name=160-POS vlan-id=160
add interface=TO-DIST-Bridge name=170-Vigilancia vlan-id=170
add interface=TO-DIST-Bridge name=180-Departamento-IT vlan-id=180
/interface vrrp
add interface=100-Usuarios-Administracion name=Gateway-Vlan100 priority=110
add interface=110-Servidores name=Gateway-Vlan110 priority=110
add interface=120-Telefonos-IP name=Gateway-Vlan120 priority=110
add interface=130-WiFi-Administracion name=Gateway-Vlan130 priority=110
add interface=140-WiFi-Inventario name=Gateway-Vlan140
add interface=150-WiFi-Invitados name=Gateway-Vlan150
add interface=160-POS name=Gateway-Vlan160
add interface=170-Vigilancia name=Gateway-Vlan170
add interface=180-Departamento-IT name=Gateway-Vlan180
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/routing ospf area
add area-id=0.0.0.100 name=SaaS-Mcy-100
/routing ospf instance
set [ find default=yes ] distribute-default=if-installed-as-type-1 router-id=10.1.0.1
/interface bridge port
add bridge=TO-DIST-Bridge interface=ether4
add bridge=TO-DIST-Bridge interface=ether5
/ip address
add address=10.1.1.1/30 interface=ether3 network=10.1.1.0
add address=10.1.1.5/30 interface=ether1 network=10.1.1.4
add address=10.1.1.13/30 interface=ether2 network=10.1.1.12
add address=10.1.0.1 interface=Loopback01 network=10.1.0.0
add address=10.1.0.130/25 interface=TO-DIST-Bridge network=10.1.0.128
add address=10.1.2.2/24 interface=100-Usuarios-Administracion network=10.1.2.0
add address=10.1.3.2/24 interface=110-Servidores network=10.1.3.0
add address=10.1.4.2/24 interface=120-Telefonos-IP network=10.1.4.0
add address=10.1.5.2/24 interface=130-WiFi-Administracion network=10.1.5.0
add address=10.1.6.2/24 interface=140-WiFi-Inventario network=10.1.6.0
add address=10.1.7.2/24 interface=150-WiFi-Invitados network=10.1.7.0
add address=10.1.8.2/24 interface=160-POS network=10.1.8.0
add address=10.1.9.2/24 interface=170-Vigilancia network=10.1.9.0
add address=10.1.10.2/24 interface=180-Departamento-IT network=10.1.10.0
add address=10.1.2.1 interface=Gateway-Vlan100 network=10.1.2.1
add address=10.1.3.1 interface=Gateway-Vlan110 network=10.1.3.1
add address=10.1.4.1 interface=Gateway-Vlan120 network=10.1.4.1
add address=10.1.5.1 interface=Gateway-Vlan130 network=10.1.5.1
add address=10.1.6.1 interface=Gateway-Vlan140 network=10.1.6.1
add address=10.1.7.1 interface=Gateway-Vlan150 network=10.1.7.1
add address=10.1.8.1 interface=Gateway-Vlan160 network=10.1.8.1
add address=10.1.9.1 interface=Gateway-Vlan170 network=10.1.9.1
add address=10.1.10.1 interface=Gateway-Vlan180 network=10.1.10.1
add address=10.1.0.129 interface=Gateway-Vlan1.mgmt network=10.1.0.129
add address=192.168.137.100/24 interface=ether8 network=192.168.137.0
/ip dhcp-relay
add dhcp-server=10.1.3.10 disabled=no interface=Gateway-Vlan100 name=relay1
/ip route
add check-gateway=ping distance=1 gateway=8.8.8.8 target-scope=19
add check-gateway=ping distance=1 dst-address=8.8.8.8/32 gateway=10.1.1.6 scope=19
/routing ospf interface
add interface=100-Usuarios-Administracion passive=yes
add interface=110-Servidores passive=yes
add interface=120-Telefonos-IP passive=yes
add interface=130-WiFi-Administracion passive=yes
add interface=140-WiFi-Inventario passive=yes
add interface=150-WiFi-Invitados passive=yes
add interface=160-POS passive=yes
add interface=170-Vigilancia passive=yes
add interface=180-Departamento-IT passive=yes
add interface=TO-DIST-Bridge passive=yes
/routing ospf network
add area=SaaS-Mcy-100 network=10.1.0.1/32
add area=SaaS-Mcy-100 network=10.1.0.128/25
add area=SaaS-Mcy-100 network=10.1.1.0/24
add area=SaaS-Mcy-100 network=10.1.2.0/24
add area=SaaS-Mcy-100 network=10.1.3.0/24
add area=SaaS-Mcy-100 network=10.1.4.0/24
add area=SaaS-Mcy-100 network=10.1.5.0/24
add area=SaaS-Mcy-100 network=10.1.6.0/24
add area=SaaS-Mcy-100 network=10.1.7.0/24
add area=SaaS-Mcy-100 network=10.1.8.0/24
add area=SaaS-Mcy-100 network=10.1.9.0/24
add area=SaaS-Mcy-100 network=10.1.10.0/24
/system identity
set name=R01
/system logging
add topics=vrrp
/tool sniffer
set filter-interface=ether5 memory-limit=1000KiB

R02 config

/interface bridge
add comment="Interfaz para administracion de dispositivo" name=Loopback01 protocol-mode=none
add admin-mac=00:00:5E:80:00:02 auto-mac=no comment="Bridge de conexion con los switches de distribucion" name=TO-DIST-Bridge protocol-mode=none
/interface ethernet
set [ find default-name=ether1 ] comment="Conexion con firewall de borde -pfsense-"
set [ find default-name=ether2 ] comment="Conexion con concentrador enlaces entre sedes"
set [ find default-name=ether3 ] comment="Conexion con SaaS.Mcy.R01"
set [ find default-name=ether4 ] comment="Conexion con SG200-SW01"
set [ find default-name=ether5 ] comment="Conexion con SG200-SW02"
/interface vrrp
add interface=TO-DIST-Bridge name=Gateway-Vlan1.mgmt
/ip neighbor discovery
set ether1 comment="Conexion con firewall de borde -pfsense-"
set ether2 comment="Conexion con concentrador enlaces entre sedes"
set ether3 comment="Conexion con SaaS.Mcy.R01"
set ether4 comment="Conexion con SG200-SW01"
set ether5 comment="Conexion con SG200-SW02"
set Loopback01 comment="Interfaz para administracion de dispositivo"
set TO-DIST-Bridge comment="Bridge de conexion con los switches de distribucion"
/interface vlan
add interface=TO-DIST-Bridge name=100-Usuarios-Administracion vlan-id=100
add interface=TO-DIST-Bridge name=110-Servidores vlan-id=110
add interface=TO-DIST-Bridge name=120-Telefonos-IP vlan-id=120
add interface=TO-DIST-Bridge name=130-WiFi-Administracion vlan-id=130
add interface=TO-DIST-Bridge name=140-WiFi-Inventario vlan-id=140
add interface=TO-DIST-Bridge name=150-WiFi-Invitados vlan-id=150
add interface=TO-DIST-Bridge name=160-POS vlan-id=160
add interface=TO-DIST-Bridge name=170-Vigilancia vlan-id=170
add interface=TO-DIST-Bridge name=180-Departamento-IT vlan-id=180
/interface vrrp
add interface=100-Usuarios-Administracion name=Gateway-Vlan100
add interface=110-Servidores name=Gateway-Vlan110
add interface=120-Telefonos-IP name=Gateway-Vlan120
add interface=130-WiFi-Administracion name=Gateway-Vlan130
add interface=140-WiFi-Inventario name=Gateway-Vlan140 priority=110
add interface=150-WiFi-Invitados name=Gateway-Vlan150 priority=110
add interface=160-POS name=Gateway-Vlan160 priority=110
add interface=170-Vigilancia name=Gateway-Vlan170 priority=110
add interface=180-Departamento-IT name=Gateway-Vlan180 priority=110
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/routing ospf area
add area-id=0.0.0.100 name=SaaS-Mcy-100
/routing ospf instance
set [ find default=yes ] distribute-default=if-installed-as-type-1 router-id=10.1.0.2
/interface bridge port
add bridge=TO-DIST-Bridge interface=ether4
add bridge=TO-DIST-Bridge interface=ether5
/ip address
add address=10.1.0.2 interface=Loopback01 network=10.1.0.2
add address=10.1.1.2/30 interface=ether3 network=10.1.1.0
add address=10.1.1.9/30 interface=ether1 network=10.1.1.8
add address=10.1.1.17/30 interface=ether2 network=10.1.1.16
add address=10.1.2.3/24 interface=100-Usuarios-Administracion network=10.1.2.0
add address=10.1.3.3/24 interface=110-Servidores network=10.1.3.0
add address=10.1.4.3/24 interface=120-Telefonos-IP network=10.1.4.0
add address=10.1.5.3/24 interface=130-WiFi-Administracion network=10.1.5.0
add address=10.1.6.3/24 interface=140-WiFi-Inventario network=10.1.6.0
add address=10.1.7.3/24 interface=150-WiFi-Invitados network=10.1.7.0
add address=10.1.8.3/24 interface=160-POS network=10.1.8.0
add address=10.1.9.3/24 interface=170-Vigilancia network=10.1.9.0
add address=10.1.10.3/24 interface=180-Departamento-IT network=10.1.10.0
add address=10.1.2.1 interface=Gateway-Vlan100 network=10.1.2.1
add address=10.1.3.1 interface=Gateway-Vlan110 network=10.1.3.1
add address=10.1.4.1 interface=Gateway-Vlan120 network=10.1.4.1
add address=10.1.5.1 interface=Gateway-Vlan130 network=10.1.5.1
add address=10.1.6.1 interface=Gateway-Vlan140 network=10.1.6.1
add address=10.1.7.1 interface=Gateway-Vlan150 network=10.1.7.1
add address=10.1.8.1 interface=Gateway-Vlan160 network=10.1.8.1
add address=10.1.9.1 interface=Gateway-Vlan170 network=10.1.9.1
add address=10.1.10.1 interface=Gateway-Vlan180 network=10.1.10.1
add address=10.1.0.129 interface=Gateway-Vlan1.mgmt network=10.1.0.129
add address=10.1.0.131/25 interface=TO-DIST-Bridge network=10.1.0.128
add address=192.168.137.101/24 interface=ether8 network=192.168.137.0
/ip dhcp-client
add disabled=no interface=ether1
/ip dhcp-relay
add dhcp-server=10.1.3.10 disabled=no interface=Gateway-Vlan100 name=relay1
/ip route
add check-gateway=ping distance=1 gateway=8.8.4.4 target-scope=19
add check-gateway=ping distance=1 dst-address=8.8.4.4/32 gateway=10.1.1.10 scope=19
/routing ospf interface
add interface=100-Usuarios-Administracion passive=yes
add interface=110-Servidores passive=yes
add interface=120-Telefonos-IP passive=yes
add interface=130-WiFi-Administracion passive=yes
add interface=140-WiFi-Inventario passive=yes
add interface=150-WiFi-Invitados passive=yes
add interface=160-POS passive=yes
add interface=170-Vigilancia passive=yes
add interface=180-Departamento-IT passive=yes
add interface=TO-DIST-Bridge passive=yes
/routing ospf network
add area=SaaS-Mcy-100 network=10.1.0.2/32
add area=SaaS-Mcy-100 network=10.1.0.128/25
add area=SaaS-Mcy-100 network=10.1.1.0/24
add area=SaaS-Mcy-100 network=10.1.2.0/24
add area=SaaS-Mcy-100 network=10.1.3.0/24
add area=SaaS-Mcy-100 network=10.1.4.0/24
add area=SaaS-Mcy-100 network=10.1.5.0/24
add area=SaaS-Mcy-100 network=10.1.6.0/24
add area=SaaS-Mcy-100 network=10.1.7.0/24
add area=SaaS-Mcy-100 network=10.1.8.0/24
add area=SaaS-Mcy-100 network=10.1.9.0/24
add area=SaaS-Mcy-100 network=10.1.10.0/24
/system identity
set name=R02
/tool sniffer
set filter-interface=ether5 memory-limit=1000KiB

My questions being:

Has anyone tried this kind of setup? If so, how did it go? currently this is just running inside a LAB, not in production yet, I’m trying to validate whether this is a viable design (on Cisco it’s, trying to validate on Mikrotik side for deployment).

Any insight would be appreciated about my config, anything that can help me achieve 100% working state of this.

Thanks in advance

mpreissner · March 8, 2016, 11:10am

Just curious why you’re using a bridge at the routers. It’s very CPU intensive since you’re actively forcing traffic to use the bridges to get from one switch to the other. You could run a link between the two switches so that anything in the same layer 2 domain wouldn’t have to cross a software bridge at the router. You’d still want to use bridges on the routers to have links from each router to both switches, but since your switch support RSTP, you probably wouldn’t even have to configure that on the routers as the switches would detect the double path and disable one of the links. Since only one link to each router would be active at any given time, the bridge interfaces on the routers wouldn’t consume as much CPU since it will never have to do any layer 2 forwarding.

Just my early morning pre-coffee thoughts…

shaoranrch · March 8, 2016, 3:45pm

mpreissner:

Just curious why you’re using a bridge at the routers. It’s very CPU intensive since you’re actively forcing traffic to use the bridges to get from one switch to the other. You could run a link between the two switches so that anything in the same layer 2 domain wouldn’t have to cross a software bridge at the router. You’d still want to use bridges on the routers to have links from each router to both switches, but since your switch support RSTP, you probably wouldn’t even have to configure that on the routers as the switches would detect the double path and disable one of the links. Since only one link to each router would be active at any given time, the bridge interfaces on the routers wouldn’t consume as much CPU since it will never have to do any layer 2 forwarding.

Just my early morning pre-coffee thoughts…

Hello,

Well bridges aren’t as cpu intensive as you may think when using ccr which are the routers for this design. In any case nowadays most traffic isnt targeted to the same vlan but rather to resources on different networks. This is a trend that I’ve seen in basically most deployments I’ve worked on.

If I added a new link between the switches STP would do its job so we have 2 scenarios:

STP disables the upstream link to the current gateway thus we get a suboptimal traffic path (again remember this is taking into account current network trends and needs)
STP disables the new link. Thus making it useless except as a backup

The purpose here is to keep an optimal traffic path. Also take account that I’m tweaking rstp on a per vlan basis on switches (thanks to pvst+)

Anyway the design is validated already an issue with one of the switches was messing with everything. Now it works as intended.

Enviado desde mi MotoE2(4G-LTE) mediante Tapatalk