Firewall not fully working with VLANs? Or could it be a (asymmetric) routing issue?

AquaL1te · Sun Jun 12, 2022 3:16 pm

Hi, based on this post I started to experiment with the Mikrotik Hex PoE firewall in combination with VLANs. But I fail to understand what to expect from this setup.

Network/Mikrotik setup
I have the following config:

/export hide-sensitive
# jun/12/2022 12:10:24 by RouterOS 7.3.1
# software id = A1GI-TFVF
#
# model = 960PGS
# serial number = 89F90861A06A
/interface bridge
add admin-mac=CC:2D:E0:81:0A:BE auto-mac=no comment=defconf name=bridge
/interface ethernet
set [ find default-name=ether2 ] poe-out=forced-on
set [ find default-name=ether3 ] poe-out=forced-on
set [ find default-name=ether4 ] poe-out=forced-on
set [ find default-name=ether5 ] poe-out=forced-on
/interface ethernet switch port
set 0 default-vlan-id=10 vlan-mode=secure
set 1 default-vlan-id=10 vlan-mode=secure
set 2 default-vlan-id=10 vlan-mode=secure
set 3 default-vlan-id=10 vlan-mode=secure
set 4 default-vlan-id=10 vlan-mode=secure
set 5 default-vlan-id=10 vlan-mode=secure
/interface list
add comment=defconf name=WAN
add comment=defconf name=LAN
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip pool
add name=default-dhcp ranges=192.168.88.10-192.168.88.254
/ip dhcp-server
add address-pool=default-dhcp disabled=yes interface=bridge name=defconf
/interface bridge port
add bridge=bridge comment=defconf interface=ether2
add bridge=bridge comment=defconf interface=ether3
add bridge=bridge comment=defconf interface=ether4
add bridge=bridge comment=defconf interface=ether5
add bridge=bridge comment=defconf interface=sfp1
/ip neighbor discovery-settings
set discover-interface-list=LAN
/ip settings
set rp-filter=strict
/interface ethernet switch vlan
add comment=native independent-learning=yes ports=ether1,ether2,ether3,ether4,ether5,switch1-cpu switch=switch1 vlan-id=10
add comment=management independent-learning=yes ports=ether2,ether3,ether4,ether5,switch1-cpu switch=switch1 vlan-id=11
add comment=replication independent-learning=yes ports=ether2,ether3,ether4,ether5,switch1-cpu switch=switch1 vlan-id=12
add comment=public independent-learning=yes ports=ether2,ether3,ether4,ether5,switch1-cpu,ether1 switch=switch1 vlan-id=13
/interface list member
add comment=defconf interface=bridge list=LAN
add comment=defconf interface=ether1 list=WAN
/ip dhcp-client
add comment=defconf interface=ether1
/ip dhcp-server network
add address=192.168.88.0/24 comment=defconf dns-server=192.168.88.1 gateway=192.168.88.1 netmask=24
/ip dns
set allow-remote-requests=yes
/ip dns static
add address=192.168.88.1 comment=defconf name=router.lan
/ip firewall filter
add action=log chain=output connection-state=new,untracked log=yes log-prefix="LOG ALL"
add action=accept chain=input comment="defconf: accept established,related,untracked" connection-state=established,related,untracked
add action=drop chain=input comment="defconf: drop invalid" connection-state=invalid log=yes
add action=log chain=input connection-state=new,untracked log=yes log-prefix="LOG ALL"
add action=accept chain=input comment="defconf: accept ICMP" protocol=icmp
add action=accept chain=input comment="defconf: accept to local loopback (for CAPsMAN)" disabled=yes dst-address=127.0.0.1
add action=drop chain=input comment="defconf: drop all not coming from LAN" disabled=yes in-interface-list=!LAN
add action=accept chain=forward comment="defconf: accept in ipsec policy" disabled=yes ipsec-policy=in,ipsec
add action=accept chain=forward comment="defconf: accept out ipsec policy" disabled=yes ipsec-policy=out,ipsec
add action=fasttrack-connection chain=forward comment="defconf: fasttrack" connection-state=established,related disabled=yes hw-offload=yes
add action=accept chain=forward comment="defconf: accept established,related, untracked" connection-state=established,related,untracked
add action=accept chain=forward connection-state=new,untracked protocol=icmp
add action=drop chain=forward comment="defconf: drop invalid" connection-state=invalid
add action=log chain=forward connection-state=new dst-address=172.27.13.0/24 dst-port=80,443 log=yes log-prefix="DROP WEB" protocol=tcp
add action=log chain=forward connection-state=new,untracked log=yes log-prefix="LOG ALL"
add action=drop chain=forward comment="defconf: drop all from WAN not DSTNATed" connection-nat-state=!dstnat connection-state=new disabled=yes in-interface-list=WAN
/ip firewall nat
add action=masquerade chain=srcnat comment="defconf: masquerade" disabled=yes ipsec-policy=out,none out-interface-list=WAN
/ip route
add disabled=no dst-address=172.27.13.0/24 gateway=ether1 routing-table=main suppress-hw-offload=no
/ipv6 firewall address-list
add address=::/128 comment="defconf: unspecified address" list=bad_ipv6
add address=::1/128 comment="defconf: lo" list=bad_ipv6
add address=fec0::/10 comment="defconf: site-local" list=bad_ipv6
add address=::ffff:0.0.0.0/96 comment="defconf: ipv4-mapped" list=bad_ipv6
add address=::/96 comment="defconf: ipv4 compat" list=bad_ipv6
add address=100::/64 comment="defconf: discard only " list=bad_ipv6
add address=2001:db8::/32 comment="defconf: documentation" list=bad_ipv6
add address=2001:10::/28 comment="defconf: ORCHID" list=bad_ipv6
add address=3ffe::/16 comment="defconf: 6bone" list=bad_ipv6
/ipv6 firewall filter
add action=accept chain=input comment="defconf: accept established,related,untracked" connection-state=established,related,untracked
add action=drop chain=input comment="defconf: drop invalid" connection-state=invalid
add action=accept chain=input comment="defconf: accept ICMPv6" protocol=icmpv6
add action=accept chain=input comment="defconf: accept UDP traceroute" port=33434-33534 protocol=udp
add action=accept chain=input comment="defconf: accept DHCPv6-Client prefix delegation." dst-port=546 protocol=udp src-address=fe80::/10
add action=accept chain=input comment="defconf: accept IKE" dst-port=500,4500 protocol=udp
add action=accept chain=input comment="defconf: accept ipsec AH" protocol=ipsec-ah
add action=accept chain=input comment="defconf: accept ipsec ESP" protocol=ipsec-esp
add action=accept chain=input comment="defconf: accept all that matches ipsec policy" ipsec-policy=in,ipsec
add action=drop chain=input comment="defconf: drop everything else not coming from LAN" in-interface-list=!LAN
add action=accept chain=forward comment="defconf: accept established,related,untracked" connection-state=established,related,untracked
add action=drop chain=forward comment="defconf: drop invalid" connection-state=invalid
add action=drop chain=forward comment="defconf: drop packets with bad src ipv6" src-address-list=bad_ipv6
add action=drop chain=forward comment="defconf: drop packets with bad dst ipv6" dst-address-list=bad_ipv6
add action=drop chain=forward comment="defconf: rfc4890 drop hop-limit=1" hop-limit=equal:1 protocol=icmpv6
add action=accept chain=forward comment="defconf: accept ICMPv6" protocol=icmpv6
add action=accept chain=forward comment="defconf: accept HIP" protocol=139
add action=accept chain=forward comment="defconf: accept IKE" dst-port=500,4500 protocol=udp
add action=accept chain=forward comment="defconf: accept ipsec AH" protocol=ipsec-ah
add action=accept chain=forward comment="defconf: accept ipsec ESP" protocol=ipsec-esp
add action=accept chain=forward comment="defconf: accept all that matches ipsec policy" ipsec-policy=in,ipsec
add action=drop chain=forward comment="defconf: drop everything else not coming from LAN" in-interface-list=!LAN
/system clock
set time-zone-name=Europe/Amsterdam
/system identity
set name=rt2
/tool mac-server
set allowed-interface-list=LAN
/tool mac-server mac-winbox
set allowed-interface-list=LAN

With this routing:

/ip/route/print 
Flags: D - DYNAMIC; A - ACTIVE; c, s, d, y - COPY
Columns: DST-ADDRESS, GATEWAY, DISTANCE
#     DST-ADDRESS     GATEWAY      DISTANCE
  DAd 0.0.0.0/0       172.27.10.1         1
  DAc 172.27.10.0/24  ether1              0
0  As 172.27.13.0/24  ether1              1

As you can see I placed some log rules in the firewall for input, output and forward. This is to identify what is picked up by the firewall and what isn't.

My network looks like this:
Workstation --> layer 2 switch --> ISP provider modem/router --> ether1 (Mikrotik) --> ether2 (Mikrotik)
As you can also see in the VLAN setup, VLAN 10 is the native VLAN. ether2-5 have VLAN 11-13 only for themselves. I only added the switch1-cpu to the VLANs for experimentation.
Native VLAN has 172.27.10.0/24 as subnet. The other VLANs have 172.27.{11,12,13}.0/24.
172.27.10.1 is my ISP modem/router. 172.27.10.3 is my Mikrotik. 172.27.10.100 is my workstation. 172.27.{10,11,12,13}.11 is a host connected to ether2.

Untitled Diagram.drawio.png

Firewall
The following is or is not registered by the firewall.
* input: in:ether1 out:(unknown 0), proto 2, 172.27.10.1->224.0.0.1
* input: in:ether1 out:(unknown 0), proto UDP, 172.27.10.1:53805->255.255.255.255:53805
* input: in:ether1 out:(unknown 0), proto ICMP (type 8, code 0), 172.27.10.100->172.27.10.3
* input: in:ether1 out:(unknown 0), proto TCP (SYN), 172.27.10.100:51176->172.27.10.3:18988 (and basically any traffic going from my workstation to ether1)
* output: in:(unknown 0) out:ether1, proto TCP (SYN), 172.27.10.3:46702->159.148.172.226:80 (and basically any traffic going out from ether1, initiated by the Mikrotik, not the connected hosts)
* An nmap scan from my workstation to 172.27.10.11 (ether2) does not show up in the firewall logs, the host is reachable though
* An nmap scan from 172.27.10.11 to my workstation does not show up in the firewall logs, host is reachable
* A ping from 172.27.10.11 to the Mikrotik (172.27.10.3) results in a (Destination Host Unreachable), the default gateway is set to 172.27.10.1 (so it should get a route and back).
* A ping from my workstation to 172.27.13.11 (ether2) also has some routing issues (see next section for details), but reaches the destination eventually, however, this is not picked up by the Mikrotik firewall either
* An nmap scan from my workstation to 172.27.13.11 (ether2) also does not result in firewall logs on the Mikrotik

It seems like ether1 to ether2 traffic is not picked up at all. Could this be part of a routing issue? Or has this to do with offloading the VLAN networks (layer 2) to hardware and thus doesn't get processed by the firewall which uses the CPU?

Routing issue
A ping from my workstation to a VLAN on ether2 is a bit flaky due to a routing issue. But eventually the ping does work. This is likely due to asymmetric routing. What would be a good fix for this in this situation?

$ ping 172.27.13.11
PING 172.27.13.11 (172.27.13.11) 56(84) bytes of data.
From 172.27.10.1 icmp_seq=1 Redirect Host(New nexthop: 172.27.10.3)
From 172.27.10.3 icmp_seq=1 Redirect Host(New nexthop: 172.27.13.11)
From 172.27.10.3 icmp_seq=2 Redirect Host(New nexthop: 172.27.13.11)
64 bytes from 172.27.13.11: icmp_seq=3 ttl=64 time=0.361 ms
64 bytes from 172.27.13.11: icmp_seq=4 ttl=64 time=0.368 ms
From 172.27.10.3 icmp_seq=1 Destination Host Unreachable
From 172.27.10.3 icmp_seq=2 Destination Host Unreachable
64 bytes from 172.27.13.11: icmp_seq=5 ttl=64 time=0.310 ms
64 bytes from 172.27.13.11: icmp_seq=6 ttl=64 time=0.342 ms
64 bytes from 172.27.13.11: icmp_seq=7 ttl=64 time=0.272 ms
^C
--- 172.27.13.11 ping statistics ---
7 packets transmitted, 5 received, +5 errors, 28.5714% packet loss, time 6161ms
rtt min/avg/max/mdev = 0.272/0.330/0.368/0.035 ms, pipe 4

My workstation has these routes:

ip route
default via 172.27.10.1 dev enp4s0u1u4 proto dhcp src 172.27.10.100 metric 100 
default via 172.27.10.1 dev wlp0s20f3 proto dhcp src 172.27.10.5 metric 600 
172.27.10.0/24 dev enp4s0u1u4 proto kernel scope link src 172.27.10.100 metric 100 
172.27.10.0/24 dev wlp0s20f3 proto kernel scope link src 172.27.10.5 metric 600

The Mikrotik has these routes:

/ip/route/print 
Flags: D - DYNAMIC; A - ACTIVE; c, s, d, y - COPY
Columns: DST-ADDRESS, GATEWAY, DISTANCE
#     DST-ADDRESS     GATEWAY      DISTANCE
  DAd 0.0.0.0/0       172.27.10.1         1
  DAc 172.27.10.0/24  ether1              0
0  As 172.27.13.0/24  ether1              1

The destination host on ether3 has these routes:

ip route
default via 172.27.10.1 dev eth0 onlink 
172.27.10.0/24 dev eth0 proto kernel scope link src 172.27.10.11 
172.27.11.0/24 dev eth0.11 proto kernel scope link src 172.27.11.11 
172.27.12.0/24 dev eth0.12 proto kernel scope link src 172.27.12.11 
172.27.13.0/24 dev eth0.13 proto kernel scope link src 172.27.13.11

The ISP modem/router is the DHCP of the 172.27.10.0/24 network and thus sets dynamic routes.
It does have a static route set:

screenshot.png

AquaL1te · Sun Jun 12, 2022 3:47 pm

I have asked this question in the past as well. I haven't touched this project in a long while. I'm not using the bridge VLAN setup anymore, that's basically the only difference. But would the solution of policy based routing still work? Because the firewall seems a bit half aware of the traffic, maybe due to the hardware offloading in the switch chip that doesn't touch the CPU now where firewall processing is done?

viewtopic.php?t=166146#p819409

mkx · Sun Jun 12, 2022 4:30 pm

The way I see it, router's L2 config is a mess. On one hand your ether1 is considered independent interface (not part of bridge, ...), but on other hand it's part of VLAN 10. Same IP subnet 172.27.10.0/24 is supposed to be both on ether1 side (ISP modem etc.) and on bridge side (R Pi).

It is fibe to have hEX configured as switch (with all ports member of VLAN 10, with most ports member of other VLANs) and as router (routing between VLANs), but not the way it is now.

As your intentions are not entirely clear to me, I'll refrain from start giving advice and I'll wait for your description of wanted setup ... expressed in plain words.

AquaL1te · Tue Jun 14, 2022 5:53 pm

The way I see it, router's L2 config is a mess. On one hand your ether1 is considered independent interface (not part of bridge, ...), but on other hand it's part of VLAN 10. Same IP subnet 172.27.10.0/24 is supposed to be both on ether1 side (ISP modem etc.) and on bridge side (R Pi).

It is fibe to have hEX configured as switch (with all ports member of VLAN 10, with most ports member of other VLANs) and as router (routing between VLANs), but not the way it is now.

As your intentions are not entirely clear to me, I'll refrain from start giving advice and I'll wait for your description of wanted setup ... expressed in plain words.

Good point, I indeed didn't explain my intent.

So the 4 PoE ports should be members of VLAN 11, 12 and 13. Where VLAN 13 should have a route to the Internet, just like VLAN 10. On VLAN 13 I'll run websites and other online services. VLAN 10 is just the default gateway to the Internet and the rest of my LAN. VLAN 11 and 12 should be closed from the Internet and the rest of the LAN.

So I want to be able to manage access to these networks. VLANs are setup, but I would prefer to also have a firewall from the Mikrotik in front of it.

If disabling the router functionality can be done and still have a L3 firewall, then that sounds great. I hope that doesn't mean I have to use the software bridge. Because if I enable VLAN filtering there, hardware offload is disabled and the performance is quite low.

AquaL1te · Wed Jun 15, 2022 10:51 pm

It is fibe to have hEX configured as switch (with all ports member of VLAN 10, with most ports member of other VLANs) and as router (routing between VLANs), but not the way it is now.

Mikrotik is still a bit confusing to me. So I did what you suggested, I added ether1 to the bridge. MSTP cannot be enabled, because for that VLAN filtering needs to be enabled on the software bridge level. So I suppose although I configured stuff on the software bridge, the VLANs are still managed by the switch chip configuration. I still get close to 1Gbit/s with iperf, so that's great. RSTP is fine on the bridge and hardware offloading with VLANs is still working. Everything is now connected on layer 2, so I suppose that's better. However, I still get these next hop messages when I ping hosts in VLAN 13 (172.27.13.0/24). So I suppose the asymmetric routing is still leading in this issue, which is on layer 3.

My L2 neighbors

[admin@MikroTik] > /ip/arp/print 
Flags: D, P - PUBLISHED; C - COMPLETE
Columns: ADDRESS, MAC-ADDRESS, INTERFACE
#    ADDRESS        MAC-ADDRESS        INTERFACE
0 DC 172.27.10.1    DC:15:C8:47:6C:88  bridge   
1 DC 172.27.10.100  70:B1:3D:E5:92:ED  bridge   
2 DC 172.27.13.11   DC:A6:32:B8:70:CE  bridge   
3 DC 172.27.13.14   DC:A6:32:44:C4:33  bridge   
4 DC 172.27.13.12   DC:A6:32:B3:6B:3F  bridge

My routes

[admin@MikroTik] > /ip/route/print 
Flags: D - DYNAMIC; A - ACTIVE; c, s, d, y - COPY
Columns: DST-ADDRESS, GATEWAY, DISTANCE
#     DST-ADDRESS      GATEWAY      DISTANCE
  DAd 0.0.0.0/0        172.27.10.1         1
  DAc 172.27.10.0/24   bridge              0
0  As 172.27.13.0/24   bridge              1
  DAc 192.168.88.0/24  bridge              0

What else could I be missing here?

[admin@MikroTik] > /export hide-sensitive 
# jun/15/2022 21:48:40 by RouterOS 7.3.1
# software id = A1GI-TFVF
#
# model = 960PGS
# serial number = 89F90861A06A
/interface bridge
add admin-mac=CC:2D:E0:81:0A:BE auto-mac=no comment=defconf name=bridge
/interface ethernet
set [ find default-name=ether2 ] poe-out=forced-on
set [ find default-name=ether3 ] poe-out=forced-on
set [ find default-name=ether4 ] poe-out=forced-on
set [ find default-name=ether5 ] poe-out=forced-on
/interface ethernet switch port
set 0 default-vlan-id=10 vlan-mode=secure
set 1 default-vlan-id=10 vlan-mode=secure
set 2 default-vlan-id=10 vlan-mode=secure
set 3 default-vlan-id=10 vlan-mode=secure
set 4 default-vlan-id=10 vlan-mode=secure
set 5 default-vlan-id=10 vlan-mode=secure
/interface list
add comment=defconf name=WAN
add comment=defconf name=LAN
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip pool
add name=default-dhcp ranges=192.168.88.10-192.168.88.254
/ip dhcp-server
add address-pool=default-dhcp interface=bridge name=defconf
/interface bridge port
add bridge=bridge comment=defconf interface=ether2
add bridge=bridge comment=defconf interface=ether3
add bridge=bridge comment=defconf interface=ether4
add bridge=bridge comment=defconf interface=ether5
add bridge=bridge comment=defconf interface=sfp1
add bridge=bridge interface=ether1
/ip neighbor discovery-settings
set discover-interface-list=LAN
/interface ethernet switch vlan
add comment=management independent-learning=yes ports=ether2,ether3,ether4,ether5 switch=switch1 vlan-id=11
add comment=replication independent-learning=yes ports=ether2,ether3,ether4,ether5 switch=switch1 vlan-id=12
add comment=public independent-learning=yes ports=ether1,ether2,ether3,ether4,ether5,switch1-cpu switch=switch1 vlan-id=13
add comment=native independent-learning=no ports=ether1,ether2,ether3,ether4,ether5,switch1-cpu switch=switch1 vlan-id=10
/interface list member
add comment=defconf interface=bridge list=LAN
add comment=defconf interface=ether1 list=WAN
/ip address
add address=192.168.88.1/24 comment=defconf interface=bridge network=192.168.88.0
/ip dhcp-client
add comment=defconf interface=bridge
/ip dhcp-server network
add address=192.168.88.0/24 comment=defconf dns-server=192.168.88.1 gateway=192.168.88.1
/ip dns
set allow-remote-requests=yes
/ip dns static
add address=192.168.88.1 comment=defconf name=router.lan
/ip firewall filter
add action=accept chain=input comment="defconf: accept established,related,untracked" connection-state=established,related,untracked
add action=drop chain=input comment="defconf: drop invalid" connection-state=invalid
add action=accept chain=input comment="defconf: accept ICMP" protocol=icmp
add action=accept chain=input comment="defconf: accept to local loopback (for CAPsMAN)" dst-address=127.0.0.1
add action=drop chain=input comment="defconf: drop all not coming from LAN" disabled=yes in-interface-list=!LAN
add action=accept chain=forward comment="defconf: accept in ipsec policy" ipsec-policy=in,ipsec
add action=accept chain=forward comment="defconf: accept out ipsec policy" ipsec-policy=out,ipsec
add action=fasttrack-connection chain=forward comment="defconf: fasttrack" connection-state=established,related hw-offload=yes
add action=accept chain=forward comment="defconf: accept established,related, untracked" connection-state=established,related,untracked
add action=drop chain=forward comment="defconf: drop invalid" connection-state=invalid
add action=drop chain=forward comment="defconf: drop all from WAN not DSTNATed" connection-nat-state=!dstnat connection-state=new disabled=yes in-interface-list=WAN
/ip firewall nat
add action=masquerade chain=srcnat comment="defconf: masquerade" ipsec-policy=out,none out-interface-list=WAN
/ip route
add disabled=no dst-address=172.27.13.0/24 gateway=bridge pref-src="" routing-table=main suppress-hw-offload=no
/ipv6 firewall address-list
add address=::/128 comment="defconf: unspecified address" list=bad_ipv6
add address=::1/128 comment="defconf: lo" list=bad_ipv6
add address=fec0::/10 comment="defconf: site-local" list=bad_ipv6
add address=::ffff:0.0.0.0/96 comment="defconf: ipv4-mapped" list=bad_ipv6
add address=::/96 comment="defconf: ipv4 compat" list=bad_ipv6
add address=100::/64 comment="defconf: discard only " list=bad_ipv6
add address=2001:db8::/32 comment="defconf: documentation" list=bad_ipv6
add address=2001:10::/28 comment="defconf: ORCHID" list=bad_ipv6
add address=3ffe::/16 comment="defconf: 6bone" list=bad_ipv6
/ipv6 firewall filter
add action=accept chain=input comment="defconf: accept established,related,untracked" connection-state=established,related,untracked
add action=drop chain=input comment="defconf: drop invalid" connection-state=invalid
add action=accept chain=input comment="defconf: accept ICMPv6" protocol=icmpv6
add action=accept chain=input comment="defconf: accept UDP traceroute" port=33434-33534 protocol=udp
add action=accept chain=input comment="defconf: accept DHCPv6-Client prefix delegation." dst-port=546 protocol=udp src-address=fe80::/10
add action=accept chain=input comment="defconf: accept IKE" dst-port=500,4500 protocol=udp
add action=accept chain=input comment="defconf: accept ipsec AH" protocol=ipsec-ah
add action=accept chain=input comment="defconf: accept ipsec ESP" protocol=ipsec-esp
add action=accept chain=input comment="defconf: accept all that matches ipsec policy" ipsec-policy=in,ipsec
add action=drop chain=input comment="defconf: drop everything else not coming from LAN" in-interface-list=!LAN
add action=accept chain=forward comment="defconf: accept established,related,untracked" connection-state=established,related,untracked
add action=drop chain=forward comment="defconf: drop invalid" connection-state=invalid
add action=drop chain=forward comment="defconf: drop packets with bad src ipv6" src-address-list=bad_ipv6
add action=drop chain=forward comment="defconf: drop packets with bad dst ipv6" dst-address-list=bad_ipv6
add action=drop chain=forward comment="defconf: rfc4890 drop hop-limit=1" hop-limit=equal:1 protocol=icmpv6
add action=accept chain=forward comment="defconf: accept ICMPv6" protocol=icmpv6
add action=accept chain=forward comment="defconf: accept HIP" protocol=139
add action=accept chain=forward comment="defconf: accept IKE" dst-port=500,4500 protocol=udp
add action=accept chain=forward comment="defconf: accept ipsec AH" protocol=ipsec-ah
add action=accept chain=forward comment="defconf: accept ipsec ESP" protocol=ipsec-esp
add action=accept chain=forward comment="defconf: accept all that matches ipsec policy" ipsec-policy=in,ipsec
add action=drop chain=forward comment="defconf: drop everything else not coming from LAN" in-interface-list=!LAN
/system clock
set time-zone-name=Europe/Amsterdam
/tool mac-server
set allowed-interface-list=LAN
/tool mac-server mac-winbox
set allowed-interface-list=LAN

I think these 2 issues are separate. One is asymmetric routing, policy routing might fix that where the route back is static by tagging traffic going a specific path (but please correct me if I'm wrong). But the other problem is still a mystery. I get that some traffic cannot be seen because it's going over the hardware offloaded VLAN, which does not come in contact with the CPU. So is it then safe to assume that only traffic directly sent to the bridge (172.27.10.3) is seen? And traffic that goes out from the bridge, to e.g. the default gateway (172.27.10.1). So basically the firewall is mostly blind. Policy routing would therefore also not work, since the firewall won't see that traffic because that's processed in the CPU.

So if my rambling is correct. It would basically mean, don't bother with a firewall too much. At least in the context of anything else than traffic to and from the bridge itself. And the asymmetric routing cannot be fixed on Mikrotik level. But probably it will be fixable on the Linux host, by doing something like this.

Is that correct?

AquaL1te · Sat Jun 18, 2022 11:43 am

Any suggestions? Or examples how other people do similar setups? I get the impression that this Mikrotik's can be great for their money. But only if you know the sweet spot with all the features, since not everything is hardware offloaded. And when hardware offloaded, it comes with downsides as well.

mkx · Sat Jun 18, 2022 1:49 pm

There seems to be a problem (or rather a bug) in ROS, reported by other forum members: if MT device, used as router, has multiple IP addresses from different IP subnets bound to same L2 interface, then it'll send out those "ICMP redirect" ... with "better gateway" address not belonging to sender's own IP subnet which then doesn't make any sense to sender. The solution is to avoid creating such scenario (i.e. always have different IP subnets on different ethernet/VLAN segments).

The other problem is routing triangle where firewall on the "half-in" device drops connection because traffic in one direction bypasses it.

I guess that your problem is the first one. Your topology chart from post #1 lacks some detail (such as: which interface of mikrotik actually connects to L2 switch, which interface of mikrotik actually connects to rPi, what about all those VLANs mentioned on rPi but not elsewhere, etc). The config you posted obviously has some problems, but to untangle it we first need to have clear picture of desired topology, then we can deal with config of MT.

AquaL1te · Mon Jun 20, 2022 12:16 pm

The solution is to avoid creating such scenario (i.e. always have different IP subnets on different ethernet/VLAN segments).

Haven't I done so already? I mean, every segment has it's own subnet. Native VLAN 10 is using 172.27.10.0/24, and then VLAN 11, 12 and 13 are using 172.27.{11,12,13}.0/24.

Your topology chart from post #1 lacks some detail (such as: which interface of mikrotik actually connects to L2 switch, which interface of mikrotik actually connects to rPi, what about all those VLANs mentioned on rPi but not elsewhere, etc).

Ether1 on the Mikrotik is connected to the L2 swtich. The Raspberry Pi in the tests is connected to ether2. But there are also Pi's on ether2-5. In the drawing I've put notes on the ether interfaces. But I agree this is a sketch at best. If needed, I can create a more detailed drawing. But if this already clears things up, then that would be great as well. Let me know if a new drawing would help.

mkx · Mon Jun 20, 2022 10:41 pm

The solution is to avoid creating such scenario (i.e. always have different IP subnets on different ethernet/VLAN segments).
Haven't I done so already?

From the config you posted:

/ip address
add address=192.168.88.1/24 comment=defconf interface=bridge network=192.168.88.0
/ip route
add disabled=no dst-address=172.27.13.0/24 gateway=bridge pref-src="" routing-table=main suppress-hw-offload=no

The static route makes ROS believe that 172.27.13.0/24 subnet is directly accesible via bridge interface (setting interface as gateway as opposed to next hop gateway's IP address actually means that). The address assignment also makes ROS believe that 192.168.88.0/24 subnet is directly accessible via bridge interface. Technically this excerpt doesn't set two IP addresses to the same interface, but it does effectively set two "native" IP subnets that should be directly accessible through that interface. Which results in ICMP redirect mesages.

AquaL1te · Tue Jun 21, 2022 12:05 am

The static route makes ROS believe that 172.27.13.0/24 subnet is directly accesible via bridge interface (setting interface as gateway as opposed to next hop gateway's IP address actually means that). The address assignment also makes ROS believe that 192.168.88.0/24 subnet is directly accessible via bridge interface. Technically this excerpt doesn't set two IP addresses to the same interface, but it does effectively set two "native" IP subnets that should be directly accessible through that interface. Which results in ICMP redirect mesages.

I've made some changes. But the native VLAN 10 also has the bridge as the gateway, this one is assigned dynamically. Which makes sense when the Mikrotik is setup as a bridge, right? Pinging VLAN 10 works without redirects. I removed the 192.168.88.1/24 IP, and with that the route also was deleted since it was dynamic.

[admin@MikroTik] > /ip/route/print detail 
Flags: D - dynamic; X - disabled, I - inactive, A - active; c - connect, s - static, r - rip, b - bgp, o - ospf, d - dhcp, v - vpn, m - modem, y - copy; H - hw-offloaded; + - ecmp 
   DAd   dst-address=0.0.0.0/0 routing-table=main pref-src="" gateway=172.27.10.1 immediate-gw=172.27.10.1%bridge distance=1 scope=30 target-scope=10 vrf-interface=bridge 
         suppress-hw-offload=no 

   DAc   dst-address=172.27.10.0/24 routing-table=main gateway=bridge immediate-gw=bridge distance=0 scope=10 suppress-hw-offload=no local-address=172.27.10.3%bridge 

 0  As   dst-address=172.27.13.0/24 routing-table=main pref-src="" gateway=172.27.10.1 immediate-gw=172.27.10.1%bridge distance=1 scope=30 target-scope=10 suppress-hw-offload=no

When I ping now from my worksation I get 100% packet loss.

$ ping -c10 172.27.13.11
PING 172.27.13.11 (172.27.13.11) 56(84) bytes of data.
From 172.27.10.1 icmp_seq=1 Redirect Host(New nexthop: 172.27.10.3)
From 172.27.10.1 icmp_seq=2 Redirect Host(New nexthop: 172.27.10.3)
From 172.27.10.1 icmp_seq=3 Time to live exceeded
From 172.27.10.1 icmp_seq=4 Time to live exceeded
From 172.27.10.1 icmp_seq=5 Time to live exceeded
From 172.27.10.1 icmp_seq=6 Time to live exceeded
From 172.27.10.1 icmp_seq=7 Time to live exceeded
From 172.27.10.1 icmp_seq=8 Time to live exceeded
From 172.27.10.1 icmp_seq=9 Time to live exceeded
From 172.27.10.1 icmp_seq=10 Time to live exceeded

--- 172.27.13.11 ping statistics ---
10 packets transmitted, 0 received, +10 errors, 100% packet loss, time 9013ms

The route on my ISP router/modem for 172.27.13.0/24 is set to 172.27.10.3, which is the IP of the bridge.

So when I ping from my workstation to 172.27.13.11 it gets routed to 172.27.10.3, which then has a route back to the ISP router/modem. Which makes sense to see this route loop, which I also see with mtr/traceroute. In the previous situation the ping reached the node because the bridge is aware where the destination is connected to on layer 2. So I'm not so sure this is the solution. I by the way implemented the bridge setup because the routing solution didn't work either. In that setup I think I probably should indeed have the route I have set now, and then the pint would arrive (I guess). But still with the redirects that I experienced in that setup as well.

All IPs in my ARP table point to the bridge, only showed the one that matters for this exercise.

[admin@MikroTik] > /ip/arp/print 
Flags: D, P - PUBLISHED; C - COMPLETE
Columns: ADDRESS, MAC-ADDRESS, INTERFACE
#    ADDRESS         MAC-ADDRESS        INTERFACE
9 DC 172.27.13.11    DC:A6:32:B8:70:CE  bridge

mkx · Tue Jun 21, 2022 7:53 am

As I wrote: your network layout and routing is a mess. You don't seem to accept my opinion, so I'm done helping in this thread. Perhaps somebody else will pass by and give you a hand sorting it out your way (which IMO isn't possible).

AquaL1te · Wed Jun 22, 2022 8:31 am

As I wrote: your network layout and routing is a mess. You don't seem to accept my opinion, so I'm done helping in this thread. Perhaps somebody else will pass by and give you a hand sorting it out your way (which IMO isn't possible).

I have accepted your advice. In routed mode I got these redirect messages, then you said that I should use the bridge mode. I setup the bridge mode, still the same issue. Then you say it's my static route, which shouldn't point to the bridge. I changed it to the next hop as you mentioned and it doesn't work. I'm communicating back to what you're suggesting. I'm willing to try any advice, but please don't blame me if your suggestion doesn't work. I don't get why you get upset with me. I really want to solve this issue and any help is appreciated!

What is exactly the mess? From the redirect messages I can concur it's not optimal. But can you please point to what exactly is the cause? You seem convinced to know the cause, please share. You also said my config is a mess, can you be more specific?

mkx · Wed Jun 22, 2022 3:04 pm

What is exactly the mess?

OK, perhaps I used too harsh words. But your network diagram from initial post is missing quite a few important details and we're pulling those out of you gradually. It is really hard to give a good advice if the (mental) picture is not clear and complete. Generally speaking the person which comes around with a problem he can't solve is hardly competent to decide which pieces of information are relevant and which are not, so it's only fair to the helpers to give out complete information (if that's not possible, then don't bother coming here to ask for help).

Our friend @anav used to have a simple initial response to such troubleshooting topics: state your requirements in plain English words.

AquaL1te · Thu Jun 23, 2022 9:06 am

But I offered to include a more detailed diagram, if the original one wasn't enough. But you asked about which interface is connected to which one. That is in the diagram, rudimentary, true, and that's why I added the offer for a more detailed diagram if needed. I'll include one in the weekend

Thanks for clarifying this!

Firewall not fully working with VLANs? Or could it be a (asymmetric) routing issue?

Firewall not fully working with VLANs? Or could it be a (asymmetric) routing issue?

Re: Firewall not fully working with VLANs? Or could it be a (asymmetric) routing issue?

Re: Firewall not fully working with VLANs? Or could it be a (asymmetric) routing issue?

Re: Firewall not fully working with VLANs? Or could it be a (asymmetric) routing issue?

Re: Firewall not fully working with VLANs? Or could it be a (asymmetric) routing issue?

Re: Firewall not fully working with VLANs? Or could it be a (asymmetric) routing issue?

Re: Firewall not fully working with VLANs? Or could it be a (asymmetric) routing issue?

Re: Firewall not fully working with VLANs? Or could it be a (asymmetric) routing issue?

Re: Firewall not fully working with VLANs? Or could it be a (asymmetric) routing issue?

Re: Firewall not fully working with VLANs? Or could it be a (asymmetric) routing issue?

Re: Firewall not fully working with VLANs? Or could it be a (asymmetric) routing issue?

Re: Firewall not fully working with VLANs? Or could it be a (asymmetric) routing issue?

Re: Firewall not fully working with VLANs? Or could it be a (asymmetric) routing issue?

Re: Firewall not fully working with VLANs? Or could it be a (asymmetric) routing issue?

Who is online