Static routing issue?

RouterOS General
1

Hi,

I have a strange routing problem.

My Mikrotik config:

[user@rt2] > export hide-sensitive
# sep/13/2020 15:59:24 by RouterOS 6.47.3
# software id = A1GI-TFVF
#
# model = 960PGS
# serial number = 89F90861A06A
/interface bridge
add admin-mac=CC:2D:E0:81:0A:BE auto-mac=no dhcp-snooping=yes ingress-filtering=yes name=br0 protocol-mode=mstp pvid=10 vlan-filtering=yes
/interface ethernet
set [ find default-name=ether2 ] poe-out=forced-on
set [ find default-name=ether3 ] poe-out=forced-on
set [ find default-name=ether4 ] poe-out=forced-on
set [ find default-name=ether5 ] poe-out=forced-on
set [ find default-name=sfp1 ] disabled=yes
/interface vlan
add interface=br0 name=management vlan-id=11
add interface=br0 name=public vlan-id=13
add interface=br0 name=replication vlan-id=12
/interface ethernet switch
set 0 name=sw1
/interface list
add name=external
add name=internal
/ip pool
add name=default-dhcp ranges=192.168.88.10-192.168.88.254
/snmp community
set [ find default=yes ] disabled=yes
/interface bridge port
add bpdu-guard=yes bridge=br0 comment=rp1 edge=yes-discover hw=no ingress-filtering=yes interface=ether2 pvid=10
add bpdu-guard=yes bridge=br0 comment=rp2 edge=yes-discover hw=no ingress-filtering=yes interface=ether3 pvid=10
add bpdu-guard=yes bridge=br0 comment=rp3 edge=yes-discover hw=no ingress-filtering=yes interface=ether4 pvid=10
add bridge=br0 comment=rt1 frame-types=admit-only-untagged-and-priority-tagged hw=no ingress-filtering=yes interface=ether1 pvid=10 trusted=yes
add bpdu-guard=yes bridge=br0 comment=rp4 edge=yes-discover hw=no ingress-filtering=yes interface=ether5 pvid=10
/interface bridge settings
set use-ip-firewall=yes use-ip-firewall-for-vlan=yes
/ip neighbor discovery-settings
set discover-interface-list=none
/interface bridge vlan
add bridge=br0 comment=native untagged=ether1,ether2,ether3,ether4,ether5 vlan-ids=10
add bridge=br0 comment=management tagged=ether2,ether3,ether4,ether5 vlan-ids=11
add bridge=br0 comment=replication tagged=ether2,ether3,ether4,ether5 vlan-ids=12
add bridge=br0 comment=public tagged=ether2,ether3,ether4,ether5 vlan-ids=13
/interface list member
add interface=ether1 list=external
add interface=ether2 list=internal
add interface=ether3 list=internal
add interface=ether4 list=internal
add interface=ether5 list=internal
/ip address
add address=172.27.11.254/24 interface=management network=172.27.11.0
add address=172.27.12.254/24 interface=replication network=172.27.12.0
add address=172.27.13.254/24 interface=public network=172.27.13.0
/ip dhcp-client
add disabled=no interface=br0
/ip firewall address-list
add address=0.0.0.0/8 comment="self-identification [rfc 3330]" list=bogon
add address=10.0.0.0/8 comment="private class a [rfc 1918]" list=bogon
add address=127.0.0.0/8 comment="loopback [rfc 3330]" list=bogon
add address=172.16.0.0/12 comment="private class b [rfc 1918]" disabled=yes list=bogon
add address=169.254.0.0/16 comment="link-local [rfc 3330]" disabled=yes list=bogon
add address=192.168.0.0/16 comment="private class c [rfc 1918]" list=bogon
add address=192.0.2.0/24 comment="test-net 1 [rfc 5737]" list=bogon
add address=192.88.99.0/24 comment="6to4 relay anycast [rfc 3068]" list=bogon
add address=198.18.0.0/15 comment="bmwg testing [rfc 6815]" list=bogon
add address=198.51.100.0/24 comment="test-net 2 [rfc 5737]" list=bogon
add address=203.0.113.0/24 comment="test-net 3 [rfc 5737]" list=bogon
add address=224.0.0.0/4 comment="private class d (multicast) [rfc 1112]" list=bogon
add address=172.27.10.0/24 list=native
add address=172.27.11.0/24 list=management
add address=172.27.12.0/24 list=replication
add address=172.27.13.0/24 list=public
/ip firewall filter
add action=drop chain=forward comment="drop invalid" connection-state=invalid log=yes log-prefix=invalid
add action=accept chain=forward comment="accept established, related, untracked (external --> internal)" connection-state=established,related,untracked
add action=accept chain=forward comment="accept new (internal --> external)" connection-state=new dst-address-list=!bogon in-bridge-port-list=internal out-bridge-port-list=external
add action=accept chain=input comment="accept ssh (native) (remove later)" connection-state=new dst-address-list=native dst-port=22,900,4443 in-interface=br0 protocol=tcp
add action=accept chain=input comment="accept ssh (management) (remove later)" connection-state=new dst-address-list=management dst-port=22,900,4443 in-interface=management protocol=tcp
add action=jump chain=forward comment="allow from external (remove later)" connection-state=new dst-address-list=native dst-port=22,900,4443 in-bridge-port-list=external jump-target=native out-bridge-port-list=internal protocol=tcp src-address-list=native
add action=jump chain=forward comment="jump to native rules" connection-state=new dst-address-list=native in-bridge-port-list=internal jump-target=native out-bridge-port-list=internal src-address-list=native
add action=accept chain=forward comment="remove (temp ssh)" connection-state=new dst-port=900 protocol=tcp
add action=jump chain=forward comment="jump to management rules" connection-state=new dst-address-list=management in-bridge-port-list=internal jump-target=management out-bridge-port-list=internal src-address-list=management
add action=jump chain=forward comment="jump to replication rules" connection-state=new dst-address-list=replication in-bridge-port-list=internal jump-target=replication out-bridge-port-list=internal src-address-list=replication
add action=jump chain=forward comment="jump to public rules" connection-state=new dst-address-list=public jump-target=public
add action=accept chain=forward comment="accept dns (external --> internal)" connection-state=new dst-port=1024-65535 in-bridge-port-list=external out-bridge-port-list=internal protocol=udp src-port=53
add action=accept chain=forward comment="accept dhcp (external --> internal)" connection-state=new dst-port=68 in-bridge-port-list=external out-bridge-port-list=internal protocol=udp src-port=67
add action=accept chain=native comment="accept ssh (remove later)" dst-port=22,900,4443 protocol=tcp
add action=accept chain=native comment="accept pacemaker (remove later)" dst-port=2224 protocol=tcp
add action=accept chain=native comment="accept mail relay, submission, imaps" dst-port=25,587,993 protocol=tcp
add action=accept chain=native comment="accept dns" dst-port=53 protocol=tcp
add action=accept chain=native comment="accept dns" dst-port=53 protocol=udp
add action=accept chain=native comment="accept http, https" dst-port=80,443 protocol=tcp
add action=accept chain=management comment="accept ssh" dst-port=22,900,4443 protocol=tcp
add action=accept chain=management comment="accept http, https" dst-port=80,443 protocol=tcp
add action=accept chain=management comment="accept snmp, snmp trap" dst-port=161,162 protocol=udp
add action=accept chain=management comment="accept pacemaker" dst-port=2224 protocol=tcp
add action=accept chain=replication comment="accept lmtp" dst-port=24 protocol=tcp
add action=accept chain=replication comment="accept sasl" dst-port=2222 protocol=tcp
add action=accept chain=replication comment="accept mysql" dst-port=3308 protocol=tcp
add action=accept chain=replication comment="accept gluster" dst-port=24007-24008,49152-49155 protocol=tcp
add action=accept chain=public comment="accept mail relay, submission, imaps" dst-port=25,587,993 protocol=tcp
add action=accept chain=public comment="accept dns" dst-port=53 protocol=tcp
add action=accept chain=public comment="accept dns" dst-port=53 protocol=udp
add action=accept chain=public comment="accept http, https" dst-port=80,443 protocol=tcp
add action=accept chain=public comment="accept vpn" dst-port=1194 protocol=tcp
add action=accept chain=public comment="accept vpn" dst-port=1194 protocol=udp
add action=accept chain=forward comment="accept ICMP" protocol=icmp
add action=accept chain=forward comment="accept broadcast" dst-address-type=broadcast
add action=accept chain=forward comment="accept multicast" dst-address-type=multicast
add action=drop chain=input comment="drop invalid" connection-state=invalid log=yes log-prefix=invalid
add action=accept chain=input comment="accept established, related, untracked" connection-state=established,related,untracked
add action=accept chain=input comment="accept to local loopback (for capsman)" dst-address=127.0.0.1
add action=accept chain=input comment="accept ICMP" protocol=icmp
add action=accept chain=input comment="accept broadcast" dst-address-type=broadcast
add action=accept chain=input comment="accept multicast" dst-address-type=multicast
add action=reject chain=input log=yes reject-with=icmp-admin-prohibited
add action=reject chain=forward log=yes reject-with=icmp-admin-prohibited
add action=drop chain=output connection-state=invalid log=yes log-prefix=invalid
add action=accept chain=output
/ip service
set telnet disabled=yes
set ftp disabled=yes
set www disabled=yes
set ssh port=900
set api disabled=yes
set winbox disabled=yes
set api-ssl disabled=yes
/ip ssh
set strong-crypto=yes
/system clock
set time-zone-name=Europe/Amsterdam
/system identity
set name=rt2
/system note
set note="UNAUTHORIZED ACCESS TO THIS NETWORK IS PROHIBITED"
/tool bandwidth-server
set enabled=no
/tool mac-server
set allowed-interface-list=none
/tool mac-server mac-winbox
set allowed-interface-list=none
/tool mac-server ping
set enabled=no

Network drawing of RT1 (my internet provider router) and RT2 (my Mikrotik) and PC1 (connected to VLAN 13 on the Mikrotik)

My static routes on RT1:

My static and dynamic routes on RT2:

[user@rt2] > /ip route print detail
Flags: X - disabled, A - active, D - dynamic, C - connect, S - static, r - rip, b - bgp, o - ospf, m - mme, B - blackhole, U - unreachable, P - prohibit 
 0 ADS  dst-address=0.0.0.0/0 gateway=172.27.10.254 gateway-status=172.27.10.254 reachable via  br0 distance=1 scope=30 target-scope=10 vrf-interface=br0 

 1 ADC  dst-address=172.27.10.0/24 pref-src=172.27.10.100 gateway=br0 gateway-status=br0 reachable distance=0 scope=10 

 2 ADC  dst-address=172.27.11.0/24 pref-src=172.27.11.254 gateway=management gateway-status=management reachable distance=0 scope=10 

 3 ADC  dst-address=172.27.12.0/24 pref-src=172.27.12.254 gateway=replication gateway-status=replication reachable distance=0 scope=10 

 4 ADC  dst-address=172.27.13.0/24 pref-src=172.27.13.254 gateway=public gateway-status=public reachable distance=0 scope=10

ARP table on RT2:

[user@rt2] > /ip arp print detail
Flags: X - disabled, I - invalid, H - DHCP, D - dynamic, P - published, C - complete 
 0 DC address=172.27.10.254 mac-address=7C:FF:4D:57:A1:F3 interface=br0 published=no 

 1 DC address=172.27.10.115 mac-address=3E:BD:84:BD:41:38 interface=br0 published=no

My static/dynamic routes on PC1:

[root@pc1 ~]# ip route
default via 172.27.10.254 dev eth0 onlink 
172.27.10.0/24 dev eth0 proto kernel scope link src 172.27.10.22 
172.27.10.0/24 dev tun0 proto kernel scope link src 172.27.10.1 
172.27.11.0/24 dev eth0.11 proto kernel scope link src 172.27.11.2 
172.27.12.0/24 dev eth0.12 proto kernel scope link src 172.27.12.2 
172.27.13.0/24 dev eth0.13 proto kernel scope link src 172.27.13.2

ARP table on PC1:

[root@pc1 ~]# ip -4 neigh show
172.27.13.254 dev eth0.13  FAILED
172.27.12.3 dev eth0.12 lladdr dc:a6:32:b3:6b:3f REACHABLE
172.27.10.100 dev eth0 lladdr cc:2d:e0:81:0a:be STALE
172.27.13.3 dev eth0.13 lladdr dc:a6:32:b3:6b:3f STALE
172.27.10.115 dev eth0 lladdr 3e:bd:84:bd:41:38 REACHABLE
172.27.12.4 dev eth0.12 lladdr dc:a6:32:1d:f4:7f REACHABLE
172.27.12.5 dev eth0.12 lladdr dc:a6:32:44:c4:33 REACHABLE
172.27.10.254 dev eth0 lladdr 7c:ff:4d:57:a1:f3 REACHABLE
172.27.10.4 dev eth0 lladdr 30:85:a9:45:e8:c4 STALE

Some ping examples

  • RT2 to RT1
  • RT2 to PC1
  • PC1 to RT1
  • PC1 to RT2
[user@rt2] > /ping 172.27.10.254 count=3
  SEQ HOST                                     SIZE TTL TIME  STATUS                                                                                                                                                                                                             
    0 172.27.10.254                              56  64 0ms  
    1 172.27.10.254                              56  64 0ms  
    2 172.27.10.254                              56  64 0ms  
    sent=3 received=3 packet-loss=0% min-rtt=0ms avg-rtt=0ms max-rtt=0ms




[user@rt2] > /ping 172.27.13.2 count=3   
  SEQ HOST                                     SIZE TTL TIME  STATUS                                                                       
    0 172.27.13.2                                             timeout                                                                      
    1 172.27.13.2                                             timeout                                                                      
    2 172.27.13.254                              84  64 987ms host unreachable                                                             
    sent=3 received=0 packet-loss=100%




[user@pc1 ~]# ping -c 3 172.27.13.254
PING 172.27.13.254 (172.27.13.254) 56(84) bytes of data.
From 172.27.13.2 icmp_seq=1 Destination Host Unreachable
From 172.27.13.2 icmp_seq=2 Destination Host Unreachable
From 172.27.13.2 icmp_seq=3 Destination Host Unreachable

--- 172.27.13.254 ping statistics ---
3 packets transmitted, 0 received, +3 errors, 100% packet loss, time 92ms


[root@pc1 ~]# ping -c 3 172.27.10.100
PING 172.27.10.100 (172.27.10.100) 56(84) bytes of data.
64 bytes from 172.27.10.100: icmp_seq=1 ttl=64 time=0.414 ms
64 bytes from 172.27.10.100: icmp_seq=2 ttl=64 time=0.231 ms
64 bytes from 172.27.10.100: icmp_seq=3 ttl=64 time=0.225 ms

--- 172.27.10.100 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 50ms
rtt min/avg/max/mdev = 0.225/0.290/0.414/0.087 ms




[user@pc1 ~]# ping -c 3 172.27.10.254
PING 172.27.10.254 (172.27.10.254) 56(84) bytes of data.
64 bytes from 172.27.10.254: icmp_seq=1 ttl=64 time=0.486 ms
64 bytes from 172.27.10.254: icmp_seq=2 ttl=64 time=0.494 ms
64 bytes from 172.27.10.254: icmp_seq=3 ttl=64 time=0.467 ms

--- 172.27.10.254 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 85ms
rtt min/avg/max/mdev = 0.467/0.482/0.494/0.021 ms

Question

Shouldn’t ping just work in all cases? Or does my VLAN config mess things up? Or are my static routes not configured right? I see no firewall logs of dropped/rejected packets, or anything else going wrong. Or is there a total different reason why this doesn’t work?

2

You can’t rely on that as some devices’ firewalls do not accept pings from other subnets than “connected” ones, but it’s not the cause of your issue.


Yes. It’s not obvious, but the /interface bridge configuration actually merges configuration items for two distinct objects - the bridge itself, and the virtual interface which is a member port of the bridge and connects the L3 stack of the router to the bridge.

So in the /interface bridge vlan configuration, you have to add the bridge (representing the virtual interface here) to the tagged list on each row. Without that, traffic is bridged among the other member ports, but the IP stack of the router doesn’t receive it (nor can it send it to the bridge).

3

Yes. It’s not obvious, but the > /interface bridge > configuration actually merges configuration items for two distinct objects - the bridge itself, and the virtual interface which is a member port of the bridge and connects the L3 stack of the router to the bridge.

Thanks! One more thing, for IPv4, the R1 applies port forwarding for e.g. UDP 1194 (OpenVPN), which listens on PC1 on 172.27.13.2/24 which is configured on eth0.13. The default route of PC1 is set on the eth0 interface.

So I guess that’s why I see martian packet warnings on PC1? Because asymmetric routing is applied. Packets come in on eth0.13 and go out on eth0. Correct?

IPv4: martian source 172.27.13.2 from 1.2.3.4, on dev eth0.13

In /etc/sysct.conf I have

net.ipv4.conf.default.rp_filter=1

set, which is the strict mode for reverse path filtering. When I configure

net.ipv4.conf.eth0/13.rp_filter=2

, it is not blocked anymore. Which makes sense, because loose mode does allow asymmetric routing.

So my final questions are:

  1. Can someone confirm that this is really the cause of the martian packets and a proper solution? It would make sense of course, since the warnings are gone and routing works now. But I think it’s always good to confirm if this is the best solution.
  2. In terms of security, this may introduce problems, like CVE-2019-14899 (never mind, that’s a client side vulnerability, not server). But according to RFC 3704, this seems like a valid thing to do. If I would switch the roles of the public VLAN with the native VLAN, then no loose reverse path filtering is needed. Because eth0 is the default gateway, so incoming packets from the port forwarding will also go out of that interface. However, then my “native” VLAN would become the public facing one. Which I also don’t find very attractive. Because everything will become a member of that VLAN by default. Of course, firewalls are suppose to provide protection in this domain, not a layer 2 solution like VLANs. Please share some critical comments. Thanks.
4

To me, a proper solution would be to set up policy routing on the PC - connection marking and associated routing marking would make sure that what comes in via eth0.13 will be responded that way, but I have no clue whether the rp_filter check interworks with the conntrack etc. deeply enough to evaluate this properly.

5

That sounds interesting. But for that solution there would still be asymmetric routing, right? So the reverse path filter would still need to be set to from “strict” to “loose”. Although policy routing sounds like an interesting thing to do, I’m not sure if it would really change much. Because the default gateway is already set to eth0. If I understand it correctly, policy routing would just do the same, but then more explicitly setting traffic to eth0.13 → eth0. The whole vlan0.13 is considered “public” and open to the internet, and the gateway back to the internet is eth0.

But if I understand this correctly, on the server side, this setup would make sense? And not open security holes? I read that the “loose” reverse path filter would cause some security issues, mainly with VPN clients.

6



  • the primary purpose of rp-filter (filtering of received packets based on reverse path availability) is to prevent packets with spoofed source addresses from being forwarded or responded without need to use firewall rules for that purpose on routers with many interfaces and routes
  • rp-filter=strict relies on exactly one route to exist per destination, so it cannot be used if more routes to the same destination exist at a time, no matter why they do

Since connection marking and the subsequent routing marking, which are necessary to choose the correct route for a response depening on via which interface the request came, is a firewall handling, the advantage of the RP-filtering being faster than firewall processing is lost anyway. So instead of setting rp-filter to strict on the server, you have to add firewall filter rules, making sure that packets received via the VPN will not be forwarded and only those towards the port(s) on which the service(s) is/are listening will be accepted locally. And if forwarding is disabled on the server itself in general, as is the default on most Linux distributions, the forwarding part is covered even without any firewall rules in chain forward of filter.