I have the following issue; the ARP table on my Miktorik hEX PoE seems wrong.
First, below is my configuration:
[user@rt2] > /export hide-sensitive
# aug/30/2020 17:01:15 by RouterOS 6.47.2
# software id = A1GI-TFVF
#
# model = 960PGS
# serial number = 89F90861A06A
/interface bridge
add admin-mac=CC:2D:E0:81:0A:BE auto-mac=no dhcp-snooping=yes ingress-filtering=yes name=br0 protocol-mode=mstp pvid=10 vlan-filtering=yes
/interface ethernet
set [ find default-name=ether2 ] poe-out=forced-on
set [ find default-name=ether3 ] poe-out=forced-on
set [ find default-name=ether4 ] poe-out=forced-on
set [ find default-name=ether5 ] poe-out=forced-on
set [ find default-name=sfp1 ] disabled=yes
/interface vlan
add interface=br0 name=management vlan-id=11
/interface ethernet switch
set 0 name=sw1
/interface list
add name=external
add name=internal
/ip pool
add name=default-dhcp ranges=192.168.88.10-192.168.88.254
/snmp community
set [ find default=yes ] disabled=yes
/interface bridge port
add bpdu-guard=yes bridge=br0 comment=rp1 edge=yes-discover hw=no ingress-filtering=yes interface=ether2 pvid=10
add bpdu-guard=yes bridge=br0 comment=rp2 edge=yes-discover hw=no ingress-filtering=yes interface=ether3 pvid=10
add bpdu-guard=yes bridge=br0 comment=rp3 edge=yes-discover hw=no ingress-filtering=yes interface=ether4 pvid=10
add bridge=br0 comment=rt1 frame-types=admit-only-untagged-and-priority-tagged hw=no ingress-filtering=yes interface=ether1 pvid=10 trusted=yes
add bpdu-guard=yes bridge=br0 comment=rp4 edge=yes-discover hw=no ingress-filtering=yes interface=ether5 pvid=10
/interface bridge settings
set use-ip-firewall=yes use-ip-firewall-for-vlan=yes
/ip neighbor discovery-settings
set discover-interface-list=none
/interface bridge vlan
add bridge=br0 comment=native untagged=ether1,ether2,ether3,ether4,ether5 vlan-ids=10
add bridge=br0 comment=management tagged=ether2,ether3,ether4,ether5 vlan-ids=11
add bridge=br0 comment=replication tagged=ether2,ether3,ether4,ether5 vlan-ids=12
add bridge=br0 comment=public tagged=ether2,ether3,ether4,ether5 vlan-ids=13
/interface list member
add interface=ether1 list=external
add interface=ether2 list=internal
add interface=ether3 list=internal
add interface=ether4 list=internal
add interface=ether5 list=internal
/ip address
add address=172.27.11.1/24 interface=management network=172.27.11.0
/ip dhcp-client
add disabled=no interface=br0
/ip firewall address-list
add address=0.0.0.0/8 comment="self-identification [rfc 3330]" list=bogon
add address=10.0.0.0/8 comment="private class a [rfc 1918]" list=bogon
add address=127.0.0.0/8 comment="loopback [rfc 3330]" list=bogon
add address=172.16.0.0/12 comment="private class b [rfc 1918]" disabled=yes list=bogon
add address=169.254.0.0/16 comment="link-local [rfc 3330]" disabled=yes list=bogon
add address=192.168.0.0/16 comment="private class c [rfc 1918]" list=bogon
add address=192.0.2.0/24 comment="test-net 1 [rfc 5737]" list=bogon
add address=192.88.99.0/24 comment="6to4 relay anycast [rfc 3068]" list=bogon
add address=198.18.0.0/15 comment="bmwg testing [rfc 6815]" list=bogon
add address=198.51.100.0/24 comment="test-net 2 [rfc 5737]" list=bogon
add address=203.0.113.0/24 comment="test-net 3 [rfc 5737]" list=bogon
add address=224.0.0.0/4 comment="private class d (multicast) [rfc 1112]" list=bogon
add address=172.27.10.0/24 list=native
add address=172.27.11.0/24 list=management
add address=172.27.12.0/24 list=replication
add address=172.27.13.0/24 list=public
/ip firewall filter
add action=drop chain=forward comment="drop invalid" connection-state=invalid log=yes log-prefix=invalid
add action=accept chain=forward comment="accept established, related, untracked (external --> internal)" connection-state=established,related,untracked
add action=accept chain=forward comment="accept new (internal --> external)" connection-state=new dst-address-list=!bogon in-bridge-port-list=internal out-bridge-port-list=external
add action=accept chain=input comment="accept ssh (native) (remove later)" connection-state=new dst-address-list=native dst-port=22,900,4443 in-interface=br0 protocol=tcp
add action=accept chain=input comment="accept ssh (management) (remove later)" connection-state=new dst-address-list=management dst-port=22,900,4443 in-interface=management protocol=tcp
add action=jump chain=forward comment="allow from external (remove later)" connection-state=new dst-address-list=native dst-port=22,900,4443 in-bridge-port-list=external jump-target=native out-bridge-port-list=internal protocol=tcp src-address-list=native
add action=jump chain=forward comment="jump to native rules" connection-state=new dst-address-list=native in-bridge-port-list=internal jump-target=native out-bridge-port-list=internal src-address-list=native
add action=accept chain=forward comment="remove (temp ssh)" connection-state=new dst-port=900 protocol=tcp
add action=jump chain=forward comment="jump to management rules" connection-state=new disabled=yes dst-address-list=management in-bridge-port-list=internal jump-target=management out-bridge-port-list=internal src-address-list=management
add action=jump chain=forward comment="jump to replication rules" connection-state=new disabled=yes dst-address-list=replication in-bridge-port-list=internal jump-target=replication out-bridge-port-list=internal src-address-list=replication
add action=jump chain=forward comment="jump to public rules" connection-state=new dst-address-list=public in-bridge-port-list=external jump-target=public out-bridge-port-list=internal
add action=accept chain=forward comment="accept dns (external --> internal)" connection-state=new dst-port=1024-65535 in-bridge-port-list=external out-bridge-port-list=internal protocol=udp src-port=53
add action=accept chain=forward comment="accept dhcp (external --> internal)" connection-state=new dst-port=68 in-bridge-port-list=external out-bridge-port-list=internal protocol=udp src-port=67
add action=accept chain=native comment="accept ssh (remove later)" dst-port=22,900,4443 protocol=tcp
add action=accept chain=native comment="accept pacemaker (remove later)" dst-port=2224 protocol=tcp
add action=accept chain=native comment="accept mail relay, submission, imaps" dst-port=25,587,993 protocol=tcp
add action=accept chain=native comment="accept dns" dst-port=53 protocol=tcp
add action=accept chain=native comment="accept dns" dst-port=53 protocol=udp
add action=accept chain=native comment="accept http, https" dst-port=80,443 protocol=tcp
add action=accept chain=management comment="accept ssh" dst-port=22,900,4443 protocol=tcp
add action=accept chain=management comment="accept http, https" dst-port=80,443 protocol=tcp
add action=accept chain=management comment="accept snmp, snmp trap" dst-port=161,162 protocol=udp
add action=accept chain=management comment="accept pacemaker" dst-port=2224 protocol=tcp
add action=accept chain=replication comment="accept lmtp" dst-port=24 protocol=tcp
add action=accept chain=replication comment="accept sasl" dst-port=2222 protocol=tcp
add action=accept chain=replication comment="accept mysql" dst-port=3308 protocol=tcp
add action=accept chain=replication comment="accept gluster" dst-port=24007-24008,49152-49155 protocol=tcp
add action=accept chain=public comment="accept mail relay, submission, imaps" dst-port=25,587,993 protocol=tcp
add action=accept chain=public comment="accept dns" dst-port=53 protocol=tcp
add action=accept chain=public comment="accept dns" dst-port=53 protocol=udp
add action=accept chain=public comment="accept http, https" dst-port=80,443 protocol=tcp
add action=accept chain=public comment="accept vpn" dst-port=1194 protocol=tcp
add action=accept chain=public comment="accept vpn" dst-port=1194 protocol=udp
add action=accept chain=forward comment="accept ICMP" protocol=icmp
add action=accept chain=forward comment="accept broadcast" dst-address-type=broadcast
add action=accept chain=forward comment="accept multicast" dst-address-type=multicast
add action=drop chain=input comment="drop invalid" connection-state=invalid log=yes log-prefix=invalid
add action=accept chain=input comment="accept established, related, untracked" connection-state=established,related,untracked
add action=accept chain=input comment="accept to local loopback (for capsman)" dst-address=127.0.0.1
add action=accept chain=input comment="accept ICMP" protocol=icmp
add action=accept chain=input comment="accept broadcast" dst-address-type=broadcast
add action=accept chain=input comment="accept multicast" dst-address-type=multicast
add action=reject chain=input log=yes reject-with=icmp-admin-prohibited
add action=reject chain=forward log=yes reject-with=icmp-admin-prohibited
add action=drop chain=output connection-state=invalid log=yes log-prefix=invalid
add action=accept chain=output
/ip route
add distance=1 dst-address=172.27.12.0/24 gateway=br0
add distance=1 dst-address=172.27.13.0/24 gateway=br0
/ip service
set telnet disabled=yes
set ftp disabled=yes
set www disabled=yes
set ssh port=900
set api disabled=yes
set winbox disabled=yes
set api-ssl disabled=yes
/ip ssh
set strong-crypto=yes
/system clock
set time-zone-name=Europe/Amsterdam
/system identity
set name=rt2
/system note
set note="UNAUTHORIZED ACCESS TO THIS NETWORK IS PROHIBITED"
/tool bandwidth-server
set enabled=no
/tool mac-server
set allowed-interface-list=none
/tool mac-server mac-winbox
set allowed-interface-list=none
/tool mac-server ping
set enabled=no
Now, the issue I’m facing is the following. I have a VLAN called “public”, which is meant to host public services (Apache, Postfix, etc.). It uses 172.27.13.0/24. These services are running on Raspberry Pi’s, which have alias interfaces that are tagging their 802.1Q traffic in that subnet. This is done by the following snippet in /etc/network/interfaces:
auto eth0.13
iface eth0.13 inet static
address 172.27.13.2/24
vlan-raw-device eth0
However, because it’s an alias, it inherits the same MAC address of eth0. This is also visible in the ARP table on the Mikrotik:
[user@rt2] > /ip arp print detail
Flags: X - disabled, I - invalid, H - DHCP, D - dynamic, P - published, C - complete
6 DC address=172.27.13.2 mac-address=DC:A6:32:B8:70:CE interface=br0 published=no
This matches with the MAC address of eth0 on this particular Raspberry Pi:
[root@rp1 ~]# ip -br link
lo UNKNOWN 00:00:00:00:00:00 <LOOPBACK,UP,LOWER_UP>
eth0 UP dc:a6:32:b8:70:ce <BROADCAST,MULTICAST,UP,LOWER_UP>
eth0.13@eth0 UP 7a:01:1d:f4:0a:8c <BROADCAST,MULTICAST,UP,LOWER_UP>
As you can see, the eth0.13 MAC address doesn’t match with the MAC address of eth0. This is because for testing purposes I configured a random MAC address for eth0.13. Before I did that the eth0.13 had the same MAC address as eth0. However, the ARP table doesn’t change. Even when I remove the records and reboot the Mikrotik router.
It does work between the Raspberry Pi’s. When I check the ARP table of the peer, I see the fake MAC address listed.
[root@rp2 ~]# ip neigh show
172.27.13.2 dev eth0.13 lladdr 7a:01:1d:f4:0a:8c STALE
But this is not the case for the Mikrotik router. This is an issue, because when I forward traffic to the 172.27.13.0/24 subnet, it should go to eth0.13, but it arrives on eth0 and gets dropped. Because it enters on the wrong interface (eth0). For some reason the Raspberry Pi’s do know to which interface to send this, but the Microtik does not. Any suggestions?