Routing question in ROS 7

RouterOS General
1

I have a bit of a lab network setup. I have a small pool of public IP addresses that I’m providing to individual subnets behind my router (RB3011). The main thing I wanted was for each subnet to have it’s own separate public IP when it connects out. I did this previously with mangle rules, but that had some limitations and it looked like Routing Tables in ROS7 would be a cleaner way to implement this. I’ve built the following config and got everything working as expected there.

I’m just a little stuck at routing between my networks. I want to be able to ping devices between 172.16.200.0/24 on ether6 and 172.16.30.0/24 devices on ether7. I made a pair of firewall rules for this and added routes to all three routing tables, but I’m unable to fully pass my traffic. I can see the stat counters go up on one side of the rule, but not the response side. I’m hoping someone can take a look at my config here to point out what I’m doing wrong.

# 2024-12-11 09:52:57 by RouterOS 7.16.2
# software id = TCVA-I1D0
#
# model = RB3011UiAS
# serial number = E7EA0EAD4BBB
/interface bridge
add admin-mac=2C:C8:1B:F5:F2:E9 auto-mac=no comment=defconf name=bridge
/interface ethernet
set [ find default-name=ether1 ] comment=wan-1
set [ find default-name=ether2 ] comment=wan-2
set [ find default-name=ether3 ] comment=wan-3
set [ find default-name=ether4 ] comment=wan-4
set [ find default-name=ether5 ] comment=client02-internal-network
set [ find default-name=ether6 ] comment=client01-internal-network
set [ find default-name=ether7 ] comment=client01-proxmox-internal
set [ find default-name=ether8 ] comment=client01-proxmox-hosted
set [ find default-name=ether10 ] comment=cellular-failover
/interface list
add comment=defconf name=WAN
add comment=defconf name=LAN
/ip pool
add name=default-dhcp ranges=192.168.88.10-192.168.88.254
add name=client01-internal ranges=172.16.200.10-172.16.200.254
add name=client02-internal ranges=192.168.1.10-192.168.1.254
add name=proxmox-network ranges=10.0.0.10-10.0.0.254
add name=proxmox-internal ranges=172.16.30.10-172.16.30.50
/ip dhcp-server
add address-pool=default-dhcp interface=bridge name=defconf
add add-arp=yes address-pool=client01-internal interface=ether6 name=\
    client01-internal
add address-pool=client02-internal interface=ether5 name=\
    client02-internal
add add-arp=yes address-pool=proxmox-internal interface=ether7 name=\
    proxmox-internal
/port
set 0 name=serial0
/routing table
add disabled=no fib name=WAN-1
add disabled=no fib name=WAN-2
add disabled=no fib name=WAN-3
add disabled=no fib name=WAN-4
/zerotier
set zt1 comment="ZeroTier Central controller - https://my.zerotier.com/" \
    name=zt1 port=9993
/disk settings
set auto-media-interface=bridge auto-media-sharing=yes auto-smb-sharing=yes
/interface bridge port
add bridge=bridge comment=defconf interface=ether9
add bridge=bridge comment=defconf interface=sfp1
/ip firewall connection tracking
set enabled=yes udp-stream-timeout=10m udp-timeout=10m
/ip neighbor discovery-settings
set discover-interface-list=LAN
/interface list member
add comment=defconf interface=bridge list=LAN
add comment=defconf interface=ether1 list=WAN
add interface=ether2 list=WAN
add interface=ether3 list=WAN
add interface=ether4 list=WAN
add interface=ether6 list=LAN
add interface=ether5 list=LAN
add interface=ether7 list=LAN
add interface=ether8 list=LAN
add comment=cellular-backup interface=ether10 list=WAN
/ip address
add address=192.168.88.1/24 comment=defconf interface=bridge network=\
    192.168.88.0
add address=172.16.200.1/24 comment=client01-internal interface=ether6 \
    network=172.16.200.0
add address=192.168.1.1/24 comment=client02-internal interface=ether5 \
    network=192.168.1.0
add address=172.16.30.1/24 comment=proxmox-internal interface=ether7 network=\
    172.16.30.0
add address=10.0.0.1/24 comment=proxmox-hosted interface=ether8 network=\
    10.0.0.0
/ip cloud
set ddns-enabled=yes ddns-update-interval=5m
/ip dhcp-client
add add-default-route=no comment=client01-wan interface=ether1
add add-default-route=no comment=spare-wan interface=ether2
add add-default-route=no comment=spare-wan interface=ether3
add add-default-route=no comment=client02-wan interface=ether4
add add-default-route=no comment=cellular-backup interface=ether10
/ip dhcp-server lease
add address=192.168.1.11 client-id=1:c4:5a:b1:9e:84:2a mac-address=\
    C4:5A:B1:9E:84:2A server=client02-internal
add address=192.168.1.12 client-id=1:0:15:5d:58:f3:0 mac-address=\
    00:15:5D:58:F3:00 server=client02-internal
add address=172.16.200.30 client-id=1:78:2b:cb:32:61:0 mac-address=\
    78:2B:CB:32:61:00 server=client01-internal
add address=172.16.200.34 client-id=1:90:b1:1c:14:70:6e mac-address=\
    90:B1:1C:14:70:6E server=client01-internal
add address=192.168.1.100 client-id=1:0:20:6b:7e:10:81 mac-address=\
    00:20:6B:7E:10:81 server=client02-internal
add address=172.16.30.48 client-id=1:fa:6e:dd:63:fa:5f mac-address=\
    FA:6E:DD:63:FA:5F server=proxmox-internal
/ip dhcp-server network
add address=10.0.0.0/24 comment=proxmox-hosted dns-server=8.8.8.8 gateway=\
    10.0.0.1
add address=172.16.30.0/24 comment=proxmox-internal dns-server=8.8.8.8 \
    gateway=172.16.30.1
add address=172.16.200.0/24 comment=client01-internal dns-server=\
    8.8.8.8,1.1.1.1 gateway=172.16.200.1 netmask=24
add address=192.168.1.0/24 comment=client02-internal dns-server=\
    192.168.1.11 gateway=192.168.1.1 netmask=24
add address=192.168.88.0/24 comment=defconf dns-server=192.168.88.1 gateway=\
    192.168.88.1
/ip dns
set servers=8.8.8.8
/ip dns static
add address=192.168.88.1 comment=defconf name=router.lan type=A
/ip firewall filter
add action=accept chain=forward comment=\
    "client01-internal <--> proxmox-internal" dst-address=172.16.30.0/24 log=\
    yes log-prefix="To-30: " src-address=172.16.200.0/24
add action=accept chain=forward comment=\
    "proxmox-internal <--> client01-internal" dst-address=172.16.200.0/24 log=\
    yes log-prefix="To-200: " src-address=172.16.30.0/24
add action=accept chain=input comment=zerotier in-interface=zerotier1
add action=accept chain=forward comment=zerotier in-interface=zerotier1
add action=accept chain=input comment=\
    "defconf: accept established,related,untracked" connection-state=\
    established,related,untracked
add action=drop chain=input comment="defconf: drop invalid" connection-state=\
    invalid
add action=accept chain=input comment="defconf: accept ICMP" disabled=yes \
    protocol=icmp
add action=accept chain=input comment=\
    "defconf: accept to local loopback (for CAPsMAN)" dst-address=127.0.0.1
add action=drop chain=input comment="defconf: drop all not coming from LAN" \
    in-interface-list=!LAN
add action=accept chain=forward comment="defconf: accept in ipsec policy" \
    ipsec-policy=in,ipsec
add action=accept chain=forward comment="defconf: accept out ipsec policy" \
    ipsec-policy=out,ipsec
add action=fasttrack-connection chain=forward comment="defconf: fasttrack" \
    connection-state=established,related disabled=yes hw-offload=yes
add action=accept chain=forward comment=\
    "defconf: accept established,related, untracked" connection-state=\
    established,related,untracked
add action=drop chain=forward comment="defconf: drop invalid" \
    connection-state=invalid
add action=drop chain=forward comment=\
    "defconf: drop all from WAN not DSTNATed" connection-nat-state=!dstnat \
    connection-state=new in-interface-list=WAN
/ip firewall nat
add action=masquerade chain=srcnat comment="defconf: masquerade" \
    ipsec-policy=out,none out-interface-list=WAN
/ip firewall service-port
set sip disabled=yes
/ip ipsec profile
set [ find default=yes ] dpd-interval=2m dpd-maximum-failures=5
/ip route
add check-gateway=ping disabled=no distance=10 dst-address=0.0.0.0/0 gateway=\
    66.183.191.1%ether1 routing-table=main suppress-hw-offload=no
add check-gateway=ping disabled=no distance=10 dst-address=0.0.0.0/0 gateway=\
    66.183.191.1%ether4 routing-table=WAN-4 scope=30 suppress-hw-offload=no \
    target-scope=10
add check-gateway=ping disabled=no distance=10 dst-address=0.0.0.0/0 gateway=\
    66.183.191.1%ether1 routing-table=WAN-1 scope=30 suppress-hw-offload=no \
    target-scope=10
add check-gateway=ping disabled=no distance=10 dst-address=0.0.0.0/0 gateway=\
    66.183.191.1%ether2 routing-table=WAN-2 scope=30 suppress-hw-offload=no \
    target-scope=10
add check-gateway=ping disabled=no distance=10 dst-address=0.0.0.0/0 gateway=\
    66.183.191.1%ether3 routing-table=WAN-3 scope=30 suppress-hw-offload=no \
    target-scope=10
add check-gateway=ping disabled=no distance=1 dst-address=172.16.30.0/24 \
    gateway=ether7 routing-table=WAN-1 suppress-hw-offload=no
add check-gateway=ping disabled=no distance=1 dst-address=172.16.200.0/24 \
    gateway=ether6 routing-table=WAN-1 suppress-hw-offload=no
add check-gateway=ping disabled=no distance=20 dst-address=0.0.0.0/0 gateway=\
    10.100.0.1%ether10 routing-table=main suppress-hw-offload=no
add check-gateway=ping disabled=no distance=1 dst-address=172.16.30.0/24 \
    gateway=ether7 routing-table=WAN-2 suppress-hw-offload=no
add check-gateway=ping disabled=no distance=1 dst-address=172.16.200.0/24 \
    gateway=ether6 routing-table=WAN-2 suppress-hw-offload=no
/ip service
set www address=172.16.200.0/24,0.0.0.0/0,172.25.0.0/16
set winbox address=0.0.0.0/0
/ipv6 firewall address-list
add address=::/128 comment="defconf: unspecified address" list=bad_ipv6
add address=::1/128 comment="defconf: lo" list=bad_ipv6
add address=fec0::/10 comment="defconf: site-local" list=bad_ipv6
add address=::ffff:0.0.0.0/96 comment="defconf: ipv4-mapped" list=bad_ipv6
add address=::/96 comment="defconf: ipv4 compat" list=bad_ipv6
add address=100::/64 comment="defconf: discard only " list=bad_ipv6
add address=2001:db8::/32 comment="defconf: documentation" list=bad_ipv6
add address=2001:10::/28 comment="defconf: ORCHID" list=bad_ipv6
add address=3ffe::/16 comment="defconf: 6bone" list=bad_ipv6
/ipv6 firewall filter
add action=accept chain=input comment=\
    "defconf: accept established,related,untracked" connection-state=\
    established,related,untracked
add action=drop chain=input comment="defconf: drop invalid" connection-state=\
    invalid
add action=accept chain=input comment="defconf: accept ICMPv6" protocol=\
    icmpv6
add action=accept chain=input comment="defconf: accept UDP traceroute" \
    dst-port=33434-33534 protocol=udp
add action=accept chain=input comment=\
    "defconf: accept DHCPv6-Client prefix delegation." dst-port=546 protocol=\
    udp src-address=fe80::/10
add action=accept chain=input comment="defconf: accept IKE" dst-port=500,4500 \
    protocol=udp
add action=accept chain=input comment="defconf: accept ipsec AH" protocol=\
    ipsec-ah
add action=accept chain=input comment="defconf: accept ipsec ESP" protocol=\
    ipsec-esp
add action=accept chain=input comment=\
    "defconf: accept all that matches ipsec policy" ipsec-policy=in,ipsec
add action=drop chain=input comment=\
    "defconf: drop everything else not coming from LAN" in-interface-list=\
    !LAN
add action=accept chain=forward comment=\
    "defconf: accept established,related,untracked" connection-state=\
    established,related,untracked
add action=drop chain=forward comment="defconf: drop invalid" \
    connection-state=invalid
add action=drop chain=forward comment=\
    "defconf: drop packets with bad src ipv6" src-address-list=bad_ipv6
add action=drop chain=forward comment=\
    "defconf: drop packets with bad dst ipv6" dst-address-list=bad_ipv6
add action=drop chain=forward comment="defconf: rfc4890 drop hop-limit=1" \
    hop-limit=equal:1 protocol=icmpv6
add action=accept chain=forward comment="defconf: accept ICMPv6" protocol=\
    icmpv6
add action=accept chain=forward comment="defconf: accept HIP" protocol=139
add action=accept chain=forward comment="defconf: accept IKE" dst-port=\
    500,4500 protocol=udp
add action=accept chain=forward comment="defconf: accept ipsec AH" protocol=\
    ipsec-ah
add action=accept chain=forward comment="defconf: accept ipsec ESP" protocol=\
    ipsec-esp
add action=accept chain=forward comment=\
    "defconf: accept all that matches ipsec policy" ipsec-policy=in,ipsec
add action=drop chain=forward comment=\
    "defconf: drop everything else not coming from LAN" in-interface-list=\
    !LAN
/lcd
set color-scheme=light
/routing rule
add action=lookup disabled=no interface=ether6 table=WAN-1
add action=lookup disabled=no interface=ether7 table=WAN-2
add action=lookup-only-in-table disabled=no interface=ether8 table=WAN-3
add action=lookup-only-in-table disabled=no interface=ether5 table=WAN-4
/system clock
set time-zone-name=America/Vancouver
/system note
set show-at-login=no
/tool mac-server
set allowed-interface-list=LAN
/tool mac-server mac-winbox
set allowed-interface-list=LAN
2

I think your missing a /routing/rule that says traffic NOT to internet should use “main” routing table (i.e. the 0.0.0.0/0 route, which is represented as min-prefix=0). This rule should be list FIRST - before the WAN-per-interface route rules - otherwise local traffic is ALSO route to WAN’s default gateway, which won’t work…

/routing/rule add min-prefix=0 action=lookup table=main

There may be other issues since I didn’t disect the rest of the config, but the above (or alternative multiple rules to with “dst-address= table=main” — but the min-prefix=0 should cover all your local subnets in one line)

3

Right on! That was exactly what I needed. Pinging between networks as expected now. Thank you!

4

Good to hear. It’s a bit confusing at first… FWIW rules run from top to bottom, first match wins. So when you create a broad rule (i.e. all traffic on an interface), it does mean send ALL traffic, including local ones, to the specific routing table=. Why adding your local routes as rule to “main” is needed (which the min-prefix=0 rule does) - since each WAN routing table does NOT have any local routes defined, anything in that goes to WAN table goes that specific WAN’s gateway… But it’s not obvious at first.