Dual WAN, same Gateway, no need for load balancing or failover, just specify which vlans use which wan port

dalben · November 23, 2023, 10:43am

UPDATED

I have dual 1 Gb WAN links from my provider. I have a single ONT from my ISP, with two ethernet cables coming from the ONT to my hAP-AX2. The 2 connections are on the same /18 address space from the ISP and as such both have the same gateway address. The WAN IP addresses are DHCPd and they can change if I disable a wan port for a couple of hours.

There is no need for load balancing nor failover. It’s a single modem from the ISP and if something goes down at the ISP end, it’ll most likely take down both circuits. Same with my ONT

I just want to predetermine which VLANs use which wan port. For now the need is 2 VLANs using wan2 and the rest using wan1

From everything I have read, and all examples I have seen, this use case seems left out. I’ve tried a few different examples and while nothing broke, the predetermined routing never worked. wan2 is currently disabled because using both causes some issues with some services I am hosting that need to be accessed from the internet. (the hosting is managed through a DDNS that updates whenever it detects a change of IP on wan1 at the moment.)

anav · November 23, 2023, 10:52am

http://forum.mikrotik.com/t/failover-between-2-isps-using-gateways-with-same-ip-was-nat-traffic-to-vrf/170381/1

dalben · November 23, 2023, 12:46pm

Thanks. I had seen that but with my limited network knowledge I was 1) intimidated, 2) scared and 3) I read it to me it requires the WAN facing IP addresses to be static which isn’t a guarantee in my case as they are DHCP clients.

But if I have read 3) wrong. I’ll go back and try to understand what it all means

dalben · December 6, 2023, 4:17am

Bumping this again and added some clarity of what I am trying to achieve. In particular I don’t need load balancing or failover, I just need to predetermine which VLANs use which wan port.

anav · December 6, 2023, 12:49pm

Post your config to show what you have setup so far…
/export file=anynameyouwish (minus router serial number, public WANIP information, keys etc.)

dalben · December 6, 2023, 2:09pm

Export attached. It should be clean of from of my attempts over the last week to get it going. Once something doesn’t work and I’ve tried fiddling, I remove everything I added. Though who knows what I have missed.

# 2023-12-06 22:00:50 by RouterOS 7.12
# software id = U2BP-38L2
# model = C52iG-5HaxD2HaxD
/interface bridge
add comment="10.10.0.1 - all Infra on this subnet" name=BR1 protocol-mode=\
    none vlan-filtering=yes
/interface ethernet
set [ find default-name=ether3 ] comment=TRUNK
set [ find default-name=ether4 ] comment="Access Port - VL110"
set [ find default-name=ether5 ] comment="MGMT Port" name=ether5-mgmt
set [ find default-name=ether1 ] l2mtu=1586 mtu=1586 name=wan1 poe-out=off
set [ find default-name=ether2 ] l2mtu=1586 mtu=1586 name=wan2
/interface vlan
add comment=RaD interface=BR1 name=VL110-RaD vlan-id=110
add comment=KaL interface=BR1 name=VL120-KaL vlan-id=120
add comment=IoT interface=BR1 name=VL130-IoT vlan-id=130
add comment=Docker interface=BR1 name=VL140-DOCK vlan-id=140
add comment=Work interface=BR1 name=VL150-WORK vlan-id=150
add comment=Guest interface=BR1 name=VL190-GUEST vlan-id=190
/interface list
add comment="wan1 & wan2" name=WAN
add comment="All VLAN & LAN" name=LAN
add comment="All VLANS" name=VLAN
add name=MGMT
add comment="VLAN for wan2" name=WORK
add comment="VLAN & LAN for wan1" name=RAD
/ip pool
add comment=INFRA name=VL100-POOL ranges=10.10.0.80-10.10.0.254
add comment=RaD name=VL110-POOL ranges=10.10.1.10-10.10.1.254
add comment=KaL name=VL120-POOL ranges=10.10.2.10-10.10.2.254
add comment=IoT name=VL130-POOL ranges=10.10.3.10-10.10.3.254
add comment=Docker name=VL140-POOL ranges=10.10.4.135-10.10.4.254
add comment=WORK name=VL150-POOL ranges=10.10.5.10-10.10.5.254
add comment=GUEST name=VL190-POOL ranges=10.10.9.10-10.10.9.254
add comment=MGMT name=VL1010-POOL ranges=10.10.10.100-10.10.10.254
add comment=ACCESS name=ether5-POOL ranges=10.1.1.10-10.1.1.254
add comment=LAN name=LAN-POOL ranges=10.0.0.100-10.0.0.254
/ip dhcp-server
add address-pool=VL110-POOL comment="10.10.1.0 - wan1" interface=VL110-RaD \
    name=VL110-DHCP
add address-pool=VL120-POOL comment="10.10.2.0 - wan1" interface=VL120-KaL \
    name=VL120-DHCP
add address-pool=VL130-POOL comment="10.10.3.0 - wan1" interface=VL130-IoT \
    name=VL130-DHCP
add address-pool=VL140-POOL comment=\
    "10.10.4.0 - wan1 for now, probably wan2 later" interface=VL140-DOCK name=\
    VL140-DHCP
add address-pool=VL150-POOL comment="10.10.5.0 - wan2" interface=VL150-WORK \
    name=VL150-DHCP
add address-pool=VL190-POOL comment="10.10.9.0 - wan2" interface=VL190-GUEST \
    name=VL190-DHCP
add address-pool=ether5-POOL comment="10.1.1.0 - SNAFU port" interface=\
    ether5-mgmt name=ether5-DHCP
add address-pool=VL100-POOL comment="10.10.0.0 - Moved to Bridge to keep UNIFI\
    \_Controller, APs and Swittches happy" interface=BR1 name=LAN-DHCP
/queue interface
set ether3 queue=ethernet-default
/interface bridge port
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    ether4 pvid=110
add bridge=BR1 interface=ether3
add bridge=BR1 interface=wifi-rad-5 pvid=1000
add bridge=BR1 interface=wifi-rad-2 pvid=1000
/ip neighbor discovery-settings
set discover-interface-list=LAN
/interface bridge vlan
add bridge=BR1 tagged=BR1,ether3 vlan-ids=100
add bridge=BR1 tagged=BR1,ether3 untagged=ether4 vlan-ids=110
add bridge=BR1 tagged=BR1,ether3 vlan-ids=120
add bridge=BR1 tagged=BR1,ether3 vlan-ids=130
add bridge=BR1 tagged=BR1,ether3 vlan-ids=140
add bridge=BR1 tagged=BR1,ether3 vlan-ids=150
add bridge=BR1 tagged=BR1,ether3 vlan-ids=190
/interface list member
add interface=VL150-WORK list=MGMT
add interface=VL110-RaD list=MGMT
add interface=ether5-mgmt list=MGMT
add interface=wan1 list=WAN
add interface=VL110-RaD list=VLAN
add interface=VL120-KaL list=VLAN
add interface=VL130-IoT list=VLAN
add interface=VL140-DOCK list=VLAN
add interface=VL150-WORK list=VLAN
add interface=VL190-GUEST list=VLAN
add interface=VL110-RaD list=LAN
add interface=VL120-KaL list=LAN
add interface=VL130-IoT list=LAN
add interface=VL140-DOCK list=LAN
add interface=VL150-WORK list=LAN
add interface=VL190-GUEST list=LAN
add interface=wan2 list=WAN
add interface=BR1 list=LAN
add interface=BR1 list=RAD
add interface=VL110-RaD list=RAD
add interface=VL120-KaL list=RAD
add interface=VL130-IoT list=RAD
add interface=VL140-DOCK list=RAD
add interface=VL150-WORK list=WORK
add interface=VL190-GUEST list=WORK
/ip address
add address=10.1.1.1/24 comment=SNAFU interface=ether5-mgmt network=10.1.1.0
add address=10.10.1.1/24 interface=VL110-RaD network=10.10.1.0
add address=10.10.2.1/24 interface=VL120-KaL network=10.10.2.0
add address=10.10.3.1/24 interface=VL130-IoT network=10.10.3.0
add address=10.10.4.1/24 interface=VL140-DOCK network=10.10.4.0
add address=10.10.5.1/24 interface=VL150-WORK network=10.10.5.0
add address=10.10.9.1/24 interface=VL190-GUEST network=10.10.9.0
add address=10.10.0.1/24 interface=BR1 network=10.10.0.0
/ip cloud
set update-time=no
/ip dhcp-client
add comment="ONT Port 1 - 1Gb" interface=wan1 use-peer-dns=no
add comment="ONT Port 2 - 1Gb" interface=wan2 use-peer-dns=no
/ip dhcp-server lease
/ip dns
set allow-remote-requests=yes servers=10.10.0.5,10.10.0.7
/ip dns static
add address=8.8.8.8 comment=defconf name=router.lan
/ip firewall address-list
add address=10.10.0.0/24 list=wan1-list
add address=10.10.1.0/24 list=wan1-list
add address=10.10.2.0/24 list=wan1-list
add address=10.10.3.0/24 list=wan1-list
add address=10.10.4.0/24 list=wan1-list
add address=10.10.5.0/24 list=wan2-list
add address=10.10.9.0/24 list=wan2-list
/ip firewall filter
add action=accept chain=input comment="Allow Established,Related,Untracked" \
    connection-state=established,related,untracked
add action=drop chain=input comment="DROP - Invalid" connection-state=invalid
add action=accept chain=input comment="Allow ICMP" protocol=icmp
add action=accept chain=input comment="Allow LAN DNS queries-UDP" dst-port=53 \
    in-interface-list=LAN protocol=udp
add action=accept chain=input comment="Allow LAN DNS queries - TCP" dst-port=\
    53 in-interface-list=LAN protocol=tcp
add action=accept chain=input comment="Allow Winbox & SSH" dst-port=8291,22 \
    protocol=tcp src-port=8291,22
add action=accept chain=input comment=\
    "** Winbox Access (Full access to Router)" in-interface=ether5-mgmt
add action=accept chain=input comment=\
    "** Winbox Access (Full access to Router)" in-interface-list=MGMT
add action=drop chain=input comment=DROP
add action=fasttrack-connection chain=forward comment=FastTrack \
    connection-state=established,related hw-offload=yes
add action=accept chain=forward comment="Allow Established,Related,Untracked" \
    connection-state=established,related,untracked
add action=drop chain=forward comment="DROP - Invalid" connection-state=\
    invalid
add action=accept chain=forward comment="Allow IPSec policy - IN" \
    ipsec-policy=in,ipsec
add action=accept chain=forward comment="Allow IPSec policy - OUT" \
    ipsec-policy=out,ipsec
add action=accept chain=forward comment="ALLOW - Internet Traffic" \
    in-interface-list=LAN out-interface-list=WAN
add action=accept chain=forward comment="ALLOW - Port Forwarding" \
    connection-nat-state=dstnat
add action=accept chain=forward comment="ALLOW - VLAN inter-VLAN routing" \
    connection-state=new in-interface-list=LAN
add action=accept chain=forward comment="Accept all that is DST NATed" \
    connection-nat-state=dstnat connection-state=new
add action=drop chain=forward comment="DROP - All from WAN not DSTNATed" \
    connection-nat-state=!dstnat connection-state=new in-interface-list=WAN
add action=drop chain=forward comment=DROP
/ip firewall nat
add action=masquerade chain=srcnat comment=radgw ipsec-policy=out,none \
    out-interface=wan1
add action=masquerade chain=srcnat comment=workgw ipsec-policy=out,none \
    out-interface=wan2
add action=dst-nat chain=dstnat comment="NGINX 80 - 80" dst-port=80 \
    in-interface=wan1 log-prefix="NGNIX 80" protocol=tcp to-addresses=\
    10.10.4.10 to-ports=80
add action=dst-nat chain=dstnat dst-port=80 in-interface=wan1 protocol=udp \
    to-addresses=10.10.4.10 to-ports=80
add action=dst-nat chain=dstnat comment="NGINX 443 - 80" dst-port=443 \
    in-interface=wan1 protocol=tcp to-addresses=10.10.4.10 to-ports=443
add action=dst-nat chain=dstnat dst-port=443 in-interface=wan1 protocol=udp \
    to-addresses=10.10.4.10 to-ports=443
add action=dst-nat chain=dstnat comment=JF-HTTP dst-port=8096 in-interface=\
    wan1 protocol=tcp to-addresses=10.10.4.40 to-ports=8096
add action=dst-nat chain=dstnat dst-port=8096 in-interface=wan1 protocol=udp \
    to-addresses=10.10.4.40 to-ports=8096
add action=dst-nat chain=dstnat comment=JF-HTTPS dst-port=8920 in-interface=\
    wan1 protocol=tcp to-addresses=10.10.4.40 to-ports=8920
add action=dst-nat chain=dstnat dst-port=8920 in-interface=wan1 protocol=udp \
    to-addresses=10.10.4.40 to-ports=8920
/ip service
set www port=8181
/ip traffic-flow
set interfaces=BR1
/ip upnp
set enabled=yes
/ip upnp interfaces
add interface=BR1 type=internal
add interface=wan1 type=external
add interface=wan2 type=external
/system ntp client
set enabled=yes
/tool mac-server
set allowed-interface-list=MGMT
/tool mac-server mac-winbox
set allowed-interface-list=MGMT

hAP20231206.rsc (9.11 KB)

jbarbieri · December 18, 2023, 12:23pm

Do both the wan ports have the same IP address on them? Or do they at least get different addresses and do you know if they are in the same network? Some providers give you a pppoe which is usually a /32, some just use straight DHCP that is why I am asking.

You should be able to use multiple routing tables and mangle rules to mark the incoming packet from a specific vlan and tag it with a connection mark. Then use the connection marks to tag which routing instance you want to use. Just make sure you also have a gateway set for packets generated from the router itself.

dalben · December 18, 2023, 12:27pm

Different IP but on the same /18 subnet so they have the same gateway, which makes gateway based routing unfeasible.

anav · December 18, 2023, 12:38pm

What does your IP routes table show…covering up enough of public IPs so not known…
Assuming in default routes in ip DHCP you have wan1 a lower distance than WAN2, so that from the get go most traffic will go to WAN1.

Not convinced routing rules will not work.
One needs to ensure the routes work is the gateway fixed and the same for both WANs…

manual entered routes, unchecked add default route in dhcp client.
add check-gateway=ping distance=2 dst-address=0.0.0.0/0 gateway=fixedIP%WAN1 table=main
add check-gateway=ping distance=4 dst-address=0.0.0.0/0 gateway=fixedIP%WAN2 table=main
add dst-address=0.0.0.0/0 gateway=fixedIP%WAN2 table=useWAN2

/routing table add fib name=useWAN2
/routing rules
add dst-address=anyother subnet? (that needs to be reached by the two vlans) action=lookup-only-in-table table=main
add dst-address=anyother subnet? (that needs to be reached by the two vlans) action=lookup-only-in-table table=main
add src-address=vlansubnet1 action=lookup table=useWAN2
add src-address=vlansubnet2 action=lookup table=useWAN2

jbarbieri · December 18, 2023, 12:40pm

I think that is where using different routing tables could help. You would need a dhcp script to put the gateway for each port into a different routing table, then act on those routing tables.

dalben · December 18, 2023, 11:23pm

Thanks.
There is no guarantee that the WAN gateway IP will remain the same. Over the couple of years I’ve had the dual links I do see the subnets change, but not often. WAN IP changes frequently if I disable a WAN interface or if I unplug for a few hours.

But I’ll try the Routing tables method again with the gateway IP I have now and see if I can get it working. Once that is working I’ll looking at a script to dynamically alter the gateway IP in the routing tables if it detects a change.

dalben · December 22, 2023, 6:27am

That didn’t seem to work. My devices are still using both WAN interfaces. Here’s an export of the config. I’m not sure if I followed the instructions properly.

/interface bridge
add comment="10.10.0.1 - all Infra on this subnet" name=BR1 port-cost-mode=\
    short protocol-mode=none vlan-filtering=yes
/interface ethernet
set [ find default-name=ether3 ] comment=TRUNK
set [ find default-name=ether4 ] comment="Access Port - VL110"
set [ find default-name=ether5 ] comment="MGMT Port" name=ether5-mgmt
set [ find default-name=ether1 ] l2mtu=1586 mtu=1586 name=wan1 poe-out=off
set [ find default-name=ether2 ] l2mtu=1586 mtu=1586 name=wan2
/interface vlan
add comment=RaD interface=BR1 name=VL110-RaD vlan-id=110
add comment=KaL interface=BR1 name=VL120-KaL vlan-id=120
add comment=IoT interface=BR1 name=VL130-IoT vlan-id=130
add comment=Docker interface=BR1 name=VL140-DOCK vlan-id=140
add comment=Work interface=BR1 name=VL150-WORK vlan-id=150
add comment=Guest interface=BR1 name=VL190-GUEST vlan-id=190
/interface list
add comment="wan1 & wan2" name=WAN
add comment="All VLAN & LAN" name=LAN
add comment="All VLANS" name=VLAN
add name=MGMT
add comment="VLAN for wan2" name=WORK
add comment="VLAN & LAN for wan1" name=RAD
/ip dhcp-server
add address-pool=VL110-POOL comment="10.10.1.0 - wan1" interface=VL110-RaD \
    name=VL110-DHCP
add address-pool=VL120-POOL comment="10.10.2.0 - wan1" interface=VL120-KaL \
    name=VL120-DHCP
add address-pool=VL130-POOL comment="10.10.3.0 - wan1" interface=VL130-IoT \
    name=VL130-DHCP
add address-pool=VL140-POOL comment=\
    "10.10.4.0 - wan1 for now, probaby wan2 later" interface=VL140-DOCK name=\
    VL140-DHCP
add address-pool=VL150-POOL comment="10.10.5.0 - wan2" interface=VL150-WORK \
    name=VL150-DHCP
add address-pool=VL190-POOL comment="10.10.9.0 - wan2" interface=VL190-GUEST \
    name=VL190-DHCP
add address-pool=ether5-POOL comment="10.1.1.0 - SNAFU port" interface=\
    ether5-mgmt name=ether5-DHCP
add address-pool=VL100-POOL comment="10.10.0.0 - Moved to Bridge to keep UNIFI\
    \_Controller, APs and Swittches happy" interface=BR1 name=LAN-DHCP
/queue interface
set ether3 queue=ethernet-default
/routing table
add disabled=no fib name=useWAN2
/interface bridge port
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    ether4 internal-path-cost=10 path-cost=10 pvid=110
add bridge=BR1 interface=ether3 internal-path-cost=10 path-cost=10
add bridge=BR1 interface=wifi-rad-5 internal-path-cost=10 path-cost=10 pvid=\
    1000
add bridge=BR1 interface=wifi-rad-2 internal-path-cost=10 path-cost=10 pvid=\
    1000
/ip neighbor discovery-settings
set discover-interface-list=LAN
/interface bridge vlan
add bridge=BR1 tagged=BR1,ether3 vlan-ids=100
add bridge=BR1 tagged=BR1,ether3 untagged=ether4 vlan-ids=110
add bridge=BR1 tagged=BR1,ether3 vlan-ids=120
add bridge=BR1 tagged=BR1,ether3 vlan-ids=130
add bridge=BR1 tagged=BR1,ether3 vlan-ids=140
add bridge=BR1 tagged=BR1,ether3 vlan-ids=150
add bridge=BR1 tagged=BR1,ether3 vlan-ids=190
/interface list member
add interface=VL150-WORK list=MGMT
add interface=VL110-RaD list=MGMT
add interface=ether5-mgmt list=MGMT
add interface=wan1 list=WAN
add interface=VL110-RaD list=VLAN
add interface=VL120-KaL list=VLAN
add interface=VL130-IoT list=VLAN
add interface=VL140-DOCK list=VLAN
add interface=VL150-WORK list=VLAN
add interface=VL190-GUEST list=VLAN
add interface=VL110-RaD list=LAN
add interface=VL120-KaL list=LAN
add interface=VL130-IoT list=LAN
add interface=VL140-DOCK list=LAN
add interface=VL150-WORK list=LAN
add interface=VL190-GUEST list=LAN
add interface=wan2 list=WAN
add interface=BR1 list=LAN
add interface=BR1 list=RAD
add interface=VL110-RaD list=RAD
add interface=VL120-KaL list=RAD
add interface=VL130-IoT list=RAD
add interface=VL140-DOCK list=RAD
add interface=VL150-WORK list=WORK
add interface=VL190-GUEST list=WORK
/ip address
add address=10.1.1.1/24 comment=SNAFU interface=ether5-mgmt network=10.1.1.0
add address=10.10.1.1/24 interface=VL110-RaD network=10.10.1.0
add address=10.10.2.1/24 interface=VL120-KaL network=10.10.2.0
add address=10.10.3.1/24 interface=VL130-IoT network=10.10.3.0
add address=10.10.4.1/24 interface=VL140-DOCK network=10.10.4.0
add address=10.10.5.1/24 interface=VL150-WORK network=10.10.5.0
add address=10.10.9.1/24 interface=VL190-GUEST network=10.10.9.0
add address=10.10.0.1/24 interface=BR1 network=10.10.0.0
/ip cloud
set update-time=no
/ip dhcp-client
add comment="ONT Port 1 - 1Gb" interface=wan1 use-peer-dns=no
add comment="ONT Port 2 - 1Gb" interface=wan2 use-peer-dns=no
/ip dns
set allow-remote-requests=yes servers=10.10.0.5,10.10.0.7
/ip dns static
add address=8.8.8.8 comment=defconf name=router.lan
/ip firewall address-list
add address=10.10.0.0/24 list=wan1-list
add address=10.10.1.0/24 list=wan1-list
add address=10.10.2.0/24 list=wan1-list
add address=10.10.3.0/24 list=wan1-list
add address=10.10.4.0/24 list=wan1-list
add address=10.10.5.0/24 list=wan2-list
add address=10.10.9.0/24 list=wan2-list
/ip firewall filter
add action=accept chain=input comment="Allow Established,Related,Untracked" \
    connection-state=established,related,untracked
add action=drop chain=input comment="DROP - Invalid" connection-state=invalid
add action=accept chain=input comment="Allow ICMP" protocol=icmp
add action=accept chain=input comment="Allow LAN DNS queries-UDP" dst-port=53 \
    in-interface-list=LAN protocol=udp
add action=accept chain=input comment="Allow LAN DNS queries - TCP" dst-port=\
    53 in-interface-list=LAN protocol=tcp
add action=accept chain=input comment="Allow Winbox & SSH" dst-port=8291,22 \
    protocol=tcp src-port=8291,22
add action=accept chain=input comment=\
    "** Winbox Access (Full access to Router)" in-interface=ether5-mgmt
add action=accept chain=input comment=\
    "** Winbox Access (Full access to Router)" in-interface-list=MGMT
add action=drop chain=input comment=DROP
add action=fasttrack-connection chain=forward comment=FastTrack \
    connection-state=established,related hw-offload=yes
add action=accept chain=forward comment="Allow Established,Related,Untracked" \
    connection-state=established,related,untracked
add action=drop chain=forward comment="DROP - Invalid" connection-state=\
    invalid
add action=accept chain=forward comment="Allow IPSec policy - IN" \
    ipsec-policy=in,ipsec
add action=accept chain=forward comment="Allow IPSec policy - OUT" \
    ipsec-policy=out,ipsec
add action=accept chain=forward comment="ALLOW - Internet Traffic" \
    in-interface-list=LAN out-interface-list=WAN
add action=accept chain=forward comment="ALLOW - Port Forwarding" \
    connection-nat-state=dstnat
add action=accept chain=forward comment="ALLOW - VLAN inter-VLAN routing" \
    connection-state=new in-interface-list=LAN
add action=accept chain=forward comment="Accept all that is DST NATed" \
    connection-nat-state=dstnat connection-state=new
add action=drop chain=forward comment="DROP - All from WAN not DSTNATed" \
    connection-nat-state=!dstnat connection-state=new in-interface-list=WAN
add action=drop chain=forward comment=DROP
/ip firewall nat
add action=masquerade chain=srcnat comment=radgw ipsec-policy=out,none \
    out-interface=wan1
add action=masquerade chain=srcnat comment=workgw ipsec-policy=out,none \
    out-interface=wan2
add action=dst-nat chain=dstnat comment="NGINX 80 - 80" dst-port=80 \
    in-interface=wan1 log-prefix="NGNIX 80" protocol=tcp to-addresses=\
    10.10.4.10 to-ports=80
add action=dst-nat chain=dstnat dst-port=80 in-interface=wan1 protocol=udp \
    to-addresses=10.10.4.10 to-ports=80
add action=dst-nat chain=dstnat comment="NGINX 443 - 80" dst-port=443 \
    in-interface=wan1 protocol=tcp to-addresses=10.10.4.10 to-ports=443
add action=dst-nat chain=dstnat dst-port=443 in-interface=wan1 protocol=udp \
    to-addresses=10.10.4.10 to-ports=443
add action=dst-nat chain=dstnat comment=JF-HTTP dst-port=8096 in-interface=\
    wan1 protocol=tcp to-addresses=10.10.4.40 to-ports=8096
add action=dst-nat chain=dstnat dst-port=8096 in-interface=wan1 protocol=udp \
    to-addresses=10.10.4.40 to-ports=8096
add action=dst-nat chain=dstnat comment=JF-HTTPS dst-port=8920 in-interface=\
    wan1 protocol=tcp to-addresses=10.10.4.40 to-ports=8920
add action=dst-nat chain=dstnat dst-port=8920 in-interface=wan1 protocol=udp \
    to-addresses=10.10.4.40 to-ports=8920
/ip route
add check-gateway=ping disabled=no distance=4 dst-address=0.0.0.0/0 gateway=\
    xx.xxx.xxx.1%wan2 pref-src="" routing-table=main scope=30 \
    suppress-hw-offload=no target-scope=10
add check-gateway=ping distance=2 dst-address=0.0.0.0/0 gateway=\
    xx.xxx.xxx.1%wan1
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway=xx.xxx.xxx.1%wan2 \
    pref-src="" routing-table=useWAN2 scope=30 suppress-hw-offload=no \
    target-scope=10
/ip service
set www port=8181
/ip traffic-flow
set interfaces=BR1
/ip upnp
set enabled=yes
/ip upnp interfaces
add interface=BR1 type=internal
add interface=wan1 type=external
add interface=wan2 type=external
/tool mac-server
set allowed-interface-list=MGMT
/tool mac-server mac-winbox
set allowed-interface-list=MGMT
/tool sniffer
set filter-interface=wan2

anav · December 22, 2023, 8:47pm

You are missing the routing rules??

dalben · December 23, 2023, 8:56am

I’m not sure how that happened. Here they are. For now I am testing with just 1 VLAN that uses 10.10.5.0/24

# 2023-12-23 16:53:24 by RouterOS 7.13
/routing table
add disabled=no fib name=useWAN2
/routing rule
add action=lookup-only-in-table disabled=no dst-address=10.10.5.0/24 table=main
add action=lookup disabled=no src-address=10.10.5.0/24 table=useWAN2

anav · December 24, 2023, 8:52pm

Nonsensical…
The use of the dst address first entry is to ensure any traffic to any other subnets, doesnt go out the tunnel. Since we are forcing anything with a source address of 10…5 out the tunnel.
So in this case the first rule makes no sense to me???

dalben · December 25, 2023, 10:42am

OK, I think I understood what you meant. Is this better?

/routing table
add disabled=no fib name=useWAN2
/routing rule
add action=lookup-only-in-table disabled=no dst-address=10.10.1.0/24 table=\
    main
add action=lookup-only-in-table disabled=no dst-address=10.10.2.0/24 table=\
    main
add action=lookup-only-in-table disabled=no dst-address=10.10.3.0/24 table=\
    main
add action=lookup-only-in-table disabled=no dst-address=10.10.4.0/24 table=\
    main
add action=lookup disabled=no src-address=10.10.5.0/24 table=useWAN2
/ip route
add check-gateway=ping disabled=no distance=4 dst-address=0.0.0.0/0 gateway=\
    27.125.128.1%wan2 pref-src="" routing-table=main scope=30 \
    suppress-hw-offload=no target-scope=10
add check-gateway=ping disabled=no distance=1 dst-address=0.0.0.0/0 gateway=\
    27.125.128.1%wan1 pref-src="" routing-table=main scope=30 \
    suppress-hw-offload=no target-scope=10
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway=27.125.128.1%wan2 \
    pref-src="" routing-table=useWAN2 scope=30 suppress-hw-offload=no \
    target-scope=10

I also noticed I had a couple of dynamic routes that were active and using vrf. Thought they may have been a throw back to a previous failed attempt so I deleted them. So far, using nothing more scientific than going to a whatsmyip website, 10.10.5.0/0 looks like it’s routing to WAN2 and the others to WAN1.

anav · December 25, 2023, 1:01pm

Yup, that is the idea as you need to consider two situations.
a. originating traffic from 10.5 to the other subnets
b. return traffic from 10.5 to those subnets.

For those subnets not supposed to talk to 10.5, they need not be included.
Remember whether or not subnets can talk to each other should be determined in the forward chain firewall rules.
What we are doing in routing rules in ensuring traffic, IF ALLOWED, does not get sucked into the wireguard tunnel.

dalben · December 25, 2023, 1:07pm

Thanks. Slowly isolating or limiting vlans is in the to-do list, after wireguard.

anav · December 25, 2023, 2:28pm

vlans are automatically eliminated by a drop all rule at the end of the forward chain.
Thus all that is required is to make allow rules for what is permitted.

dalben · January 20, 2024, 8:30am

Reviving this thread again.

I realised today that devices that are on the VLANs being redirected to wan2 can no longer use the DNS servers I have setup on my LAN.

wan1 LAN
10.10.0.0

wan1 VLANs
10.10.1.0
10.10.2.0
10.10.3.0

wan2 VLANs
10.10.4.0
10.10.5.0

DNS Servers are on 10.10.0.0.
From a 10.10.4.xx or 10.10.5.xx device, I can’t ping the DNS servers nor is DNS resolution working. Config below. I’ve looked at the firewall rules and I can’t see what could be preventing this. Inter-VLAN routing rule is present. It’s possible the interVLAN routing never worked between vlans assigned to wan1 or wan2. It’s not something I would have tested as the vlan for wan2 was dedicated to work. Any pointers would be most appreciated.

/interface bridge
add comment="10.10.0.1 - all Infra on this subnet" name=BR1 port-cost-mode=\
    short protocol-mode=none vlan-filtering=yes
/interface ethernet
set [ find default-name=ether3 ] comment=TRUNK
set [ find default-name=ether4 ] comment="Access Port - VL110"
set [ find default-name=ether5 ] comment="MGMT Port" name=ether5-mgmt
set [ find default-name=ether1 ] l2mtu=1586 mtu=1586 name=wan1 poe-out=off
set [ find default-name=ether2 ] l2mtu=1586 mtu=1586 name=wan2
/interface vlan
add comment=RaD interface=BR1 name=VL110-RaD vlan-id=110
add comment=KaL interface=BR1 name=VL120-KaL vlan-id=120
add comment=IoT interface=BR1 name=VL130-IoT vlan-id=130
add comment=Docker interface=BR1 name=VL140-DOCK vlan-id=140
add comment=Work interface=BR1 name=VL150-WORK vlan-id=150
add comment=Guest interface=BR1 name=VL190-GUEST vlan-id=190
/interface list
add comment="wan1 & wan2" name=WAN
add comment="All VLAN & LAN" name=LAN
add comment="All VLANS" name=VLAN
add name=MGMT
/ip pool
add comment=INFRA name=VL100-POOL ranges=10.10.0.80-10.10.0.254
add comment=RaD name=VL110-POOL ranges=10.10.1.10-10.10.1.254
add comment=KaL name=VL120-POOL ranges=10.10.2.10-10.10.2.254
add comment=IoT name=VL130-POOL ranges=10.10.3.10-10.10.3.254
add comment=Docker name=VL140-POOL ranges=10.10.4.135-10.10.4.254
add comment=WORK name=VL150-POOL ranges=10.10.5.10-10.10.5.254
add comment=GUEST name=VL190-POOL ranges=10.10.9.10-10.10.9.254
add comment=MGMT name=VL1010-POOL ranges=10.10.10.100-10.10.10.254
add comment=ACCESS name=ether5-POOL ranges=10.1.1.10-10.1.1.254
add comment=LAN name=LAN-POOL ranges=10.0.0.100-10.0.0.254
/ip dhcp-server
add address-pool=VL110-POOL comment="10.10.1.0 - wan1" interface=VL110-RaD \
    lease-time=1d name=VL110-DHCP
add address-pool=VL120-POOL comment="10.10.2.0 - wan1" interface=VL120-KaL \
    lease-time=1d name=VL120-DHCP
add address-pool=VL130-POOL comment="10.10.3.0 - wan1" interface=VL130-IoT \
    lease-time=1d name=VL130-DHCP
add address-pool=VL140-POOL comment=\
    "10.10.4.0 - wan1 for now, probaby wan2 later" interface=VL140-DOCK \
    lease-time=1d name=VL140-DHCP
add address-pool=VL150-POOL comment="10.10.5.0 - wan2" interface=VL150-WORK \
    lease-time=1d name=VL150-DHCP
add address-pool=VL190-POOL comment="10.10.9.0 - wan2" interface=VL190-GUEST \
    lease-time=1d name=VL190-DHCP
add address-pool=ether5-POOL comment="10.1.1.0 - SNAFU port" interface=\
    ether5-mgmt name=ether5-DHCP
add address-pool=VL100-POOL comment="10.10.0.0 - Moved to Bridge to keep UNIFI\
    \_Controller, APs and Swittches happy" interface=BR1 lease-time=1d name=\
    LAN-DHCP
/queue interface
set ether3 queue=ethernet-default
/routing table
add disabled=no fib name=useWAN2
/interface bridge port
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    ether4 internal-path-cost=10 path-cost=10 pvid=110
add bridge=BR1 interface=ether3 internal-path-cost=10 path-cost=10
add bridge=BR1 interface=wifi-rad-5 internal-path-cost=10 path-cost=10 pvid=\
    1000
add bridge=BR1 interface=wifi-rad-2 internal-path-cost=10 path-cost=10 pvid=\
    1000
/ip neighbor discovery-settings
set discover-interface-list=LAN
/interface bridge vlan
add bridge=BR1 tagged=BR1,ether3 vlan-ids=100
add bridge=BR1 tagged=BR1,ether3 untagged=ether4 vlan-ids=110
add bridge=BR1 tagged=BR1,ether3 vlan-ids=120
add bridge=BR1 tagged=BR1,ether3 vlan-ids=130
add bridge=BR1 tagged=BR1,ether3 vlan-ids=140
add bridge=BR1 tagged=BR1,ether3 vlan-ids=150
add bridge=BR1 tagged=BR1,ether3 vlan-ids=190
/interface list member
add interface=VL150-WORK list=MGMT
add interface=VL110-RaD list=MGMT
add interface=ether5-mgmt list=MGMT
add interface=wan1 list=WAN
add interface=VL110-RaD list=VLAN
add interface=VL120-KaL list=VLAN
add interface=VL130-IoT list=VLAN
add interface=VL140-DOCK list=VLAN
add interface=VL150-WORK list=VLAN
add interface=VL190-GUEST list=VLAN
add interface=VL110-RaD list=LAN
add interface=VL120-KaL list=LAN
add interface=VL130-IoT list=LAN
add interface=VL140-DOCK list=LAN
add interface=VL150-WORK list=LAN
add interface=VL190-GUEST list=LAN
add interface=wan2 list=WAN
add interface=BR1 list=LAN
/ip address
add address=10.1.1.1/24 comment=SNAFU interface=ether5-mgmt network=10.1.1.0
add address=10.10.1.1/24 interface=VL110-RaD network=10.10.1.0
add address=10.10.2.1/24 interface=VL120-KaL network=10.10.2.0
add address=10.10.3.1/24 interface=VL130-IoT network=10.10.3.0
add address=10.10.4.1/24 interface=VL140-DOCK network=10.10.4.0
add address=10.10.5.1/24 interface=VL150-WORK network=10.10.5.0
add address=10.10.9.1/24 interface=VL190-GUEST network=10.10.9.0
add address=10.10.0.1/24 interface=BR1 network=10.10.0.0
/ip cloud
set update-time=no
/ip dhcp-client
add comment="ONT Port 1 - 1Gb" interface=wan1 use-peer-dns=no
add comment="ONT Port 2 - 1Gb" interface=wan2 use-peer-dns=no
/ip dhcp-server network
add address=10.1.1.0/24 comment=access domain=rdkl.lan gateway=10.1.1.1
add address=10.10.0.0/24 comment=infra dns-server=10.10.0.5,10.10.0.7 gateway=10.10.0.1
add address=10.10.1.0/24 comment=rad dns-server=10.10.0.5,10.10.0.7 gateway=10.10.1.1
add address=10.10.2.0/24 comment=kal dns-server=10.10.0.5,10.10.0.7 gateway=10.10.2.1
add address=10.10.3.0/24 comment=iot dns-server=10.10.0.7,10.10.0.5 gateway=10.10.3.1
add address=10.10.4.0/24 comment=dock dns-server=10.10.0.7,10.10.0.5 gateway=10.10.4.1
add address=10.10.5.0/24 comment=work dns-server=8.8.8.8,1.1.1.1 gateway=10.10.5.1
add address=10.10.9.0/24 comment=guest dns-server=10.10.0.5,10.10.0.7 gateway=10.10.9.1
/ip dns
set allow-remote-requests=yes servers=10.10.0.5,10.10.0.7
/ip dns static
add address=8.8.8.8 comment=defconf name=router.lan
/ip firewall address-list
add address=10.10.0.0/24 list=wan1-list
add address=10.10.1.0/24 list=wan1-list
add address=10.10.2.0/24 list=wan1-list
add address=10.10.3.0/24 list=wan1-list
add address=10.10.4.0/24 list=wan1-list
add address=10.10.5.0/24 list=wan2-list
add address=10.10.9.0/24 list=wan2-list
/ip firewall filter
add action=accept chain=input comment="Allow Established,Related,Untracked" \
    connection-state=established,related,untracked
add action=drop chain=input comment="DROP - Invalid" connection-state=invalid
add action=accept chain=input comment="Allow ICMP" protocol=icmp
add action=accept chain=input comment="Allow LAN DNS queries-UDP" dst-port=53 \
    in-interface-list=LAN protocol=udp
add action=accept chain=input comment="Allow LAN DNS queries - TCP" dst-port=\
    53 in-interface-list=LAN protocol=tcp
add action=accept chain=input comment="Allow Winbox & SSH" dst-port=8291,22 \
    protocol=tcp src-port=8291,22
add action=accept chain=input comment=\
    "** Winbox Access (Full access to Router)" in-interface=ether5-mgmt
add action=accept chain=input comment=\
    "** Winbox Access (Full access to Router)" in-interface-list=MGMT
add action=drop chain=input comment=DROP
add action=fasttrack-connection chain=forward comment=FastTrack \
    connection-state=established,related hw-offload=yes
add action=accept chain=forward comment="Allow Established,Related,Untracked" \
    connection-state=established,related,untracked
add action=drop chain=forward comment="DROP - Invalid" connection-state=\
    invalid
add action=accept chain=forward comment="Allow IPSec policy - IN" \
    ipsec-policy=in,ipsec
add action=accept chain=forward comment="Allow IPSec policy - OUT" \
    ipsec-policy=out,ipsec
add action=accept chain=forward comment="ALLOW - Internet Traffic" \
    in-interface-list=LAN out-interface-list=WAN
add action=accept chain=forward comment="ALLOW - Port Forwarding" \
    connection-nat-state=dstnat
add action=accept chain=forward comment="ALLOW - VLAN inter-VLAN routing" \
    connection-state=new in-interface-list=LAN
add action=accept chain=forward comment="Accept all that is DST NATed" \
    connection-nat-state=dstnat connection-state=new
add action=drop chain=forward comment="DROP - All from WAN not DSTNATed" \
    connection-nat-state=!dstnat connection-state=new in-interface-list=WAN
add action=drop chain=forward comment=DROP
/ip firewall nat
add action=masquerade chain=srcnat comment=radgw ipsec-policy=out,none \
    out-interface=wan1
add action=masquerade chain=srcnat comment=workgw ipsec-policy=out,none \
    out-interface=wan2
add action=dst-nat chain=dstnat comment="NGINX 80 - 80" dst-port=80 \
    in-interface=wan1 log-prefix="NGNIX 80" protocol=tcp to-addresses=\
    10.10.4.10 to-ports=80
add action=dst-nat chain=dstnat dst-port=80 in-interface=wan1 protocol=udp \
    to-addresses=10.10.4.10 to-ports=80
add action=dst-nat chain=dstnat comment="NGINX 443 - 80" dst-port=443 \
    in-interface=wan1 protocol=tcp to-addresses=10.10.4.10 to-ports=443
add action=dst-nat chain=dstnat dst-port=443 in-interface=wan1 protocol=udp \
    to-addresses=10.10.4.10 to-ports=443
add action=dst-nat chain=dstnat comment=JF-HTTP dst-port=8096 in-interface=\
    wan1 protocol=tcp to-addresses=10.10.4.40 to-ports=8096
add action=dst-nat chain=dstnat dst-port=8096 in-interface=wan1 protocol=udp \
    to-addresses=10.10.4.40 to-ports=8096
add action=dst-nat chain=dstnat comment=JF-HTTPS dst-port=8920 in-interface=\
    wan1 protocol=tcp to-addresses=10.10.4.40 to-ports=8920
add action=dst-nat chain=dstnat dst-port=8920 in-interface=wan1 protocol=udp \
    to-addresses=10.10.4.40 to-ports=8920
add action=dst-nat chain=dstnat dst-port=33443 in-interface=wan1 protocol=tcp \
    to-addresses=10.10.0.10 to-ports=8443
/ip route
add check-gateway=ping disabled=no distance=4 dst-address=0.0.0.0/0 gateway=\
    xx.xxx.xxx.1%wan2 pref-src="" routing-table=main scope=30 \
    suppress-hw-offload=no target-scope=10
add check-gateway=ping disabled=no distance=1 dst-address=0.0.0.0/0 gateway=\
    xx.xxx.xxx.1%wan1 pref-src="" routing-table=main scope=30 \
    suppress-hw-offload=no target-scope=10
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway=xx.xxx.xxx.1%wan2 \
    pref-src="" routing-table=useWAN2 scope=30 suppress-hw-offload=no \
    target-scope=10
/ip traffic-flow
set interfaces=BR1
/ip upnp
set enabled=yes
/ip upnp interfaces
add interface=BR1 type=internal
add interface=wan1 type=external
add interface=wan2 type=external
/ipv6 firewall filter
add action=accept chain=input comment=\
    "defconf: accept established,related,untracked" connection-state=\
    established,related,untracked
add action=drop chain=input comment="defconf: drop invalid" connection-state=\
    invalid
add action=accept chain=input comment="defconf: accept ICMPv6" protocol=\
    icmpv6
add action=accept chain=input comment="defconf: accept UDP traceroute" port=\
    33434-33534 protocol=udp
add action=accept chain=input comment=\
    "defconf: accept DHCPv6-Client prefix delegation." dst-port=546 protocol=\
    udp src-address=fe80::/10
add action=accept chain=input comment="defconf: accept IKE" dst-port=500,4500 \
    protocol=udp
add action=accept chain=input comment="defconf: accept ipsec AH" protocol=\
    ipsec-ah
add action=accept chain=input comment="defconf: accept ipsec ESP" protocol=\
    ipsec-esp
add action=accept chain=input comment=\
    "defconf: accept all that matches ipsec policy" ipsec-policy=in,ipsec
add action=drop chain=input comment=\
    "defconf: drop everything else not coming from LAN" in-interface-list=\
    !LAN
add action=accept chain=forward comment=\
    "defconf: accept established,related,untracked" connection-state=\
    established,related,untracked
add action=drop chain=forward comment="defconf: drop invalid" \
    connection-state=invalid
add action=drop chain=forward comment=\
    "defconf: drop packets with bad src ipv6" src-address-list=bad_ipv6
add action=drop chain=forward comment=\
    "defconf: drop packets with bad dst ipv6" dst-address-list=bad_ipv6
add action=drop chain=forward comment="defconf: rfc4890 drop hop-limit=1" \
    hop-limit=equal:1 protocol=icmpv6
add action=accept chain=forward comment="defconf: accept ICMPv6" protocol=\
    icmpv6
add action=accept chain=forward comment="defconf: accept HIP" protocol=139
add action=accept chain=forward comment="defconf: accept IKE" dst-port=\
    500,4500 protocol=udp
add action=accept chain=forward comment="defconf: accept ipsec AH" protocol=\
    ipsec-ah
add action=accept chain=forward comment="defconf: accept ipsec ESP" protocol=\
    ipsec-esp
add action=accept chain=forward comment=\
    "defconf: accept all that matches ipsec policy" ipsec-policy=in,ipsec
add action=drop chain=forward comment=\
    "defconf: drop everything else not coming from LAN" in-interface-list=\
    !LAN
/routing rule
add action=lookup-only-in-table disabled=no dst-address=10.10.1.0/24 table=\
    main
add action=lookup-only-in-table disabled=no dst-address=10.10.2.0/24 table=\
    main
add action=lookup-only-in-table disabled=no dst-address=10.10.3.0/24 table=\
    main
add action=lookup-only-in-table disabled=no dst-address=10.10.4.0/24 \
    src-address="" table=main
add action=lookup disabled=yes src-address=10.10.4.0/24 table=useWAN2
add action=lookup disabled=no src-address=10.10.5.0/24 table=useWAN2
/tool mac-server
set allowed-interface-list=MGMT
/tool mac-server mac-winbox
set allowed-interface-list=MGMT
/tool sniffer
set filter-interface=wan2