Wireguard WAN but no LAN access

FlippinTurt · August 20, 2023, 8:14pm

Hey all

As the title suggests, I have WAN working fine on my router, but can’t access any lan devices, have looked through the forums and setup firewall rules which should allow the traffic but doesn’t.
Server is setup to allow 10.6.0.0/24 & 10.20.20.0/24
Peer is setup to allow 0.0.0.0/0

Network setup:
Dual WAN,
1 4G connection with static IP, DMZ’d to the TIK
1 5G connection behind a CGNAT, mainly used for dhcp clients
main route is set out WAN1 (Spark), to allow wireguard to connect properly, everything else is set to 2d(WAN2)

# 2023-08-21 07:50:14 by RouterOS 7.11
# software id = **ELIDED**
#
# model = C53UiG+5HPaxD2HPaxD
# serial number = **ELIDED**
/interface bridge
add name=IoT_Bridge
add name=network_out_bridge
/interface ethernet
set [ find default-name=ether1 ] mtu=1560 name=ether1-WAN-SPARK poe-out=off
set [ find default-name=ether2 ] mtu=1560 name=ether2-WAN-2D
set [ find default-name=ether3 ] name=ether3-network_out
set [ find default-name=ether4 ] name=ether4-network_out
set [ find default-name=ether5 ] name=ether5-network_out
/interface wireguard
add listen-port=51856 mtu=1404 name=wireguard1
/interface list
add name=wan
add name=lan
/interface wifiwave2 security
add authentication-types=wpa2-psk,wpa3-psk disabled=no name=2ghz wps=disable
add authentication-types=wpa2-psk,wpa3-psk disabled=no name=5ghz
add authentication-types=wpa2-psk disabled=no name=IoT
/interface wifiwave2
set [ find default-name=wifi2 ] channel.band=2ghz-n .width=20mhz \
    configuration.mode=ap .ssid=$$$$$$$$$$$$$ disabled=no name=AC_2.4 security=\
    2ghz
add configuration.hide-ssid=yes .mode=ap .ssid=IoT_SBS disabled=no \
    mac-address=AA:BB:CC:DD:EE:FF master-interface=AC_2.4 name=AC_2.4_IoT \
    security=IoT
set [ find default-name=wifi1 ] channel.band=5ghz-ax .width=20/40/80mhz \
    configuration.mode=ap .ssid=$$$$$$$$$$$$ disabled=no name=AX_5 \
    security=5ghz
/ip pool
add name=dhcp_pool1 ranges=10.20.20.68-10.20.20.200
add name=IoT_Pool ranges=10.30.30.20-10.30.30.50
/ip dhcp-server
add address-pool=dhcp_pool1 interface=network_out_bridge lease-time=1d name=\
    dhcp1
add address-pool=IoT_Pool interface=IoT_Bridge name=IoT_dhcp
/routing table
add disabled=no fib name=spark
add disabled=no fib name=WireG
add disabled=no fib name=2D
/system logging action
add name=MinervaSyslog remote=10.20.20.10 src-address=10.20.20.1 target=\
    remote
/zerotier
set zt1 comment="ZeroTier Central controller - https://my.zerotier.com/" \
    interfaces=ether2-WAN-2D name=zt1 port=9993
/zerotier interface
add allow-default=no allow-global=no allow-managed=yes disabled=no instance=\
    zt1 name=AtlasV2_Link network=###############
/interface bridge port
add bridge=network_out_bridge ingress-filtering=no interface=\
    ether3-network_out
add bridge=network_out_bridge ingress-filtering=no interface=\
    ether4-network_out
add bridge=network_out_bridge interface=ether5-network_out
add bridge=IoT_Bridge interface=AC_2.4_IoT
add bridge=network_out_bridge interface=AC_2.4
add bridge=network_out_bridge interface=AX_5
add bridge=network_out_bridge disabled=yes interface=AtlasV2_Link
/ip firewall connection tracking
set tcp-established-timeout=1h
/ip neighbor discovery-settings
set discover-interface-list=!wan
/ip settings
set max-neighbor-entries=8192
/ipv6 settings
set disable-ipv6=yes max-neighbor-entries=8192
/interface list member
add interface=ether1-WAN-SPARK list=wan
add interface=ether2-WAN-2D list=wan
add interface=wireguard1 list=lan
add interface=network_out_bridge list=lan
/interface ovpn-server server
set auth=sha1,md5
/interface wireguard peers
add allowed-address=10.6.0.2/32 interface=wireguard1 public-key=\
    "#######################################"
/ip address
add address=10.20.20.1/24 interface=network_out_bridge network=10.20.20.0
add address=192.168.10.10/24 comment=WAN1 interface=ether1-WAN-SPARK network=\
    192.168.10.0
add address=192.168.11.10/24 comment=WAN2 interface=ether2-WAN-2D network=\
    192.168.11.0
add address=10.30.30.1/24 interface=IoT_Bridge network=10.30.30.0
add address=10.6.0.1/24 interface=wireguard1 network=10.6.0.0
/ip dhcp-client
add disabled=yes interface=ether2-WAN-2D
add disabled=yes interface=ether1-WAN-SPARK
/ip dhcp-server network
add address=10.20.20.0/24 dns-server=10.20.20.6,10.20.20.7 gateway=10.20.20.1
add address=10.30.30.0/24 dns-server=1.1.1.1,1.0.0.1 gateway=10.30.30.1
/ip dns
set allow-remote-requests=yes max-udp-packet-size=512 servers=\
    10.20.20.6,10.20.20.7
/ip firewall address-list
add address=10.20.20.0/24 list=trusted_admin
add address=0.0.0.0/8 comment="Self-Identification [RFC 3330]" list=bogons
add address=127.0.0.0/8 comment="Loopback [RFC 3330]" list=bogons
add address=169.254.0.0/16 comment="Link Local [RFC 3330]" list=bogons
add address=172.16.0.0/12 comment="Private[RFC 1918] - CLASS B # Check if you \
    need this subnet before enable it" list=bogons
add address=192.0.2.0/24 comment="Reserved - IANA - TestNet1" list=bogons
add address=192.88.99.0/24 comment="6to4 Relay Anycast [RFC 3068]" list=\
    bogons
add address=198.18.0.0/15 comment="NIDB Testing" list=bogons
add address=198.51.100.0/24 comment="Reserved - IANA - TestNet2" list=bogons
add address=203.0.113.0/24 comment="Reserved - IANA - TestNet3" list=bogons
add address=224.0.0.0/4 comment=\
    "MC, Class D, IANA # Check if you need this subnet before enable it" \
    list=bogons
add address=10.30.30.10 comment=Bulb list=Trusted_IoT
add address=10.20.20.200 comment=TurtBook list=my_devices
add address=10.20.20.201 comment=TurtMax list=my_devices
add address=10.20.20.2 comment=SWAG list=spark_wan
add address=10.20.20.9 comment=PLEX list=spark_wan
add address=10.20.20.31 comment=Smokeping-Spark list=spark_wan
/ip firewall filter
add action=accept chain=input comment="Allow ZeroTier" in-interface=\
    AtlasV2_Link
add action=accept chain=forward comment="Allow ZeroTier" in-interface=\
    AtlasV2_Link
add action=drop chain=input log-prefix=>>INBLOCK>> src-address-list=\
    CountryIPBlocks
add action=drop chain=output dst-address-list=CountryIPBlocks log=yes \
    log-prefix=>>OUTBLOCK>>
add action=accept chain=input comment="allow WireGuard traffic" src-address=\
    10.6.0.0/24
add action=drop chain=output comment="Drop Invalid Out" connection-state=\
    invalid
add action=accept chain=input comment=Wireguard dst-port=51856 in-interface=\
    ether1-WAN-SPARK log-prefix=WG-IN>> protocol=udp
add action=accept chain=output comment=Wireguard disabled=yes log-prefix=\
    WG-OUT>> protocol=udp src-port=51856
add action=accept chain=output comment="Accept Established Out" \
    connection-state=established disabled=yes
add action=accept chain=output comment="Accept Related Out" connection-state=\
    related disabled=yes
add action=drop chain=forward comment="Drop Invalid Forward" \
    connection-state=invalid
add action=drop chain=input comment="Accept ICMP" protocol=icmp \
    src-address-list=!trusted_admin
add action=drop chain=input comment="Drop invalid" connection-state=invalid
add action=drop chain=forward comment="Drop invalid" connection-state=invalid
add action=fasttrack-connection chain=forward comment=Fasttrack \
    connection-state=established,related hw-offload=yes
add action=accept chain=forward comment="Accept Established Forward" \
    connection-state=established log-prefix="Forward "
add action=accept chain=forward comment="Accept Related Forward" \
    connection-state=related
add action=accept chain=input comment="Accept SSH from trusted_admin" \
    dst-port=22 protocol=tcp src-address-list=trusted_admin
add action=accept chain=input comment="Accept WinBox from trusted_admin" \
    dst-port=8298 protocol=tcp src-address-list=trusted_admin
add action=drop chain=input comment="Drop WinBox" dst-port=8298 log=yes \
    log-prefix=CPE-DROP-WINBOX protocol=tcp
add action=drop chain=input comment="Drop WinBox" dst-port=8291 log-prefix=\
    CPE-DROP-WINBOX protocol=tcp
add action=drop chain=input comment="Drop SNMP" dst-port=161-162 log-prefix=\
    CPE-DROP-SNMP protocol=udp
add action=drop chain=input comment="Drop SSH" dst-port=22 log-prefix=\
    CPE-DROP-SSH protocol=tcp
add action=drop chain=input comment="Drop TELNET" dst-port=23 log-prefix=\
    CPE-DROP-TELNET protocol=tcp
add action=add-src-to-address-list address-list=Syn_Flooder \
    address-list-timeout=30m chain=input comment=\
    "Add Syn Flood IP to the list" connection-limit=30,32 protocol=tcp \
    tcp-flags=syn
add action=drop chain=input comment="Drop to syn flood list" \
    src-address-list=Syn_Flooder
add action=add-src-to-address-list address-list=Port_Scanner \
    address-list-timeout=1w chain=input comment="Port Scanner Detect" \
    protocol=tcp psd=21,3s,3,1
add action=drop chain=input comment="Drop to port scan list" \
    src-address-list=Port_Scanner
add action=jump chain=input comment="Jump for icmp input flow" jump-target=\
    ICMP protocol=icmp
add action=add-src-to-address-list address-list=spammers \
    address-list-timeout=3h chain=forward comment=\
    "Add Spammers to the list for 3 hours" connection-limit=30,32 dst-port=\
    25,587 limit=30/1m,0 protocol=tcp
add action=drop chain=forward comment="Drop to bogon list" dst-address-list=\
    bogons
add action=drop chain=forward comment="Avoid spammers action" dst-port=25,587 \
    protocol=tcp src-address-list=spammers
add action=accept chain=input comment="Accept DNS - UDP" port=53 protocol=udp
add action=accept chain=input comment="Accept DNS - TCP" port=53 protocol=tcp
add action=accept chain=input comment="Accept to established connections" \
    connection-state=established
add action=accept chain=input comment="Accept to related connections" \
    connection-state=related
add action=accept chain=input comment="Allow Plex GDM" in-interface=\
    network_out_bridge log-prefix=PlexGDM>> port=32412-32414 protocol=udp \
    src-address=10.20.20.9
add action=drop chain=forward comment="Drop non lan forwards" dst-address=\
    !10.20.20.0/24 in-interface=network_out_bridge log-prefix=NON_LAN>> \
    src-address=!10.20.20.0/24
add action=drop chain=input comment="Drop anything else!" log-prefix=\
    DROPALL>>
add action=accept chain=ICMP comment=\
    "Echo request - Avoiding Ping Flood, adjust the limit as needed" \
    icmp-options=8:0 limit=2,5 protocol=icmp
add action=accept chain=ICMP comment="Echo reply" icmp-options=0:0 protocol=\
    icmp
add action=accept chain=ICMP comment="Time Exceeded" icmp-options=11:0 \
    protocol=icmp
add action=accept chain=ICMP comment="Destination unreachable" icmp-options=\
    3:0-1 protocol=icmp
add action=accept chain=ICMP comment=PMTUD icmp-options=3:4 protocol=icmp
add action=drop chain=ICMP comment="Drop to the other ICMPs" protocol=icmp
add action=jump chain=output comment="Jump for icmp output" jump-target=ICMP \
    protocol=icmp
add action=reject chain=forward comment="Drop IoT" log-prefix=IoTDrop>> \
    reject-with=icmp-network-unreachable src-address=10.30.30.0/24 \
    src-address-list=!Trusted_IoT
add action=accept chain=forward comment="AllowCAM to my_devices" \
    dst-address-list=my_devices src-address=10.30.30.5
add action=accept chain=forward comment="AllowCAM to my_devices" \
    dst-address=10.30.30.5 src-address-list=my_devices
add action=reject chain=forward comment="Drop IoT" dst-address=10.20.20.0/24 \
    log=yes log-prefix=IoT>LAN>> reject-with=icmp-network-unreachable \
    src-address=10.30.30.0/24 src-address-list=!Trusted_IoT
add action=reject chain=forward comment="Drop IoT" dst-address=10.30.30.0/24 \
    dst-address-list=!Trusted_IoT log=yes log-prefix=LAN>IoT>> reject-with=\
    icmp-network-unreachable src-address=10.20.20.0/24
/ip firewall mangle
add action=mark-connection chain=prerouting connection-mark=no-mark \
    in-interface=ether1-WAN-SPARK new-connection-mark=spark_conn passthrough=\
    yes
add action=mark-connection chain=prerouting connection-mark=no-mark \
    in-interface=ether2-WAN-2D new-connection-mark=2d_conn passthrough=yes
add action=mark-routing chain=prerouting comment=spark connection-mark=\
    spark_conn in-interface-list=!wan new-routing-mark=spark passthrough=no
add action=mark-routing chain=prerouting comment=2d connection-mark=2d_conn \
    in-interface-list=!wan new-routing-mark=2D passthrough=no
add action=mark-connection chain=output comment=2d connection-mark=no-mark \
    new-connection-mark=2d_conn out-interface=ether2-WAN-2D passthrough=yes
add action=mark-connection chain=output comment=spark new-connection-mark=\
    spark_conn out-interface=ether1-WAN-SPARK passthrough=yes
add action=mark-routing chain=output comment=Spark connection-mark=spark_conn \
    new-routing-mark=spark passthrough=no
add action=mark-routing chain=output connection-mark=2d_conn \
    new-routing-mark=2D passthrough=no
/ip firewall nat
add action=masquerade chain=srcnat comment="MASQ Spark" out-interface=\
    ether1-WAN-SPARK
add action=masquerade chain=srcnat comment="MASQ 2Deg" out-interface=\
    ether2-WAN-2D
add action=masquerade chain=srcnat comment="MASQ IoT" out-interface=\
    IoT_Bridge
add action=dst-nat chain=dstnat comment=SWAG dst-port=443 in-interface=\
    ether1-WAN-SPARK protocol=tcp to-addresses=10.20.20.2 to-ports=443
add action=dst-nat chain=dstnat comment=PLEX dst-port=32504 in-interface=\
    ether1-WAN-SPARK protocol=tcp to-addresses=10.20.20.9 to-ports=32400
add action=dst-nat chain=dstnat comment=MINERVA_WG disabled=yes dst-port=\
    51856 in-interface=ether1-WAN-SPARK log=yes log-prefix=DST-WG>> protocol=\
    udp to-addresses=10.20.20.10 to-ports=51856
/ip route
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway=192.168.10.254 \
    pref-src="" routing-table=spark scope=30 suppress-hw-offload=no \
    target-scope=10
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway=192.168.10.254 \
    pref-src="" routing-table=main scope=30 suppress-hw-offload=no \
    target-scope=10
add disabled=yes distance=1 dst-address=10.253.0.0/24 gateway=10.20.20.10 \
    pref-src="" routing-table=main scope=30 suppress-hw-offload=no \
    target-scope=10
add disabled=yes distance=1 dst-address=192.168.191.0/24 gateway=AtlasV2_Link \
    pref-src="" routing-table=main scope=30 suppress-hw-offload=no \
    target-scope=10
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway=192.168.11.1 \
    pref-src="" routing-table=2D scope=30 suppress-hw-offload=no \
    target-scope=10
add disabled=no distance=5 dst-address=0.0.0.0/0 gateway=192.168.11.1 \
    routing-table=main scope=30 suppress-hw-offload=no target-scope=10
/ip service
set telnet disabled=yes
set ftp disabled=yes
set www address=10.20.20.0/24 disabled=yes port=82
set ssh address=10.20.20.0/24
set www-ssl address=10.20.20.0/24
set api disabled=yes
set winbox address=10.20.20.0/24 port=8298
set api-ssl disabled=yes
/routing rule
add action=lookup-only-in-table comment=SWAG disabled=no src-address=\
    10.20.20.2/32 table=spark
add action=lookup-only-in-table comment=PLEX disabled=no src-address=\
    10.20.20.9/32 table=spark
add action=lookup-only-in-table comment=SMOKEPING disabled=no src-address=\
    10.20.20.31/32 table=spark
add action=lookup-only-in-table comment=Wireguard disabled=no src-address=\
    10.6.0.1/24 table=spark
add action=lookup-only-in-table comment=MACBOOK disabled=yes src-address=\
    10.20.20.200/32 table=spark
add action=lookup-only-in-table disabled=yes src-address=10.20.20.200/32 \
    table=2D
add action=lookup-only-in-table disabled=no src-address=10.20.20.3/32 table=\
    2D
add action=lookup-only-in-table disabled=no src-address=10.20.20.4/30 table=\
    2D
add action=lookup-only-in-table disabled=no src-address=10.20.20.8/29 table=\
    2D
add action=lookup-only-in-table disabled=no src-address=10.20.20.16/28 table=\
    2D
add action=lookup-only-in-table disabled=no src-address=10.20.20.32/27 table=\
    2D
add action=lookup-only-in-table disabled=no src-address=10.20.20.64/26 table=\
    2D
add action=lookup-only-in-table disabled=no src-address=10.20.20.128/25 \
    table=2D
/system clock
set time-zone-name=Pacific/Auckland
/system identity
set name=AtlasV2
/system logging
add action=MinervaSyslog topics=warning
add action=MinervaSyslog topics=critical
add action=MinervaSyslog topics=error
add action=MinervaSyslog topics=interface
add action=MinervaSyslog topics=system
add action=MinervaSyslog topics=firewall
add action=MinervaSyslog topics=wireguard
add action=MinervaSyslog topics=info
add disabled=yes topics=debug
add topics=wireguard
add disabled=yes topics=wireless
/system note
set show-at-login=no
/system ntp client
set enabled=yes
/system ntp client servers
add address=101.100.146.146
add address=202.68.92.244
add address=43.252.70.34
add address=162.159.200.123
/tool bandwidth-server
set enabled=no

Any help is greatly appreciated,
I am aware there are a lot of firewall rules, and can probably be cleaned up a little

anav · August 21, 2023, 1:56pm

First, thing is One bridge and use vlans for different subnets.
Yes, FW cleanup a good idea.
Why do you limit udp packet size??
What is the purpose of mangling? Just to be clear you do not have any LAN servers people are using from external sites??
(aka all you are doing externally is moblile user wishing to access router for config purposes and LANs as well )

Okay so you claim wan1 is for wireguard but you are port forwarding servers via WAN1…
Will assume this is intentional and thus the issue is you have LAN users going out WAN2, but these servers are getting queries via WAN1.
So you need the responses to back out WAN1 vice WAN2.

Get rid of two of your dstnat port forwardings… the one for wireguard is nonsensical.
The one for port 443, why?? You have wireguard access now…
Why is there no source address for those coming in to access servers? Must have for servers as a minimum for security.
They either have fixed WANIPs OR can acquire a free dyndns name for their dynamic IP,
Lastly consider their use of wireguard to access LAN services…

Access from two IPs on trusted LAN to CAM 10.30.30.5, only should need access to the dst address.
Any return traffic from those queries is allowed and thus two rules not required.
To be clear, which side originates traffic, one or both??

Assumed the trusted IOT IP, needs full access to Home LAN??

Very confusing for mangles and routing rules. I gather the issue is wireguard needs WAN1 but all the port forwardings are on WAN2.

No requirement to sourcenat Iot subnet.

Your routes to some unknowns 10.253 192.168.192 make no sense whatsoever…

Simplified mangling plus removed all routing rules.
We needed mangling to ensure wg return traffic goes out Wan1, and to ensure any Server Returns for plex go out WAN1 ( assuming they were supposed to come in WAN1 ).
The rest of the LAN traffic will use WAN2 solely need to use distance ( 5 for WAN2, 10 for WAN1 ).

The question is do you want users to use WAN1 if WAN2 is down/not working??? Assumed yes in config. If not suspect just remove main table route for wan1.

model = C53UiG+5HPaxD2HPaxD
# serial number = **ELIDED**
/interface bridge
add name=BR1  vlan-filitering=yes
/interface vlan
add interface=BR1  name=vHOME  vlan-ids=20
add interface=BR1  name=vIOT  vlan-ids=30
/interface ethernet
set [ find default-name=ether1 ] mtu=1560 name=ether1-WAN-SPARK poe-out=off
set [ find default-name=ether2 ] mtu=1560 name=ether2-WAN-2D
set [ find default-name=ether3 ] name=eth3-no
set [ find default-name=ether4 ] name=eth4-no
set [ find default-name=ether5 ] name=eth5-no
/interface wireguard
add listen-port=51856 mtu=1404 name=wireguard1
/interface list
add name=WAN
add name=LAN
/interface wifiwave2 security
add authentication-types=wpa2-psk,wpa3-psk disabled=no name=2ghz wps=disable
add authentication-types=wpa2-psk,wpa3-psk disabled=no name=5ghz
add authentication-types=wpa2-psk disabled=no name=IoT
/interface wifiwave2
set [ find default-name=wifi2 ] channel.band=2ghz-n .width=20mhz \
    configuration.mode=ap .ssid=$$$$$$$$$$$$$ disabled=no name=AC_2.4 security=\
    2ghz
add configuration.hide-ssid=yes .mode=ap .ssid=IoT_SBS disabled=no \
    mac-address=AA:BB:CC:DD:EE:FF master-interface=AC_2.4 name=AC_2.4_IoT \
    security=IoT
set [ find default-name=wifi1 ] channel.band=5ghz-ax .width=20/40/80mhz \
    configuration.mode=ap .ssid=$$$$$$$$$$$$ disabled=no name=AX_5 \
    security=5ghz
/ip pool
add name=dhcp_pool1 ranges=10.20.20.68-10.20.20.200
add name=IoT_Pool ranges=10.30.30.20-10.30.30.50
/ip dhcp-server
add address-pool=dhcp_pool1 interface=vHOME lease-time=1d name=\
    dhcp1
add address-pool=IoT_Pool interface=vIOT name=IoT_dhcp
/routing table
add disabled=no fib name=spark
add disabled=no fib name=WireG
add disabled=no fib name=2D
/system logging action
add name=MinervaSyslog remote=10.20.20.10 src-address=10.20.20.1 target=\
    remote
/interface bridge port
add bridge=BR1 ingress-filtering=yes frame-types=admit-priority-and-untagged interface=eth3-no  pvid=20
add bridge=BR1 ingress-filtering=yes frame-types=admit-priority-and-untagged interface=eth4-no  pvid=20
add bridge=BR1 ingress-filtering=yes frame-types=admit-priority-and-untagged interface=eth5-no  pvid=20
add bridge=BR1 ingress-filtering=yes frame-types=admit-priority-and-untagged interface=AC_2.4   pvid=20
add bridge=BR1 ingress-filtering=yes frame-types=admit-priority-and-untagged interface=AX_5   pvid=20
add bridge=BR1 ingress-filtering=yes frame-types=admit-priority-and-untagged interface=AC_2.4_IoT   pvid=30
/interface bridge vlan
add bridge=BR1 tagged=BR1 untagged=eth3-no,eth4-no.eth5-no,AC_2.4,AX_5  vlan-ids=20
add bridge=BR1 tagged=BR1 untagged=AC_2.4_IoT
/ip neighbor discovery-settings
set discover-interface-list=LAN
/ip settings
set max-neighbor-entries=8192
/ipv6 settings
set disable-ipv6=yes max-neighbor-entries=8192
/interface list member
add interface=ether1-WAN-SPARK list=WAN
add interface=ether2-WAN-2D list=WAN
add interface=wireguard1 list=LAN
add interface=vHOME  list=LAN
and interface=vIOT list=LAN
/interface ovpn-server server
set auth=sha1,md5
/interface wireguard peers
add allowed-address=10.6.0.2/32 interface=wireguard1 public-key=\
    "#######################################"
/ip address
add address=192.168.10.10/24 comment=WAN1 interface=ether1-WAN-SPARK network=\
    192.168.10.0
add address=192.168.11.10/24 comment=WAN2 interface=ether2-WAN-2D network=\
    192.168.11.0
add address=10.20.20.1/24 interface=vHOME network=10.20.20.0
add address=10.30.30.1/24 interface=vIOT network=10.30.30.0
add address=10.6.0.1/24 interface=wireguard1 network=10.6.0.0
/ip dhcp-client
add disabled=yes interface=ether2-WAN-2D
add disabled=yes interface=ether1-WAN-SPARK
/ip dhcp-server network
add address=10.20.20.0/24 dns-server=10.20.20.6,10.20.20.7 gateway=10.20.20.1
add address=10.30.30.0/24 dns-server=1.1.1.1,1.0.0.1 gateway=10.30.30.1
/ip dns
set allow-remote-requests=yes max-udp-packet-size=512 servers=\
    10.20.20.6,10.20.20.7
/ip firewall address-list
add address=10.20.20.X/32 list=trusted_admin  comment="admin_desktop"
add address=10.20.20.Y/32 list=trusted_admin comment="admin_laptop"
add address=10.20.20.Z/32   list=trusted_admin comment="admin_smartphone"
add address=10.6.0.2/32  list=trusted_admin  comment=Mobile_Laptop-WG
add address=10.6.0.3/32 list=trusted_admin comment=Mobile_Smartphone-WG
add address=10.30.30.10 comment=Bulb list=Trusted_IoT
add address=10.20.20.200 comment=TurtBook list=my_devices
add address=10.20.20.201 comment=TurtMax list=my_devices
/ip firewall filter
{Input Chain}
add action=accept chain=input comment="defconf: accept established,related,untracked" connection-state=established,related,untracked
add action=drop chain=input comment="defconf: drop invalid" connection-state=invalid
add action=accept chain=input comment="defconf: accept ICMP" protocol=icmp
add action=accept chain=input comment="defconf: accept to local loopback (for CAPsMAN)" dst-address=127.0.0.1
add action=accept chain=input comment-="WG handshake"  in-interface=ether1-WAN-SPARK  dst-port=51856 protocol=udp
add action=accept chain=input in-interface-list=LAN  src-address-list=trusted_admin
add action=accept chain=input comment="Allow LAN DNS queries-UDP"  and NTP  services"
dst-port=53,123 in-interface-list=LAN protocol=udp
add action=accept chain=input comment="Allow LAN DNS queries - TCP" \
dst-port=53 in-interface-list=LAN protocol=tcp
add action=drop chain=input comment="drop all else"
{forward chain}
add action=fasttrack-connection chain=forward comment="defconf: fasttrack" connection-state=established,related
add action=accept chain=forward comment="defconf: accept established,related, untracked" connection-state=established,related,untracked
add action=drop chain=forward comment="defconf: drop invalid" connection-state=invalid
add action=accept chain=forward comment="allow internet traffic" in-interface-list=LAN out-interface-list=WAN
add action=accept chain=forward comment="lan access - WG" in-interface=wireguard1  out-interface-list=LAN
add action=accept chain=forward comment="port forwarding" connection-nat-state=dstnat
add action=accept chain=forward comment="access cam"  src-address-list=my-devices dst-address=10.30.30.5
add action=accept chain=forward comment="Iot to Home LAN"  src-address-list=Trusted_IoT out-interface=vHOME
add action=drop chain=forward comment="drop all else
/ip firewall mangle
add action=mark-connection chain=prerouting connection-mark=no-mark \
    in-interface=ether1-WAN-SPARK new-connection-mark=spark_conn passthrough=\
    yes
add action=mark-routing chain=prerouting connection-mark=spark_conn  \
   in-interface-list=LAN  new-routing-mark=useWAN1  passthrough=yes  \
comment="Server Return Traffic"
add action=mark-routing chain=output connection-mark=spark_conn \
   new-routing-mark=useWAN1  passthrough=no \ 
comment="Wireguard Return Traffic"
/ip firewall nat
add action=masquerade chain=srcnat comment="MASQ Spark" out-interface=\
    ether1-WAN-SPARK
add action=masquerade chain=srcnat comment="MASQ 2Deg" out-interface=\
    ether2-WAN-2D
add action=dst-nat chain=dstnat comment=PLEX dst-port=32504 in-interface=\
    ether1-WAN-SPARK protocol=tcp to-addresses=10.20.20.9 to-ports=32400
/ip route
add distance=10 dst-address=0.0.0.0/0 gateway=192.168.10.254 \
    routing-table=main \
add check-gateway=ping distance=5 dst-address=0.0.0.0/0 gateway=192.168.11.1 \
    routing-table=main
add  dst-address=0.0.0.0/0 gateway=192.168.10.254 \
    routing-table=useWAN1
/ip service
set telnet disabled=yes
set ftp disabled=yes
set www  disable=yes        {  not secure do not use }
set ssh address=10.20.20.0/24
set www-ssl address=10.20.20.0/24
set api disabled=yes
set winbox address=10.20.20.0/24 port=8298
set api-ssl disabled=yes
/system clock
set time-zone-name=Pacific/Auckland
/system identity
set name=AtlasV2
/system logging
add action=MinervaSyslog topics=warning
add action=MinervaSyslog topics=critical
add action=MinervaSyslog topics=error
add action=MinervaSyslog topics=interface
add action=MinervaSyslog topics=system
add action=MinervaSyslog topics=firewall
add action=MinervaSyslog topics=wireguard
add action=MinervaSyslog topics=info
add disabled=yes topics=debug
add topics=wireguard
add disabled=yes topics=wireless
/system note
set show-at-login=no
/system ntp client
set enabled=yes
/system ntp client servers
add address=101.100.146.146
add address=202.68.92.244
add address=43.252.70.34
add address=162.159.200.123
/tool bandwidth-server
set enabled=no

FlippinTurt · August 21, 2023, 8:29pm

First up,
Thank you for your reply, and a nicer looking firewall, much appreciated

Answers to your questions;
443 is for an external site (SWAG is a reverse proxy, and has it’s own geoip / oauth protections in place)
Mangling was in an attempt to get this to work earlier, which it didn’t
The wireguard dstnat was also an attempt which failed, and left disabled, not removed
The LAN 2 CAM, 10.30.30.5 is the originator

The 191 route is / was for zero tier, and the 253 was an attempt to get another wireguard server working inside the LAN / on a server.

Onto the config, it looks like even with your mangle rules the wireguard return traffic is trying to go out WAN2.
I resolved this before by making WAN1 the primary, and setup routing rules for every IP not 10.20.20.1 to go out the WAN2 connection (which is why there were so many)

Any idea why the return traffic isn’t following the mangle rules? (I can see the traffic is getting marked)

WGReturn>> output: in:(unknown 0) out:ether2-WAN-2D, connection-mark:spark_conn connection-state:related proto ICMP (type 3, code 3), 192.168.10.10->118.149.79.156, len 204

anav · August 21, 2023, 10:16pm

Hi there, I would have to see your latest config to make any comments with respect to things working or not working.
Even if you think you did nothing, evidence is the best medicine. It is a rare OP, that actually gets the details right the first time.

Also you didnt answer the question//////////// is WAN1 to be available to WAN2 users if WAN2 is not working ??

FlippinTurt · August 21, 2023, 10:34pm

Apologies, and totally understand lol

Regarding the failover, not hugely fussed, I have removed the check gateway though as I will likely play around with scripting to ping a different IP. Currently everything is going out WAN2, apart from the ip’s specified in the routing rules (10.20.20.2, 10.20.20.9, 10.20.20.31) - that shouldn’t stop the mangle rules working though?

/interface bridge
add name=BR1 vlan-filtering=yes
/interface ethernet
set [ find default-name=ether1 ] mtu=1560 name=ether1-WAN-SPARK poe-out=off
set [ find default-name=ether2 ] mtu=1560 name=ether2-WAN-2D
set [ find default-name=ether3 ] name=ether3-network_out
set [ find default-name=ether4 ] name=ether4-network_out
set [ find default-name=ether5 ] name=ether5-network_out
/interface wireguard
add disabled=no listen-port=51856 mtu=1404 name=wireguard1
/interface vlan
add interface=BR1 name=vHOME vlan-id=20
add interface=BR1 name=vIOT vlan-id=30
/interface list
add name=wan
add name=lan
/interface wifiwave2 security
add authentication-types=wpa2-psk,wpa3-psk disabled=no name=2ghz wps=disable
add authentication-types=wpa2-psk,wpa3-psk disabled=no name=5ghz
add authentication-types=wpa2-psk disabled=no name=IoT
/interface wifiwave2
set [ find default-name=wifi2 ] channel.band=2ghz-n .width=20mhz \
    configuration.mode=ap .ssid=$$$$$$$$$$$ disabled=no name=AC_2.4 security=\
    2ghz
add configuration.hide-ssid=yes .mode=ap .ssid=IoT_SBS disabled=no \
    mac-address=AA:BB:CC:DD:EE:FF master-interface=AC_2.4 name=AC_2.4_IoT \
    security=IoT
set [ find default-name=wifi1 ] channel.band=5ghz-ax .width=20/40/80mhz \
    configuration.mode=ap .ssid=$$$$$$$$$$$$ disabled=no name=AX_5 \
    security=5ghz
/ip pool
add name=dhcp_pool1 ranges=10.20.20.68-10.20.20.200
add name=IoT_Pool ranges=10.30.30.20-10.30.30.50
/ip dhcp-server
add address-pool=dhcp_pool1 interface=vHOME lease-time=1d name=dhcp1
add address-pool=IoT_Pool interface=vIOT name=IoT_dhcp
/routing table
add disabled=no fib name=spark
add disabled=no fib name=WireG
add disabled=no fib name=2D
add disabled=no fib name=useWAN1
/system logging action
add name=MinervaSyslog remote=10.20.20.10 src-address=10.20.20.1 target=\
    remote
/zerotier
set zt1 comment="ZeroTier Central controller - https://my.zerotier.com/" \
    interfaces=ether2-WAN-2D,BR1 name=zt1 port=9993
/zerotier interface
add allow-default=no allow-global=no allow-managed=yes disabled=no instance=\
    zt1 name=AtlasV2_Link network=###########
/interface bridge port
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    ether3-network_out pvid=20
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    ether4-network_out pvid=20
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    ether5-network_out pvid=20
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    AC_2.4_IoT pvid=30
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    AC_2.4 pvid=20
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    AX_5 pvid=20
add bridge=BR1 disabled=yes interface=AtlasV2_Link
/ip firewall connection tracking
set tcp-established-timeout=1h
/ip neighbor discovery-settings
set discover-interface-list=lan
/ip settings
set max-neighbor-entries=8192
/ipv6 settings
set disable-ipv6=yes max-neighbor-entries=8192
/interface bridge vlan
add bridge=BR1 tagged=BR1 untagged=\
    ether3-network_out,ether4-network_out,ether5-network_out,AC_2.4,AX_5 \
    vlan-ids=20
add bridge=BR1 tagged=BR1 untagged=AC_2.4_IoT vlan-ids=30
/interface list member
add interface=ether1-WAN-SPARK list=wan
add interface=ether2-WAN-2D list=wan
add interface=vHOME list=lan
add interface=vIOT list=lan
add interface=wireguard1 list=lan
/interface ovpn-server server
set auth=sha1,md5
/interface wireguard peers
add allowed-address=10.6.0.2/32 disabled=no interface=wireguard1 public-key=\
    "########################"
/ip address
add address=10.20.20.1/24 interface=vHOME network=10.20.20.0
add address=192.168.10.10/24 comment=WAN1 interface=ether1-WAN-SPARK network=\
    192.168.10.0
add address=192.168.11.10/24 comment=WAN2 interface=ether2-WAN-2D network=\
    192.168.11.0
add address=10.30.30.1/24 interface=vIOT network=10.30.30.0
add address=10.6.0.1/24 disabled=no interface=wireguard1 network=10.6.0.0
/ip dhcp-client
add disabled=yes interface=ether2-WAN-2D
add disabled=yes interface=ether1-WAN-SPARK
/ip dhcp-server network
add address=10.20.20.0/24 dns-server=10.20.20.6,10.20.20.7 gateway=10.20.20.1
add address=10.30.30.0/24 dns-server=1.1.1.1,1.0.0.1 gateway=10.30.30.1
/ip dns
set allow-remote-requests=yes max-udp-packet-size=512 servers=\
    10.20.20.6,10.20.20.7
/ip firewall address-list
add address=10.20.20.0/24 list=trusted_admin
add address=10.30.30.10 comment=Bulb list=Trusted_IoT
add address=10.20.20.200 comment=TurtBook list=my_devices
add address=10.20.20.201 comment=TurtMax list=my_devices
add address=10.20.20.2 comment=SWAG list=spark_wan
add address=10.20.20.9 comment=PLEX list=spark_wan
add address=10.20.20.31 comment=Smokeping-Spark list=spark_wan
/ip firewall filter
add action=accept chain=input comment=\
    "defconf: accept established,related,untracked" connection-state=\
    established,related,untracked
add action=drop chain=input comment="defconf: drop invalid" connection-state=\
    invalid
add action=accept chain=input comment="defconf: accept ICMP" protocol=icmp \
    src-address-list=trusted_admin
add action=accept chain=input comment=\
    "defconf: accept to local loopback (for CAPsMAN)" dst-address=127.0.0.1
add action=accept chain=input comment="WG handshake" dst-port=51856 \
    in-interface=ether1-WAN-SPARK protocol=udp
add action=accept chain=input in-interface-list=lan src-address-list=\
    trusted_admin
add action=accept chain=input comment=\
    "Allow lan DNS queries-UDP and NTP  services" dst-port=53,123 \
    in-interface-list=lan protocol=udp
add action=accept chain=input comment="Allow lan DNS queries - TCP" dst-port=\
    53 in-interface-list=lan protocol=tcp
add action=drop chain=input comment="drop all else"
add action=fasttrack-connection chain=forward comment="defconf: fasttrack" \
    connection-state=established,related hw-offload=yes
add action=accept chain=forward comment=\
    "defconf: accept established,related, untracked" connection-state=\
    established,related,untracked
add action=drop chain=forward comment="defconf: drop invalid" \
    connection-state=invalid
add action=accept chain=forward comment="allow internet traffic" \
    in-interface-list=lan out-interface-list=wan
add action=accept chain=forward comment="lan access - WG" in-interface=\
    wireguard1 out-interface-list=lan
add action=accept chain=forward comment="port forwarding" \
    connection-nat-state=dstnat
add action=accept chain=forward comment="access cam" dst-address=10.30.30.5 \
    src-address-list=my_devices
add action=accept chain=forward comment="Iot to Home lan" out-interface=vHOME \
    src-address-list=Trusted_IoT
add action=drop chain=forward comment="drop all else"
/ip firewall mangle
add action=mark-connection chain=prerouting connection-mark=no-mark \
    in-interface=ether1-WAN-SPARK new-connection-mark=spark_conn passthrough=\
    yes
add action=mark-routing chain=prerouting comment="Server Return Traffic" \
    connection-mark=spark_conn in-interface-list=lan log-prefix=ServReturn>> \
    new-routing-mark=useWAN1 passthrough=yes
add action=mark-routing chain=output comment="Wireguard Return Traffic" \
    connection-mark=spark_conn log=yes log-prefix=WGReturn>> \
    new-routing-mark=useWAN1 passthrough=no
/ip firewall nat
add action=masquerade chain=srcnat comment="MASQ Spark" out-interface=\
    ether1-WAN-SPARK
add action=masquerade chain=srcnat comment="MASQ 2Deg" out-interface=\
    ether2-WAN-2D
add action=dst-nat chain=dstnat comment=SWAG-Proxy dst-port=443 in-interface=\
    ether1-WAN-SPARK protocol=tcp to-addresses=10.20.20.2 to-ports=443
add action=dst-nat chain=dstnat comment=PLEX dst-port=32504 in-interface=\
    ether1-WAN-SPARK protocol=tcp to-addresses=10.20.20.9 to-ports=32400
/ip route
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway=192.168.10.254 \
    pref-src="" routing-table=useWAN1 scope=30 suppress-hw-offload=no \
    target-scope=10
add disabled=no distance=10 dst-address=0.0.0.0/0 gateway=192.168.10.254 \
    pref-src="" routing-table=main scope=30 suppress-hw-offload=no \
    target-scope=10
add disabled=no distance=5 dst-address=0.0.0.0/0 gateway=192.168.11.1 \
    pref-src="" routing-table=main scope=30 suppress-hw-offload=no \
    target-scope=10
/ip service
set telnet disabled=yes
set ftp disabled=yes
set www address=10.20.20.0/24 disabled=yes port=82
set ssh address=10.20.20.0/24
set www-ssl address=10.20.20.0/24
set api disabled=yes
set winbox address=10.20.20.0/24 port=8298
set api-ssl disabled=yes
/routing rule
add action=lookup-only-in-table comment=SWAG disabled=no src-address=\
    10.20.20.2/32 table=useWAN1
add action=lookup-only-in-table comment=PLEX disabled=no src-address=\
    10.20.20.9/32 table=useWAN1
add action=lookup-only-in-table comment=SMOKEPING disabled=no src-address=\
    10.20.20.31/32 table=useWAN1
add action=lookup-only-in-table comment=Wireguard disabled=yes src-address=\
    10.6.0.1/24 table=spark
add action=lookup-only-in-table comment=MACBOOK disabled=yes src-address=\
    10.20.20.200/32 table=spark
add action=lookup-only-in-table disabled=yes src-address=10.20.20.200/32 \
    table=2D
add action=lookup-only-in-table disabled=yes src-address=10.20.20.3/32 table=\
    2D
add action=lookup-only-in-table disabled=yes src-address=10.20.20.4/30 table=\
    2D
add action=lookup-only-in-table disabled=yes src-address=10.20.20.8/29 table=\
    2D
add action=lookup-only-in-table disabled=yes src-address=10.20.20.16/28 \
    table=2D
add action=lookup-only-in-table disabled=yes src-address=10.20.20.32/27 \
    table=2D
add action=lookup-only-in-table disabled=yes src-address=10.20.20.64/26 \
    table=2D
add action=lookup-only-in-table disabled=yes src-address=10.20.20.128/25 \
    table=2D
/system clock
set time-zone-name=Pacific/Auckland
/system identity
set name=AtlasV2
/system logging
add action=MinervaSyslog topics=warning
add action=MinervaSyslog topics=critical
add action=MinervaSyslog topics=error
add action=MinervaSyslog topics=interface
add action=MinervaSyslog topics=system
add action=MinervaSyslog topics=firewall
add action=MinervaSyslog disabled=yes topics=wireguard
add action=MinervaSyslog topics=info
add disabled=yes topics=debug
add disabled=yes topics=wireguard
add disabled=yes topics=wireless
/system note
set show-at-login=no
/system ntp client
set enabled=yes
/system ntp client servers
add address=101.100.146.146
add address=202.68.92.244
add address=43.252.70.34
add address=162.159.200.123
/tool bandwidth-server
set enabled=no
/tool mac-server
set allowed-interface-list=lan
/tool mac-server mac-winbox
set allowed-interface-list=lan

anav · August 22, 2023, 1:44pm

I do not understand the reasons for your routing rules… In fact, I dont see any reasons for any routing rules. Perhaps you can explain?
I provided simple mangling rules that:

a. marked any incoming WAN traffic on WAN1.
(i) thus any wireguard handshake traffic is marked and will use the output routing-mark to ensure handshake goes out WAN1
(ii) thus any remote users coming in WAN1 for LAN servers, will use the prerouting routing-mark to ensure traffic goes back out WAN1

b. created the Route and table for WAN1, which both the output and prerouting rules will point traffic too.

+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
For example, why does 10.20.20.2 need access to WAN1, if it is not originating traffic…
The same for all the rules!!
/routing rule
add action=lookup-only-in-table comment=SWAG disabled=no src-address=
10.20.20.2/32 table=useWAN1

The only one I dont quite understand is the reverse thingy with port 443>
Assuming that is a server on the LAN and it actually originates traffic then perhaps…
would agree that particular LANIP should have a routing rule, first in the order…
add src-address=LANIP action=lookup-only-in-table table=main

FlippinTurt · August 22, 2023, 9:11pm

Yeah of course

10.20.20.2 - reverse proxy, early days it used http validation for a domain so had to go out the same ip it came in on, now uses dns validation so probably isn’t needed to ‘originate’ traffic anymore
10.20.20.9 - plex, only have that so the plex server knows what it’s external IP is - else the server will try use WAN2’s ip and then use their relay. Now thinking about it, the srcnat should actually do the same job
10.20.20.31 - smokeping server, I have one on WAN1 and one on WAN2, testing pings to different locations from both wans - ICMP and curls

I will remove the .2 & .9 from the routing list

Apart from that though, do you see any reason why the tagged traffic still appears to go out WAN2?

anav · August 22, 2023, 10:21pm

What do you mean the plex will use the wrong WAN?
The question is simple does it originate traffic outbound and to whom/where… if not it only responds to external requests and is covered already assuming those requests come in on WAN1.

Would need to see your latest config to make any new assessments.

FlippinTurt · August 22, 2023, 10:34pm

If Plex doesn’t route out WAN1, it will by default route out WAN2. In doing this Plex will see it’s IP as the CGNAT IP and not the static IP of WAN1, This will make the server think there is no port forward available and route via their relay servers and not use the port forward. As it is a plex server it originates a connection to check what streaming capabilities it has (my understanding at least)

Same config as in post #5 http://forum.mikrotik.com/t/wireguard-wan-but-no-lan-access/168938/7
I don’t see how plex having it’s own route to WAN1 is making the WAN1 mangle route marked connection go out the wrong wan though

anav · August 22, 2023, 10:57pm

Understood, the plex needs to have access out WAN1 for some testing purposes and this is also the WAN that external users will use to access the plex.

WHY Do you have two addresses for WAN1??
add address=192.168.10.10/24 comment=WAN1 interface=ether1-WAN-SPARK network=
192.168.10.0
add address=192.168.11.10/24 comment=WAN2 interface=ether2-WAN-2D network=
192.168.11.0
add address=10.30.30.1/24 interface=vIOT network=10.30.30.0
add address=10.6.0.1/24 disabled=no interface=wireguard1 network=10.6.0.0
add address=210.54.89.78 disabled=yes interface=ether1-WAN-SPARK network=
210.54.89.78

At least for now, testing, remove the restriction, not required…

TWO OPTIONS.

Routing Rule, disable all routing rules keep this rule.
/routing rule
add action=lookup-only-in-table comment=PLEX disabled=no src-address=
10.20.20.9/32 table=useWAN1
Mangle Rules… disable all routing rules
add chain=prerouting action=mark-connection in-interface-list=lan src-address=10.20.20.9/32
connection-mark=no-mark new-connection-mark=internal_plex passthrough=yes
add chain=prerouting action=mark-routing connection-mark=internal-plex
new routing-mark=useWAN1 passthrough=yes

We could have done it with one rule, but I wanted to mark connection the traffic so it follows our fastrack rule…

FlippinTurt · August 23, 2023, 3:55am

RE two IP’s for WAN1, that was an earlier attempt at getting stuff to work, just messing around and have kept it disabled since, and have now removed it completely

I have gone with your option 1 and only left Plex (.9) in the routing rules section
WG traffic is still going out the second IP - in connections table it shows the correct connection mark but coming out WAN2 as well - thus never connects to client/phone

anav · August 23, 2023, 12:40pm

During testing at least remove this restriction it is not required anyway…
add action=accept chain=input comment=“defconf: accept ICMP” protocol=icmp
src-address-list=trusted_admin

This makes no sense to me,
a. you have the correct port set on the input chain rule and your test phone (using cellular) is connecting to WAN1 in its settings… ( do you do this by IP address or by dyndns name? )
b. you can confirm a handshake occurs on the input chain rule?
++++++++++++++++++++++++++++++++++++++++++++++++++++++++
c. we have mangling to mark the WAN port on the way in.
d. we output chain the marked route so the response goes back out WAN1.

It should work,
Please post your latest config this time.
Including the client settings, minus the actual public WANIP info

FlippinTurt · August 23, 2023, 10:33pm

Also very confused why it doesn’t work.
The client fails to handshake, I can see the connection come into the router, can see it get the connection mark, and can see it going out WAN2 with the connection park for WAN1.

The only way I got this to work was by making WAN1 primary, and routing everything out WAN2 with rules - seems maybe system services prefer the primary WAN?
Doing this got me WAN access through the vpn, but not LAN no matter what I did - as stated in earlier post, so same issue

/interface bridge
add name=BR1 vlan-filtering=yes
/interface ethernet
set [ find default-name=ether1 ] mtu=1560 name=ether1-WAN-SPARK poe-out=off
set [ find default-name=ether2 ] mtu=1560 name=ether2-WAN-2D
set [ find default-name=ether3 ] name=ether3-network_out
set [ find default-name=ether4 ] name=ether4-network_out
set [ find default-name=ether5 ] name=ether5-network_out
/interface wireguard
add listen-port=51856 mtu=1404 name=wireguard1
/interface vlan
add interface=BR1 name=vHOME vlan-id=20
add interface=BR1 name=vIOT vlan-id=30
/interface list
add name=wan
add name=lan
/interface wifiwave2 security
add authentication-types=wpa2-psk,wpa3-psk disabled=no name=2ghz wps=disable
add authentication-types=wpa2-psk,wpa3-psk disabled=no name=5ghz
add authentication-types=wpa2-psk disabled=no name=IoT
/interface wifiwave2
set [ find default-name=wifi2 ] channel.band=2ghz-n .width=20mhz \
    configuration.mode=ap .ssid=$$$$$$$$$ disabled=no name=AC_2.4 security=\
    2ghz
add configuration.hide-ssid=yes .mode=ap .ssid=IoT_SBS disabled=no \
    mac-address=11:22:33:44:55:66 master-interface=AC_2.4 name=AC_2.4_IoT \
    security=IoT
set [ find default-name=wifi1 ] channel.band=5ghz-ax .width=20/40/80mhz \
    configuration.mode=ap .ssid=$$$$$$$$$$$ disabled=no name=AX_5 \
    security=5ghz
/ip pool
add name=dhcp_pool1 ranges=10.20.20.68-10.20.20.200
add name=IoT_Pool ranges=10.30.30.20-10.30.30.50
/ip dhcp-server
add address-pool=dhcp_pool1 interface=vHOME lease-time=1d name=dhcp1
add address-pool=IoT_Pool interface=vIOT name=IoT_dhcp
/routing table
add disabled=no fib name=spark
add disabled=no fib name=WireG
add disabled=no fib name=2D
add disabled=no fib name=useWAN1
/system logging action
add name=MinervaSyslog remote=10.20.20.10 src-address=10.20.20.1 target=\
    remote
/zerotier
set zt1 comment="ZeroTier Central controller - https://my.zerotier.com/" \
    interfaces=ether2-WAN-2D,BR1 name=zt1 port=9993
/zerotier interface
add allow-default=no allow-global=no allow-managed=yes disabled=no instance=\
    zt1 name=AtlasV2_Link network=##############
/interface bridge port
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    ether3-network_out pvid=20
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    ether4-network_out pvid=20
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    ether5-network_out pvid=20
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    AC_2.4_IoT pvid=30
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    AC_2.4 pvid=20
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    AX_5 pvid=20
add bridge=BR1 disabled=yes interface=AtlasV2_Link
/ip firewall connection tracking
set tcp-established-timeout=1h
/ip neighbor discovery-settings
set discover-interface-list=lan
/ip settings
set max-neighbor-entries=8192
/ipv6 settings
set disable-ipv6=yes max-neighbor-entries=8192
/interface bridge vlan
add bridge=BR1 tagged=BR1 untagged=\
    ether3-network_out,ether4-network_out,ether5-network_out,AC_2.4,AX_5 \
    vlan-ids=20
add bridge=BR1 tagged=BR1 untagged=AC_2.4_IoT vlan-ids=30
/interface list member
add interface=ether1-WAN-SPARK list=wan
add interface=ether2-WAN-2D list=wan
add interface=vHOME list=lan
add interface=vIOT list=lan
add interface=wireguard1 list=lan
/interface ovpn-server server
set auth=sha1,md5
/interface wireguard peers
add allowed-address=10.6.0.2/32 interface=wireguard1 public-key=\
    "#####################"
/ip address
add address=10.20.20.1/24 interface=vHOME network=10.20.20.0
add address=192.168.10.10/24 comment=WAN1 interface=ether1-WAN-SPARK network=\
    192.168.10.0
add address=192.168.11.10/24 comment=WAN2 interface=ether2-WAN-2D network=\
    192.168.11.0
add address=10.30.30.1/24 interface=vIOT network=10.30.30.0
add address=10.6.0.1/24 interface=wireguard1 network=10.6.0.0
/ip dhcp-server network
add address=10.20.20.0/24 dns-server=10.20.20.6,10.20.20.7 gateway=10.20.20.1
add address=10.30.30.0/24 dns-server=1.1.1.1,1.0.0.1 gateway=10.30.30.1
/ip dns
set allow-remote-requests=yes max-udp-packet-size=512 servers=\
    10.20.20.6,10.20.20.7
/ip firewall address-list
add address=10.20.20.0/24 list=trusted_admin
add address=10.30.30.10 comment=Bulb list=Trusted_IoT
add address=10.20.20.200 comment=TurtBook list=my_devices
add address=10.20.20.201 comment=TurtMax list=my_devices
add address=10.20.20.2 comment=SWAG list=spark_wan
add address=10.20.20.9 comment=PLEX list=spark_wan
add address=10.20.20.31 comment=Smokeping-Spark list=spark_wan
/ip firewall filter
add action=accept chain=input comment=\
    "defconf: accept established,related,untracked" connection-state=\
    established,related,untracked
add action=drop chain=input comment="defconf: drop invalid" connection-state=\
    invalid log-prefix=IN-INVLD>>
add action=accept chain=input comment="defconf: accept ICMP" protocol=icmp
add action=accept chain=input comment=\
    "defconf: accept to local loopback (for CAPsMAN)" dst-address=127.0.0.1
add action=accept chain=input comment="WG handshake" dst-port=51856 \
    in-interface=ether1-WAN-SPARK protocol=udp
add action=accept chain=input in-interface-list=lan src-address-list=\
    trusted_admin
add action=accept chain=input comment=\
    "Allow lan DNS queries-UDP and NTP  services" dst-port=53,123 \
    in-interface-list=lan protocol=udp
add action=accept chain=input comment="Allow lan DNS queries - TCP" dst-port=\
    53 in-interface-list=lan protocol=tcp
add action=drop chain=input comment="drop all else" log-prefix=IN_DROP>>
add action=fasttrack-connection chain=forward comment="defconf: fasttrack" \
    connection-state=established,related hw-offload=yes
add action=accept chain=forward comment=\
    "defconf: accept established,related, untracked" connection-state=\
    established,related,untracked
add action=drop chain=forward comment="defconf: drop invalid" \
    connection-state=invalid log-prefix=FWD-INVLD>>
add action=accept chain=forward comment="allow internet traffic" \
    in-interface-list=lan out-interface-list=wan
add action=accept chain=forward comment="lan access - WG" in-interface=\
    wireguard1 out-interface-list=lan
add action=accept chain=forward comment="port forwarding" \
    connection-nat-state=dstnat
add action=accept chain=forward comment="access cam" dst-address=10.30.30.5 \
    src-address-list=my_devices
add action=accept chain=forward comment="Iot to Home lan" out-interface=vHOME \
    src-address-list=Trusted_IoT
add action=drop chain=forward comment="drop all else" log=yes log-prefix=\
    FWD-DROP>>
/ip firewall mangle
add action=mark-connection chain=prerouting comment="Spark Mark" \
    connection-mark=no-mark in-interface=ether1-WAN-SPARK \
    new-connection-mark=spark_conn passthrough=yes
add action=mark-routing chain=prerouting comment="Spark Return Mark" \
    connection-mark=spark_conn in-interface-list=lan log-prefix=ServReturn>> \
    new-routing-mark=spark passthrough=yes
add action=mark-routing chain=output comment="Spark Return Traffic" \
    connection-mark=spark_conn log=yes log-prefix=WGReturn>> \
    new-routing-mark=spark passthrough=no
/ip firewall nat
add action=masquerade chain=srcnat comment="MASQ Spark" out-interface=\
    ether1-WAN-SPARK
add action=masquerade chain=srcnat comment="MASQ 2Deg" out-interface=\
    ether2-WAN-2D
add action=dst-nat chain=dstnat comment=SWAG-Proxy dst-port=443 in-interface=\
    ether1-WAN-SPARK protocol=tcp to-addresses=10.20.20.2 to-ports=443
add action=dst-nat chain=dstnat comment=PLEX dst-port=32504 in-interface=\
    ether1-WAN-SPARK protocol=tcp to-addresses=10.20.20.9 to-ports=32400
/ip route
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway=192.168.10.254 \
    pref-src="" routing-table=spark scope=30 suppress-hw-offload=no \
    target-scope=10
add disabled=no distance=10 dst-address=0.0.0.0/0 gateway=192.168.10.254 \
    pref-src="" routing-table=main scope=30 suppress-hw-offload=no \
    target-scope=10
add disabled=no distance=5 dst-address=0.0.0.0/0 gateway=192.168.11.1 \
    pref-src="" routing-table=main scope=30 suppress-hw-offload=no \
    target-scope=10
/ip service
set telnet disabled=yes
set ftp disabled=yes
set www address=10.20.20.0/24 disabled=yes port=82
set ssh address=10.20.20.0/24
set www-ssl address=10.20.20.0/24 disabled=yes
set api disabled=yes
set winbox address=10.20.20.0/24 port=8298
set api-ssl disabled=yes
/routing rule
add action=lookup-only-in-table comment=PLEX disabled=no src-address=\
    10.20.20.9/32 table=spark
add action=lookup-only-in-table comment=SMOKEPING disabled=no src-address=\
    10.20.20.31/32 table=spark
/system clock
set time-zone-name=Pacific/Auckland
/system identity
set name=AtlasV2
/system logging
add action=MinervaSyslog topics=warning
add action=MinervaSyslog topics=critical
add action=MinervaSyslog topics=error
add action=MinervaSyslog topics=interface
add action=MinervaSyslog topics=system
add action=MinervaSyslog topics=firewall
add action=MinervaSyslog disabled=yes topics=wireguard
add action=MinervaSyslog topics=info
add disabled=yes topics=debug
add disabled=yes topics=wireguard
add disabled=yes topics=wireless
/system note
set show-at-login=no
/system ntp client
set enabled=yes
/system ntp client servers
add address=101.100.146.146
add address=202.68.92.244
add address=43.252.70.34
add address=162.159.200.123
/tool bandwidth-server
set enabled=no
/tool mac-server
set allowed-interface-list=lan
/tool mac-server mac-winbox
set allowed-interface-list=lan

Client is on a phone atm, so it doesn’t look fancy but thats what it is setup as.

WireGuard
Name	Atlas
Public Key	#################
Addresses	10.6.0.2/32
DNS servers	10.6.0.1
PEER	
Public Key	#################
Preshared key	Enabled
Endpoint	1.2.3.4:51856
Allowed IP’s	0.0.0.0/0

anav · August 23, 2023, 10:38pm

CLient Wireguard Settings:

The client is missing persistent-keep alive?
Also try changing client DNS to 10.20.20.6 or just 1.1.1.1, not saying what you have is wrong but willing to try anything at this point.
Ensure client wireguard MTU settings match that of the wireguard settings on the routers’ wireguard setting.

Make sure the public key you have in PEER settings is the one from the Router.

Router:
Why do you have MTU settings on your WAN connections???
Why are you monkeying with Wireguard MTU settings aka changing them from default???

Make sure the Public key in the peer settings is the public key from the Phone (the one on the phones interface settings)

MAJOR PROBLEM.
The traffic is not hitting the input chain rule for wireguard. That tells me that the WAN1 is not a public IP address directly connected to the router.
Otherwise the rule would go up by one, as there is nothing stopping that traffic unless its not a public IP…

DIFFERENT ISSUE - Something seems off on your mangling.

Also forget spark and smoke ping for now, it does nothing for us but potentially get in the way…
I dont understand why you are mangling and Routing Rules …

+++++++++++++++++
First you have not adjusted your fastrack rule it needs an addition!
add action=fasttrack-connection chain=forward comment=“defconf: fasttrack”
connection-state=established,related hw-offload=yes mark=no-mark

Now try these mangle rules and NO routing rules.
/ip firewall mangle
add action=mark-connection chain=prerouting comment=“Identify incoming WAN1”
connection-mark=no-mark in-interface=ether1-WAN-SPARK
new-connection-mark=spark_conn passthrough=yes
add action=mark-routing chain=output comment=“ensure services go out same WAN1”
connection-mark=spark_conn new-routing-mark=useWAN passthrough=no
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
add action=mark-connection chain=prerouting comment=“Identify outgoing plex”
connection-mark=no-mark src-address=10.20.20.9/32
new-connection-mark=plex_conn passthrough=yes
add action=mark-routing chain=prerouting comment=“routing for plex”"
connection-mark=plex_conn new-routing-mark=useWAN1 passthrough=yes

If you Want to use a mix of mangle and Routing rules.
Ditch the second half of the mangles ( the top one ensures Router answers services on same WAN)

Add add this routing rule.
add src-address=10.20.20.9/32 action=lookup table=useWAN1