Messed up routing between multiple wireguard tunnels

andrey555 · August 14, 2024, 10:48am

I have issues with setting up wireguard routing through another wireguard tunnel and the ISP’s default gateway. The main goal is to enable WireGuard clients (wg3) to connect to hap ac3 and route all client traffic as if the mobile client is simply within the hap ac3 network so I can access geo restricted services.

On hap ac3 (gwd), there are two WireGuard tunnels, wg0 (192.168.20.0/24) and wg1,(192.168.30.0/24) to VPN servers, with all outgoing traffic routed through one of these tunnels. Additionally, BGP is used to obtain network addresses that should be routed through the ISP’s gateway on ether1. There’s also a wg2 (192.168.10.0/24) interface connected to another hap ax3, located at your parents’ home.

When connecting to hap ac3 from a mobile client, I can see all networks: 192.168.2.0/24 (hap ac3’s local network), 192.168.1.0/24 (hap ax3 network via wg2), and both VPN server networks. However, external traffic from the mobile client does not leave hap ac3, as if the routing table is not being used. Traffic from clients directly connected to hap ac3 is successfully routed either through VPN servers or the ISP gateway.

I can add masquerade rule:

add action=masquerade chain=srcnat log=yes log-prefix="masq wg3-wg0" \
    out-interface=wg0 src-address-list=wireguard-clients

In this case, all outgoing traffic is routed through the VPN, ignoring routes obtained via BGP.
It seems I’ve overcomplicated the configuration but can’t find where (probably everywhere)

Some map
network.drawio-2.png

routeros config
wg0, wg1 - client to external vpn servers
wg2 - client to another hap ax3
wg3 - server for mobile clients, 192.168.15.0/24 subnet, test client have 192.168.15.5 address, AllowedIPs in config is 0.0.0.0/0
ether1 - ISP WAN

# 2024-08-14 11:57:50 by RouterOS 7.15.3
# software id = 3B4V-DQ23
#
# model = RBD53iG-5HacD2HnD
# serial number = [REDACTED]

/interface wireguard
add comment=wg-2 listen-port=16948 mtu=1420 name=wg0
add comment=wg-1 listen-port=41378 mtu=1420 name=wg1
add comment=to-d1 listen-port=41388 mtu=1420 name=wg2
add comment="d32 wg incoming" listen-port=51821 mtu=1420 name=wg3
/interface list
add comment=defconf name=WAN
add comment=defconf name=LAN

/routing bgp template
set default as=[REDACTED] hold-time=4m keepalive-time=1m multihop=yes \
    output.network=bgp-networks .no-client-to-client-reflection=yes \
    router-id=[REDACTED]
/routing bgp connection
add as=[REDACTED] connect=yes disabled=no input.filter=bgp-in listen=yes \
    local.address=[REDACTED] .role=ebgp multihop=yes name=\
    bgp-peer0 output.filter-chain=discard remote.address=\
    [REDACTED] .as=[REDACTED] .port=179 router-id=[REDACTED] \
    routing-table=main templates=default
/routing filter rule
add chain=discard disabled=no rule="reject;"
add chain=bgp-in disabled=no rule="set gw ether1; accept;"    
/routing table
add fib name=direct
/interface bridge port
add bridge=bridge comment=defconf interface=ether2
add bridge=bridge comment=defconf interface=ether3
add bridge=bridge comment=defconf interface=ether4
add bridge=bridge comment=defconf interface=ether5
add bridge=bridge comment=defconf interface=wifi1
add bridge=bridge comment=defconf interface=wifi2
/interface list member
add comment=defconf interface=bridge list=LAN
add comment=defconf interface=ether1 list=WAN
add interface=wg2 list=LAN
add interface=wg3 list=LAN
add interface=wg0 list=LAN
/interface wireguard peers
add allowed-address=0.0.0.0/0 comment=vpn2 endpoint-address=\
    vpn-1.[REDACTED] endpoint-port=51826 interface=wg0 name=vpn-1 \
    persistent-keepalive=10s public-key=\
    "[REDACTED]"
add allowed-address=0.0.0.0/0 comment=vpn2 endpoint-address=\
    vpn-2.[REDACTED] endpoint-port=51826 interface=wg1 name=vpn-2 \
    persistent-keepalive=10s public-key=\
    "[REDACTED]"
add allowed-address=0.0.0.0/0 comment="to-d1" endpoint-address=\
    vpn-d1.[REDACTED] endpoint-port=59321 interface=wg2 name=gw-d1 \
    persistent-keepalive=15s preshared-key=\
    "[REDACTED]" public-key=\
    "[REDACTED]"
add allowed-address=192.168.15.5/32 comment=iphone endpoint-address=\
    192.168.15.5 interface=wg3 is-responder=yes name=peer6 preshared-key=\
    "[REDACTED]" public-key=\
    "[REDACTED]"
/ip address
add address=192.168.2.1/24 comment=defconf interface=bridge network=\
    192.168.2.0
add address=192.168.20.3/24 interface=wg0 network=192.168.20.0
add address=192.168.30.3/24 interface=wg1 network=192.168.30.0
add address=192.168.10.2 interface=wg2 network=192.168.10.0
add address=192.168.15.1/24 interface=wg3 network=192.168.15.0
/ip dhcp-client
add add-default-route=no comment=defconf interface=ether1 use-peer-dns=no
/ip dhcp-server network
add address=192.168.2.0/24 comment=defconf dns-server=192.168.2.10 domain=\
    local gateway=192.168.2.1 netmask=24
/ip dns
set allow-remote-requests=yes servers=172.64.36.1,172.64.36.2 use-doh-server=\
    https://[REDACTED].cloudflare-gateway.com/dns-query verify-doh-cert=yes
/ip dns static
add address=192.168.2.1 comment=defconf name=gwd.lan
add address=162.159.0.0 name=[REDACTED].cloudflare-gateway.com
add address=162.159.0.0 name=[REDACTED].cloudflare-gateway.com

/ip firewall address-list
add address=192.168.2.10 list=dns-servers
add address=192.168.2.1 list=dns-servers
add address=192.168.1.2 list=dns-servers
add address=192.168.1.3 list=dns-servers
add address=192.168.1.4 list=dns-servers
add address=192.168.1.5 list=dns-servers
add address=192.168.20.1 list=dns-servers
add address=192.168.30.1 list=dns-servers
add address=192.168.15.0/24 list=wireguard-clients
/ip firewall filter
add action=accept chain=input comment="Allow Wireguard from All" dst-port=\
    51821 log-prefix=WG_In protocol=udp
add action=accept chain=input comment=\
    "defconf: accept established,related,untracked" connection-state=\
    established,related,untracked
add action=drop chain=input comment="defconf: drop invalid" connection-state=\
    invalid
add action=accept chain=input comment="defconf: accept ICMP" protocol=icmp
add action=accept chain=input comment=\
    "defconf: accept to local loopback (for CAPsMAN)" dst-address=127.0.0.1
add action=drop chain=input comment="defconf: drop all not coming from LAN" \
    in-interface-list=!LAN log-prefix=drop-input
add action=accept chain=forward comment="defconf: accept in ipsec policy" \
    ipsec-policy=in,ipsec
add action=accept chain=forward in-interface=wg3
add action=accept chain=forward comment="wg rw to outside" in-interface=wg3 \
    log=yes log-prefix=wg3_to_wg0 out-interface=wg0
add action=accept chain=forward comment="wg rw to outside" in-interface=wg3 \
    log=yes log-prefix=wg3_to_ether1 out-interface-list=WAN
add action=accept chain=forward comment="defconf: accept out ipsec policy" \
    ipsec-policy=out,ipsec
add action=fasttrack-connection chain=forward comment="defconf: fasttrack" \
    connection-state=established,related hw-offload=yes
add action=accept chain=forward comment=\
    "defconf: accept established,related, untracked" connection-state=\
    established,related,untracked
add action=drop chain=forward comment="defconf: drop invalid" \
    connection-state=invalid
add action=drop chain=forward comment=\
    "defconf: drop all from WAN not DSTNATed" connection-nat-state=!dstnat \
    connection-state=new in-interface-list=WAN
add action=drop chain=input dst-port=53 in-interface-list=WAN log=yes \
    log-prefix=DNS_In protocol=udp
add action=drop chain=input dst-port=53 in-interface-list=WAN log=yes \
    log-prefix=DNS_In protocol=tcp
add action=drop chain=forward dst-port=53 log=yes log-prefix=DNSBlock_Out \
    out-interface-list=WAN protocol=tcp src-address-list=!dns-servers
add action=drop chain=forward dst-port=53 log=yes log-prefix=DNSBlock_Out \
    out-interface-list=WAN protocol=udp src-address-list=!dns-servers
add action=drop chain=forward dst-port=53 log=yes log-prefix=DNSBlock_Out \
    out-interface=wg0 protocol=udp src-address-list=!dns-servers
add action=accept chain=input comment="Allow DNS from Wireguard Users" \
    dst-port=53 in-interface=wg3 protocol=udp
/ip firewall mangle
add action=change-mss chain=forward new-mss=clamp-to-pmtu out-interface=wg0 \
    passthrough=yes protocol=tcp tcp-flags=syn
add action=change-mss chain=forward new-mss=clamp-to-pmtu out-interface=wg1 \
    passthrough=yes protocol=tcp tcp-flags=syn
add action=change-mss chain=forward new-mss=clamp-to-pmtu out-interface=wg1 \
    passthrough=yes protocol=tcp tcp-flags=syn
add action=change-mss chain=forward new-mss=clamp-to-pmtu out-interface=wg3 \
    passthrough=yes protocol=tcp tcp-flags=syn
/ip firewall nat
add action=masquerade chain=srcnat comment="defconf: masquerade" \
    ipsec-policy=out,none out-interface-list=WAN
add action=dst-nat chain=dstnat dst-address-list=!dns-servers dst-port=53 \
    in-interface-list=LAN log=yes log-prefix=DNS_Redir_DST_TCP protocol=tcp \
    src-address-list=!dns-servers to-addresses=192.168.2.10 to-ports=53
add action=dst-nat chain=dstnat dst-address-list=!dns-servers dst-port=53 \
    in-interface-list=LAN log=yes log-prefix=DNS_Redir_DST_UDP protocol=udp \
    src-address-list=!dns-servers to-addresses=192.168.2.10 to-ports=53
add action=masquerade chain=srcnat dst-port=53 log=yes log-prefix=\
    DNS_Redir_MASQ_UDP protocol=udp src-address-list=!dns-servers

/ip firewall raw
add action=drop chain=prerouting in-interface-list=WAN log=yes log-prefix=\
    DNS_RAW_UDP port=53 protocol=udp
add action=drop chain=prerouting in-interface-list=WAN log=yes log-prefix=\
    DNS_RAW_UDP port=53 protocol=tcp
/ip route
add dst-address=192.168.20.0/24 gateway=wg0
add comment=d1 dst-address=[wg2 remote server addr] gateway=ether1
add disabled=no distance=1 dst-address=0.0.0.0/1 gateway=wg0
add disabled=no distance=1 dst-address=128.0.0.0/1 gateway=wg0
add comment=vpn1 dst-address=[wg0 remote server addr] gateway=ether1
add comment=vpn2 dst-address=[wg1 remote server addr] gateway=ether1
add disabled=no distance=2 dst-address=0.0.0.0/1 gateway=wg1
add disabled=no distance=2 dst-address=128.0.0.0/1 gateway=wg1
add dst-address=192.168.1.0/24 gateway=wg2
add disabled=yes distance=10 dst-address=0.0.0.0/0 gateway=ether1


/system identity
set name=gwd.lan
/system ntp client servers
add address=time.cloudflare.com

anav · August 14, 2024, 2:16pm

I am lost in your explanation.
Let me state what I think is true.

You have ISP1 router/modem connected to the hapax3.
Does the hapax get a public IP or private IP but you can forward port from ISP device to AX3

You have ISP2 router/modem connected to the hapac.
Does the hapac get a public IP or private IP but you can forward port from ISP device to AC

You have three wireguard subnets identified
192.168.10.0
192.168.20.0
192.168.30.0

Which Router is the client SERVER for handshake and which Router is the client Peer for handshake for each of the three subnets.
Note if using a third party VPN provider for one of them, then state which Router is the Client Peer for handshake.

Finally mobile wireguard clients.
Which users are connecting to which wireguard tunnel and for what purposes…

When directing local users or local subnets on a router into wireguard, state clearly which subnets and users…
State, what happens if the wireguard tunnel fails…should these local users get local WAN access if the use of the tunnel is for internet??

Larsa · August 14, 2024, 3:16pm

And what role does BGP play in all of this? Is iBGP used for internal routing?

anav · August 14, 2024, 3:20pm

Hopefully very little or at least it can be ignored as part of the bread and butter of getting wireguard working first.
Otherwise it may be Larsa on his own.

andrey555 · August 14, 2024, 4:13pm

Yes, configuration is complicated, and it evolved gradually, starting with a tunnel to one of the VPN servers, which is functioning well.

BGP can be completely disabled, relying solely on static routes through ether1 (ISP gateway). It used just to get geo restricted subnets.

Both Mikrotik devices have public IPs.

hapax3 is the WireGuard server (192.168.10.1), wireguard subnet 192.168.10.0/24.

hapac2 is a client to hapax3 with the address 192.168.10.2/32.
Everything is working perfectly here, with clients behind hapac2 able to see clients behind hapax3 and vice versa.

There are two VPN servers (VPS using Debian) with wireguard set up as the server:

vpn1 - 192.168.20.0/24
vpn2 - 192.168.30.0/24

hapac2 acts as a client to these servers:

1   192.168.20.3/24   192.168.20.0  wg0
2   192.168.30.3/24   192.168.30.0  wg1

Static routes are configured as follows:

vpn1_addr via ether1
vpn2_addr via ether1
0.0.0.0/0 via wg0 distance 1
0.0.0.0/0 via wg1 distance 2

Up to this, everything works perfectly, with routes for specific addresses going through ether1, and all other traffic through wg0.

hapac2 is configured as a wireguard server (192.168.15.0/24) for mobile clients. The clients are configured to route all traffic through hapac2.

The mobile client (192.168.15.5/32) has access to other clients behind hapac2 and hapax3, but internet traffic doesn’t pass through either ether1 or wg0. The traceroute fails at the first hop, 192.168.15.1 (hapac2 wg address).

State, what happens if the wireguard tunnel fails…should these local users get local WAN access if the use of the tunnel is for internet??

Right now local users will loose access to internet, I was planning to add netwatch script to disable routes via failed tunnel, failing back to ISP gateway as last resort

hapax3 actually don’t needed for all configuration and can be perfectly disconneced, it just shows that at least some routing works as desired and without NAT:
mobile client (192.168.15.5) → hapac2 wireguard server (192.168.15.1) → hapax3 wireguard server (192.168.10.1) → client behind hapax3 (192.168.1.3)

I’ll try to backup hapac2 and configure it from scratch, removing BGP, DNS redirect to local server, leaving just minimal wireguard tunnels setup. but right now as I understand what to do that this will likely result in a similar configuration. Each part of config works fine on its own, but not together

Larsa · August 14, 2024, 5:06pm

This seems like a classic case of an XY problem, made even harder to understand due to an overly complicated network diagram and an even more confusing technical walkthrough.

My understanding is that you want all clients from site 1 to route to the internet via site 2. Is that correct?

anav · August 14, 2024, 5:44pm

Almost there…
The VPS VPN servers, do you mean in the cloud somewhere?
The purpose of this is to access internet from a different location for clients ( hapac or mobile ) ??

Why dont you use CHR in the cloud LOL.

anav · August 14, 2024, 6:07pm

New Questions.
Do you also have mobile clients going directly the hapax3 as well? ( at least the admin for config purposes or to reachLAN, two devices being common, laptop and smartphone/ipad )???

Understand mobile clients connecting directly to hapac are for two purposes.
a. some to go through the internet of hapac
b. admin to be able to configure hapac remotely.

Understand LAN clients on both routers need to reach each other and do so on the hapax wireguard server subnet with no issue.

What is not clear to me is which sets of users need to go out the two debian VPS wireguard (server) connections.
Assuming these connections are established from the hapac to the VPS.
Is it just the mobile clients connecting direclty to the hapac via wg (assuming yes).

Will need to see both MT router configs and wireguard settings on vps.
WHY TWO Wireguard VPNS on VPS, that was also confusing.

anav · August 14, 2024, 6:27pm

brief comments on hapac config shown.
Suggesting modify interface list members ( plus missing wg1 )

/interface list member
add comment=defconf interface=bridge list=LAN
add comment=defconf interface=ether1 list=WAN
add interface=wg0 list=WAN comment=“WG1 VPS”
add interface=wg1 list=WAN comment=“WG2 VPS”
add interface=wg2 list=LAN comment=“client to ax3”
add interface=wg3 list=LAN comment=“server for mobile”

Modify PEERS
-First comment should be vpn1 not vpn2
-No such use of pre-shared key, remove!!!
-no need for endpoint address on wg3

/interface wireguard peers
add allowed-address=0.0.0.0/0 comment=vpn**1** endpoint-address=
vpn-1.[REDACTED] endpoint-port=51826 interface=wg0 name=vpn-1 persistent-keepalive=10s public-key=“-”
add allowed-address=0.0.0.0/0 comment=vpn2 endpoint-address=vpn-2.[REDACTED]
endpoint-port=51826 interface=wg1 name=vpn-2 persistent-keepalive=10s public-key=“[–”
add allowed-address=0.0.0.0/0 comment=“to-d1” endpoint-address=vpn-d1.[REDACTED]
endpoint-port=59321 interface=wg2 name=gw-d1 persistent-keepalive=15s public-key=“__”
add allowed-address=192.168.15.5/32 comment=iphone interface=wg3 is-responder=yes name=peer6 public-key=“++”

Why are you using 0.0.0.0/0 for allowed IPs on the hapac for the wg2 connection to the hapAX3 server.
Do you intend for anyone in particular to be using the WAN side of the hapax3 ???

I would have expected
add allowed-address=192.168.10.0/24,192.168.1.0/24 comment=“to-d1” endpoint-address=hapax3

anav · August 14, 2024, 6:30pm

Looking at routes… the only people being forced are mobile clients on wg3…
Also still dont know why you have VPN1 and VPN2 ???

/routing table
add fib name=direct1
add fib name=direct2

/ip route
add dst-address=0.0.0.0/0 gateway=ether1-gatewayIP routing-table=main
add dst-address=192.168.1.0/24 gateway=wg2 routing-table=main
++++++++++++++++++++++++++++++++++++++++++++++++++
add distance=2 dst-address=0.0.0.0/0 gateway=1.1.1.1 scope=10 target-scope=12
add distance=2 dst-address=1.1.1.1/32 gateway=wg0 scope=10 target-scope=11
add distance=4 dst-address=0.0.0.0/0 gateway=9.9.9.9 scope=10 target-scope=12
add distance=4 dst-address=9.9.9.9/32 gateway=wg1 scope=10 target-scope=11
++++++++++++
add dst-address=0.0.0.0/0 gateway=wg0 routing-table=direct1
add dst-address=0.0.0.0/0 gateway=wg1 routing-table=direct2

/ip routing rule
add dst-address=192.168.2.0/24 action=lookup-only-in-table table=main { remove if these clients .15, should not access local hapac LAN }
add dst-address=192.168.1.0/24 action=lookup-only-in-table table=main { remove if these clients should not access AX3 }
add src-address=192.168.15.0/24 action=lookup table=direct1
add src-address=192..168.15.0/24 action=lookup table=direct2

andrey555 · August 14, 2024, 6:30pm

I think Larsa right and I should start from what configuration i want to have working

What i want to do is to be able being in another country virtually being “at home”. Have all access to network behind home hapac2 and have routes same as every device behind ac2 have.

phone -- wireguard --> hapac2 ----> internet via cloud vps vpn (1)
						|---> geo restricted sites (goverment, etc) via ISP gw (2)
					        |---> local devices (3)
					        |--- tunnel to hapax3 --> devices behind  hapax3 (4)

(3) and (4) works perfectly, but (1) and (2) - no. It looks like hapac2 don’t know where to send this packets next

At least

The VPS VPN servers, do you mean in the cloud somewhere?

yes, and masquerading configured and of course moblie clients can have internet acces via them. But they don’t have network access to geo restricted service and ISP gateway - has.
CHR whould be just drop-in replacement for installed debian, but maybe someday )

anav · August 14, 2024, 6:33pm

Didnt understand a word of that post, please answer questions asked and/or confirum assumptions made.
Also, the routes were a mess, surprised anything worked.

andrey555 · August 14, 2024, 6:56pm

Actually yes to all

Do you also have mobile clients going directly the hapax3 as well? ( at least the admin for config purposes or to reachLAN, two devices being common, laptop and smartphone/ipad )???
you mean without any tunnels, just using ethernet/wifi on hapax3? yes, of course

Understand mobile clients connecting directly to hapac are for two purposes. > yes
a. some to go through the internet of hapac
b. admin to be able to configure hapac remotely.

Understand LAN clients on both routers need to reach each other and do so on the hapax wireguard server subnet with no issue. > yes

What is not clear to me is which sets of users need to go out the two debian VPS wireguard (server) connections. > all users behind two mikrotiks and all mobile clinets
Assuming these connections are established from the hapac to the VPS. > yes
Is it just the mobile clients connecting direclty to the hapac via wg (assuming yes). > yes

I reviewed configuration again and found issue, I completely forgot to include mobile clients wg subnet to allowed ips. After adding everything works smoothly

Thanks for you suggestions, 0.0.0.0/0 and other left from my initial attempts, will change it with removing preshared keys
Maybe last question - why “no need for endpoint address on wg3” ? If I remove it I wouldn’t have any traffic between hapac2 and hapax3.
If I trace from client behind hapac2 another client behind hapax3 I see wg3 address as intermediate host

 
 andrey@rpi5 ~> ip -br a sh eth0
eth0             UP             192.168.2.10/24
andrey@rpi5 ~> traceroute 192.168.1.3
traceroute to 192.168.1.3 (192.168.1.3), 30 hops max, 60 byte packets
 1  router.lan (192.168.2.1)  0.385 ms  0.359 ms  0.347 ms
 2  192.168.10.1 (192.168.10.1)  7.276 ms  7.414 ms  7.403 ms
 3  192.168.1.3 (192.168.1.3)  7.587 ms  7.576 ms  7.955 ms

Routing… yes, messed up, it was in plan to refactor it next. I will try splitting to different tables and using rules. Again thanks for help
second VPN is just for backup purposes..

anav · August 14, 2024, 8:52pm

Okay starting to get a clearer picture LOL
You dont need endpoint address on wg3, because the hapac is the SERVER for this wireguard network. Only client peers for handshake require to put the endpoint it.
Wg3 has nothing to do with the hapax3. That wg interface connectiong to the axe3 is wg2, where the hapac is a client to the ax3 server and does need endpoint address.
Not my fault your weird naming structure confuses you LOL.

Okay, so in SUMMARY
You have two ROUTERS, that need to be connected via Wireguard, such that admins can configure either router from either location and where subnets should be able to reach each other.
You have mobile road warriors that need to be connected via Wireguard and should be able to reach both router LANs and also the configs of Both Router.
This Wireguard requires a single interface but will use a different subnet for the mobile clients for clear distinction, and fw rules.

All users, subnets on both routers and mobile road warrior SHOULD get their internet from VPN1, VPN2.
If Both VPN1 and VPN2 are not available Then please confirm:
a. Router1 subnet users should get local internet from Router 1 Y/N
b. Router 2 subnet users should get local internet from Router 2 Y/N
c. Where should the mobile warriors get their internet from if VPS is not available, Assuming routerhapac local WAN???

By the way the reason to have multiple wireguard interfaces is too avoid conflicts since ALL USers need to have 0.0.0.0/0 in their allowed IPs for internet access via VPN1 VPN2.

You need a different wireguard for VPN1 Both Routers will be clients on this wg
You need a different wireguard for VPN2 Both routers will be clients on this wg.
In other words, its less efficient and too complex to try to create a wireguard tunnel within a wireguard tunnel when its not required.
Mobile wireguard clients will connect to the hapac router, and then from there will be directed to the VPN1 or VPN2 for internet.

anav · August 14, 2024, 8:52pm

I am still waiting for the hapax3 config !!!

andrey555 · August 15, 2024, 9:29am

Okay, so in SUMMARY

You have two ROUTERS, that need to be connected via Wireguard, such that admins can configure either router from either location and where subnets should be able to reach each other.

yes

You have mobile road warriors that need to be connected via Wireguard and should be able to reach both router LANs and also the configs of Both Router.

yes

This Wireguard requires a single interface but will use a different subnet for the mobile clients for clear distinction, and fw rules.

All users, subnets on both routers and mobile road warrior SHOULD get their internet from VPN1, VPN2.

by default gateway should be VPN1,VPN2, and static routes for specific subnets via IPS gateway via ether1

If Both VPN1 and VPN2 are not available Then please confirm:
a. Router1 subnet users should get local internet from Router 1

yes

b. Router 2 subnet users should get local internet from Router 2

yes

c. Where should the mobile warriors get their internet from if VPS is not available, Assuming routerhapac local WAN???

if no matches to static routes via isp gateway then no internet would be available.

By the way the reason to have multiple wireguard interfaces is too avoid conflicts since ALL USers need to have 0.0.0.0/0 in their allowed IPs for internet access via VPN1 VPN2.

You need a different wireguard for VPN1 Both Routers will be clients on this wg
You need a different wireguard for VPN2 Both routers will be clients on this wg.

Maybe I don't understand you correctly, but that's what i already have. wg0, wg1 interfaces on both mikrotiks

Mobile wireguard clients will connect to the hapac router, and then from there will be directed to the VPN1 or VPN2 for internet.

yes, thats what I'm trying to do, not wireguard inside wireguard

hapax3 config:

# 2024-08-15 11:58:14 by RouterOS 7.15.2
# software id = 0YNX-ERAG
#
# model = C53UiG+5HPaxD2HPaxD
# serial number = [REDACTED]
/interface wireguard
add comment=vpn1 listen-port=44302 mtu=1420 name=wg0
add comment=vpn2 listen-port=44303 mtu=1420 name=wg1
add comment=hapac2-d32 listen-port=59321 mtu=1420 name=wg2
/interface vlan
add interface=ether1 name=vlan1 vlan-id=3
/interface list
add comment=defconf name=WAN
add comment=defconf name=LAN
/ip pool
add name=dhcp ranges=192.168.1.20-192.168.1.99

/routing table
add fib name=direct

/ip firewall connection tracking
set udp-timeout=10s
/ip neighbor discovery-settings
set discover-interface-list=LAN
/interface list member
add comment=defconf interface=bridge list=LAN
add comment=defconf interface=ether1 list=WAN
add interface=vlan1 list=WAN
add interface=wg2 list=LAN
add interface=wg0 list=LAN
add interface=wg1 list=LAN
/interface wireguard peers
add allowed-address=0.0.0.0/0 comment=vpn1 endpoint-address=\
    [vpn1_addr] endpoint-port=51826 interface=wg0 name=vpn1 \
    persistent-keepalive=10s public-key=\
    "[REDACTED]"
add allowed-address=0.0.0.0/0 comment=vpn2 endpoint-address=\
    [vpn2_addr] endpoint-port=51826 interface=wg1 name=vpn2 \
    persistent-keepalive=10s public-key=\
    "[REDACTED]"
add allowed-address=192.168.10.0/24,192.168.2.0/24,192.168.15.0/24 comment=\
    gw-hapac2-d32 interface=wg2 is-responder=yes name=gw-d32 public-key=\
    "[REDACTED]"

/ip address
add address=192.168.1.1/24 comment=defconf interface=bridge network=\
    192.168.1.0
add address=192.168.20.2/24 interface=wg0 network=192.168.20.0
add address=192.168.30.2/24 interface=wg1 network=192.168.30.0
add address=192.168.10.1/24 interface=wg2 network=192.168.10.0
/ip dhcp-client
add add-default-route=no interface=vlan1 use-peer-dns=no use-peer-ntp=no

/ip dns
set allow-remote-requests=yes doh-max-concurrent-queries=1000 \
    doh-max-server-connections=100 servers=172.64.36.1,172.64.36.2 \
    use-doh-server=https://[REDACTED].cloudflare-gateway.com/dns-query \
    verify-doh-cert=yes
/ip dns static
add address=192.168.1.1 comment=defconf name=router.lan
add address=[REDACTED] name=[vpn1_address]
add address=[REDACTED] name=[vpn2_address]

/ip firewall address-list
add address=192.168.1.2 list=dns-servers
add address=192.168.1.3 list=dns-servers
add address=192.168.1.4 list=dns-servers
add address=192.168.1.5 list=dns-servers
add address=192.168.1.6 comment="torrent client" list=direct-hosts
add address=192.168.1.225 list=dns-servers

/ip firewall filter
add action=accept chain=input comment="wireguard in" dst-port=51826 \
    in-interface-list=WAN log=yes log-prefix=WG_In protocol=udp
add action=accept chain=input comment=\
    "defconf: accept established,related,untracked" connection-state=\
    established,related,untracked
add action=drop chain=input comment=antifilter.network connection-state=\
    invalid
add action=accept chain=input comment="defconf: accept ICMP" protocol=icmp
add action=accept chain=input comment=\
    "defconf: accept to local loopback (for CAPsMAN)" dst-address=127.0.0.1
add action=drop chain=input comment="defconf: drop all not coming from LAN" \
    in-interface-list=!LAN
add action=drop chain=input connection-nat-state=dstnat connection-state="" \
    dst-port=53 in-interface=vlan1 log=yes log-prefix=DNS_IN_vlan1 protocol=\
    udp
add action=drop chain=input comment="block incoming dns" dst-port=53 \
    in-interface-list=WAN log=yes log-prefix=DNS_In protocol=tcp
add action=drop chain=input comment="block incoming dns" dst-port=53 \
    in-interface-list=WAN log=yes log-prefix=DNS_In protocol=udp
add action=accept chain=forward comment="defconf: accept in ipsec policy" \
    ipsec-policy=in,ipsec
add action=accept chain=forward comment="defconf: accept out ipsec policy" \
    ipsec-policy=out,ipsec
add action=fasttrack-connection chain=forward comment="defconf: fasttrack" \
    connection-state=established,related hw-offload=yes
add action=accept chain=forward comment=\
    "defconf: accept established,related, untracked" connection-state=\
    established,related,untracked
add action=drop chain=forward comment="defconf: drop invalid" \
    connection-state=invalid
add action=drop chain=forward comment=\
    "defconf: drop all from WAN not DSTNATed" connection-nat-state=!dstnat \
    connection-state=new in-interface-list=WAN
add action=reject chain=output comment="block DNS-over-TLS" dst-port=853 \
    out-interface-list=WAN protocol=tcp src-address-list=!dns-servers
add action=reject chain=forward comment="block DNS-over-TLS" dst-port=853 \
    out-interface-list=WAN protocol=tcp src-address-list=!dns-servers
add action=drop chain=forward comment="block outgoing dns" dst-port=53 log=\
    yes log-prefix=DNS_Out out-interface-list=WAN protocol=tcp \
    src-address-list=!dns-servers
add action=drop chain=forward comment="block outgoing dns" dst-port=53 log=\
    yes log-prefix=DNS_Out out-interface-list=WAN protocol=udp \
    src-address-list=!dns-servers
add action=drop chain=forward comment="block outgoing dns" dst-port=53 log=\
    yes log-prefix=DNS_Out out-interface=wg0 protocol=tcp src-address-list=\
    !dns-servers
add action=drop chain=forward comment="block outgoing dns" dst-port=53 log=\
    yes log-prefix=DNS_Out out-interface=wg0 protocol=udp src-address-list=\
    !dns-servers
add action=reject chain=output comment="block DNS-over-TLS" dst-port=853 \
    out-interface=wg0 protocol=tcp src-address-list=!dns-servers
add action=reject chain=forward comment="block DNS-over-TLS" dst-port=853 \
    out-interface=wg0 protocol=tcp src-address-list=!dns-servers
/ip firewall mangle
add action=mark-routing chain=prerouting disabled=yes dst-address-list=\
    youtube new-routing-mark=direct passthrough=yes
add action=change-mss chain=forward new-mss=clamp-to-pmtu out-interface=wg0 \
    passthrough=yes protocol=tcp tcp-flags=syn
add action=change-mss chain=forward new-mss=clamp-to-pmtu out-interface=wg1 \
    passthrough=yes protocol=tcp tcp-flags=syn
add action=change-mss chain=forward new-mss=clamp-to-pmtu out-interface=wg2 \
    passthrough=yes protocol=tcp tcp-flags=syn
/ip firewall nat
add action=masquerade chain=srcnat comment="defconf: masquerade" \
    ipsec-policy=out,none out-interface-list=WAN
add action=dst-nat chain=dstnat comment="redirect dns" dst-port=53 \
    in-interface-list=LAN log=yes log-prefix=DNS_Redir_DST_UDP protocol=udp \
    src-address-list=!dns-servers to-addresses=192.168.1.5 to-ports=53
add action=dst-nat chain=dstnat comment="redirect dns" dst-port=53 \
    in-interface-list=LAN log=yes log-prefix=DNS_Redir_DST_TCP protocol=tcp \
    src-address-list=!dns-servers to-addresses=192.168.1.5 to-ports=53
add action=masquerade chain=srcnat comment="masquerade dns" dst-address=\
    192.168.1.5 dst-port=53 log=yes log-prefix=DNS_Redir_MASQ_UDP protocol=\
    udp src-address-list=!dns-servers
add action=masquerade chain=srcnat comment="masquerade dns" dst-address=\
    192.168.1.5 dst-port=53 log=yes log-prefix=DNS_Redir_MASQ_TCP protocol=\
    tcp src-address-list=!dns-servers
add action=dst-nat chain=output comment="redirect dns" dst-port=53 log=yes \
    log-prefix=DNS_Redir_WG_DST_UDP protocol=udp src-address-list=\
    !dns-servers to-addresses=192.168.1.5 to-ports=53
/ip firewall raw
add action=drop chain=prerouting in-interface-list=WAN log=yes log-prefix=\
    DNS_RAW_UDP port=53 protocol=udp
add action=drop chain=prerouting in-interface-list=WAN log=yes log-prefix=\
    DNS_RAW_TCP port=53 protocol=tcp
/ip route
add disabled=no dst-address=[vpn1_addr]/32 gateway=vlan1
add distance=10 gateway=vlan1 routing-table=direct
add dst-address=172.64.36.2/32 gateway=wg0 routing-table=direct
add dst-address=172.64.36.1/32 gateway=wg0 routing-table=direct
add dst-address=192.168.20.0/24 gateway=wg0 routing-table=direct
add disabled=no dst-address=[vpn2_addr]/32 gateway=vlan1
add check-gateway=none disabled=no distance=2 dst-address=0.0.0.0/1 gateway=\
    wg1
add disabled=no distance=2 dst-address=128.0.0.0/1 gateway=wg1
add dst-address=192.168.21.0/24 gateway=wg1
add disabled=no dst-address=0.0.0.0/1 gateway=wg0
add disabled=no dst-address=128.0.0.0/1 gateway=wg0
add dst-address=192.168.2.0/24 gateway=wg2
add comment=gw-hapac2-d32 dst-address=[REDACTED] gateway=vlan1
add disabled=no distance=2 dst-address=172.64.36.1/32 gateway=wg1
add disabled=no distance=2 dst-address=172.64.36.2/32 gateway=wg1
add comment="hapac2-d32 wg ingress network" dst-address=192.168.15.0/24 gateway=\
    192.168.10.2
/ip upnp
set enabled=yes
/ip upnp interfaces
add interface=bridge type=internal
add interface=vlan1 type=external

/routing rule
add action=lookup comment="rpi direct access interface" src-address=192.168.1.6/32 table=\
    direct
add action=lookup comment=macbookpro disabled=yes src-address=192.168.1.20 \
    table=direct
add action=lookup comment=iPadAir-Valentina disabled=yes src-address=\
    192.168.1.27 table=direct
add action=lookup comment="lg tv" src-address=192.168.1.33 table=direct
/system clock
set time-zone-name=Europe/Helsinki
/system identity
set name=gw
/system logging
add disabled=yes topics=bgp
add topics=wireguard
add disabled=yes topics=dhcp
add topics=upnp
add disabled=yes topics=route
/system ntp client
set enabled=yes
/system ntp client servers
add address=time.cloudflare.com

VPN servers config same, only difference is subnet (192.168.20.0/24 and 192.168.30.0/24):

[Interface]
Address = 192.168.20.1/24
ListenPort = 51826
PrivateKey = [REDACTED]
MTU = 1420
PostUp = iptables -A FORWARD -i %i -j ACCEPT; iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
PostDown = iptables -D FORWARD -i %i -j ACCEPT; iptables -t nat -D POSTROUTING -o eth0 -j MASQUERADE
Table = auto

[Peer]

gw-hapax2-d1

AllowedIPs = 192.168.20.2/32, 192.168.1.0/24, 192.168.10.0/24
PublicKey = [REDACTED]
PersistentKeepalive = 10

[Peer]

gw-hapac2-d32

AllowedIPs = 192.168.20.3/32, 192.168.2.0/24, 192.168.15.0/24
PublicKey = [REDACTED]
PersistentKeepalive = 10

[Peer]

macbook-andrey

AllowedIPs = 192.168.20.4/32
PublicKey = [REDACTED]
...

more direct configs for mobile clients