Hi Dears,
I am struggling for more than one week now on this problem.
Here is my network:
I have 2 wan connections:
WAN1 with its own Public IP address
WAN2 That goes through an orange box.
I use PCC in order to load balance between both WAN1 and WAN2
And I use recursive routing in order to make sure that internet is available on both WANs.
I also have 2 VPN connections assigned to WAN1:
- SSTP for historical reasons
- Wireguard
I have 50+ remote sites connected via VPN I just show 2 of them here.
I have several DST-NAT redirections, I just show NTP (UDP123)
here is a part of my config:
/interface bridge
add admin-mac=DC:2C:6E:28:EB:FB auto-mac=no comment=defconf name=bridge \
port-cost-mode=short
add name=bridgeKAZ port-cost-mode=short
add igmp-snooping=yes name=bridgeLAN port-cost-mode=short
/interface ethernet
set [ find default-name=ether1 ] comment=SRR l2mtu=1514 name=WAN1
set [ find default-name=ether2 ] comment=Orange l2mtu=1514 name=WAN2
set [ find default-name=ether3 ] l2mtu=1514
set [ find default-name=ether4 ] l2mtu=1514
set [ find default-name=ether5 ] l2mtu=1514
set [ find default-name=ether6 ] l2mtu=1514
set [ find default-name=ether7 ] l2mtu=1514
set [ find default-name=ether8 ] l2mtu=1514
set [ find default-name=sfp-sfpplus1 ] l2mtu=1514
/interface wireguard
add listen-port=13231 mtu=1420 name=wireguard1
/interface list
add comment=defconf name=WAN
add comment=defconf name=LAN
/routing table
add disabled=no fib name=to_WAN1
add disabled=no fib name=to_WAN2
/interface bridge port
add bridge=bridge comment=defconf interface=ether8 internal-path-cost=10 \
path-cost=10
add bridge=bridgeLAN interface=sfp-sfpplus1 internal-path-cost=10 path-cost=\
10
add bridge=bridgeLAN hw=no interface=ether4 internal-path-cost=10 path-cost=\
10
add bridge=bridgeLAN hw=no interface=ether6 internal-path-cost=10 path-cost=\
10
add bridge=bridgeLAN interface=ether7 internal-path-cost=10 path-cost=10
/interface list member
add interface=WAN1 list=WAN
add interface=WAN2 list=WAN
add interface=bridgeLAN list=LAN
add interface=wireguard1 list=LAN
/interface sstp-server server
set authentication=mschap1,mschap2 certificate=Server enabled=yes port=444 \
verify-client-certificate=yes
/interface wireguard peers
add allowed-address=192.168.4.10/32,10.40.0.0/16 comment="Client Portable" \
interface=wireguard1 name=gigi public-key=\
""
/ip address
add address=XXX.XXX.XXX.3/24 interface=WAN1 network=MikrotXXX.XXX.XXX.0
add address=192.168.50.254/24 interface=WAN2 network=192.168.50.0
add address=192.168.0.1/24 interface=bridgeLAN network=192.168.0.0
add address=192.168.4.1/23 interface=wireguard1 network=192.168.4.0
/ip dns
set allow-remote-requests=yes servers=192.168.0.254
/ip firewall filter
add action=accept chain=input comment=\
"defconf: accept established,related,untracked" connection-state=\
established,related,untracked
add action=drop chain=input comment="defconf: drop invalid" connection-state=\
invalid
add action=accept chain=input comment="defconf: accept ICMP" protocol=icmp
add action=accept chain=input comment="Accept WIREGUARD 13231" dst-port=13231 \
protocol=udp
add action=accept chain=input comment="Accept SSTP 444" dst-port=444 \
protocol=tcp
add action=accept chain=input comment="Allow All from LANDomot" \
in-interface-list=LAN
add action=accept chain=forward comment=\
"defconf: accept established,related, untracked" connection-state=\
established,related,untracked
add action=accept chain=output comment=\
"defconf: accept established,related, untracked" connection-state=\
established,related,untracked
add action=drop chain=input comment="defconf: drop all not coming from LAN" \
in-interface-list=!LAN
add action=drop chain=forward comment="defconf: drop invalid" \
connection-state=invalid
add action=drop chain=forward comment=\
"defconf: drop all from WAN not DSTNATed" connection-nat-state=!dstnat \
connection-state=new in-interface-list=WAN
/ip firewall mangle
add action=accept chain=prerouting comment="Allow LAN to WAN1" \
dst-address=XXX.XXX.XXX.0/24 in-interface=bridgeLAN
add action=accept chain=prerouting comment="Allow LAN to WAN2" \
dst-address=192.168.50.0/24 in-interface=bridgeLAN
add action=mark-connection chain=prerouting comment=\
"Mark Incoming Packet to WAN1" connection-mark=no-mark \
connection-state="" in-interface=WAN1 new-connection-mark=WAN1_conn
add action=mark-connection chain=prerouting comment=\
"Mark Incomming Packet to WAN2" connection-mark=no-mark \
connection-state="" in-interface=WAN2 new-connection-mark=WAN2_conn
add action=mark-connection chain=prerouting comment="PCC WAN1" \
connection-mark=no-mark dst-address-type=!local in-interface-list=LAN \
new-connection-mark=WAN1_conn per-connection-classifier=\
both-addresses:2/0
add action=mark-connection chain=prerouting comment="PCC WAN2" \
connection-mark=no-mark dst-address-type=!local in-interface-list=LAN \
new-connection-mark=WAN2_conn per-connection-classifier=\
both-addresses:2/1
add action=mark-connection chain=prerouting comment=\
"Remove PCC for Wireguard" dst-address=192.168.4.0/24 in-interface=\
bridgeLAN new-connection-mark=no-mark
add action=mark-connection chain=prerouting comment="Remove PCC for SSTP" \
dst-address=10.40.0.0/24 in-interface=bridgeLAN new-connection-mark=\
no-mark
add action=mark-connection chain=prerouting comment=\
"Remove PCC for SSTP NEtworks" dst-address=10.41.0.0/16 in-interface=\
bridgeLAN new-connection-mark=no-mark
add action=mark-connection chain=prerouting comment=\
"Remove PCC for Wireguard to LAN" dst-address=192.168.0.0/24 in-interface=\
wireguard1 new-connection-mark=no-mark
add action=mark-routing chain=prerouting comment="Mark routing to WAN1" \
connection-mark=WAN1_conn in-interface-list=LAN new-routing-mark=to_WAN1 \
passthrough=no
add action=mark-routing chain=prerouting comment="Mark routing to WAN2" \
connection-mark=WAN2_conn in-interface-list=LAN new-routing-mark=to_WAN2 \
passthrough=no
add action=mark-routing chain=output comment="Output Mark Routing to WAN1" \
connection-mark=WAN1_conn new-routing-mark=to_WAN1 passthrough=no
add action=mark-routing chain=output comment="Output Mark Routing to WAN2" \
connection-mark=WAN2_conn new-routing-mark=to_WAN2 passthrough=no
/ip firewall nat
add action=masquerade chain=srcnat comment="masquerade to WANs" \
out-interface-list=WAN
add action=dst-nat chain=dstnat comment="redirect NTP u123" dst-port=123 log=\
yes protocol=udp to-addresses=192.168.0.253
/ip route
add check-gateway=ping comment="Route marked to WAN1" disabled=no distance=1 \
dst-address=0.0.0.0/0 gateway=1.1.1.1 pref-src="" routing-table=to_WAN1 \
scope=30 target-scope=11
add check-gateway=ping comment="Route Marked to WAN2" disabled=no distance=1 \
dst-address=0.0.0.0/0 gateway=8.8.4.4 pref-src="" routing-table=to_WAN2 \
scope=30 target-scope=11
add disabled=no distance=11 dst-address=0.0.0.0/0 gateway=192.168.50.1 \
pref-src="" routing-table=main scope=30 target-scope=10
add comment="route to remote site" disabled=no distance=1 dst-address=\
10.41.10.0/24 gateway=wireguard1 routing-table=main scope=10 \
target-scope=10
add comment="recusive route 1.1.1.1 WAN1" disabled=no distance=1 dst-address=\
1.1.1.1/32 gateway=XXX.XXX.XXX.254 routing-table=main scope=10 \
target-scope=10
add comment="recursive route 8.8.4.4 WAN2" disabled=no distance=1 dst-address=\
8.8.4.4/32 gateway=192.168.50.1 routing-table=main scope=30 target-scope=\
10
add check-gateway=ping comment="Route marked to WAN1" disabled=no distance=2 \
dst-address=0.0.0.0/0 gateway=8.8.4.4 pref-src="" routing-table=to_WAN1 \
scope=30 target-scope=11
add check-gateway=ping comment="Route Marked to WAN2" disabled=no distance=2 \
dst-address=0.0.0.0/0 gateway=1.1.1.1 pref-src="" routing-table=to_WAN2 \
scope=30 target-scope=11
add disabled=no distance=10 dst-address=0.0.0.0/0 gateway=XXX.XXX.XXX.254 \
routing-table=main
Fail-over works great, if I unplug WAN1 or WAN2, traffic is automatically redirected to the other interface.
But Load balancing does not work.
When both WAN1 and WAN2 are connected, all traffic goes to WAN1. And i can not ping recursive router 8.8.4.4.
Moreover, all traffic coming to WAN2 from internet, can not reach my LAN.
here are some packets captures:
ping 1.1.1.1 from 192.168.0.24/24
this works as expected:
My computer is plugged to Ether4, when I ping recursive route via WAN1,
Packet first comes on ether4 then bridegLAN, then goes to WAN1 via masquerading.
response is received on WAN1 and is redirected via masquerading to my computer going first to bridgeLAN then ether4.
Now ping 8.8.4.4 form 192.168.0.24/24
This is awkward. As 8.8.4.4 is defined as my recursive route via WAN2, I expect all packets going through this interface.
But first packet (line 1-4) is sent via WAN1
It is first recevied on ether 4 goes to bridegLAN and goes out to WAN1.
Then response is received on WAN1 but is not transfered to bridgeLAN NOR Ether4
the second packet (line5-10) seems to be sent via both WAN1 and WAN2
it comes from ether4 goes through bridgeLAN
then first goes via WAN1 (line 7-8)
then via WAN2 (line 9-10)
The response packet both comes on WAN1 and WAN2, but never reaches Ether4 nor BridgeLAN.
What am I doing wrong?