I have a big problem and don't know why.
Here is what I like to do:
WAN1 -> DSL 4 Mbit fix IP Range
WAN2 -> LTE fix IP
WAN3 -> LTE fix IP
WAN4 -> LTE fix IP
WAN5 -> LTE fix I
All Modems are bridged, the static IP in available on the WAN ports
I setup the Mange in preroute chain and mark every incoming connection.
I setup mark routing.
I setup destNat for Port forwarding
Scenario a 5Webserver, here a docker container is availiable on each ip address.
And now the difficult to understand for me:
I setup the first WAN everything works, great.
I setup the second WAN, everything works.
I setup WAN 3 and the Webserver from WAN 2 is not responding, or better, I can see the connection come in, is correctly tagged but the return path is wrong?
I see a syn State on the connection but it will not establish?
If I disconnect WAN3, WAN 2 is working again?
Is it possible to debug the package flow?
How can I see or trace what's going wrong?
I think there is something wrong With the way back, but I can't see what's wrong.
I try to bring it up with Router OS 6.49 and with 7.2 but still the same.
The same config with the same hardware is running on a Zywall 11000 as a WAN trunk.
Any help / hint would be great.
config:
Code: Select all
/interface ethernet
set [ find default-name=ether1 ] name=WAN1
set [ find default-name=ether2 ] name=WAN2
set [ find default-name=ether3 ] name=WAN3
set [ find default-name=ether4 ] name=WAN4
set [ find default-name=ether5 ] name=WAN5
set [ find default-name=ether10 ] name="ether10 INT"
set [ find default-name=ether11 ] name=ether11-LAN
/interface list
add name=WAN-list
add name=LAN-list
/interface lte apn
set [ find default=yes ] ip-type=ipv4 use-network-apn=no
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip pool
add name=dhcp_pool0 ranges=10.0.0.100-10.0.0.200
/ip dhcp-server
add address-pool=dhcp_pool0 interface=ether11-LAN lease-time=10m name=dhcp1
/port
set 0 name=serial0
set 1 name=serial1
/routing bgp template
set default disabled=no output.network=bgp-networks
/routing ospf instance
add disabled=no name=default-v2
/routing ospf area
add disabled=yes instance=default-v2 name=backbone-v2
/routing table
add fib name=toWan1
add fib name=toWan2
add fib name=toWan3
add fib name=toWan4
add fib name=toWan5
/ip settings
set max-neighbor-entries=8192
/ipv6 settings
set disable-ipv6=yes max-neighbor-entries=8192
/interface list member
add interface=WAN1 list=WAN-list
add interface=WAN2 list=WAN-list
add interface=WAN3 list=WAN-list
add interface=WAN4 list=WAN-list
add interface=WAN5 list=WAN-list
add interface=ether11-LAN list=LAN-list
add interface="ether10 INT" list=LAN-list
/interface ovpn-server server
set auth=sha1,md5
/ip address
add address=81.223.59.xxx/29 interface=WAN1 network=81.223.59.xxx
add address=10.0.0.1/24 interface=ether11-LAN network=10.0.0.0
add address=212.95.26.xxx/30 disabled=yes interface=WAN2 network=212.95.26.xxx
add address=212.95.26.xxx/29 disabled=yes interface=WAN3 network=212.95.26.xxx
add address=212.95.27.xxx/24 disabled=yes interface=WAN4 network=212.95.27.xxx
add address=176.66.79.xxx/16 disabled=yes interface=WAN5 network=176.66.0.xxx
add address=10.0.0.95/24 interface="ether10 INT" network=10.0.0.0
/ip dhcp-client
add add-default-route=no interface=WAN5 use-peer-ntp=no
add add-default-route=no interface=WAN2 use-peer-ntp=no
add add-default-route=no interface=WAN3 use-peer-ntp=no
add add-default-route=no interface=WAN4 use-peer-ntp=no
/ip dhcp-server lease
add address=10.0.0.144 client-id=1:0:50:56:93:f0:9a mac-address=\
00:50:56:93:F0:9A server=dhcp1
add address=10.0.0.122 client-id=1:0:23:a4:2:36:86 mac-address=\
00:23:A4:02:36:86 server=dhcp1
/ip dhcp-server network
add address=10.0.0.0/24 dns-server=10.0.0.25,10.0.0.1 domain=dom-01.net \
gateway=10.0.0.1
/ip dns
set allow-remote-requests=yes servers=10.0.0.25
/ip dns static
add address=8.8.8.8 name=dns.google
add address=8.8.4.4 name=dns.google
/ip firewall address-list
add address=10.0.0.0/8 list=RFC1918
add address=172.16.0.0/12 list=RFC1918
add address=192.168.0.0/16 list=RFC1918
/ip firewall filter
add action=drop chain=input comment="Drop Invalid connections" \
connection-state=invalid
add action=accept chain=input comment="access only from inside" dst-port=\
8291,22,80 in-interface-list=LAN-list protocol=tcp
add action=accept chain=input comment=\
"Allow Established/Related/Untracked connections" connection-state=\
established,related
add action=accept chain=input comment="Allow ICMP" protocol=icmp
add action=accept chain=input comment="DNS nicht von extern" dst-port=53 \
protocol=tcp src-address-list=RFC1918
add action=accept chain=input dst-port=53 protocol=udp src-address-list=\
RFC1918
add action=drop chain=input comment="Drop everything else" log-prefix=\
_DROP_IN_
add action=accept chain=forward comment="allow port forwarding" \
connection-nat-state=dstnat connection-state=new in-interface-list=\
WAN-list
add action=accept chain=forward comment="forward established" \
connection-state=established,related
add action=accept chain=forward comment="allow internet traffic" \
in-interface-list=LAN-list out-interface-list=WAN-list
add action=drop chain=forward comment="Forward Drop invalid " \
connection-state=invalid
add action=drop chain=forward comment="forward drop anything"
/ip firewall mangle
add action=accept chain=prerouting comment=0001 dst-address-list=RFC1918 \
src-address-list=RFC1918
add action=accept chain=prerouting comment=0002 dst-address=81.223.59.xxx/29 \
in-interface-list=LAN-list
add action=accept chain=prerouting dst-address=212.95.26.xxx/30 \
in-interface-list=LAN-list
add action=accept chain=prerouting dst-address=212.95.26.xxx/29 \
in-interface-list=LAN-list
add action=accept chain=prerouting dst-address=212.95.27.xxx/24 \
in-interface-list=LAN-list
add action=accept chain=prerouting dst-address=176.66.0.0/xxx \
in-interface-list=LAN-list
add action=mark-connection chain=prerouting comment="Incomming WAN marking" \
connection-mark=no-mark in-interface=WAN1 new-connection-mark=WAN1_conn \
passthrough=no
add action=mark-connection chain=prerouting connection-mark=no-mark \
in-interface=WAN2 new-connection-mark=WAN2_conn passthrough=no
add action=mark-connection chain=prerouting connection-mark=no-mark \
in-interface=WAN3 new-connection-mark=WAN3_conn passthrough=no
add action=mark-connection chain=prerouting connection-mark=no-mark \
in-interface=WAN4 log-prefix=_WAN4_inMark new-connection-mark=WAN4_conn \
passthrough=no
add action=mark-connection chain=prerouting connection-mark=no-mark \
in-interface=WAN5 new-connection-mark=WAN5_conn passthrough=no
add action=mark-connection chain=prerouting comment="Loadbalancing WAN 2- 5" \
connection-mark=no-mark disabled=yes dst-address-type=!local \
in-interface-list=LAN-list new-connection-mark=WAN2_conn passthrough=yes \
per-connection-classifier=both-addresses:4/0
add action=mark-connection chain=prerouting connection-mark=no-mark disabled=\
yes dst-address-type=!local in-interface-list=LAN-list log-prefix=\
mark_conn3 new-connection-mark=WAN3_conn passthrough=yes \
per-connection-classifier=both-addresses:4/1
add action=mark-connection chain=prerouting connection-mark=no-mark disabled=\
yes dst-address-type=!local in-interface-list=LAN-list log-prefix=\
mark_conn4 new-connection-mark=WAN4_conn passthrough=yes \
per-connection-classifier=both-addresses:4/2
add action=mark-connection chain=prerouting connection-mark=no-mark disabled=\
yes dst-address-type=!local in-interface-list=LAN-list \
new-connection-mark=WAN5_conn passthrough=yes per-connection-classifier=\
both-addresses:4/3
add action=mark-routing chain=prerouting comment="outgoing routing" \
connection-mark=WAN1_conn in-interface-list=LAN-list new-routing-mark=\
toWan1 passthrough=no
add action=mark-routing chain=prerouting connection-mark=WAN2_conn \
in-interface-list=LAN-list new-routing-mark=toWan2 passthrough=no
add action=mark-routing chain=prerouting connection-mark=WAN3_conn \
in-interface-list=LAN-list new-routing-mark=toWan3 passthrough=no
add action=mark-routing chain=prerouting connection-mark=WAN4_conn \
in-interface-list=LAN-list new-routing-mark=toWan4 passthrough=no
add action=mark-routing chain=prerouting connection-mark=WAN5_conn \
in-interface-list=LAN-list new-routing-mark=toWan5 passthrough=no
add action=mark-routing chain=output comment="outgoing WAN routing" \
connection-mark=WAN1_conn new-routing-mark=toWan1 passthrough=no
add action=mark-routing chain=output connection-mark=WAN2_conn \
new-routing-mark=toWan2 passthrough=no
add action=mark-routing chain=output connection-mark=WAN3_conn log=yes \
log-prefix=out_W3 new-routing-mark=toWan3 passthrough=no
add action=mark-routing chain=output connection-mark=WAN4_conn log=yes \
log-prefix=out_W4 new-routing-mark=toWan4 passthrough=no
add action=mark-routing chain=output connection-mark=WAN5_conn \
new-routing-mark=toWan5 passthrough=no
/ip firewall nat
add action=dst-nat chain=dstnat comment="testwebsite W5" dst-address=\
176.66.79.xxx dst-port=80 protocol=tcp to-addresses=10.0.0.33
add action=dst-nat chain=dstnat comment="test destnat wan4" dst-address=\
212.95.27.xxx dst-port=80 protocol=tcp to-addresses=10.0.0.33
add action=dst-nat chain=dstnat comment="test website W3" dst-address=\
212.95.26.xxx dst-port=80 protocol=tcp to-addresses=10.0.0.33
add action=dst-nat chain=dstnat comment="test website W2" dst-address=\
212.95.26.xxx dst-port=80 protocol=tcp to-addresses=10.0.0.33
add action=dst-nat chain=dstnat comment="testwebsite W1" dst-address=\
81.223.59.xxx dst-port=80 protocol=tcp to-addresses=10.0.0.33 to-ports=\
4400
add action=masquerade chain=srcnat comment="WAN Masquerade" \
out-interface-list=WAN-list
/ip route
add comment="Standard WAN1" disabled=no distance=2 dst-address=0.0.0.0/0 \
gateway=81.223.59.xxx%WAN1
add comment="Standard WAN2" disabled=no distance=2 dst-address=0.0.0.0/0 \
gateway=212.95.26.xxx%WAN2
add comment="Standard WAN3" disabled=no distance=2 dst-address=0.0.0.0/0 \
gateway=212.95.26.xxx%WAN3
add comment="Standard WAN4" disabled=no distance=2 dst-address=0.0.0.0/0 \
gateway=212.95.27.xxx%WAN4 pref-src="" routing-table=main scope=30 \
suppress-hw-offload=no target-scope=10
add comment="Standard WAN5" disabled=no distance=2 dst-address=0.0.0.0/0 \
gateway=176.66.0.1%WAN5
add comment="EXPL WAN1" disabled=no dst-address=0.0.0.0/0 gateway=\
81.223.59.xxx%WAN1 routing-table=toWan1
add comment="EXPL WAN2" disabled=no dst-address=0.0.0.0/0 gateway=\
212.95.26.xxx%WAN2 routing-table=toWan2
add comment="EXPL WAN3" disabled=no dst-address=0.0.0.0/0 gateway=\
212.95.26.xxx%WAN3 routing-table=toWan3
add comment="EXPL WAN4" disabled=no distance=1 dst-address=0.0.0.0/0 gateway=\
212.95.27.xxx%WAN4 pref-src="" routing-table=toWan4 scope=30 \
suppress-hw-offload=no target-scope=10
add comment="EXPL WAN5" disabled=no dst-address=0.0.0.0/0 gateway=\
176.66.0.1%WAN5 routing-table=toWan5
/routing bfd configuration
add disabled=no interfaces=all min-rx=200ms min-tx=200ms multiplier=5
/routing rule
add action=lookup disabled=no src-address=::/0 table=main
/system clock
set time-zone-name=Europe/Vienna
/system note
set show-at-login=no
/system ntp client
set enabled=yes
/system ntp client servers
add address=10.0.0.25
/tool sniffer
set streaming-enabled=yes streaming-server=10.0.0.122
Oliver
