Hi
i need some help, i’m stuck with a problem giving me headaches
so, i have more than 1 connection: i have 3 optical fiber + 1 LTE connection
Optical fiber “free” has metric 2
Optical fiber “Orange” has metric 3
Optical fiber “Alpha” has metric 4
LTE has metric 10
so all data go through Free by default. fine.
but i would like to use our servers (FTP, iperf, wireguard) from any WAN.
so i did this:
i added each connection to WAN interface list
i added dst-NAT to my servers internally
and i added mangle rules to mark connection based on the incoming interface
i tried with FTP server and it was apparently working fine: connection was OK
but i also have a wireguard server running in my LAN (not on the mikrotik)
and, it does not work when trying to access to wireguard
Same with iperf: when trying to connect to our iperf , using any WAN interface, i get either packet loss, or no connection at all 
if i remove the connection marking features, and connect using wan “FREE” (so the default metric connection), it works fine
if i enable the connection marking feature, and connect using any WAN, it does not work
do you have any clue about the problem ?
below an extract of the configuration:
# Here are 4 ethernet connections.
/interface ethernet
set [ find default-name=ether1 ] name="ether1 Fibre_Orange"
set [ find default-name=ether2 ] name="ether2 Fibre_Free"
set [ find default-name=ether3 ] name="ether3 Backup_4G"
set [ find default-name=ether4 ] name="ether4 Fibre_Alpha"
# Orange optical fiber need to set vlan-id
/interface vlan
add interface="ether1 Fibre_Orange" name=vlan-orange vlan-id=835
# Orange and Alpha are pppoe client.
/interface pppoe-client
add add-default-route=yes default-route-distance=4 disabled=no interface="ether4 Fibre_Alpha" name=pppoe-Alpha user=xxxxxxxxxxxxxxxxxx
add add-default-route=yes default-route-distance=3 disabled=no interface=vlan-orange max-mtu=1492 name=pppoe-orange user=yyyyyyyyyyyy
/interface list
add comment=defconf name=WAN
add comment=defconf name=LAN
# I use routing table for the marking connection feature.
/routing table
add disabled=no fib name=VersOrange
add disabled=no fib name=VersFree
add disabled=no fib name=Vers4G
add disabled=no fib name=VersAlpha
# so, ethernet 2, 3 and pppoe-clients are WAN. bridge is LAN
/interface list member
add interface=bridge list=LAN
add interface="ether2 Fibre_Free" list=WAN
add interface="ether3 Backup_4G" list=WAN
add interface=pppoe-orange list=WAN
add interface=pppoe-Alpha list=WAN
# Set some IP addresses... our local network in 192.168.11.0
/ip address
add address=192.168.11.1/24 interface=bridge network=192.168.11.0
add address=10.10.10.254/24 interface=bridge network=10.10.10.0
add address=192.168.5.1/24 interface=bridge network=192.168.5.0
add address=10.11.11.2/24 interface="ether3 Backup_4G" network=10.11.11.0
# some basic firewall rules. this is more or less the default + some extra config
/ip firewall filter
add action=accept chain=input comment="defconf: accept ICMP" in-interface-list=WAN log-prefix=ping-ipc protocol=icmp
add action=accept chain=input comment="defconf: accept established,related,untracked" connection-state=established,related,untracked
add action=accept chain=input comment="Acces Winbox distant" dst-port=8291 log=yes log-prefix=winbox-ipc protocol=tcp
add action=drop chain=input comment="defconf: drop invalid" connection-state=invalid
add action=accept chain=input comment="defconf: accept to local loopback (for CAPsMAN)" dst-address=127.0.0.1
add action=accept chain=input comment="Bandwitch test server TCP" dst-port=2000 in-interface="ether2 Fibre_Free" protocol=tcp
add action=drop chain=input comment="defconf: drop all not coming from LAN" in-interface-list=!LAN
add action=accept chain=forward comment="defconf: accept in ipsec policy" ipsec-policy=in,ipsec
add action=accept chain=forward comment="defconf: accept out ipsec policy" ipsec-policy=out,ipsec
add action=fasttrack-connection chain=forward comment="defconf: fasttrack" connection-state=established,related hw-offload=yes
add action=accept chain=forward comment="defconf: accept established,related, untracked" connection-state=established,related,untracked
add action=drop chain=forward comment="defconf: drop invalid" connection-state=invalid
add action=drop chain=forward comment="defconf: drop all from WAN not DSTNATed" connection-nat-state=!dstnat connection-state=new in-interface-list=WAN
# Here is the interesting part. I want to mark connection based of the in-interface.
/ip firewall mangle
# When coming from alpha, mark as "alpha"
# if connection mark alpha is set, use alpha routing table.
add action=mark-connection chain=input connection-mark=no-mark in-interface=pppoe-Alpha new-connection-mark=Alpha passthrough=no
add action=mark-connection chain=forward connection-mark=no-mark in-interface=pppoe-Alpha new-connection-mark=Alpha passthrough=no
add action=mark-routing chain=output connection-mark=Alpha new-routing-mark=VersAlpha out-interface-list=WAN passthrough=no
add action=mark-routing chain=prerouting connection-mark=Alpha in-interface=bridge new-routing-mark=VersAlpha passthrough=no
# When coming from Orange, mark as "orange"
# if connection mark orange is set, use orange routing table.
add action=mark-connection chain=input connection-mark=no-mark in-interface=pppoe-orange new-connection-mark=ORANGE passthrough=yes
add action=mark-connection chain=forward connection-mark=no-mark in-interface=pppoe-orange new-connection-mark=ORANGE passthrough=yes
add action=mark-routing chain=output connection-mark=ORANGE new-routing-mark=VersOrange out-interface-list=WAN passthrough=yes
add action=mark-routing chain=prerouting connection-mark=ORANGE in-interface=bridge new-routing-mark=VersOrange passthrough=yes
# When coming from Free, mark as "free"
# if connection mark free is set, use free routing table.
add action=mark-connection chain=input connection-mark=no-mark in-interface="ether2 Fibre_Free" new-connection-mark=FREE passthrough=yes
add action=mark-connection chain=forward connection-mark=no-mark in-interface="ether2 Fibre_Free" new-connection-mark=FREE passthrough=yes
add action=mark-routing chain=output connection-mark=FREE new-routing-mark=VersFree out-interface-list=WAN passthrough=yes
add action=mark-routing chain=prerouting connection-mark=FREE in-interface=bridge new-routing-mark=VersFree passthrough=yes
# The NAT rules. the interesting one is my wireguard VPN running on 192.168.11.43
/ip firewall nat
add action=masquerade chain=srcnat comment="NAT Loopback rule" out-interface=bridge protocol=tcp src-address=192.168.11.0/24
add action=masquerade chain=srcnat comment="NAT loopback rule" log-prefix=SRCNAT out-interface=bridge protocol=udp src-address=192.168.11.0/24
add action=masquerade chain=srcnat out-interface-list=WAN
add action=dst-nat chain=dstnat comment="Wireguard VPN" dst-port=51820 in-interface-list=WAN protocol=udp to-addresses=192.168.11.43 to-ports=51820
add action=dst-nat chain=dstnat comment="Iperf TCP" dst-port=5201-5209 in-interface-list=WAN protocol=tcp to-addresses=192.168.11.42
add action=dst-nat chain=dstnat comment="Iperf UDP" dst-port=5201-5209 in-interface-list=WAN protocol=udp to-addresses=192.168.11.42
# routes.
/ip route
add disabled=no distance=10 dst-address=0.0.0.0/0 gateway="ether3 Backup_4G" pref-src="" routing-table=main scope=30 suppress-hw-offload=no target-scope=10
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway=pppoe-orange pref-src="" routing-table=VersOrange scope=30 suppress-hw-offload=no target-scope=10
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway="ether2 Fibre_Free" pref-src="" routing-table=VersFree scope=30 suppress-hw-offload=no target-scope=10
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway="ether3 Backup_4G" pref-src="" routing-table=Vers4G scope=30 suppress-hw-offload=no target-scope=10
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway=pppoe-Alpha pref-src="" routing-table=VersAlpha scope=30 suppress-hw-offload=no target-scope=10
# i just have 1 rule, one server must use orange connexion by default.
# remember: others will use free (as metric is lower), or will use the connection base on connection-mark
/routing rule
add action=lookup disabled=no src-address=192.168.11.100/32 table=VersOrange