Mangle rule for forward traffic on secondary routing table

RouterOS General
1

Good morning everyone.
I have a routerboard with a configuration like this: two internet connections via pppoe, one in fiber optic and one in FWA Wireless. The fiber optic has distance 1, while the FWA line has distance 2 so that if the fiber optics fails, the system will use the FWA line to navigate. A VoIP ATA is connected to the routerboard (with IP: 10.100.3.20) and a third Firewall (with IP: 192.168.3.2).
Here is the complete configuration:

/interface bridge
add frame-types=admit-only-vlan-tagged name=bridge1 pvid=100 vlan-filtering=yes
/interface ethernet
set [ find default-name=ether2 ] comment=Firewall
set [ find default-name=ether4 ] comment=VoIP
set [ find default-name=ether5 ] comment=Management
set [ find default-name=ether7 ] comment=Fiber Pppoe Port
set [ find default-name=ether8 ] comment=FWA Pppoe Port
/interface pppoe-client
add add-default-route=yes comment="Fiber" disabled=no interface=\
    ether7 name=pppoe-out1 use-peer-dns=yes user=********
add add-default-route=yes comment="FWA" default-route-distance=2 \
    disabled=no interface=ether8 name=pppoe-out2 user=********
/interface wireguard
add listen-port=18731 mtu=1420 name=wireguard1
/interface vlan
add comment=Management interface=bridge1 name=vlan100 vlan-id=100
add comment="To Firewall" interface=bridge1 name=vlan200 vlan-id=200
add comment=VoIP interface=bridge1 name=vlan700 vlan-id=700
/interface list
add name=WAN
add name=LAN
add name=MANAGEMENT
/ip dhcp-server
add interface=vlan700 name=server700
/ip pool
add name=dhcp_pool0 ranges=10.100.100.100-10.100.100.200
/ip dhcp-server
add address-pool=dhcp_pool0 interface=vlan100 name=dhcp1
/port
set 0 name=serial0
/routing table
add disabled=no fib name=secondaria
/interface bridge port
add bridge=bridge1 interface=ether1
add bridge=bridge1 frame-types=admit-only-untagged-and-priority-tagged \
    interface=ether2 pvid=200
add bridge=bridge1 interface=ether3
add bridge=bridge1 frame-types=admit-only-untagged-and-priority-tagged \
    interface=ether4 pvid=700
add bridge=bridge1 frame-types=admit-only-untagged-and-priority-tagged \
    interface=ether5 pvid=100
/ip neighbor discovery-settings
set discover-interface-list=MANAGEMENT
/interface bridge vlan
add bridge=bridge1 tagged=bridge1 vlan-ids=100
/interface list member
add interface=pppoe-out1 list=WAN
add interface=pppoe-out2 list=WAN
add interface=vlan100 list=LAN
add interface=vlan200 list=LAN
add interface=vlan700 list=LAN
add interface=vlan100 list=MANAGEMENT
/interface wireguard peers
add allowed-address=10.100.90.2/32 interface=wireguard1 name=Simone public-key=\
    "********************"
/ip address
add address=10.100.100.1/24 interface=vlan100 network=10.100.100.0
add address=10.100.3.1/24 interface=vlan700 network=10.100.3.0
add address=192.168.3.1/24 interface=vlan200 network=192.168.3.0
add address=10.100.90.1/24 interface=wireguard1 network=10.100.90.0
/ip dhcp-server lease
add address=10.100.3.20 comment="VoIP ATA" mac-address=***********
/ip dhcp-server network
add address=10.100.3.0/24 dns-server=10.100.3.1 gateway=10.100.3.1
add address=10.100.100.0/24 dns-server=10.100.100.1 gateway=10.100.100.1
/ip dns
set allow-remote-requests=yes cache-max-ttl=1d servers=8.8.8.8,8.8.4.4
/ip firewall address-list
add address=************** list="Simone IP"
add address=************** list="Simone IP"
add address=************** list=VoipVoice
/ip firewall filter
add action=accept chain=input comment="Accept Established, Related, Untracked" \
    connection-state=established,related
add action=accept chain=input comment="Accept ICMP from Simone IP" \
    protocol=icmp src-address-list="Simone IP"
add action=accept chain=input comment="Accept WinBox from Simone IP" \
    dst-port=8291 protocol=tcp src-address-list="Simone IP"
add action=accept chain=input comment="Accept www-ssl" dst-port=**** protocol=\
    tcp
add action=accept chain=input comment="Accept DNS requests from LAN" dst-port=\
    53 in-interface-list=LAN protocol=tcp
add action=accept chain=input dst-port=53 in-interface-list=LAN protocol=udp
add action=accept chain=input comment="Accept Wireguard" dst-port=***** \
    protocol=udp
add action=accept chain=input comment="Accept from Simone Wireguard IP" \
    in-interface=wireguard1 src-address=10.100.90.2
add action=drop chain=input
add action=fasttrack-connection chain=forward hw-offload=yes
/ip firewall mangle
add action=mark-connection chain=input comment=\
    "Marca tutte le connessioni in ingresso dall'interfaccia WAN2" \
    in-interface=pppoe-out2 new-connection-mark=route-secondaria-conn
add action=mark-routing chain=output comment="Sposta il routing in output sulla \
    tabella di route secondaria di tutti i pacchetti in ingresso dalla WAN2" \
    connection-mark=route-secondaria-conn new-routing-mark=secondaria \
    passthrough=no
/ip firewall nat
add action=masquerade chain=srcnat out-interface-list=WAN
add action=accept chain=dstnat comment="Accept ICMP from Simone IP" \
    in-interface-list=WAN protocol=icmp src-address-list="Simone IP"
add action=accept chain=dstnat comment="Accept WinBox from Simone IP" \
    dst-port=8291 protocol=tcp src-address-list="Simone IP"
add action=accept chain=dstnat comment="Accept www-ssl" dst-port=***** \
    in-interface-list=WAN protocol=tcp
add action=accept chain=dstnat comment="Accept Wireguard" dst-port=***** \
    in-interface-list=WAN protocol=udp
add action=dst-nat chain=dstnat comment="Accept VoipVoip SIP " dst-port=5060 \
    in-interface-list=WAN protocol=tcp src-address-list=VoipVoice to-addresses=\
    10.100.3.20
add action=dst-nat chain=dstnat dst-port=5060 in-interface-list=WAN protocol=\
    udp src-address-list=VoipVoice to-addresses=10.100.3.20
add action=accept chain=dstnat comment="Accept DNS Requests" dst-port=53 \
    in-interface-list=LAN protocol=tcp
add action=accept chain=dstnat dst-port=53 in-interface-list=LAN protocol=udp
add action=dst-nat chain=dstnat comment="DMZ to Firewall" in-interface-list=WAN \
    to-addresses=192.168.3.2
/ip firewall service-port
set sip disabled=yes
/ip ipsec profile
set [ find default=yes ] dpd-interval=2m dpd-maximum-failures=5
/ip route
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway=pppoe-out2 \
    routing-table=secondaria suppress-hw-offload=no
/ip service
set telnet disabled=yes
set ftp disabled=yes
set www disabled=yes
set ssh disabled=yes
set www-ssl certificate="cert1 www-ssl" disabled=no port=******
set api disabled=yes
set api-ssl disabled=yes
/system clock
set time-zone-name=Europe/Rome
/system identity
set name=******************
/system note
set show-at-login=no
/system routerboard settings
set enter-setup-on=delete-key
/tool mac-server
set allowed-interface-list=MANAGEMENT
/tool mac-server mac-winbox
set allowed-interface-list=MANAGEMENT

Everything works perfectly. I had to create this new routing table:

/routing table
add disabled=no fib name=secondaria

These two mangle rules:

/ip firewall mangle
add action=mark-connection chain=input comment=\
    "Marca tutte le connessioni in ingresso dall'interfaccia WAN2" \
    in-interface=pppoe-out2 new-connection-mark=route-secondaria-conn
add action=mark-routing chain=output comment="Sposta il routing in output sulla \
    tabella di route secondaria di tutti i pacchetti in ingresso dalla WAN2" \
    connection-mark=route-secondaria-conn new-routing-mark=secondaria \
    passthrough=no

And this static routing rule:

/ip route
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway=pppoe-out2 \
    routing-table=secondaria suppress-hw-offload=no

Since I wanted to reach the routerboard also from the secondary WAN regardless of whether the primary WAN was Up or not.
Now the router responds correctly even on the secondary WAN, with the exception of Wireguard.
Wireguard seems to always follow the main routing table, I don’t understand why.
Furthermore, I would like to make the firewall reachable from the secondary WAN, regardless of whether the primary WAN was Up or not. In this case I tried to use mark connection rules in prerouting and mark routing, always in prerouting, but it doesn’t seem to work.
Does anyone have any advice to give me?
Thank you very much

2

hello simone,

Now the router responds correctly even on the secondary WAN, with the exception of Wireguard.
Wireguard seems to always follow the main routing table, I don’t understand why.

is the first pppoe in the main table?

try to make 3 routing table.
1 main
2 pppoe1
3 pppoe2

for each pppoe you need to track down their own request reply sessions for traffic to flow through its correct incoming interface.

the same goes for firewall as well.

@anav have plenty working examples to achieve your goals.

3

Hi

some feedback:

Wireguard seems to always follow the main routing table, I don’t understand why.

currently wg will use main table, so if wan1 is up (distance=1), it will use it, if wan1 not there, it will use wan2 (distance=2)
Switching may require purging conntrack data, some have observed.

In this case I tried to use mark connection rules in prerouting and mark routing, always in prerouting, but it doesn’t seem to work.

Because of this:
add action=fasttrack-connection chain=forward hw-offload=yes

short: Mangling <> fasttrack are incompatible…
long: fasttrack skips firewall for majority of packet, hence mangling is not possible then

4

Your mangle rules mark traffic going to/from the router itself, not forwarded traffic. Fasttrack only works on forwarded traffic.

A side note, you are fasttracking everything, not just established connections. This effectively disables the forward firewall (sounds like this is intentional).

add action=fasttrack-connection chain=forward hw-offload=yes

If you want to keep using fasttrack while still having the ability to mark some packets, you can exclude all marked packets from fasttrack.

/ip firewall filter
add action=fasttrack-connection chain=forward hw-offload=yes packet-mark=no-mark

But it doesn’t apply in your case since you are not marking any forwarded packets to begin with.

5

good catch on the “forward” chain

Just to correct/clarify:
fasttracking doesn’'t disable firewall, it optimises it
https://help.mikrotik.com/docs/spaces/ROS/pages/130220087/Connection+tracking#Connectiontracking-FastTrack

further

you can exclude all marked packets from fasttrack

once a connection is fasttracked, it can not be un-fasttracked or selectively skipped

6

Correct, but once a packet matches the fasttrack rule, no other rules are processed (with the exceptions of random packets and non-TCP/UDP). Since the OP has a rule that fasttracks everything in the forward chain without any filters (as opposed to established connections only), it effectively works as “accept all”. This might not be an issue as I didn’t see any forward rules. I just wanted to point it out.

True, but when using the fasttrack rule properly (connection-state=established,related), it won’t touch new connections, giving the mangle rule a chance to mark the packet. Since fasttrack rule uses connection table, which is where packet marks are kept, it will not apply to marked packets (that is, when using “packet-mark=no-mark”).

Bottom line, you decide what traffic you want to fasttrack. Like you said, once it is fasttracked, you can’t undo do it. But it just means you need to configure rules in such a way that excludes specific traffic from the start.

I use this trick to send select traffic to the slow path so that I can keep marking it for my purposes (queues).

7

Trick??
Fastrack will not work in traffic in certain circumstances, such as queuing and mangling so its not trick LOL, you have no choice but to disable fastrack for certain traffic.

8

And how would you do that? By changing the rule that sends the packet down the fast path. You didn’t like the word “trick”, fine. My point is that it’s not all-or-nothing like some think. You don’t have to give up fasttrack completely just because certain traffic isn’t compatible with it.

It’s actually not that fasttrack won’t work with queues (as an example), it’s the opposite: queues won’t work on fasttracked traffic since it bypasses them unless configured not to bypass. As the documentation puts it:

it is up to the administrator to make sure FastTrack does not interfere with other configuration!

https://help.mikrotik.com/docs/spaces/ROS/pages/328227/Packet+Flow+in+RouterOS#PacketFlowinRouterOS-FastTrack

By the way, speaking of queues, I’ve been successfully using interface queues along with fasttrack for many years (other queues won’t work).

9

Correct, but once a packet matches the fasttrack rule, no other rules are processed
it effectively works as “accept all”

not quite true, see previous link

agreed for the rest

10

Now the router responds correctly even on the secondary WAN, with the exception of Wireguard.

Furthermore, I would like to make the firewall reachable from the secondary WAN, regardless of whether the primary WAN was Up or not.

These seem to contradict each other?

11

Concur Sebastia, an incomplete or confusing set of requirements will lead to unneeded series of posts.

As for trick, yes, it was incorrectly worded, its simply a normal process within mt ROS to ensure certain traffic types avoid fastrack, ( known limitation).

One can use traffic with NO connection marks, for example in mangling situations to avoid fastrack rule ( just add connection-mark=no-mark to the rule )
For others one can simply accept the non-friendly fasttrack traffic PRIOR to fastrack in the forward chain (be sure to cover both incoming and outgoing associated traffic aka normally two rules).

12

A quote from that link:

FastTrack-ed packets are bypassing:
firewall

How is that different in the end result from “accept” (with the exception on non-TCP/UDP and some select packets)?

13

It’s not really like “accept” because the fasttrack rule still has something like a hidden passthrough=yes. That’s why if you place your fasttrack rule immediately above the accept established,related rule, you’ll normally see them having the same counter values. After hitting the fasttrack rule, the next rule is still processed. It normally stops there because usually the next rule is the accept rule. But you can still put a drop rule right there after the fasttrack rule and see what happens.

First with the next packet after this does fasttrack really kicks-in and bypasses the firewall.

fasttrack-counter.png

14

My understanding is that if a packet hits the fasttrack rule, it’s already in the slow path and will go through all firewall rules until a match. That’s why the counters match (if using default rules), since they both use identical matchers. Traffic that shows there is new connections, random packets to maintain conntrack state, non-TCP/UDP traffic and maybe some other traffic. The actual fasttracked packets don’t make it even to rule #1, that’s why there is a dummy rule at position 0 to show the counters. And those counters are much higher.

Good clarification point that fasttrack is really like a marking rule with a passthrough (the docs use the word “mark” but don’t explicitly say traffic is passed through).

Here is how the documentation explains the reason for the accept rule below the fasttrack one:

Notice that the first rule marks established/related connections as fast-tracked, the second rule is still required to accept packets belonging to those connections. The reason for this is that, as was mentioned earlier, some random packets from fast-tracked connections are still sent the slow pathway and only UDP and TCP are fast-tracked, but we still want to accept packets for other protocols.

So, the rule is there not to accept fasttracked connections (they never reach this rule) but to accept anything that falls through.

But I agree, what I said before isn’t entirely correct like @sebastia pointed out, the fasttrack rule by itself doesn’t jump the firewall. I was thrown off by the modified rule with no matchers at all that the OP has:

add action=fasttrack-connection chain=forward hw-offload=yes

But as I think more, it’s probably not that much different from the default one. Since for a connection to be marked as fasttracked, it has to be tracked first, meaning at least the first packet goes through the firewall. As it does with the normal “established,related” rule. Still, I wouldn’t suggest anyone using the rule above.

15

Thank you for your reply.
From what I understand from your text, Wireguard always uses the primary routing table, is there no way to make it work on the secondary one too?
Regarding the incompatibility of mangles with fasttrack, thank you very much, I will do some tests with fasttrack disabled.
Thanks

16

You can make WG use other routing tables if you apply the NAT trick first, see this thread http://forum.mikrotik.com/t/routeros-blatantly-ignores-pref-src-can-this-really-be-a-bug/180360/48

After doing that you can use mark-connection/mark-routing with the WG traffic.

17

Be careful with the word “trick”, @anav will be coming after you saying “it’s simply a normal process within ROS”, no tricks! :laughing:

18

Well its tricking the router so to speak LOL. We call it a hack! A huge workaround because the normal behaviour of the router is not following our standard rules for mangling and routing.

The example linked to is a very complex dual WAN with dual WANIPs at each wan interface… I get confused looking at… for a simple example prefer this:
A generic solution that fixes wireguard specifically LOL. ( note will not work if one expects to use wireguard on WAN1 )

/ip firewall nat
chain=dstnat dst-address-type=local in-interface=WANX protocol=udp dst-port=YYYYY action=dst-nat to-addresses=ip.of.wan.PRIMARY

WANX = any WAN other than wanPRIMARY and of course being secondary or tertiary (non-primary) WAN route.
YYYYY = the port used for wireguard handshake.

Which basically tells the router, anything leaving the PRIMARY WAN, based on the wireguard port, was actually dstnatted ( even though our traffic was actually was) and thus needs to exit from the router from (un destnatted) to the WANIP of WANX.


+++++++++++++++++++++++++++

Apparently the issues surrounding the source addresses supplied by the router when replying to a client peer handshake attempt is EASILY handled in linux by the subject property (fwmark) that is found within normal linux wireguard structure. This property allows all packets emitted by wg to be marked on creation. We stumble upon this issue during RoS as most people do not know that the wireguard interface does not apply/decide source IP like other protocols.

I am not sure of exactly the right or best ref so here are a few:
https://www.linux.org/docs/man8/tc-fw.html
https://man7.org/linux/man-pages/man8/tc-fw.8.html

The problem is the source address of the return handshake traffic heading to the remote router is the WAN1 IP.
Because in wireguard, UNLIKE other structures, the source address of the wireguard response is decided by the most appropriate interface and this happens to be WAN1 with lower distance.
A work around from Sindy, used a DSTNAT CHAIN Rule to state traffic Headed for WAN2 should be changed to the WANIP of WAN1. Thus any traffic associated with WAN1 would be un-dstnated to WAN2.
What this does effectively is in the output chain (in route adjustment) by conntrak, is to see that the source address applied to the reply wireguard traffic ( decided by the router to be WAN1) and ensure that this traffic is un-dstnatted to the IP address of WAN2 ) when leaving the router using the WAN2 route ( on wan2 due to mangling and routing mark).

This works like a charm, however, what happens if WAN1 is hard down. The dstnat rule is no longer valid and in fact interferes with a wireguard connection. So a better work around had to be devised.
Suffice to say it involves using a faux bridge and address and a dstnat rule to change endpoint address WAN2, to faux address of bridge and a sourcenat rule on the input chain to change the original source address (remote router IP) to another IP on the bridge………. This IP being the source IP becomes the destination IP of the return traffic from the wireguard RoS process, and the router in choosing the source address for this local traffic chooses the best suitable local interface and in this time its not WAN1 but the bridge itself………

Conntrak then un-dstnatas and un-srcnats the return wireguard traffic from source interface IP (the bridge) to the destination address ( other bridge address) into source address WAN2 IP and destination address Remote Router IP.

Issue resolved by trickery ( heheh) etc.

What would be much cleaner is allowing the admin to capture the reply traffic leaving the wireguard process and ensuring it goes out the right WAN route BUT also with the right source IP address.
This is were fwmark would come in handy!!

19

ok, I tried to reach the http port of the firewall located downstream of the router from the secondary WAN interface and it does not respond.
From torch, filtering the WAN2 input interface and the src-address my home public IP, I see this, so the request arrives on port 80 (correct) but does not receive a response

44 Csd tcp  [my public ip]:55838    [WAN2 Public IP]:80     syn-sent     4s       0bps       0bps  2  0  1

Here are the mangle rules applied:

/ip firewall mangle
add action=mark-connection chain=input in-interface=pppoe-out3 new-connection-mark=route-secondaria-conn
add action=mark-connection chain=prerouting in-interface=pppoe-out3 new-connection-mark=route-secondaria-conn
add action=mark-routing chain=output connection-mark=route-secondaria-conn new-routing-mark=secondaria passthrough=no
add action=mark-routing chain=prerouting connection-mark=route-secondaria-conn \
    new-routing-mark=secondaria passthrough=no

I specify that all requests coming from WAN2 and directed to the router (icmp, winbox, webfig etc..) work correctly thanks to the input and output mangle rules.
The two prerouting rules, increase in the packets column.

I tried to disable WAN1, so as to make WAN2 the default gateway of the main routing table and in this way I can reach the firewall http page also from WAN2, so I’m extremely sure it’s a mangle problem, since they don’t move the routing table of traffic from WAN2 and not destined for the router, but I really don’t understand where I’m going wrong.

20

please also post rest of /ip firewall config as it’s relevant

(or just the full config minus sensitive stuff)