Asterisk and double NAT through mangle rules

wassy83 · March 12, 2018, 9:53am

Hi to all,
this is my situation:

4 vdsl 100/30Mb provided by 4 different internet suppliers with stock routers( I cannot change them) and I have disabled firewall AGL parameters and all other security options
of these 3 VDSL only one supplier gives me a mask of public IPs, the other 2 have only 1 static IP so everything is behind NAT
these 4 routers are connected to the 4 wan interfaces of a CCR1009-7G-1C-1S+ cloud router like this

eth1 is connected to provider 1 vdsl router 192.168.30.1 - eth1 mikrotik address 192.168.30.2/24 (main:fibra-telecom)
eth2 is connected to provider 2 vdsl router 192.168.130.1 - eth2 mikrotik address 192.168.130.2/24 (fibra-backup)
eth3 is connected to provider 3 vdsl router 192.168.230.1 - eth2 mikrotik address 109.234.60.236/29 (this is the public one: fibra-ehiweb)
eth4 is connected to provider 4 vdsl router 192.168.231.1 - eth2 mikrotik address 192.168.231.2/24 (TIM4G)
eth7 is my local lan 192.168.25.0/24

eth1 is the main interface with the highest priority, anything else is selected trough mangle rules and mark routing to a specific IP (in example mail server is on fibra-backup with ip 192.168.25.5, and asterisk is on fibra-ehiweb with ip 192.168.25.200)

so.. My asterisk Freepbx is hosted on a local HA cluster and is receving the public IP through a mangle rule, this is working very well and stable.
What I want to do is an internet backup in case of failure simply changing the freepbx magle rule from the provider 3 to any other provider.
I know that double NAt is always a problem with freepbx, but my sip trunks uses chan_pjsip driver and there are a lot of rules that must help me to fix the problem. But I cannot figure out of this, maybe is something missing in my mikrotik to makes things works. The strange thing is that using the main 192.168.30.1 as mark route sometimes works but is very unstable, no way with the other providers.
This is my configuration I hope to find a solution, cause I think that is very interesting to have a backup for a VOIP production environment.

# mar/12/2018 10:36:24 by RouterOS 6.36.4
# software id = BXAR-5YJB
#
/interface ethernet
set [ find default-name=ether1 ] name="ether1-fibra telecom"
set [ find default-name=ether2 ] name="ether2-fibra backup"
set [ find default-name=ether3 ] name="ether3-fibra ehiweb"
set [ find default-name=ether4 ] name=ether4-TIM4G
set [ find default-name=ether7 ] arp=proxy-arp name="ether7-local lan"
/interface gre
add allow-fast-path=no disabled=yes !keepalive local-address=192.168.30.2 \
    name=anek remote-address=aaa.aaa.aaa.aaa
/ip ipsec proposal
set [ find default=yes ] enc-algorithms=aes-128-cbc,3des pfs-group=none
add enc-algorithms=3des name=proposal1 pfs-group=none
add auth-algorithms=md5 enc-algorithms=des name=superfast pfs-group=none
add enc-algorithms=3des name=proposal2 pfs-group=none
/ip pool
add name=dhcp_pool1 ranges=192.168.25.201-192.168.25.250
/ip dhcp-server
add address-pool=dhcp_pool1 disabled=no interface="ether7-local lan" name=\
    dhcp1
/interface l2tp-server server
set enabled=yes
/ip address
add address=192.168.25.62/24 interface="ether7-local lan" network=\
    192.168.25.0
add address=192.168.233.54/30 interface=anek network=192.168.233.52
add address=192.168.30.2/25 interface="ether1-fibra telecom" network=\
    192.168.30.0
add address=10.1.2.1/30 network=10.1.2.0
add address=172.0.0.1/24 comment="lan per cisco" interface=ether6 network=\
    172.0.0.0
add address=192.168.130.2/24 interface="ether2-fibra backup" network=\
    192.168.130.0
add address=bbb.bbb.bbb.bbb/29 interface="ether3-fibra ehiweb" network=\
    ccc.ccc.ccc.ccc
add address=192.168.230.2/24 interface="ether3-fibra ehiweb" network=\
    192.168.230.0
add address=192.168.231.2/24 interface=ether4-TIM4G network=192.168.231.0
add address=ddd.ddd.ddd.ddd/29 disabled=yes interface="ether3-fibra ehiweb" \
    network=ccc.ccc.ccc.ccc
/ip dhcp-client
add add-default-route=no dhcp-options=hostname,clientid interface=ether5
/ip dhcp-server lease
add block-access=yes mac-address=2A:BD:6F:6B:00:B3
/ip dhcp-server network
add address=192.168.25.0/24 dns-server=192.168.25.62 gateway=192.168.25.62 \
    netmask=24
/ip dns
set allow-remote-requests=yes servers=8.8.8.8,8.8.4.4
/ip dns static
add address=192.168.25.5 name=mail.anekitalia.com
add address=192.168.25.126 name=crm.anekitalia.com
/ip firewall address-list
add address=eee.eee.eee.eee list=toplevel
add address=192.168.27.0/24 list=venezia
add address=ddd.ddd.ddd.ddd list=vtecrm
/ip firewall filter
add action=drop chain=forward connection-state=invalid,new disabled=yes \
    in-interface="ether2-fibra backup" src-address=0.0.0.0/0
add action=accept chain=forward in-interface="ether1-fibra telecom" protocol=\
    tcp src-port=1723
add action=accept chain=forward in-interface="ether1-fibra telecom" protocol=\
    gre
add action=accept chain=input dst-port=500 protocol=udp
add action=accept chain=input dst-port=4500 protocol=udp
add action=accept chain=input dst-port=1701 protocol=udp
add action=accept chain=input protocol=ipsec-ah
add action=accept chain=input protocol=ipsec-esp
add action=accept chain=input in-interface="ether1-fibra telecom" \
    src-address=79.58.72.190
add action=accept chain=input in-interface="ether1-fibra telecom" \
    src-address=93.42.154.226
add action=accept chain=input src-address=aaa.aaa.aaa.aaa
add action=accept chain=input src-address=212.205.152.194
add action=accept chain=input src-address=78.134.7.223
add action=accept chain=input src-address=8.8.8.8
add action=accept chain=input src-address=8.8.4.4
add action=accept chain=input src-address=83.211.227.21
add action=drop chain=input in-interface="ether1-fibra telecom"
add action=drop chain=input in-interface="ether2-fibra backup"
add action=drop chain=input in-interface="ether3-fibra ehiweb"
add action=drop chain=input in-interface=ether4-TIM4G
/ip firewall mangle
add action=mark-routing chain=prerouting new-routing-mark=fibra-backup \
    passthrough=yes src-address=192.168.25.208
add action=mark-routing chain=prerouting dst-address=!192.168.26.0/24 \
    dst-address-list=!venezia new-routing-mark=fibra-ehiweb passthrough=yes \
    src-address=192.168.25.200
add action=mark-routing chain=prerouting dst-address=!192.168.26.0/24 \
    dst-address-list=!venezia new-routing-mark=fibra-backup passthrough=yes \
    src-address=192.168.25.5
add action=mark-routing chain=prerouting new-routing-mark=fibra-backup \
    passthrough=yes src-address=192.168.25.75
add action=accept chain=prerouting
add action=accept chain=prerouting
/ip firewall nat
add action=masquerade chain=srcnat disabled=yes src-address=192.168.26.0/24
add action=src-nat chain=srcnat disabled=yes out-interface=\
    "ether3-fibra ehiweb" src-address=192.168.25.47 to-addresses=\
    ddd.ddd.ddd.ddd
add action=dst-nat chain=dstnat disabled=yes dst-address=ddd.ddd.ddd.ddd \
    in-interface="ether3-fibra ehiweb" to-addresses=192.168.25.6
add action=masquerade chain=srcnat dst-address=10.1.1.0/24
add action=masquerade chain=srcnat dst-address=10.1.150.128
add action=masquerade chain=srcnat comment="Fibra Telecom main" \
    out-interface="ether1-fibra telecom"
add action=masquerade chain=srcnat out-interface="ether2-fibra backup"
add action=masquerade chain=srcnat comment=eihweb out-interface=\
    "ether3-fibra ehiweb"
add action=masquerade chain=srcnat out-interface=ether4-TIM4G
add action=masquerade chain=srcnat disabled=yes src-address=192.168.25.192/27
add action=dst-nat chain=dstnat dst-port=22 in-interface=\
    "ether1-fibra telecom" protocol=tcp src-address-list=vtecrm to-addresses=\
    192.168.25.126 to-ports=22
add action=dst-nat chain=dstnat dst-port=80 in-interface=\
    "ether1-fibra telecom" protocol=tcp src-address-list=vtecrm to-addresses=\
    192.168.25.126 to-ports=80
add action=dst-nat chain=dstnat dst-port=8080 in-interface=\
    "ether1-fibra telecom" protocol=tcp src-address-list=vtecrm to-addresses=\
    192.168.25.126 to-ports=8080
add action=dst-nat chain=dstnat dst-port=1194 in-interface=\
    "ether1-fibra telecom" protocol=udp to-addresses=192.168.25.7 to-ports=\
    1194
add action=dst-nat chain=dstnat dst-port=1195 in-interface=\
    "ether1-fibra telecom" protocol=udp to-addresses=192.168.25.76 to-ports=\
    1195
add action=dst-nat chain=dstnat dst-port=8070 protocol=tcp to-addresses=\
    192.168.25.109 to-ports=80
add action=dst-nat chain=dstnat dst-port=443 in-interface=\
    "ether2-fibra backup" protocol=tcp to-addresses=192.168.25.5 to-ports=443
add action=dst-nat chain=dstnat dst-port=110 in-interface=\
    "ether2-fibra backup" protocol=tcp to-addresses=192.168.25.5 to-ports=110
add action=dst-nat chain=dstnat dst-port=995 in-interface=\
    "ether2-fibra backup" protocol=tcp to-addresses=192.168.25.5 to-ports=995
add action=dst-nat chain=dstnat dst-port=143 in-interface=\
    "ether2-fibra backup" protocol=tcp to-addresses=192.168.25.5 to-ports=143
add action=dst-nat chain=dstnat dst-port=993 in-interface=\
    "ether2-fibra backup" protocol=tcp to-addresses=192.168.25.5 to-ports=993
add action=dst-nat chain=dstnat dst-port=80 in-interface=\
    "ether2-fibra backup" protocol=tcp to-addresses=192.168.25.5 to-ports=80
add action=dst-nat chain=dstnat disabled=yes dst-port=5060 in-interface=\
    "ether2-fibra backup" protocol=udp to-addresses=192.168.25.6 to-ports=\
    5060
add action=dst-nat chain=dstnat disabled=yes dst-port=5061 in-interface=\
    "ether2-fibra backup" protocol=udp to-addresses=192.168.25.6 to-ports=\
    5061
add action=dst-nat chain=dstnat disabled=yes dst-port=5160 in-interface=\
    "ether2-fibra backup" protocol=udp to-addresses=192.168.25.6 to-ports=\
    5160
add action=dst-nat chain=dstnat disabled=yes dst-port=5151 in-interface=\
    "ether2-fibra backup" protocol=udp to-addresses=192.168.25.6 to-ports=\
    5161
add action=dst-nat chain=dstnat disabled=yes dst-port=5065 in-interface=\
    "ether2-fibra backup" protocol=udp to-addresses=192.168.25.6 to-ports=\
    5065
add action=dst-nat chain=dstnat disabled=yes dst-port=10000-20000 \
    in-interface="ether2-fibra backup" protocol=udp to-addresses=192.168.25.6 \
    to-ports=10000-20000
add action=dst-nat chain=dstnat dst-port=25 in-interface=\
    "ether2-fibra backup" protocol=tcp to-addresses=192.168.25.75 to-ports=25
add action=dst-nat chain=dstnat dst-port=587 in-interface=\
    "ether2-fibra backup" protocol=tcp to-addresses=192.168.25.5 to-ports=587
add action=accept chain=srcnat disabled=yes
/ip firewall service-port
set sip disabled=yes
/ip route
add distance=1 gateway=eee.eee.eee.eee routing-mark=fibra-ehiweb
add distance=3 gateway=192.168.130.1 routing-mark=fibra-backup
add distance=100 gateway=192.168.231.1 routing-mark=TIM4G
add distance=1 gateway=192.168.30.1
add distance=2 gateway=192.168.130.1
add distance=4 gateway=192.168.230.1
add distance=1 dst-address=10.1.1.0/24 gateway=192.168.25.106
add distance=1 dst-address=10.1.150.128/32 gateway=192.168.25.106
add distance=1 dst-address=10.1.150.201/32 gateway=192.168.25.106
add distance=1 dst-address=10.250.0.2/32 gateway=192.168.25.106
add distance=1 dst-address=79.58.72.190/32 gateway=192.168.30.1
add distance=1 dst-address=192.168.0.0/24 gateway=192.168.25.106
add distance=1 dst-address=192.168.1.0/24 gateway=192.168.25.106
add distance=1 dst-address=192.168.2.0/24 gateway=192.168.25.106
add distance=1 dst-address=192.168.10.0/24 gateway=192.168.25.106
/ip route rule
add action=lookup-only-in-table disabled=yes table=main
/ip service
set telnet disabled=yes
set ftp disabled=yes
set www port=8089
set ssh disabled=yes
set api disabled=yes
set api-ssl disabled=yes
/lcd
set time-interval=hour
/ppp secret
add local-address=1.3.3.1 name=Portoanek password=Portoanek13579 \
    remote-address=1.3.3.2 routes="192.168.26.0/24 1.3.3.2 1" service=l2tp
add local-address=1.3.3.3 name=venezia password=venezia13579@ remote-address=\
    1.3.3.4 routes="192.168.27.0/24 1.3.3.4 3" service=l2tp
add disabled=yes local-address=10.10.10.210 name=toplevel password=\
    toplevelts12@ remote-address=10.10.10.211 routes=\
    "192.168.0.0/24 10.10.10.211 210" service=l2tp
/system clock
set time-zone-name=Europe/Rome
/system routerboard settings
set protected-routerboot=disabled

wassy83 · March 18, 2018, 4:13pm

Just want to share this.. After changing the mangle rule, let’s say from voip-fiber to voip-4gLTE, it’s necessary to reboot the mikrotik or you will continue to receive errors like “no response from sip provider” in the asterisk-cli I think that this is related to the already enstablished connections.. I tried to delete from IP–>firewall–>connections the related asterisk local ip address with no success. Honestly this is very annoying cause I have a lot of services on this mikrotik that are in a production environment, and even if the Mikrotik takes only few seconds to reboot, this action will crash all the applications that are currently connected.. but at least I found a solution.. yes it was stupid but nevermind that mikrotik needs to be reboot after applying a mangle rule.

CZFan · March 18, 2018, 11:22pm

I don’t think the problem was mikrotik here, but how networking protocol is working.

If you restarted the asterisk box, I suspect you would have came to the same end result

sindy · March 19, 2018, 11:47am

I’m afraid that restart of Asterisk may simply take log enough so that the established connections would decay in Mikrotik and it would seem as if restart of Asterisk was necessary while the reason would be the Mikrotik WAN backup handling.

If I were to deal with this kind of trouble, I would first of all sniff simultaneously at the interface between the Asterisk and the Mikrotik and on both WAN uplinks of the Mikrotik when putting down the primary uplink, to see what is going on there. pjsip somehow detects the actual public address it has from the 200 response to REGISTER, so one possible explanation may be that it then does not like the change.

But first things first, do the WAN interfaces have static addresses? If so, use ****

action=src-nat to-addresses=<the.static.ip.address>

instead of

action=masquerade

.