Our primary router is a CCR1016-12G and we used to have 5 different ISPs, and load balancing worked great. Unfortunately, 4 of these ISPs were acquired by the same company that uses CGNAT/LSNAT; so, we are down to two ISPs.
WANs 1-4 are with the CGNAT carrier, all are 100Mbps down / 5 Mbps up. WAN 5 is only 10 Mbps down / 0.5 Mbps up.
The contracted static public IP addresses for WANs 1-4 have yet to be assigned to us (ISPs merger issues).
We are very aware of all that CGNAT/LSNAT breaks (particularly in our case, where the ISPs incorrectly implemented it: they are using RFC1918 addresses for their CGNAT/LSNAT instead of using RFC6598 addresses, with the corresponding consequences).
In fact, I'm 99% certain that the ISP's incorrect CGNAT/LSNAT implementation is what is causing most of the issues below (except that ether6_WAN5, the different ISP is not responding at times); but, I'd really appreciate all the help the forum cares to give me:
-
From within the CCR, pings and traceroutes timeout except through a single WAN interface (WAN 1-4), most of the time. Frequently all WAN interfaces time out (except that the CCR and all devices connected to it always can ping successfully all gateways, and the connected devices can also ping the CCR.
-
Frequent extremely slow or no fqdn resolution from devices directly connected to the CCR through a switch (this gigabit switch has been replaced to rule it out), and extremely slow loading pages, incoming email timeouts and outgoing email unable to find server.
-
When the connection doesn't timeout, we can successfully work with secure sites (web and email), despite the CGNAT/LSNAT shortcomings.
This is the export for the CCR config (the disabled interfaces were intentionally disabled while we troubleshoot the issues):
mar/08/2016 10:26:41 by RouterOS 6.10
software id = YEP1-9633
/interface ethernet
set [ find default-name=ether1 ] name=ether1_WAN1
set [ find default-name=ether2 ] name=ether2_LAN1
set [ find default-name=ether3 ] name=ether3_WAN2
set [ find default-name=ether4 ] name=ether4_WAN3
set [ find default-name=ether5 ] name=ether5_WAN4
set [ find default-name=ether6 ] name=ether6_WAN5
set [ find default-name=ether10 ] name=ether10_LAN
set [ find default-name=ether11 ] name=ether11_PuntaMita
/ip hotspot user profile
set [ find default=yes ] idle-timeout=none keepalive-timeout=2m
mac-cookie-timeout=3d
/ip pool
add name=dhcp_pool1 ranges=192.168.120.1-192.168.120.253
add name=dhcp_pool2 ranges=192.168.203.1-192.168.203.253
add name=pool4 ranges=172.16.1.24-172.16.1.29
add name=dhcp_pool3 ranges=172.16.0.30-172.16.0.50
/ip dhcp-server
add address-pool=dhcp_pool1 disabled=no interface=ether10_LAN name=dhcp1
add address-pool=dhcp_pool2 disabled=no interface=ether9 name=dhcp2
add address-pool=pool4 disabled=no interface=ether11_PuntaMita name=LAN11
add address-pool=dhcp_pool3 disabled=no interface=ether2_LAN1 lease-time=15m
name=dhcp3
/port
set 0 name=serial0
set 1 name=serial1
/ip address
add address=192.168.120.254/24 disabled=yes interface=ether10_LAN network=
192.168.120.0
add address=192.168.203.254/24 disabled=yes interface=ether9 network=
192.168.203.0
add address=172.16.0.254/24 interface=ether2_LAN1 network=172.16.0.0
add address=172.16.1.1/24 disabled=yes interface=ether11_PuntaMita network=
172.16.1.0
/ip dhcp-client
add add-default-route=no dhcp-options=hostname,clientid disabled=no
interface=ether3_WAN2
add add-default-route=no dhcp-options=hostname,clientid disabled=no
interface=ether1_WAN1
add add-default-route=no dhcp-options=hostname,clientid disabled=no
interface=ether4_WAN3
add add-default-route=no dhcp-options=hostname,clientid disabled=no
interface=ether5_WAN4
add add-default-route=no dhcp-options=hostname,clientid disabled=no
interface=ether6_WAN5
/ip dhcp-server network
add address=172.16.0.0/24 dns-server=172.16.0.254 gateway=172.16.0.254
add address=192.168.120.0/24 gateway=192.168.120.254
add address=192.168.203.0/24 gateway=192.168.203.254
/ip dns
set allow-remote-requests=yes cache-size=5000KiB max-udp-packet-size=512
servers=
192.168.2.254,192.168.0.254,192.168.1.254,192.168.73.1,192.168.72.1
/ip firewall address-list
add address=172.16.0.30 list=MBPR
/ip firewall filter
add action=drop chain=forward dst-address=192.168.1.0/24 src-address=
192.168.73.0/24
add action=drop chain=forward dst-address=192.168.2.0/24 src-address=
192.168.73.0/24
add action=drop chain=forward dst-address=192.168.0.0/24 src-address=
192.168.73.0/24
add action=drop chain=forward dst-address=192.168.72.0/24 src-address=
192.168.73.0/24
add action=drop chain=forward dst-address=192.168.73.0/24 src-address=
192.168.1.0/24
add action=drop chain=forward dst-address=192.168.2.0/24 src-address=
192.168.1.0/24
add action=drop chain=forward dst-address=192.168.0.0/24 src-address=
192.168.1.0/24
add action=drop chain=forward dst-address=192.168.72.0/24 src-address=
192.168.1.0/24
add action=drop chain=forward dst-address=192.168.73.0/24 src-address=
192.168.2.0/24
add action=drop chain=forward dst-address=192.168.1.0/24 src-address=
192.168.2.0/24
add action=drop chain=forward dst-address=192.168.0.0/24 src-address=
192.168.2.0/24
add action=drop chain=forward dst-address=192.168.72.0/24 src-address=
192.168.2.0/24
add action=drop chain=forward dst-address=192.167.73.0/24 src-address=
192.168.0.0/24
add action=drop chain=forward dst-address=192.167.1.0/24 src-address=
192.168.0.0/24
add action=drop chain=forward dst-address=192.167.2.0/24 src-address=
192.168.0.0/24
add action=drop chain=forward dst-address=192.167.72.0/24 src-address=
192.168.0.0/24
add action=drop chain=forward dst-address=192.168.73.0/24 src-address=
192.168.72.0/24
add action=drop chain=forward dst-address=192.168.1.0/24 src-address=
192.168.72.0/24
add action=drop chain=forward dst-address=192.168.2.0/24 src-address=
192.168.72.0/24
add action=drop chain=forward dst-address=192.168.0.0/24 src-address=
192.168.72.0/24
add action=drop chain=input dst-port=53 in-interface=ether1_WAN1 protocol=udp
add action=drop chain=input dst-port=53 in-interface=ether1_WAN1 protocol=tcp
add action=drop chain=input dst-port=53 in-interface=ether3_WAN2 protocol=udp
add action=drop chain=input dst-port=53 in-interface=ether3_WAN2 protocol=tcp
add action=drop chain=input dst-port=53 in-interface=ether4_WAN3 protocol=udp
add action=drop chain=input dst-port=53 in-interface=ether4_WAN3 protocol=tcp
add action=drop chain=input dst-port=53 in-interface=ether5_WAN4 protocol=udp
add action=drop chain=input dst-port=53 in-interface=ether5_WAN4 protocol=tcp
add action=drop chain=input dst-port=53 in-interface=ether6_WAN5 protocol=udp
add action=drop chain=input dst-port=53 in-interface=ether6_WAN5 protocol=tcp
/ip firewall mangle
add action=mark-connection chain=input connection-state=new in-interface=
ether1_WAN1 new-connection-mark=ether1_WAN1_conn passthrough=no
add action=mark-connection chain=input connection-state=new in-interface=
ether3_WAN2 new-connection-mark=ether3_WAN2_conn passthrough=no
add action=mark-connection chain=input connection-state=new in-interface=
ether4_WAN3 new-connection-mark=ether4_WAN3_conn passthrough=no
add action=mark-connection chain=input connection-state=new in-interface=
ether5_WAN4 new-connection-mark=ether5_WAN4_conn passthrough=no
add action=mark-connection chain=input connection-state=new in-interface=
ether6_WAN5 new-connection-mark=ether6_WAN5_conn passthrough=no
add action=mark-routing chain=output connection-mark=ether1_WAN1_conn
new-routing-mark=to_ether1_WAN1 passthrough=no
add action=mark-routing chain=output connection-mark=ether3_WAN2_conn
new-routing-mark=to_ether3_WAN2 passthrough=no
add action=mark-routing chain=output connection-mark=ether4_WAN3_conn
new-routing-mark=to_ether4_WAN3 passthrough=no
add action=mark-routing chain=output connection-mark=ether5_WAN4_conn
new-routing-mark=to_ether5_WAN4 passthrough=no
add action=mark-routing chain=output connection-mark=ether6_WAN5_conn
new-routing-mark=to_ether6_WAN5 passthrough=no
add chain=prerouting dst-address=192.168.2.0/24 in-interface=ether2_LAN1
add chain=prerouting dst-address=192.168.0.0/24 in-interface=ether2_LAN1
add chain=prerouting dst-address=192.168.1.0/24 in-interface=ether2_LAN1
add chain=prerouting dst-address=192.168.73.0/24 in-interface=ether2_LAN1
add chain=prerouting dst-address=192.168.72.0/24 in-interface=ether2_LAN1
add action=mark-connection chain=prerouting dst-address-type=!local
in-interface=ether2_LAN1 new-connection-mark=ether4_WAN3_conn
per-connection-classifier=src-address:8/0
add action=mark-connection chain=prerouting dst-address-type=!local
in-interface=ether2_LAN1 new-connection-mark=ether4_WAN3_conn
per-connection-classifier=src-address:8/1
add action=mark-connection chain=prerouting dst-address-type=!local
in-interface=ether2_LAN1 new-connection-mark=ether5_WAN4_conn
per-connection-classifier=src-address:8/2
add action=mark-connection chain=prerouting dst-address-type=!local
in-interface=ether2_LAN1 new-connection-mark=ether5_WAN4_conn
per-connection-classifier=src-address:8/3
add action=mark-connection chain=prerouting dst-address-type=!local
in-interface=ether2_LAN1 new-connection-mark=ether3_WAN2_conn
per-connection-classifier=src-address:8/4
add action=mark-connection chain=prerouting dst-address-type=!local
in-interface=ether2_LAN1 new-connection-mark=ether3_WAN2_conn
per-connection-classifier=src-address:8/5
add action=mark-connection chain=prerouting dst-address-type=!local
in-interface=ether2_LAN1 new-connection-mark=ether1_WAN1_conn
per-connection-classifier=src-address:8/6
add action=mark-connection chain=prerouting dst-address-type=!local
in-interface=ether2_LAN1 new-connection-mark=ether6_WAN5_conn
per-connection-classifier=src-address:8/7
add action=mark-routing chain=prerouting connection-mark=ether4_WAN3_conn
in-interface=ether2_LAN1 new-routing-mark=to_ether4_WAN3 passthrough=no
add action=mark-routing chain=prerouting connection-mark=ether5_WAN4_conn
in-interface=ether2_LAN1 new-routing-mark=to_ether5_WAN4 passthrough=no
add action=mark-routing chain=prerouting connection-mark=ether3_WAN2_conn
in-interface=ether2_LAN1 new-routing-mark=to_ether3_WAN2 passthrough=no
add action=mark-routing chain=prerouting connection-mark=ether1_WAN1_conn
in-interface=ether2_LAN1 new-routing-mark=to_ether1_WAN1 passthrough=no
add action=mark-routing chain=prerouting connection-mark=ether6_WAN5_conn
in-interface=ether2_LAN1 new-routing-mark=to_ether6_WAN5 passthrough=no
/ip firewall nat
add action=masquerade chain=srcnat out-interface=ether1_WAN1
add action=masquerade chain=srcnat out-interface=ether3_WAN2
add action=masquerade chain=srcnat out-interface=ether4_WAN3
add action=masquerade chain=srcnat out-interface=ether5_WAN4
add action=masquerade chain=srcnat out-interface=ether6_WAN5
/ip route
add check-gateway=ping distance=1 gateway=192.168.73.1 routing-mark=
to_ether1_WAN1
add check-gateway=ping distance=1 gateway=192.168.1.254 routing-mark=
to_ether3_WAN2
add check-gateway=ping distance=1 gateway=192.168.2.254 routing-mark=
to_ether4_WAN3
add check-gateway=ping distance=1 gateway=192.168.0.254 routing-mark=
to_ether5_WAN4
add check-gateway=ping distance=1 gateway=192.168.72.1 routing-mark=
to_ether6_WAN5
add check-gateway=ping distance=1 gateway=192.168.2.254
add check-gateway=ping distance=2 gateway=192.168.0.254
add check-gateway=ping distance=3 gateway=192.168.1.254
add check-gateway=ping distance=4 gateway=192.168.73.1
add check-gateway=ping distance=5 gateway=192.168.72.1
/ip upnp
set allow-disable-external-interface=no
/lcd interface
set ether1_WAN1 interface=ether1_WAN1
set ether2_LAN1 interface=ether2_LAN1
set ether3_WAN2 interface=ether3_WAN2
set ether4_WAN3 interface=ether4_WAN3
set ether5_WAN4 interface=ether5_WAN4
set ether6_WAN5 interface=ether6_WAN5
set ether7 interface=ether7
set ether8 interface=ether8
set ether9 interface=ether9
set ether10_LAN interface=ether10_LAN
set ether11_PuntaMita interface=ether11_PuntaMita
set ether12 interface=ether12
/system clock
set time-zone-name=America/Mexico_City
/system identity
set name=5WAN_Working_Newest
/system ntp client
set enabled=yes mode=unicast primary-ntp=17.253.4.253 secondary-ntp=
17.253.34.253
/system routerboard settings
set cpu-frequency=1200MHz memory-frequency=1066DDR
Again, all your help will be greatly appreciated. TIA.
Luis.