Recursive Failover - unstable

densukke · Tue Jul 05, 2022 12:51 am

Hello folks!

Me again bugging, if someone is kind enough to share some wisdom on a very annoying problem i´ve been dealing with, on my RB2011UiAS-

first things first my config:

# jul/04/2022 18:36:11 by RouterOS 7.3.1
# software id = B2RC-819H
#
# model = RB2011UiAS
# serial number = 00000000000
/caps-man channel
add band=2ghz-g/n control-channel-width=20mhz extension-channel=disabled \
    frequency=2422 name=channel1
add band=2ghz-g/n control-channel-width=20mhz extension-channel=disabled \
    frequency=2432 name=channel6
add band=2ghz-g/n control-channel-width=20mhz extension-channel=disabled \
    frequency=2447 name=channel9
add band=2ghz-g/n control-channel-width=20mhz extension-channel=disabled \
    frequency=2457 name=channel12
/interface bridge
add admin-mac=11:22:33:AA:BB:CC auto-mac=no name=bridge1
/interface ethernet
set [ find default-name=ether1 ] comment="WAN1 - IPLAN" loop-protect=on \
    loop-protect-disable-time=1m
set [ find default-name=ether2 ] comment="WAN2 - FIBERCORP" loop-protect=on
set [ find default-name=ether5 ] comment="LAN - switch Trunk to TPLINK" \
    loop-protect=on
/caps-man configuration
add channel.band=2ghz-g/n .control-channel-width=20mhz country=argentina \
    datapath.bridge=bridge1 .client-to-client-forwarding=no \
    .local-forwarding=no name=configuracion_barentz \
    security.authentication-types=wpa-psk,wpa2-psk .encryption=aes-ccm,tkip \
    .group-key-update=1d ssid="Wifi Barentz"
/caps-man interface
add channel=channel6 configuration=configuracion_barentz disabled=no l2mtu=\
    1600 mac-address=DC:2C:6E:64:9E:27 master-interface=none name=cap1 \
    radio-mac=DC:2C:6E:64:9E:27 radio-name=DC2C6E649E27
add channel=channel12 configuration=configuracion_barentz disabled=no l2mtu=\
    1600 mac-address=DC:2C:6E:64:A0:55 master-interface=none name=cap2 \
    radio-mac=DC:2C:6E:64:A0:55 radio-name=DC2C6E64A055
add channel=channel1 channel.frequency=2412 configuration=\
    configuracion_barentz disabled=no l2mtu=1600 mac-address=\
    DC:2C:6E:64:9F:57 master-interface=none name=cap4 radio-mac=\
    DC:2C:6E:64:9F:57 radio-name=DC2C6E649F57
add channel=channel9 configuration=configuracion_barentz disabled=no l2mtu=\
    1600 mac-address=DC:2C:6E:64:9F:33 master-interface=none name=cap5 \
    radio-mac=DC:2C:6E:64:9F:33 radio-name=DC2C6E649F33
/interface lte apn
set [ find default=yes ] ip-type=ipv4 use-network-apn=no
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip pool
add name=dhcp_pool1 ranges=192.168.101.2-192.168.101.254
/ip dhcp-server
add address-pool=dhcp_pool1 interface=bridge1 lease-time=1d name=dhcp1
/port
set 0 name=serial0
/queue type
add kind=pcq name=DOWN pcq-classifier=dst-address
add kind=pcq name=UP pcq-classifier=src-address
/queue tree
add name="WAN1 DOWN" parent=global queue=default
add name="WAN1 UP" parent=ether1 queue=default
add name="WAN1 - WEB - rx" packet-mark=web-wan1 parent="WAN1 DOWN" priority=3 \
    queue=DOWN
add name="WAN1 - WEB- tx" packet-mark=web parent="WAN1 UP" priority=3 queue=\
    UP
add name="WAN1 - DNS - rx" packet-mark=dns-wan1 parent="WAN1 DOWN" priority=2 \
    queue=DOWN
add name="WAN1 - DNS - tx" packet-mark=dns parent="WAN1 UP" priority=2 queue=\
    UP
add name="WAN1 - ICMP -rx" packet-mark=icmp-wan1 parent="WAN1 DOWN" priority=\
    1 queue=DOWN
add name="WAN1 - ICMP - tx" packet-mark=icmp parent="WAN1 UP" priority=1 \
    queue=UP
add name="WAN1 - QUIC - rx" packet-mark=quic-wan1 parent="WAN1 DOWN" \
    priority=5 queue=DOWN
add name="WAN1 - QUIC -tx" packet-mark=quic parent="WAN1 UP" priority=5 \
    queue=UP
add name="WAN2 DOWN" parent=global queue=default
add name="WAN2 UP" parent=ether2 queue=default
add name="WAN2- DNS - rx" packet-mark=dns-wan2 parent="WAN2 DOWN" priority=2 \
    queue=DOWN
add name="WAN2 - DNS -tx" packet-mark=dns parent="WAN2 UP" priority=2 queue=\
    UP
add name="WAN2 - ICMP - rx" packet-mark=icmp-wan2 parent="WAN2 DOWN" \
    priority=1 queue=DOWN
add name="WAN2 - ICMP -tx" packet-mark=icmp parent="WAN2 UP" priority=1 \
    queue=UP
add name="WAN2 - QUIC - rx" packet-mark=quic-wan2 parent="WAN2 DOWN" \
    priority=5 queue=DOWN
add name="WAN1 - QUIC - tx" packet-mark=quic parent="WAN2 UP" priority=5 \
    queue=UP
add name="WAN2 - RESTO - rx" packet-mark=resto-wan2 parent="WAN2 DOWN" queue=\
    DOWN
add name="WAN2 - RESTO - tx" packet-mark=resto parent="WAN2 UP" queue=UP
add name="WAN2 - WEB - rx" packet-mark=web-wan2 parent="WAN2 DOWN" priority=3 \
    queue=DOWN
add name="WAN2 - WEB - tx" packet-mark=web parent="WAN2 UP" priority=3 queue=\
    UP
add name="WAN1 - Resto -rx" packet-mark=resto-wan1 parent="WAN1 DOWN" queue=\
    DOWN
add name="WAN1 - Resto - tx" packet-mark=resto parent="WAN1 UP" queue=UP
/routing bgp template
set default disabled=no output.network=bgp-networks
/routing table
add disabled=yes fib name=to_ISP1
add disabled=yes fib name=to_ISP2
/caps-man manager
set enabled=yes
/caps-man manager interface
set [ find default=yes ] forbid=yes
add disabled=no interface=bridge1
/caps-man provisioning
add action=create-dynamic-enabled master-configuration=configuracion_barentz
/interface bridge port
add bridge=bridge1 ingress-filtering=no interface=ether3
add bridge=bridge1 ingress-filtering=no interface=ether4
add bridge=bridge1 ingress-filtering=no interface=ether5
add bridge=bridge1 ingress-filtering=no interface=ether6
add bridge=bridge1 ingress-filtering=no interface=ether7
add bridge=bridge1 ingress-filtering=no interface=ether8
add bridge=bridge1 ingress-filtering=no interface=ether9
add bridge=bridge1 ingress-filtering=no interface=ether10
/ip neighbor discovery-settings
set discover-interface-list=all
/ip settings
set max-neighbor-entries=8192
/ipv6 settings
set disable-ipv6=yes max-neighbor-entries=8192
/interface ovpn-server server
set auth=sha1,md5
/ip address
add address=192.168.101.1/24 comment="LAN SUBNET" interface=bridge1 network=\
    192.168.101.0
add address=x.x.x.x/24 comment="IPLAN STATIC IP" interface=ether1 \
    network=x.x.x.x
/ip cloud
set ddns-enabled=yes ddns-update-interval=1m
/ip dhcp-client
add add-default-route=no disabled=yes interface=ether2 use-peer-dns=no \
    use-peer-ntp=no
/ip dhcp-server lease
add address=192.168.101.249 client-id=1:dc:2c:6e:64:a0:54 mac-address=\
    DC:2C:6E:64:A0:54 server=dhcp1
add address=192.168.101.247 client-id=1:14:eb:b6:ce:de:ff mac-address=\
    14:EB:B6:CE:DE:FF server=dhcp1
add address=192.168.101.251 client-id=1:dc:2c:6e:64:9e:26 mac-address=\
    DC:2C:6E:64:9E:26 server=dhcp1
add address=192.168.101.248 client-id=1:dc:2c:6e:64:9f:32 mac-address=\
    DC:2C:6E:64:9F:32 server=dhcp1
add address=192.168.101.253 client-id=1:48:5b:39:a3:ed:a3 mac-address=\
    48:5B:39:A3:ED:A3 server=dhcp1
add address=192.168.101.250 client-id=1:dc:2c:6e:64:9f:56 mac-address=\
    DC:2C:6E:64:9F:56 server=dhcp1
/ip dhcp-server network
add address=192.168.101.0/24 dns-server=192.168.101.1 gateway=192.168.101.1
/ip dns
set allow-remote-requests=yes max-udp-packet-size=512 query-server-timeout=1s \
    servers=8.8.4.4,8.8.8.8
/ip firewall address-list
add address=192.168.101.0/24 list=LocalLan
add address=x.x.x.x/24 list=SubnetWAN1
add address=y.y.y.y/24 list=SubnetWAN2
add address=cloud.mikrotik.com list=Cloud
add address=cloud2.mikrotik.com list=Cloud2
/ip firewall filter
add action=accept chain=input dst-port=8291 protocol=tcp
add action=accept chain=input comment="Alow access Router from LAN" \
    src-address-list=LocalLan
add action=drop chain=forward comment="drop invalid" connection-state=invalid
add action=accept chain=forward comment="allow conn from LAN" \
    connection-state=new in-interface=bridge1
add action=accept chain=forward comment="allow established" connection-state=\
    established
add action=accept chain=forward comment="allow related" connection-state=\
    related
add action=drop chain=forward comment="drop all fwd"
add action=accept chain=input comment="allow established to router" \
    connection-state=established
add action=accept chain=input comment="allow related to router" \
    connection-state=related
add action=drop chain=input comment="Dropp all to router"
/ip firewall mangle
add action=mark-connection chain=input comment="Mark Routing  - WAN1" \
    in-interface=ether1 new-connection-mark=WAN1 passthrough=yes
add action=mark-routing chain=output connection-mark=WAN1 new-routing-mark=\
    main passthrough=no
add action=mark-connection chain=input comment="Mark Routing - WAN2" \
    in-interface=ether2 new-connection-mark=WAN2 passthrough=yes
add action=mark-routing chain=output connection-mark=WAN2 new-routing-mark=\
    main passthrough=no
add action=mark-connection chain=prerouting comment="Mark WEB" \
    new-connection-mark=web port=80,443 protocol=tcp
add action=mark-packet chain=prerouting connection-mark=web in-interface=\
    ether1 new-packet-mark=web-wan1 passthrough=no
add action=mark-packet chain=prerouting connection-mark=web in-interface=\
    ether2 new-packet-mark=web-wan2 passthrough=no
add action=mark-packet chain=prerouting connection-mark=web new-packet-mark=\
    web passthrough=no
add action=mark-connection chain=prerouting comment="Mark DNS" \
    new-connection-mark=dns port=53 protocol=udp
add action=mark-connection chain=prerouting new-connection-mark=dns port=53 \
    protocol=tcp
add action=mark-packet chain=prerouting connection-mark=dns in-interface=\
    ether1 new-packet-mark=dns-wan1 passthrough=no
add action=mark-packet chain=prerouting connection-mark=dns in-interface=\
    ether2 new-packet-mark=dns-wan2 passthrough=no
add action=mark-packet chain=prerouting connection-mark=dns new-packet-mark=\
    dns passthrough=no
add action=mark-connection chain=prerouting comment="Mark ICMP" \
    new-connection-mark=icmp protocol=icmp
add action=mark-packet chain=prerouting connection-mark=icmp in-interface=\
    ether1 new-packet-mark=icmp-wan1 passthrough=no protocol=icmp
add action=mark-packet chain=prerouting connection-mark=icmp in-interface=\
    ether2 new-packet-mark=icmp-wan2 passthrough=no protocol=icmp
add action=mark-packet chain=prerouting connection-mark=icmp new-packet-mark=\
    icmp passthrough=no protocol=icmp
add action=mark-connection chain=prerouting comment="Mark QUIC" \
    new-connection-mark=quic port=443 protocol=udp
add action=mark-packet chain=prerouting connection-mark=quic in-interface=\
    ether1 new-packet-mark=quic-wan1 passthrough=no protocol=udp
add action=mark-packet chain=prerouting connection-mark=quic new-packet-mark=\
    quic passthrough=no protocol=udp
add action=mark-packet chain=prerouting connection-mark=quic in-interface=\
    ether2 new-packet-mark=quic-wan2 passthrough=no protocol=udp
add action=mark-connection chain=prerouting comment="Mark RESTO" \
    new-connection-mark=resto
add action=mark-packet chain=prerouting connection-mark=resto in-interface=\
    ether1 new-packet-mark=resto-wan1 passthrough=no
add action=mark-packet chain=prerouting connection-mark=resto \
    new-packet-mark=resto passthrough=no
add action=mark-packet chain=prerouting connection-mark=resto in-interface=\
    ether2 new-packet-mark=resto-wan2 passthrough=no
/ip firewall nat
add action=masquerade chain=srcnat comment="Masquerade - WAN1" out-interface=\
    ether1 src-address-type=""
add action=masquerade chain=srcnat comment="Masquerade - WAN2" out-interface=\
    ether2
/ip route
add comment="Monitor Ext Host Via ISP1" disabled=yes distance=1 dst-address=\
    1.1.1.1/32 gateway=x.x.x.x pref-src=0.0.0.0 routing-table=main \
    scope=10 suppress-hw-offload=no target-scope=10
add comment="Monitor Ext Host Via ISP2" disabled=yes distance=1 dst-address=\
    1.0.0.1/32 gateway=y.y.y.y pref-src=0.0.0.0 routing-table=main \
    scope=10 suppress-hw-offload=no target-scope=10
add comment="Default Route - Main" disabled=no distance=1 dst-address=\
    0.0.0.0/0 gateway=x.x.x.x pref-src=0.0.0.0 routing-table=main \
    scope=30 suppress-hw-offload=no target-scope=10
add check-gateway=ping comment="Default Route - Backup" disabled=yes \
    distance=2 dst-address=0.0.0.0/0 gateway=1.0.0.1 pref-src=0.0.0.0 \
    routing-table=main scope=30 suppress-hw-offload=no target-scope=11
add comment="Monitor Ext Host Via ISP1*" disabled=yes distance=1 dst-address=\
    208.67.222.222/32 gateway=x.x.x.x pref-src="" routing-table=main \
    scope=10 suppress-hw-offload=no target-scope=10
add comment="Monitor Ext Host Via ISP2*" disabled=yes distance=1 dst-address=\
    208.67.220.220/32 gateway=y.y.y.y pref-src="" routing-table=main \
    scope=10 suppress-hw-offload=no target-scope=10
/lcd
set default-screen=informative-slideshow
/system clock
set time-zone-name=America/Argentina/Buenos_Aires
/system identity
set name=Barentz
/system logging
add action=disk topics=critical
add action=disk topics=error
add topics=event
add action=disk topics=event
add action=disk topics=info
add topics=interface
add action=disk topics=interface
add topics=script
add action=disk topics=script
add topics=state
add action=disk topics=state
add action=disk topics=warning
add topics=caps
add action=disk topics=caps
add prefix="Route Changes" topics=route
add action=disk prefix="Route Changes" topics=route
/system ntp client
set enabled=yes
/system ntp client servers
add address=104.171.113.34
add address=162.159.200.1

my current config is hardcoded to NO check-gateway ping, since its unbearebly inconsistent... as for the 2nd ISP, its was on DHCP (ISP hasnt decided to provide a static IP yet) and it was making my config even more complicated to manage, so its single threaded as of today.

am i missing anything obvious? the setup look correct in theory (help Mikrotik, google, guides, etc) but this really throws me off, and im unable to understand what is wrong here:

Route Changes: 2.2 Merge forwarding path updates
Route Changes: Prepare queued IP/1.1.1.1/30-11/2
Route Changes: Disqualified fwp IP/1.1.1.1/30-11/2
Route Changes: Resolving IP/1.1.1.1/30-11/2
Route Changes: Resolve as unreachable, gateway is not active

2 min later it recovers...

Route Changes: Resolved link IP/1.1.1.1/30-11/2 via 1.1.1.1->IP/x.x.x.x/11-10/0 FLD{1} rr tr has metric BEST/32

I cant possible fhatom the idea of Softlayer DNS being THIS unstable...

any idea or help is appreciated.

Thanks!

sindy · Tue Jul 05, 2022 8:53 am

I cant possible fhatom the idea of Softlayer DNS being THIS unstable...

any idea or help is appreciated.

Nothing obviously wrong in your configuration.

It may not be the Softlayer DNS itself, it may well be something on the network path between your WAN and the nearest Softlayer DNS server. What does /tool/traceroute 208.67.220.220 and /tool/traceroute 208.67.222.222 show if you let each run for a while?

On the other hand, as several users here have pointed out, there is no obligation of the DNS providers to respond to pings, so maybe they got fed up with Mikrotik users around the world using their hardware to check their network connectivity and want to hint us this way to stop doing that

?

Also, I can see nothing in your configuration that would require RouterOS 7, so maybe it is worth trying the same setup using 6.48.6.

Regarding DHCP on WAN 2 - it only makes sense to monitor transparency of the WAN 2 uplink if you want to use it not only as a backup but also as a preferred route for some traffic. If you do:

if the ISP changes the address but not the gateway, no special measures need to be taken
if they do change the gateway, the DHCP lease script updating the route is quite a simple one and does not need to interact with anything else, the only mandatory condition is that you are able to identify the correct route to update - matching on a specific value of comment is the most generic way, but here in particular you can match on the dst-address as the only routes you need to update are the lowest ones in the recursive hierarchy. Or, to allow changing the dst-address at a single place (the routes themselves), you can match on the prefix length of the dst-address (32) and some specific distance value (distance only diferentiates between routes with identical dst-address).

densukke · Tue Jul 05, 2022 7:08 pm

Hey Sindy, thanks for looking at this: here the first part :

[XXX@Barentz] > /tool/traceroute 208.67.220.222
Columns: ADDRESS, LOSS, SENT, LAST, AVG, BEST, WORST, STD-DEV
 #  ADDRESS          LOSS  SENT  LAST    AVG   BEST  WORST  STD-DEV
 1  190.210.230.254  0%     563  2.5ms   3.7   1.9   80.3   3.8    
 2  190.210.118.106  0%     563  5.9ms   6.3   1.5   21.5   2.6    
 3  190.210.118.162  0%     563  2.9ms   3     1.7   14.7   1.1    
 4  200.51.217.105   0%     563  3ms     3.4   2.3   35.8   2      
 5  81.173.106.61    0%     563  3ms     3.3   2     23.1   1.6    
 6  213.140.39.118   0%     563  2.6ms   3.4   2.3   25.2   1.5    
 7  5.53.7.242       0%     563  8.1ms   13.2  7.2   77.2   9.1    
 8  94.142.107.121   0%     563  8ms     10.7  7.3   46.5   4.4    
 9  4.69.220.18      5.9%   563  34.9ms  38.6  34.4  78.8   6.7    
10  8.243.35.230     0%     563  35.2ms  37    34.4  80.2   5.2    
11  208.67.222.222   0%     563  34.8ms  35.6  34.4  59.1   1.5    
[XXX@Barentz] >

the other DNS

[xxx@Barentz] > /tool/traceroute 208.67.222.220
Columns: ADDRESS, LOSS, SENT, LAST, AVG, BEST, WORST, STD-DEV
 #  ADDRESS          LOSS  SENT  LAST    AVG   BEST  WORST  STD-DEV
 1  190.210.230.254  0%     564  2.8ms   3.1   2     15.7   0.9    
 2  190.210.118.106  0%     564  5.2ms   6.4   1.8   22     2.5    
 3  190.210.118.162  0%     564  2.4ms   2.9   1.9   13.2   0.9    
 4  200.51.217.105   0%     564  2.9ms   3.2   2.2   14.3   1      
 5  81.173.106.61    0.4%   564  5.7ms   3.2   2.2   12.1   1      
 6  213.140.39.118   0%     564  3.3ms   3.3   2.4   15.5   1.1    
 7  5.53.7.242       0%     564  6.7ms   5.7   2.4   95     6.8    
 8  94.142.107.121   0%     564  4.1ms   4.6   2     26.2   3.6    
 9  4.69.220.18      6.2%   564  44.4ms  33.6  30.2  118.2  7      
10  8.243.35.230     0%     564  31ms    32.2  30    78.2   5.1    
11  208.67.222.220   0%     564  30.4ms  30.5  29.2  43.5   1.4    

[xxx@Barentz] >

I have seen this consistent PKL in hop 9, which starts at 1% and consistently goes up as the trace continues... maybe you are right and Mikrotik's are being filtered out?

Will look into the older code config, whenever I can get confirmation the ISP is clean E2E.

Thanks for the feedback on DHCP for WAN2, Whenever it becomes usable again, i will implement , albeit they are not switching things on me again, any of the 2 suggestions.

Gracias!

sindy · Tue Jul 05, 2022 8:52 pm

I have seen this consistent PKL in hop 9, which starts at 1% and consistently goes up as the trace continues... maybe you are right and Mikrotik's are being filtered out?

Nope, no hop except the first one knows that the pings come from a Mikrotik, so the DNS servers would have to simply ignore all pings from time to time.

Also, there was no loss at the actual destination, so either you were lucky to run the traceroute during a period when the DNS servers were fine, or the issue has a different root cause.

So the next step is to set /tool sniffer set file-name=check-gateway.pcap, configure some route to an unused destination (dst-address=1.2.3.4/32) as a recursive one with 208.67.222.222 as gateway, and run /tool sniffer quick ip-protocol=icmp ip-address=208.67.222.222 until you can see the log messages you've shown in the OP and a return back to normal. You should then look into the .pcap file using Wireshark to see what was going on just before the outage and once it has ended.

/tool sniffer packet print will show you quite a lot too, even without Wireshark, but the data get erased some minutes after you stop sniffing. The advantage is that you can see interface names here, which is not the case with the .pcap file - there, you have to look at MAC addresses. Why I mention this is that the check-gateway pings may be sent but through a wrong interface.

densukke · Fri Jul 08, 2022 11:52 pm

apologies for the late turnaround, but i had to give up on ROS7, and downgraded to 6.49.6, and got recursive working on the first try.

not always the new and shinny work as intendeed, and i should know this by now, but i was hopeful, call me a dreamer.

many thanks Sindy, and hope this helps others, to NOT trust build Codes over configs...

Thanks!

cyayon · Mon Dec 26, 2022 10:39 pm

Hi,

Same issue here.
First primary link is direct DHCP and secondary is LTE via a dedicated router.
I do not understand why it is declared as faulty and recovered every two or three minutes.

I am on a CCR 2004 ROS 7.6.

cyayon · Sat Dec 31, 2022 2:21 am

Hello,

Did you try again with ROS 7 ?

rackonesbrains · Tue Jul 18, 2023 9:38 pm

Hello to all.

Im having the same problem.

Any chance that you can share the solution please?

Recursive Failover - unstable [SOLVED]

Recursive Failover - unstable

Re: Recursive Failover - unstable

Re: Recursive Failover - unstable

Re: Recursive Failover - unstable [SOLVED]

Re: Recursive Failover - unstable

Re: Recursive Failover - unstable

Re: Recursive Failover - unstable

Re: Recursive Failover - unstable

Who is online