Hello,
Setting up a ECMP link balance with BGP as per the below configuration guide with both interfaces being GRE tunnels…
https://wiki.mikrotik.com/wiki/Manual:BGP_Load_Balancing_with_two_interfaces
Right now, I have the configuration working however I have noticed some weird behaviour.
Before I get into it, here is the topology…
Two routers, client and server. Client with single ISP connection connects to server with two ISP connections over two IPSEC/GRE tunnels. BGP multipath set up between loopbacks on each end. On the client end, all LAN traffic is mangle marked to route out a marked default interface (installed by BGP). When both GRE tunnels are up, the load tends to be distributed and a machine on the client mikrotik LAN side can browse via the default gateway (through the tunnel pair).
When GRE tunnel A is offline, everything seems to be broken and no traffic flows. When GRE tunnel B is offline, nothing flows over the tunnel and traffic ends up NAT-ing out the ether1 interface of the client Mikrotik (the regular internet connection). I can’t seem to find anything to explain this behaviour…
Server Config:
# nov/06/2018 08:40:44 by RouterOS 6.43.4
# software id =
#
# model = RouterBOARD 1100x4
# serial number =
/interface bridge
add fast-forward=no name=bri-CTL_01A
add fast-forward=no name=bri-CTL_01B
add fast-forward=no name=bri-CTL_01BGP
/interface ethernet
set [ find default-name=ether3 ] disabled=yes
/interface gre
add keepalive=5s local-address=10.0.10.1 name=gre-CTL_01A remote-address=10.0.11.1
add keepalive=5s local-address=10.0.12.1 name=gre-CTL_01B remote-address=10.0.13.1
/interface list
add name=WAN
add name=LAN
add name=GRE_Out
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip ipsec peer profile
set [ find default=yes ] enc-algorithm=aes-128 hash-algorithm=sha256
add enc-algorithm=aes-128 hash-algorithm=sha256 name=profile1
/ip ipsec policy group
add name=VPN_01
add name=VPN_02
/ip ipsec proposal
set [ find default=yes ] disabled=yes
add auth-algorithms=sha256 enc-algorithms=aes-128-cbc name=proposal1 pfs-group=none
/routing bgp instance
set default as=65000 router-id=10.255.255.0
/ip neighbor discovery-settings
set discover-interface-list=GRE_Out
/interface list member
add interface=ether1 list=WAN
add interface=ether2 list=LAN
add interface=gre-CTL_01A list=GRE_Out
add interface=gre-CTL_01B list=GRE_Out
/ip address
add address=10.0.0.1/24 comment="Management LAN" interface=ether2 network=10.0.0.0
add address=210.xxx.xxx.230/30 interface=ether1 network=210.xxx.xxx.228
add address=10.0.10.1 interface=bri-CTL_01A network=10.0.10.1
add address=10.0.12.1 interface=bri-CTL_01B network=10.0.12.1
add address=10.10.1.1/30 interface=gre-CTL_01A network=10.10.1.0
add address=68.xxx.xxx.94/30 interface=ether1 network=68.xxx.xxx.92
add address=10.123.123.1 interface=bri-CTL_01BGP network=10.123.123.1
add address=10.10.1.5/30 interface=gre-CTL_01B network=10.10.1.4
/ip dns
set servers=1.1.1.1,1.0.0.1
/ip firewall address-list
add address=10.0.0.0/24 list=admin_management
/ip firewall filter
add action=accept chain=input connection-state=established,related
add action=accept chain=input src-address-list=admin_management
add action=accept chain=input protocol=icmp
add action=accept chain=input dst-port=500,4500 protocol=udp
add action=accept chain=input protocol=gre
add action=accept chain=input protocol=ipsec-esp
add action=accept chain=input protocol=ipsec-ah
add action=drop chain=input dst-port=53 protocol=tcp
add action=drop chain=input dst-port=53 protocol=udp
add action=drop chain=input log=yes
/ip firewall mangle
add action=change-mss chain=forward new-mss=clamp-to-pmtu out-interface-list=GRE_Out passthrough=yes protocol=tcp tcp-flags=syn
/ip firewall nat
add action=accept chain=srcnat dst-address=10.0.11.0/24 src-address=10.0.10.0/24
add action=accept chain=srcnat dst-address=10.0.13.0/24 src-address=10.0.12.0/24
add action=masquerade chain=srcnat out-interface-list=WAN
/ip ipsec peer
add address=0.0.0.0/0 auth-method=pre-shared-key-xauth exchange-mode=ike2 generate-policy=port-override local-address=210.xxx.xxx.230 passive=yes \
policy-template-group=VPN_01 profile=profile1 secret=REDACTED send-initial-contact=no
# This entry is unreachable
add address=0.0.0.0/0 auth-method=pre-shared-key-xauth exchange-mode=ike2 generate-policy=port-override local-address=68.xxx.xxx.94 passive=yes \
policy-template-group=VPN_02 profile=profile1 secret=REDACTED send-initial-contact=no
/ip ipsec policy
set 0 disabled=yes
add dst-address=10.0.11.0/24 group=VPN_01 proposal=proposal1 src-address=10.0.10.0/24 template=yes
add dst-address=10.0.13.0/24 group=VPN_02 proposal=proposal1 src-address=10.0.12.0/24 template=yes
/ip ipsec user
add name=658b9b password=password
add name=658b9b_1 password=password
/ip route
add check-gateway=ping distance=1 gateway=210.xxx.xxx.229
add check-gateway=ping distance=1 dst-address=10.123.123.2/32 gateway=10.10.1.2,10.10.1.6
/ip service
set telnet disabled=yes
set ftp disabled=yes
set ssh disabled=yes
set www-ssl disabled=no
set api disabled=yes
set api-ssl disabled=yes
/routing bgp network
add network=10.0.0.0/24 synchronize=no
/routing bgp peer
add default-originate=always name=CTL_01 remote-address=10.123.123.2 remote-as=65000 ttl=default update-source=bri-CTL_01BGP
/system clock
set time-zone-name=
/system routerboard settings
set silent-boot=no
Client Config:
# nov/06/2018 16:51:31 by RouterOS 6.43.4
# software id =
#
# model = RouterBOARD 750G r3
# serial number =
/interface bridge
add fast-forward=no name=bri-CTL_01A
add fast-forward=no name=bri-CTL_01B
add admin-mac=CC:2D:E0:65:8B:9C auto-mac=no comment=defconf name=bridge
add fast-forward=no name=loopback
/interface gre
add keepalive=5s local-address=10.0.11.1 name=gre-CTL_01A remote-address=10.0.10.1
add keepalive=5s local-address=10.0.13.1 name=gre-CTL_01B remote-address=10.0.12.1
/interface list
add comment=defconf name=WAN
add comment=defconf name=LAN
add name=GRE_Mangle
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip hotspot profile
set [ find default=yes ] html-directory=flash/hotspot
/ip ipsec peer profile
set [ find default=yes ] enc-algorithm=aes-128 hash-algorithm=sha256
/ip ipsec proposal
set [ find default=yes ] auth-algorithms=sha256 enc-algorithms=aes-128-cbc pfs-group=none
/ip pool
add name=dhcp ranges=10.100.1.2-10.100.1.6
/ip dhcp-server
add address-pool=dhcp disabled=no interface=bridge name=defconf
/queue simple
add disabled=yes max-limit=5M/5M name=queue1 target=10.100.1.0/24
/routing bgp instance
set default as=65000 router-id=10.255.255.1
/interface bridge port
add bridge=bridge comment=defconf interface=ether2
add bridge=bridge comment=defconf interface=ether3
add bridge=bridge comment=defconf interface=ether4
add bridge=bridge comment=defconf interface=ether5
/ip neighbor discovery-settings
set discover-interface-list=LAN
/interface list member
add comment=defconf interface=bridge list=LAN
add comment=defconf interface=ether1 list=WAN
add interface=gre-CTL_01A list=LAN
add interface=gre-CTL_01B list=LAN
add interface=gre-CTL_01A list=GRE_Mangle
add interface=gre-CTL_01B list=GRE_Mangle
/ip address
add address=10.100.1.1/29 comment=defconf interface=ether2 network=10.100.1.0
add address=10.0.11.1 interface=bri-CTL_01A network=10.0.11.1
add address=10.10.1.2/30 interface=gre-CTL_01A network=10.10.1.0
add address=10.0.13.1 interface=bri-CTL_01B network=10.0.13.1
add address=10.10.1.6/30 interface=gre-CTL_01B network=10.10.1.4
add address=10.123.123.2 interface=loopback network=10.123.123.2
/ip dhcp-client
add comment=defconf dhcp-options=hostname,clientid disabled=no interface=ether1
/ip dhcp-server network
add address=10.100.1.0/29 comment=defconf gateway=10.100.1.1 netmask=29
/ip dns
set allow-remote-requests=yes
/ip dns static
add address=10.100.1.1 name=router.lan
/ip firewall filter
add action=accept chain=input comment="defconf: accept established,related,untracked" connection-state=established,related,untracked
add action=drop chain=input comment="defconf: drop invalid" connection-state=invalid
add action=accept chain=input comment="defconf: accept ICMP" protocol=icmp
add action=drop chain=input comment="defconf: drop all not coming from LAN" in-interface-list=!LAN
add action=accept chain=forward comment="defconf: accept in ipsec policy" ipsec-policy=in,ipsec
add action=accept chain=forward comment="defconf: accept out ipsec policy" ipsec-policy=out,ipsec
add action=accept chain=forward comment="defconf: accept established,related, untracked" connection-state=established,related,untracked
add action=drop chain=forward comment="defconf: drop invalid" connection-state=invalid
add action=drop chain=forward comment="defconf: drop all from WAN not DSTNATed" connection-nat-state=!dstnat connection-state=new \
in-interface-list=WAN
/ip firewall mangle
add action=mark-routing chain=prerouting comment="GRE Route Packet Marking" new-routing-mark=GRE passthrough=yes src-address=10.100.1.0/24
add action=change-mss chain=forward comment="GRE MSS Clamping" new-mss=clamp-to-pmtu out-interface-list=GRE_Mangle passthrough=yes protocol=tcp \
src-address=10.100.1.0/24 tcp-flags=syn
/ip firewall nat
add action=accept chain=srcnat comment="IPSEC NAT Bypass" dst-address=10.0.10.0/24 src-address=10.0.11.0/24
add action=accept chain=srcnat dst-address=10.0.12.0/24 src-address=10.0.13.0/24
add action=accept chain=srcnat comment="VPN NAT Bypass" out-interface=gre-CTL_01A src-address=10.100.1.0/24
add action=accept chain=srcnat out-interface=gre-CTL_01B src-address=10.100.1.0/24
add action=masquerade chain=srcnat comment="defconf: masquerade" ipsec-policy=out,none out-interface-list=WAN
/ip ipsec peer
add address=210.xxx.xxx.230/32 auth-method=pre-shared-key-xauth exchange-mode=ike2 generate-policy=port-override secret=REDACTED \
send-initial-contact=no xauth-login=658b9b xauth-password=password
add address=68.xxx.xxx.94/32 auth-method=pre-shared-key-xauth exchange-mode=ike2 generate-policy=port-override secret=REDACTED send-initial-contact=\
no xauth-login=658b9b_1 xauth-password=password
/ip ipsec policy
set 0 disabled=yes
add dst-address=10.0.10.1/32 level=unique sa-dst-address=210.xxx.xxx.230 sa-src-address=0.0.0.0 src-address=10.0.11.1/32 tunnel=yes
add dst-address=10.0.12.1/32 level=unique sa-dst-address=68.xxx.xxx.94 sa-src-address=0.0.0.0 src-address=10.0.13.1/32 tunnel=yes
/ip route
add check-gateway=ping disabled=yes distance=10 gateway=10.10.1.1 routing-mark=GRE
add check-gateway=ping disabled=yes distance=1 dst-address=10.0.0.0/24 gateway=10.10.1.1,10.10.1.5
add check-gateway=ping distance=1 dst-address=10.123.123.1/32 gateway=10.10.1.5,10.10.1.1
/ip service
set telnet disabled=yes
set ftp disabled=yes
set ssh disabled=yes
set api disabled=yes
set api-ssl disabled=yes
/routing bgp network
add network=10.100.1.0/29 synchronize=no
/routing bgp peer
add in-filter=Mark-GRE keepalive-time=10s name=peer1 remote-address=10.123.123.1 remote-as=65000 ttl=default update-source=loopback
/routing filter
add chain=Mark-GRE prefix=0.0.0.0/0 set-routing-mark=GRE
/system clock
set time-zone-autodetect=no time-zone-name=UTC
/system identity
set name=CTL_01
/system routerboard settings
set silent-boot=no
/tool bandwidth-server
set enabled=no
/tool mac-server
set allowed-interface-list=LAN
/tool mac-server mac-winbox
set allowed-interface-list=LAN