Multiple WAN with PCC issues

Factor · October 4, 2017, 9:08am

Hi community,
I’m trying for couple of days now to make PCC working but no luck so far (no need to say that this topic was/is often discussed). I would like to ask for your help and opinion

Now here is export

# oct/04/2017 05:28:46 by RouterOS 6.38.5

/interface bridge
add name=lan-bridge
add name=mng-bridge

/interface ethernet
set [ find default-name=combo1 ] comment=satelite1 name=ether1
set [ find default-name=ether1 ] comment=satelite2 name=ether2
set [ find default-name=ether2 ] comment=satelite3 name=ether3
set [ find default-name=ether3 ] comment=satelite4 name=ether4
set [ find default-name=ether4 ] comment="To server" name=ether5
set [ find default-name=ether5 ] comment="To outside" name=ether6
set [ find default-name=ether6 ] comment="To mainlodge" name=ether7
set [ find default-name=ether7 ] comment="MNG service port" name=ether8

/interface vlan
add interface=ether5 name=vlan10-server vlan-id=10
add interface=ether7 name=vlan20-mainLodge vlan-id=20
add interface=ether6 name=vlan20-outside vlan-id=20

/interface list
add name=bridges

/ip hotspot profile
add dns-name=gateway.somewhere.com hotspot-address=10.1.48.1 login-by=http-chap,trial name=hsprof1 trial-uptime-limit=0s

/ip hotspot user profile
set [ find default=yes ] keepalive-timeout=5m mac-cookie-timeout=1d rate-limit="128k/512k 512k/2048k 96k/384k 8/8" shared-users=unlimited

/ip pool
add name=mng_pool ranges=192.168.10.2-192.168.10.126
add name=lan_pool ranges=10.1.48.2-10.1.51.254

/ip dhcp-server
add add-arp=yes address-pool=lan_pool disabled=no interface=lan-bridge lease-time=5m name=lan-dhcp
add add-arp=yes address-pool=mng_pool disabled=no interface=mng-bridge lease-time=5m name=mng-dhcp

/ip hotspot
add address-pool=lan_pool disabled=no interface=lan-bridge name=hotspot1 profile=hsprof1

/interface bridge port
add bridge=lan-bridge interface=vlan20-outside
add bridge=lan-bridge interface=vlan20-mainLodge
add bridge=mng-bridge interface=vlan10-server
add bridge=mng-bridge interface=ether6
add bridge=mng-bridge interface=ether7
add bridge=mng-bridge interface=ether8

/interface bridge settings
set allow-fast-path=no use-ip-firewall=yes use-ip-firewall-for-vlan=yes

/interface list member
add interface=lan-bridge list=bridges
add interface=mng-bridge list=bridges

/ip address
add address=192.168.10.1/25 interface=mng-bridge network=192.168.10.0
add address=10.1.48.1/22 interface=lan-bridge network=10.1.48.0

/ip dhcp-client
add dhcp-options=hostname,clientid disabled=no interface=ether1 use-peer-dns=no use-peer-ntp=no
add dhcp-options=hostname,clientid disabled=no interface=ether2 use-peer-dns=no use-peer-ntp=no
add dhcp-options=hostname,clientid disabled=no interface=ether3 use-peer-dns=no use-peer-ntp=no
add dhcp-options=hostname,clientid disabled=no interface=ether4 use-peer-dns=no	use-peer-ntp=no

/ip dhcp-server network
add address=10.1.48.0/22 dns-server=10.1.48.1 gateway=10.1.48.1
add address=192.168.10.0/25 dns-server=192.168.10.1 gateway=192.168.10.1

/ip dns
set allow-remote-requests=yes servers=8.8.8.8,8.8.4.4

/ip dns static
add address=10.1.48.1 name=gateway.somewhere.com

/ip firewall address-list
add list=local address=192.168.10.1-192.168.10.126 
add list=local address=10.1.48.1-10.1.51.254 

/ip firewall mangle
add action=mark-connection chain=input in-interface=ether1 new-connection-mark=WAN1conn
add action=mark-connection chain=input in-interface=ether3 new-connection-mark=WAN3conn
add action=mark-routing chain=output connection-mark=WAN1conn new-routing-mark=toWAN1
add action=mark-routing chain=output connection-mark=WAN3conn new-routing-mark=toWAN3
add action=mark-connection chain=prerouting dst-address-type=!local in-interface=mng-bridge new-connection-mark=WAN1conn passthrough=yes per-connection-classifier=both-addresses:2/0
add action=mark-connection chain=prerouting dst-address-type=!local in-interface=mng-bridge new-connection-mark=WAN3conn passthrough=yes per-connection-classifier=both-addresses:2/1
add action=mark-connection chain=prerouting dst-address-type=!local in-interface=lan-bridge new-connection-mark=WAN1conn passthrough=yes per-connection-classifier=both-addresses:2/0
add action=mark-connection chain=prerouting dst-address-type=!local in-interface=lan-bridge new-connection-mark=WAN3conn passthrough=yes per-connection-classifier=both-addresses:2/1
add action=mark-routing chain=prerouting connection-mark=WAN1conn in-interface=mng-bridge new-routing-mark=toWAN1
add action=mark-routing chain=prerouting connection-mark=WAN3conn in-interface=mng-bridge new-routing-mark=toWAN3
add action=mark-routing chain=prerouting connection-mark=WAN1conn in-interface=lan-bridge new-routing-mark=toWAN1
add action=mark-routing chain=prerouting connection-mark=WAN3conn in-interface=lan-bridge new-routing-mark=toWAN3

/ip firewall nat
add action=passthrough chain=unused-hs-chain comment="place hotspot rules here" disabled=yes
add action=masquerade chain=srcnat out-interface=ether1 routing-mark=toWAN1 src-address-list=local
add action=masquerade chain=srcnat out-interface=ether3 routing-mark=toWAN3 src-address-list=local

As you can see I have 2 local networks and only 2/4 WAN connected for testing purposes. Problems here I have are:

Connection distribution between WAN ports are not even nearly equal - after couple of hours and about 100 devices connected over time share is like 1/200
HTTPS is not working - I found topic related to this issue where someone explaining that src-address at NAT is required in combination with PCC but it won’t work for me too

Thank you for any help or hint!

Factor · October 7, 2017, 8:38am

Ok, I was able to make HTTPS works but problem with loadbalance persist

Here is updated export

/ip dhcp-client
add add-default-route=no disabled=no interface=ether1 use-peer-dns=no use-peer-ntp=no
add add-default-route=no disabled=no interface=ether2 use-peer-dns=no use-peer-ntp=no
add add-default-route=no disabled=no interface=ether3 use-peer-dns=no use-peer-ntp=no
add add-default-route=no disabled=no interface=ether4 use-peer-dns=no use-peer-ntp=no

/ip firewall nat
add action=masquerade chain=srcnat out-interface=ether1
add action=masquerade chain=srcnat out-interface=ether2
add action=masquerade chain=srcnat out-interface=ether3
add action=masquerade chain=srcnat out-interface=ether4

/ip firewall mangle
add action=mark-connection chain=prerouting connection-state=new in-interface=ether1 new-connection-mark=ether1_conn
add action=mark-connection chain=prerouting connection-state=new in-interface=ether2 new-connection-mark=ether2_conn
add action=mark-connection chain=prerouting connection-state=new in-interface=ether3 new-connection-mark=ether3_conn
add action=mark-connection chain=prerouting connection-state=new in-interface=ether4 new-connection-mark=ether4_conn

add action=mark-routing chain=output connection-mark=ether1_conn new-routing-mark=to_ether1
add action=mark-routing chain=output connection-mark=ether2_conn new-routing-mark=to_ether2
add action=mark-routing chain=output connection-mark=ether3_conn new-routing-mark=to_ether3
add action=mark-routing chain=output connection-mark=ether4_conn new-routing-mark=to_ether4

add action=mark-connection chain=prerouting connection-state=new dst-address-type=!local in-interface=mng-bridge new-connection-mark=ether1_conn per-connection-classifier=both-ports:4/0
add action=mark-connection chain=prerouting connection-state=new dst-address-type=!local in-interface=mng-bridge new-connection-mark=ether2_conn per-connection-classifier=both-ports:4/1
add action=mark-connection chain=prerouting connection-state=new dst-address-type=!local in-interface=mng-bridge new-connection-mark=ether3_conn per-connection-classifier=both-ports:4/1
add action=mark-connection chain=prerouting connection-state=new dst-address-type=!local in-interface=mng-bridge new-connection-mark=ether4_conn per-connection-classifier=both-ports:4/1

add action=mark-connection chain=prerouting connection-state=new dst-address-type=!local in-interface=lan-bridge new-connection-mark=ether1_conn per-connection-classifier=both-ports:4/0
add action=mark-connection chain=prerouting connection-state=new dst-address-type=!local in-interface=lan-bridge new-connection-mark=ether2_conn per-connection-classifier=both-ports:4/1
add action=mark-connection chain=prerouting connection-state=new dst-address-type=!local in-interface=lan-bridge new-connection-mark=ether3_conn per-connection-classifier=both-ports:4/1
add action=mark-connection chain=prerouting connection-state=new dst-address-type=!local in-interface=lan-bridge new-connection-mark=ether4_conn per-connection-classifier=both-ports:4/1

add action=mark-routing chain=prerouting connection-mark=ether1_conn in-interface=mng-bridge new-routing-mark=to_ether1
add action=mark-routing chain=prerouting connection-mark=ether2_conn in-interface=mng-bridge new-routing-mark=to_ether2
add action=mark-routing chain=prerouting connection-mark=ether3_conn in-interface=mng-bridge new-routing-mark=to_ether3
add action=mark-routing chain=prerouting connection-mark=ether4_conn in-interface=mng-bridge new-routing-mark=to_ether4

add action=mark-routing chain=prerouting connection-mark=ether1_conn in-interface=lan-bridge new-routing-mark=to_ether1
add action=mark-routing chain=prerouting connection-mark=ether2_conn in-interface=lan-bridge new-routing-mark=to_ether2
add action=mark-routing chain=prerouting connection-mark=ether3_conn in-interface=lan-bridge new-routing-mark=to_ether3
add action=mark-routing chain=prerouting connection-mark=ether4_conn in-interface=lan-bridge new-routing-mark=to_ether4

/ip route
add check-gateway=ping comment="ether1-mark" distance=1 gateway=192.168.1.1 routing-mark=to_ether1
add check-gateway=ping comment="ether2-mark" distance=1 gateway=192.168.2.1 routing-mark=to_ether2
add check-gateway=ping comment="ether3-mark" distance=1 gateway=192.168.3.1 routing-mark=to_ether3
add check-gateway=ping comment="ether4-mark" distance=1 gateway=192.168.4.1 routing-mark=to_ether4
add distance=1 gateway=192.168.1.1 comment="ether1-static"
add distance=2 gateway=192.168.2.1 comment="ether2-static"
add distance=3 gateway=192.168.3.1 comment="ether3-static"
add distance=4 gateway=192.168.4.1 comment="ether4-static"


### SCRIPT for updating gateways ###
{
	:local static "static"
	:local mark "mark"
	:foreach dhcp in=[/ip dhcp-client find status=bound] do={
		:local interface [/ip dhcp-client get $dhcp interface]
		:local commentActive "$interface-$static"
		:local commentMark "$interface-$mark"
		:local activeGW [/ip route get [find comment=$commentActive] gateway]
		:local newGW [/ip dhcp-client get $dhcp gateway]
		:if ($newGW != $activeGW) do={
			/ip route set [find comment=$commentActive] gateway=$newGW
			/ip route set [find comment=$commentMark] gateway=$newGW
		}
	}
}

/interface stats

 #     NAME                  RX-BYTE           TX-BYTE     	RX-PACKET     TX-PACKET
 0  R  ;;; WAN1
       ether1                158 360 026       20 668 576       140 870       73 159
 1  R  ;;; WAN2
       ether2                40 952            25 514           345           238

Factor · October 8, 2017, 5:57pm

OK I was finally able to make it works. Needless to say that I had wrong PCCs among other things.
So for those who will deal with same issues here is my export which works flawlessly:

tags: PCC with multiple WAN, WAN DHCP Clients, loadbalance, fail-over, hotspot, multiple LAN

/interface bridge
add name=lan-bridge
add name=mng-bridge

/ip hotspot profile
add dns-name=gateway.somewhere.com hotspot-address=10.1.48.1 login-by=http-chap,trial name=hsprof1 trial-uptime-limit=0s

/ip pool
add name=mng_pool ranges=192.168.10.2-192.168.10.126
add name=lan_pool ranges=10.1.48.2-10.1.51.254

/ip dhcp-server
add add-arp=yes address-pool=lan_pool disabled=no interface=lan-bridge name=lan-dhcp
add add-arp=yes address-pool=mng_pool disabled=no interface=mng-bridge lease-time=5m name=mng-dhcp

/ip hotspot
add address-pool=lan_pool disabled=no interface=lan-bridge name=hotspot1 profile=hsprof1

/interface bridge settings
set allow-fast-path=no use-ip-firewall=yes use-ip-firewall-for-vlan=yes

/ip address
add address=192.168.10.1/25 interface=mng-bridge network=192.168.10.0
add address=10.1.48.1/22 interface=lan-bridge network=10.1.48.0

/ip dhcp-client
add add-default-route=no dhcp-options=hostname,clientid disabled=no interface=ether2 use-peer-dns=no use-peer-ntp=no
add add-default-route=no dhcp-options=hostname,clientid disabled=no interface=ether3 use-peer-dns=no use-peer-ntp=no
add add-default-route=no dhcp-options=hostname,clientid disabled=no interface=ether4 use-peer-dns=no use-peer-ntp=no
add add-default-route=no dhcp-options=hostname,clientid disabled=no interface=ether1 use-peer-dns=no use-peer-ntp=no

/ip dhcp-server network
add address=10.1.48.0/22 dns-server=10.1.48.1 gateway=10.1.48.1
add address=192.168.10.0/25 dns-server=192.168.10.1 gateway=192.168.10.1

/ip dns
set allow-remote-requests=yes servers=8.8.8.8,8.8.4.4

/ip firewall mangle
add action=mark-connection chain=prerouting connection-state=new in-interface=ether1 new-connection-mark=ether1_conn
add action=mark-connection chain=prerouting connection-state=new in-interface=ether2 new-connection-mark=ether2_conn
add action=mark-connection chain=prerouting connection-state=new disabled=yes in-interface=ether3 new-connection-mark=ether3_conn
add action=mark-connection chain=prerouting connection-state=new disabled=yes in-interface=ether4 new-connection-mark=ether4_conn
add action=mark-routing chain=output connection-mark=ether1_conn new-routing-mark=to_ether1
add action=mark-routing chain=output connection-mark=ether2_conn new-routing-mark=to_ether2
add action=mark-routing chain=output connection-mark=ether3_conn disabled=yes new-routing-mark=to_ether3
add action=mark-routing chain=output connection-mark=ether4_conn disabled=yes new-routing-mark=to_ether4
add action=mark-connection chain=prerouting connection-state=new dst-address-type=!local hotspot=auth in-interface=lan-bridge new-connection-mark=ether1_conn per-connection-classifier=both-addresses:2/0
add action=mark-connection chain=prerouting connection-state=new dst-address-type=!local hotspot=auth in-interface=lan-bridge new-connection-mark=ether2_conn per-connection-classifier=both-addresses:2/1
add action=mark-connection chain=prerouting connection-state=new disabled=yes dst-address-type=!local hotspot=auth in-interface=lan-bridge new-connection-mark=ether3_conn per-connection-classifier=both-addresses:4/2
add action=mark-connection chain=prerouting connection-state=new disabled=yes dst-address-type=!local hotspot=auth in-interface=lan-bridge new-connection-mark=ether4_conn per-connection-classifier=both-addresses:4/3
add action=mark-connection chain=prerouting connection-state=new dst-address-type=!local in-interface=mng-bridge new-connection-mark=ether1_conn per-connection-classifier=both-addresses:2/0
add action=mark-connection chain=prerouting connection-state=new dst-address-type=!local in-interface=mng-bridge new-connection-mark=ether2_conn per-connection-classifier=both-addresses:2/1
add action=mark-connection chain=prerouting connection-state=new disabled=yes dst-address-type=!local in-interface=mng-bridge new-connection-mark=ether3_conn per-connection-classifier=both-addresses:4/2
add action=mark-connection chain=prerouting connection-state=new disabled=yes dst-address-type=!local in-interface=mng-bridge new-connection-mark=ether4_conn per-connection-classifier=both-addresses:4/2
add action=mark-routing chain=prerouting connection-mark=ether1_conn in-interface=mng-bridge new-routing-mark=to_ether1
add action=mark-routing chain=prerouting connection-mark=ether2_conn in-interface=mng-bridge new-routing-mark=to_ether2
add action=mark-routing chain=prerouting connection-mark=ether3_conn disabled=yes in-interface=mng-bridge new-routing-mark=to_ether3
add action=mark-routing chain=prerouting connection-mark=ether4_conn disabled=yes in-interface=mng-bridge new-routing-mark=to_ether4
add action=mark-routing chain=prerouting connection-mark=ether1_conn in-interface=lan-bridge new-routing-mark=to_ether1
add action=mark-routing chain=prerouting connection-mark=ether2_conn in-interface=lan-bridge new-routing-mark=to_ether2
add action=mark-routing chain=prerouting connection-mark=ether3_conn disabled=yes in-interface=lan-bridge new-routing-mark=to_ether3
add action=mark-routing chain=prerouting connection-mark=ether4_conn disabled=yes in-interface=lan-bridge new-routing-mark=to_ether4

/ip firewall nat
add action=masquerade chain=srcnat out-interface=ether1
add action=masquerade chain=srcnat out-interface=ether2
add action=masquerade chain=srcnat out-interface=ether3
add action=masquerade chain=srcnat out-interface=ether4

/ip route
add check-gateway=ping comment=ether1-mark distance=1 gateway=192.168.1.1 routing-mark=to_ether1
add check-gateway=ping comment=ether2-mark distance=1 gateway=192.168.2.1 routing-mark=to_ether2
add check-gateway=ping comment=ether3-mark distance=1 gateway=192.168.3.1 routing-mark=to_ether3
add check-gateway=ping comment=ether4-mark distance=1 gateway=192.168.4.1 routing-mark=to_ether4
add comment=ether1-static distance=1 gateway=192.168.1.1
add comment=ether2-static distance=2 gateway=192.168.2.1
add comment=ether3-static distance=3 gateway=192.168.3.1
add comment=ether4-static distance=4 gateway=192.168.4.1

/system scheduler
add interval=5s name=set-gateways on-event="/system script run assign-gateways" policy=ftp,reboot,read,write,policy,test,password,sniff,sensitive,romon start-date=oct/07/2017 start-time=05:17:30

/system script
add name=assign-gateways owner=factor policy=ftp,reboot,read,write,policy,test,password,sniff,sensitive,romon source="# Assign new gateways base on DHCP clients and predefined static routes\
    \n# /ip dhcp-client add-default-route have to be disabled for used dhcp-clients\
    \n{\
    \n    :local static \"static\"\
    \n    :local mark \"mark\"\
    \n    :foreach dhcp in=[/ip dhcp-client find status=bound] do={\
    \n        :local interface [/ip dhcp-client get \$dhcp interface]\
    \n        :local commentActive \"\$interface-\$static\"\
    \n        :local commentMark \"\$interface-\$mark\"\
    \n        :local activeGW [/ip route get [find comment=\$commentActive] gateway]\
    \n        :local newGW [/ip dhcp-client get \$dhcp gateway]\
    \n\
    \n        :if (\$newGW != \$activeGW) do={\
    \n            /ip route set [find comment=\$commentActive] gateway=\$newGW\
    \n            /ip route set [find comment=\$commentMark] gateway=\$newGW\
    \n        }\
    \n    }\
    \n}"

Hope this will help to those who try to setup similar behavior
If any questions, hints or anything, feel free to ask/add.

Cheers,
Factor