Hello,
I am trying to load balance 6 gateways and have done so using PCC.
I also have 3 levels of subscribers, bronze, silver, gold. 1M, 1.5M, and 3M.
Here are my mangle rules, routes, and queues.
Mangle
/ip firewall mangle
add action=mark-connection chain=prerouting comment="Mark Bronze Connections" \
disabled=no new-connection-mark=traffic-out passthrough=yes src-address=\
10.0.0.0/8
add action=mark-packet chain=prerouting comment="Mark Bronze Packets" \
connection-mark=traffic-out disabled=no new-packet-mark=outbound \
passthrough=yes
add action=mark-connection chain=prerouting comment="Mark Silver Connections" \
disabled=no new-connection-mark=silver-conn passthrough=yes src-address=\
10.0.0.0/8 src-address-list=Silver-Customer
add action=mark-packet chain=prerouting comment="Mark Silver Packets" \
connection-mark=silver-conn disabled=no new-packet-mark=silver-client \
passthrough=no
add action=mark-connection chain=prerouting comment="Mark Gold Connections" \
disabled=no new-connection-mark=gold-conn passthrough=yes src-address=\
10.0.0.0/8 src-address-list=Gold-Customer
add action=mark-packet chain=prerouting comment="Mark Gold Client Packets" \
connection-mark=gold-conn disabled=no new-packet-mark=gold-client \
passthrough=no src-address=10.0.0.0/8
add action=mark-connection chain=prerouting comment="" disabled=no \
new-connection-mark=1st_conn passthrough=yes per-connection-classifier=\
both-addresses-and-ports:6/0
add action=mark-connection chain=prerouting comment="" disabled=no \
new-connection-mark=2nd_conn passthrough=yes per-connection-classifier=\
both-addresses-and-ports:6/1
add action=mark-connection chain=prerouting comment="" disabled=no \
new-connection-mark=3rd_conn passthrough=yes per-connection-classifier=\
both-addresses-and-ports:6/2
add action=mark-connection chain=prerouting comment="" disabled=no \
new-connection-mark=4th_conn passthrough=yes per-connection-classifier=\
both-addresses-and-ports:6/3
add action=mark-connection chain=prerouting comment="" disabled=no \
new-connection-mark=5th_conn passthrough=yes per-connection-classifier=\
both-addresses-and-ports:6/4
add action=mark-connection chain=prerouting comment="" disabled=no \
new-connection-mark=6th_conn passthrough=yes per-connection-classifier=\
both-addresses-and-ports:6/5
add action=mark-routing chain=prerouting comment="" connection-mark=1st_conn \
disabled=no new-routing-mark=1st_route passthrough=yes
add action=mark-routing chain=prerouting comment="" connection-mark=2nd_conn \
disabled=no new-routing-mark=2nd_route passthrough=yes
add action=mark-routing chain=prerouting comment="" connection-mark=3rd_conn \
disabled=no new-routing-mark=3rd_route passthrough=yes
add action=mark-routing chain=prerouting comment="" connection-mark=4th_conn \
disabled=no new-routing-mark=4th_route passthrough=yes
add action=mark-routing chain=prerouting comment="" connection-mark=5th_conn \
disabled=no new-routing-mark=5th_route passthrough=yes
add action=mark-routing chain=prerouting comment="" connection-mark=6th_conn \
disabled=no new-routing-mark=6th_route passthrough=yes
Routes
/ip route
add check-gateway=ping comment="" disabled=no distance=1 dst-address=\
0.0.0.0/0 gateway=10.10.1.1 routing-mark=1st_route scope=30 target-scope=\
10
add check-gateway=ping comment="" disabled=no distance=1 dst-address=\
0.0.0.0/0 gateway=10.10.1.2 routing-mark=2nd_route scope=30 target-scope=\
10
add check-gateway=ping comment="" disabled=no distance=1 dst-address=\
0.0.0.0/0 gateway=10.10.1.3 routing-mark=3rd_route scope=30 target-scope=\
10
add check-gateway=ping comment="" disabled=no distance=1 dst-address=\
0.0.0.0/0 gateway=10.10.1.4 routing-mark=4th_route scope=30 target-scope=\
10
add check-gateway=ping comment="" disabled=no distance=1 dst-address=\
0.0.0.0/0 gateway=10.10.1.6 routing-mark=5th_route scope=30 target-scope=\
10
add check-gateway=ping comment="" disabled=no distance=1 dst-address=\
0.0.0.0/0 gateway=10.10.1.7 routing-mark=6th_route scope=30 target-scope=\
10
add check-gateway=ping comment="" disabled=no distance=2 dst-address=\
0.0.0.0/0 gateway=\
10.10.1.7,10.10.1.3,10.10.1.1,10.10.1.2,10.10.1.6,10.10.1.4 scope=30 \
target-scope=10
Queues
/queue tree
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
max-limit=36864k name="Basic Customer" packet-mark=outbound parent=\
"Bandwidth Management" priority=7 queue="PCQ_BASIC_IN 1M"
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
max-limit=12048k name="Silver Package" packet-mark=silver-client parent=\
"Bandwidth Management" priority=5 queue="PCQ_SILVER_IN 1.5"
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
max-limit=12048k name="Gold Clients" packet-mark=gold-client parent=\
"Bandwidth Management" priority=3 queue="PCQ_GOLD_IN 3M"
/queue interface
set ether1 queue=ethernet-default
set ether2 queue=ethernet-default
set ether3 queue=ethernet-default
Queue Types
/queue type
set default kind=pfifo name=default pfifo-limit=50
set ethernet-default kind=pfifo name=ethernet-default pfifo-limit=50
set wireless-default kind=sfq name=wireless-default sfq-allot=1514 \
sfq-perturb=5
set synchronous-default kind=red name=synchronous-default red-avg-packet=1000 \
red-burst=20 red-limit=60 red-max-threshold=50 red-min-threshold=10
set hotspot-default kind=sfq name=hotspot-default sfq-allot=1514 sfq-perturb=\
5
add kind=pcq name="PCQ_BASIC_IN 1M" pcq-classifier=dst-address,dst-port \
pcq-limit=40 pcq-rate=1024000 pcq-total-limit=7000
add kind=pcq name="PCQ_SILVER_IN 1.5" pcq-classifier=dst-port pcq-limit=40 \
pcq-rate=1536000 pcq-total-limit=7000
add kind=pcq name="PCQ_GOLD_IN 3M" pcq-classifier=dst-port pcq-limit=15 \
pcq-rate=3072000 pcq-total-limit=500
set default-small kind=pfifo name=default-small pfifo-limit=10
My load balancing works great.
Does anyone see any problems I have here and how I might resolve it so that my throttling + bandwidth balancing would work?
fewi
August 3, 2010, 1:49pm
2
a) you’re using connection marks twice. Connections can only have one connection mark at a time. While this can potentially work if you use the marks before they are overwritten it is also potentially very confusing
Ditch the rules that mark connections as gold, silver or bronze. They are unnecessary. Keep the rules that mark the packets as gold, silver or bronze and modify them all to have passthrough set to ‘yes’. That should take care of the issue.
fewi
August 3, 2010, 2:30pm
3
Now that I’m at a different machine, here the changed mangle ruleset:
/ip firewall mangle
add action=mark-packet chain=prerouting comment="Mark Bronze Packets" src-address=10.0.0.0/8 in-interface=LAN passthrough=yes
add action=mark-packet chain=prerouting comment="Mark Silver Packets" new-packet-mark=silver-client src-address-list=Silver-Customer in-interface=LAN passthrough=yes
add action=mark-packet chain=prerouting comment="Mark Gold Client Packets" new-packet-mark=gold-client src-address-list=Gold-Customer in-interface=LAN passthrough=yes
add action=mark-connection chain=prerouting new-connection-mark=1st_conn passthrough=yes per-connection-classifier=both-addresses-and-ports:6/0
add action=mark-connection chain=prerouting new-connection-mark=2nd_conn passthrough=yes per-connection-classifier=both-addresses-and-ports:6/1
add action=mark-connection chain=prerouting new-connection-mark=3rd_conn passthrough=yes per-connection-classifier=both-addresses-and-ports:6/2
add action=mark-connection chain=prerouting new-connection-mark=4th_conn passthrough=yes per-connection-classifier=both-addresses-and-ports:6/3
add action=mark-connection chain=prerouting new-connection-mark=5th_conn passthrough=yes per-connection-classifier=both-addresses-and-ports:6/4
add action=mark-connection chain=prerouting new-connection-mark=6th_conn passthrough=yes per-connection-classifier=both-addresses-and-ports:6/5
add action=mark-routing chain=prerouting connection-mark=1st_conn new-routing-mark=1st_route passthrough=no
add action=mark-routing chain=prerouting connection-mark=2nd_conn new-routing-mark=2nd_route passthrough=no
add action=mark-routing chain=prerouting connection-mark=3rd_conn new-routing-mark=3rd_route passthrough=no
add action=mark-routing chain=prerouting connection-mark=4th_conn new-routing-mark=4th_route passthrough=no
add action=mark-routing chain=prerouting connection-mark=5th_conn new-routing-mark=5th_route passthrough=no
add action=mark-routing chain=prerouting connection-mark=6th_conn new-routing-mark=6th_route passthrough=no
That’s assuming traffic out to the Internet will enter the router through an in-interface of ‘LAN’. First mark the packets that come into that interface based on source address, passing them through to further rules in the mangle ruleset. Then mark the connections via PCC, passing through again. Then mark routing based on connection marks, passthrough is set to no here so you can bail out the moment routing is set because you’re done processing that packet at that point.
That still doesn’t account for the rules shown in the PCC wiki that ensure that incoming connections to the router itself are treated properly, and doesn’t account for directly connected networks. I don’t know your network topology so I can’t account for that.
Here are my mangle rules now and they appear to be working.
PCC load balancing is definitely working across my 6 gateways, although throttling is still a bit iffy.
/ip firewall mangle
add action=mark-connection chain=prerouting comment="" disabled=no \
new-connection-mark=1st-conn passthrough=yes per-connection-classifier=\
both-addresses-and-ports:6/0
add action=mark-connection chain=prerouting comment="" disabled=no \
new-connection-mark=2nd-conn passthrough=yes per-connection-classifier=\
both-addresses-and-ports:6/1
add action=mark-connection chain=prerouting comment="" disabled=no \
new-connection-mark=3rd-conn passthrough=yes per-connection-classifier=\
both-addresses-and-ports:6/2
add action=mark-connection chain=prerouting comment="" disabled=no \
new-connection-mark=4th-conn passthrough=yes per-connection-classifier=\
both-addresses-and-ports:6/3
add action=mark-connection chain=prerouting comment="" disabled=no \
new-connection-mark=5th-conn passthrough=yes per-connection-classifier=\
both-addresses-and-ports:6/4
add action=mark-connection chain=prerouting comment="" disabled=no \
new-connection-mark=6th-conn passthrough=yes per-connection-classifier=\
both-addresses-and-ports:6/5
add action=mark-packet chain=prerouting comment="" disabled=no \
new-packet-mark=gold-traffic passthrough=yes src-address-list=\
Gold-Customer
add action=mark-packet chain=prerouting comment="" disabled=no \
new-packet-mark=silver-traffic passthrough=yes src-address-list=\
Silver-Customer
add action=mark-packet chain=prerouting comment="" disabled=no \
new-packet-mark=bronze-traffic passthrough=yes
add action=mark-routing chain=prerouting comment="" connection-mark=1st-conn \
disabled=no new-routing-mark=1st_route passthrough=yes
add action=mark-routing chain=prerouting comment="" connection-mark=2nd-conn \
disabled=no new-routing-mark=2nd_route passthrough=yes
add action=mark-routing chain=prerouting comment="" connection-mark=3rd-conn \
disabled=no new-routing-mark=3rd_route passthrough=yes
add action=mark-routing chain=prerouting comment="" connection-mark=4th-conn \
disabled=no new-routing-mark=4th_route passthrough=yes
add action=mark-routing chain=prerouting comment="" connection-mark=5th-conn \
disabled=no new-routing-mark=5th_route passthrough=yes
add action=mark-routing chain=prerouting comment="" connection-mark=6th-conn \
disabled=no new-routing-mark=6th_route passthrough=yes
[jeff@SEI Communciations Router] >
My network consists of Wireless Tranzeo/Canopy equipment Natted on a 10.0.0.0/8
Example:
gateway1 10.10.1.1
Clinet1 10.10.9.1
ap1 10.10.8.1
Each gateway has it’s own static public ip address and we’re load balancing across 6 of them.
Everything is sitting behind the router/gateways, in order to forward traffic I have to send it from the gateway to the router and then push the port to my clients which is no big deal.
Basically, if a customer is not on the ‘silver-client’ or ‘gold-client’ list, they get 1M down.
My queue tree is still the same if you want to reference it in my previous post.
Am I correct in my decision to set max-limit to 36M ?
I have 6 gateways, each has 6M link.
PCQ limit is 1024k 1536k 3072k respectively.
fewi
August 3, 2010, 4:24pm
5
Wouldn’t it make more sense to split the 36 megs you have over the max-limits applied to the three queues with the PCQ sub-types? Right now the first one could eat the entire 36 megs.
That does make sense.
Any idea why my PCQ limit has to be half of the value I wish to limit?
For instance, if I want to limit them at 1024k down, I have to put in 512k.
If I want to limit them at 1536k, I have to put in 768k
I’ve tried it and the PCQ gives them up to double what I put in.
Here’s my PCQ settings:
/queue tree
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
max-limit=0 name="Bandwidth Management" parent=ether2 priority=8
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
max-limit=18432k name="Basic Customer" packet-mark=bronze-traffic parent=\
"Bandwidth Management" priority=7 queue="PCQ_BASIC_IN 1M"
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
max-limit=9216k name="Silver Package" packet-mark=silver-traffic parent=\
"Bandwidth Management" priority=5 queue="PCQ_SILVER_IN 1.5"
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
max-limit=9216k name="Gold Clients" packet-mark=gold-traffic parent=\
"Bandwidth Management" priority=3 queue="PCQ_GOLD_IN 3M"
/queue type
set default kind=pfifo name=default pfifo-limit=50
set ethernet-default kind=pfifo name=ethernet-default pfifo-limit=50
set wireless-default kind=sfq name=wireless-default sfq-allot=1514 \
sfq-perturb=5
set synchronous-default kind=red name=synchronous-default red-avg-packet=1000 \
red-burst=20 red-limit=60 red-max-threshold=50 red-min-threshold=10
set hotspot-default kind=sfq name=hotspot-default sfq-allot=1514 sfq-perturb=\
5
add kind=pcq name="PCQ_BASIC_IN 1M" pcq-classifier=dst-port pcq-limit=40 \
pcq-rate=512000 pcq-total-limit=7000
add kind=pcq name="PCQ_SILVER_IN 1.5" pcq-classifier=dst-port pcq-limit=40 \
pcq-rate=768000 pcq-total-limit=7000
add kind=pcq name="PCQ_GOLD_IN 3M" pcq-classifier=dst-port pcq-limit=15 \
pcq-rate=1536000 pcq-total-limit=500
set default-small kind=pfifo name=default-small pfifo-limit=10
[jeff@SEI Communciations Router] >
Any ideas on that? Perhaps I’m doing something wrong?
It is routing my traffic correctly across all 6 gateways and is limited how I want it.. so it’s not ‘broken’ but I’m curious if anyone can explain why I have to set my limit at half to get the desired results?
fewi
August 3, 2010, 7:37pm
7
You’re using the wrong classifier.
Assuming an interface named LAN (I think that’s ‘ether2’ for you, but it’s not entirely clear from what you posted) that the users are behind:
/ip firewall mangle
add action=mark-packet chain=prerouting new-packet-mark=gold-traffic-up passthrough=yes src-address-list=Gold-Customer in-interface=LAN
add action=mark-packet chain=prerouting new-packet-mark=silver-traffic-up passthrough=yes src-address-list=Silver-Customer in-interface=LAN
add action=mark-packet chain=prerouting new-packet-mark=bronze-traffic-up passthrough=yes in-interface=LAN
add action=mark-packet chain=postrouting new-packet-mark=gold-traffic-down passthrough=no dst-address-list=Gold-Customer out-interface=LAN
add action=mark-packet chain=postrouting new-packet-mark=silver-traffic-down passthrough=no dst-address-list=Silver-Customer out-interface=LAN
add action=mark-packet chain=postrouting new-packet-mark=bronze-traffic-down passthrough=no out-interface=LAN
/queue type
add kind=pcq name="bronze-up" pcq-classifier=src-address pcq-limit=40 pcq-rate=512000 pcq-total-limit=7000
add kind=pcq name="silver-up" pcq-classifier=src-address pcq-limit=40 pcq-rate=768000 pcq-total-limit=7000
add kind=pcq name="gold-up" pcq-classifier=src-address pcq-limit=15 pcq-rate=1536000 pcq-total-limit=500
add kind=pcq name="bronze-down" pcq-classifier=dst-address pcq-limit=40 pcq-rate=512000 pcq-total-limit=7000
add kind=pcq name="silver-down" pcq-classifier=dst-address pcq-limit=40 pcq-rate=768000 pcq-total-limit=7000
add kind=pcq name="gold-down" pcq-classifier=dst-address pcq-limit=15 pcq-rate=1536000 pcq-total-limit=500
/queue tree
add name="Bandwidth Management" parent=ether2 priority=8
add max-limit=18432k name="Bronze Package" packet-mark=bronze-traffic-up parent="Bandwidth Management" priority=7 queue=bronze-up
add max-limit=9216k name="Silver Package" packet-mark=silver-traffic-up parent="Bandwidth Management" priority=5 queue=silver-up
add max-limit=9216k name="Gold Clients" packet-mark=gold-traffic-up parent="Bandwidth Management" priority=3 queue=gold-up
add max-limit=18432k name="Bronze Package" packet-mark=bronze-traffic-down parent="Bandwidth Management" priority=7 queue=bronze-down
add max-limit=9216k name="Silver Package" packet-mark=silver-traffic-down parent="Bandwidth Management" priority=5 queue=silver-down
add max-limit=9216k name="Gold Clients" packet-mark=gold-traffic-down parent="Bandwidth Management" priority=3 queue=gold-down
Of course make sure that the mangle rules are sorted correctly. I’d probably put the packet-marking in the prerouting queue at the very top.
That worked great! Thanks, you were very helpful +karma for you sir.
Hi guys, would you please reply to my question. I don’t know how to make a queue tree. please help,my question must be very simple for you.
http://forum.mikrotik.com/t/how-to-make-queue-trees/39901/1
thanks