PCQ problem, packet loss

NikolayN · September 16, 2009, 6:33am

Hello!

I have strange problem with PCQ, when i enable even one queue with PCQ i got ping loss to MT (1%-5%). Even on packet not going to this queue. Total bandwidth about 170Mbit/s, 30000 pps. But i got drops even at 75Mbit/s. CPU 2 x Dual Core Opteron, 2 x Gigabit Broadcom NIC. When i disable all queue with PCQ, all go fine without packet loss, even on 700Mbit/s.
PS. Try 3.13, 3.22, 3.28 same results.

# sep/16/2009 14:35:27 by RouterOS 3.28
# software id = S7BB-SP5D
#
/queue type
set default kind=sfq name=default sfq-allot=1514 sfq-perturb=5
set ethernet-default kind=sfq name=ethernet-default sfq-allot=1514 \
    sfq-perturb=5
set wireless-default kind=sfq name=wireless-default sfq-allot=1514 \
    sfq-perturb=5
set synchronous-default kind=red name=synchronous-default red-avg-packet=1000 \
    red-burst=20 red-limit=60 red-max-threshold=50 red-min-threshold=10
set hotspot-default kind=sfq name=hotspot-default sfq-allot=1514 sfq-perturb=\
    5
add kind=pcq name=unlim_128_download pcq-classifier=dst-address pcq-limit=20 \
    pcq-rate=128000 pcq-total-limit=2000
add kind=pcq name=unlim_256_download pcq-classifier=dst-address pcq-limit=20 \
    pcq-rate=256000 pcq-total-limit=2000
add kind=pcq name=unlim_512_download pcq-classifier=dst-address pcq-limit=20 \
    pcq-rate=512000 pcq-total-limit=2000
add kind=pcq name=unlim_128_upload pcq-classifier=src-address pcq-limit=20 \
    pcq-rate=128000 pcq-total-limit=2000
add kind=pcq name=unlim_256_upload pcq-classifier=src-address pcq-limit=20 \
    pcq-rate=256000 pcq-total-limit=2000
add kind=pcq name=unlim_512_upload pcq-classifier=src-address pcq-limit=20 \
    pcq-rate=512000 pcq-total-limit=2000
add kind=pcq name=unlim_1024_upload pcq-classifier=src-address pcq-limit=20 \
    pcq-rate=1024000 pcq-total-limit=2000
add kind=pcq name=unlim_1024_download pcq-classifier=dst-address pcq-limit=20 \
    pcq-rate=1024000 pcq-total-limit=2000
add kind=pcq name=unlim_2048_download pcq-classifier=dst-address pcq-limit=20 \
    pcq-rate=2048000 pcq-total-limit=2000
add kind=pcq name=unlim_2048_upload pcq-classifier=src-address pcq-limit=20 \
    pcq-rate=2048000 pcq-total-limit=2000
add kind=pcq name=unlim_640_upload pcq-classifier=src-address pcq-limit=20 \
    pcq-rate=640000 pcq-total-limit=2000
add kind=pcq name=unlim_640_download pcq-classifier=dst-address pcq-limit=20 \
    pcq-rate=640000 pcq-total-limit=2000
add kind=pcq name=unlim_1536_upload pcq-classifier=src-address pcq-limit=20 \
    pcq-rate=1536000 pcq-total-limit=2000
add kind=pcq name=unlim_1536_download pcq-classifier=dst-address pcq-limit=20 \
    pcq-rate=1536000 pcq-total-limit=2000
add kind=pcq name=unlim_2560_upload pcq-classifier=src-address pcq-limit=20 \
    pcq-rate=2560000 pcq-total-limit=2000
add kind=pcq name=unlim_2560_download pcq-classifier=dst-address pcq-limit=20 \
    pcq-rate=2560000 pcq-total-limit=2000
add kind=pcq name=unlim_308_upload pcq-classifier=src-address pcq-limit=20 \
    pcq-rate=308000 pcq-total-limit=2000
add kind=pcq name=unlim_308_download pcq-classifier=dst-address pcq-limit=20 \
    pcq-rate=308000 pcq-total-limit=2000
add kind=pcq name=unlim_615_upload pcq-classifier=src-address pcq-limit=20 \
    pcq-rate=615000 pcq-total-limit=2000
add kind=pcq name=unlim_615_download pcq-classifier=dst-address pcq-limit=20 \
    pcq-rate=615000 pcq-total-limit=2000
add kind=pcq name=unlim_2458_upload pcq-classifier=src-address pcq-limit=20 \
    pcq-rate=2458000 pcq-total-limit=2000
add kind=pcq name=unlim_2458_download pcq-classifier=dst-address pcq-limit=20 \
    pcq-rate=2458000 pcq-total-limit=2000
add kind=pcq name=unlim_3072_upload pcq-classifier=src-address pcq-limit=20 \
    pcq-rate=3072000 pcq-total-limit=2000
add kind=pcq name=unlim_3072_download pcq-classifier=dst-address pcq-limit=20 \
    pcq-rate=3072000 pcq-total-limit=2000
add kind=pcq name=unlim_3687_upload pcq-classifier=src-address pcq-limit=20 \
    pcq-rate=3687000 pcq-total-limit=2000
add kind=pcq name=unlim_3687_download pcq-classifier=dst-address pcq-limit=20 \
    pcq-rate=3687000 pcq-total-limit=2000
add kind=pcq name=unlim_6144_upload pcq-classifier=src-address pcq-limit=20 \
    pcq-rate=6144000 pcq-total-limit=2000
add kind=pcq name=unlim_6144_download pcq-classifier=dst-address pcq-limit=20 \
    pcq-rate=6144000 pcq-total-limit=2000
add kind=pcq name=unlim_1230_upload pcq-classifier=src-address pcq-limit=20 \
    pcq-rate=1230000 pcq-total-limit=2000
add kind=pcq name=unlim_1230_download pcq-classifier=dst-address pcq-limit=20 \
    pcq-rate=1230000 pcq-total-limit=2000
set default-small kind=pfifo name=default-small pfifo-limit=10

# sep/16/2009 14:33:34 by RouterOS 3.28
# software id = S7BB-SP5D
#
/queue tree
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_128_download packet-mark=unlim_128_download \
    parent=vlan7 priority=8 queue=unlim_128_download
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_128_upload packet-mark=unlim_128_upload parent=\
    vlan102 priority=8 queue=unlim_128_upload
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_256_download packet-mark=unlim_256_download \
    parent=vlan7 priority=8 queue=unlim_256_download
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_256_upload packet-mark=unlim_256_upload parent=\
    vlan102 priority=8 queue=unlim_256_upload
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_512_download packet-mark=unlim_512_download \
    parent=vlan7 priority=8 queue=unlim_512_download
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_512_upload packet-mark=unlim_512_upload parent=\
    vlan102 priority=8 queue=unlim_512_upload
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_1024_download packet-mark=unlim_1024_download \
    parent=vlan7 priority=8 queue=unlim_1024_download
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_1024_upload packet-mark=unlim_1024_upload parent=\
    vlan102 priority=8 queue=unlim_1024_upload
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_2048_download packet-mark=unlim_2048_download \
    parent=vlan7 priority=8 queue=unlim_2048_download
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_2048_upload packet-mark=unlim_2048_upload parent=\
    vlan102 priority=8 queue=unlim_2048_upload
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_640_download packet-mark=unlim_640_download \
    parent=vlan7 priority=8 queue=unlim_640_download
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_640_upload packet-mark=unlim_640_upload parent=\
    vlan102 priority=8 queue=unlim_640_upload
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_308_download packet-mark=unlim_308_download \
    parent=vlan7 priority=8 queue=unlim_308_download
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_308_upload packet-mark=unlim_308_upload parent=\
    vlan102 priority=8 queue=unlim_308_upload
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_615_download packet-mark=unlim_615_download \
    parent=vlan7 priority=8 queue=unlim_615_download
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_615_upload packet-mark=unlim_615_upload parent=\
    vlan102 priority=8 queue=unlim_615_upload
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_2458_download packet-mark=unlim_2458_download \
    parent=vlan7 priority=8 queue=unlim_2458_download
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_2458_upload packet-mark=unlim_2458_upload parent=\
    vlan102 priority=8 queue=unlim_2458_upload
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_3072_download packet-mark=unlim_3072_download \
    parent=vlan7 priority=8 queue=unlim_3072_download
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_3072_upload packet-mark=unlim_3072_upload parent=\
    vlan102 priority=8 queue=unlim_3072_upload
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_3687_download packet-mark=unlim_3687_download \
    parent=vlan7 priority=8 queue=unlim_3687_download
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_3687_upload packet-mark=unlim_3687_upload parent=\
    vlan102 priority=8 queue=unlim_3687_upload
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_1230_download packet-mark=unlim_1230_download \
    parent=vlan7 priority=8 queue=unlim_1230_download
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no limit-at=0 \
    max-limit=0 name=unlim_1230_upload packet-mark=unlim_1230_upload parent=\
    vlan102 priority=8 queue=unlim_1230_upload
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=yes limit-at=30M \
    max-limit=30M name=test packet-mark=unlim_1230_download parent=global-in \
    priority=8 queue=ethernet-default
# sep/16/2009 14:36:11 by RouterOS 3.28
# software id = S7BB-SP5D
#
/ip firewall mangle
add action=change-mss chain=forward comment="" disabled=yes new-mss=1360 \
    protocol=tcp src-address=x.x.x.0/21 tcp-flags=syn tcp-mss=1361-65535
add action=change-mss chain=forward comment="" disabled=yes dst-address=\
    x.x.x.0/21 new-mss=1360 protocol=tcp tcp-flags=syn tcp-mss=\
    1361-65535
add action=change-mss chain=forward comment="" disabled=yes new-mss=1360 \
    protocol=tcp src-address=x.x.x.0/21 tcp-flags=syn tcp-mss=1361-65535
add action=change-mss chain=forward comment="" disabled=yes dst-address=\
    x.x.x.0/21 new-mss=1360 protocol=tcp tcp-flags=syn tcp-mss=\
    1361-65535
add action=accept chain=prerouting comment=local disabled=yes src-address=\
    10.0.0.0/8
add action=mark-packet chain=prerouting comment=Speed_Limit_640 disabled=yes \
    new-packet-mark=unlim_640_upload passthrough=no src-address-list=\
    Speed_Limit_640
add action=mark-packet chain=prerouting comment="" disabled=yes \
    dst-address-list=Speed_Limit_640 new-packet-mark=unlim_640_download \
    passthrough=no
add action=mark-packet chain=prerouting comment=Speed_Limit_615 disabled=yes \
    new-packet-mark=unlim_615_upload passthrough=no src-address-list=\
    Speed_Limit_615
add action=mark-packet chain=prerouting comment="" disabled=yes \
    dst-address-list=Speed_Limit_615 new-packet-mark=unlim_615_download \
    passthrough=no
add action=mark-packet chain=prerouting comment=Speed_Limit_3687 disabled=yes \
    new-packet-mark=unlim_3687_upload passthrough=no src-address-list=\
    Speed_Limit_3687
add action=mark-packet chain=prerouting comment="" disabled=yes \
    dst-address-list=Speed_Limit_3687 new-packet-mark=unlim_3687_download \
    passthrough=no
add action=mark-packet chain=prerouting comment=Speed_Limit_308 disabled=yes \
    new-packet-mark=unlim_308_upload passthrough=no src-address-list=\
    Speed_Limit_308
add action=mark-packet chain=prerouting comment="" disabled=yes \
    dst-address-list=Speed_Limit_308 new-packet-mark=unlim_308_download \
    passthrough=no
add action=mark-packet chain=prerouting comment=Speed_Limit_3072 disabled=yes \
    new-packet-mark=unlim_3072_upload passthrough=no src-address-list=\
    Speed_Limit_3072
add action=mark-packet chain=prerouting comment="" disabled=yes \
    dst-address-list=Speed_Limit_3072 new-packet-mark=unlim_3072_download \
    passthrough=no
add action=mark-packet chain=prerouting comment=Speed_Limit_512 disabled=no \
    new-packet-mark=unlim_512_upload passthrough=no src-address-list=\
    Speed_Limit_512
add action=mark-packet chain=prerouting comment="" disabled=no \
    dst-address-list=Speed_Limit_512 new-packet-mark=unlim_512_download \
    passthrough=no
add action=mark-packet chain=prerouting comment=Speed_Limit_256 disabled=no \
    new-packet-mark=unlim_256_upload passthrough=no src-address-list=\
    Speed_Limit_256
add action=mark-packet chain=prerouting comment="" disabled=no \
    dst-address-list=Speed_Limit_256 new-packet-mark=unlim_256_download \
    passthrough=no
add action=mark-packet chain=prerouting comment=Speed_Limit_2458 disabled=no \
    new-packet-mark=unlim_2458_upload passthrough=no src-address-list=\
    Speed_Limit_2458
add action=mark-packet chain=prerouting comment="" disabled=no \
    dst-address-list=Speed_Limit_2458 new-packet-mark=unlim_2458_download \
    passthrough=no
add action=mark-packet chain=prerouting comment=Speed_Limit_2048 disabled=no \
    new-packet-mark=unlim_2048_upload passthrough=no src-address-list=\
    Speed_Limit_2048
add action=mark-packet chain=prerouting comment="" disabled=no \
    dst-address-list=Speed_Limit_2048 new-packet-mark=unlim_2048_download \
    passthrough=no
add action=mark-packet chain=prerouting comment=Speed_Limit_128 disabled=no \
    new-packet-mark=unlim_128_upload passthrough=no src-address-list=\
    Speed_Limit_128
add action=mark-packet chain=prerouting comment="" disabled=no \
    dst-address-list=Speed_Limit_128 new-packet-mark=unlim_128_download \
    passthrough=no
add action=mark-packet chain=prerouting comment=Speed_Limit_1230 disabled=no \
    new-packet-mark=unlim_1230_upload passthrough=no src-address-list=\
    Speed_Limit_1230
add action=mark-packet chain=prerouting comment="" disabled=no \
    dst-address-list=Speed_Limit_1230 new-packet-mark=unlim_1230_download \
    passthrough=no
add action=mark-packet chain=prerouting comment=Speed_Limit_1024 disabled=no \
    new-packet-mark=unlim_1024_upload passthrough=no src-address-list=\
    Speed_Limit_1024
add action=mark-packet chain=prerouting comment="" disabled=no \
    dst-address-list=Speed_Limit_1024 new-packet-mark=unlim_1024_download \
    passthrough=no

macgaiver · September 17, 2009, 9:19am

make sure that pcq-total-limit=20*(number of clients in this queue) at least.

omidkosari · September 17, 2009, 8:07pm

I have same problem in addition of another serious problems with PCQ .

i have more than that .

PCQ is buggy specially when using pppoe and radius .

Chupaka · September 17, 2009, 8:35pm

radius?..

NikolayN · September 18, 2009, 7:29am

Already try to change pcq-total-limit and pcq-limit, same results.

NikolayN · September 18, 2009, 7:34am

I have PPTP + radius. But problem not in PPTP or radius. I move PCQ Shaper from PPTP MT to dedicated MT where i run only PCQ Shaper. Same problem with packet loss to PCQ Shaper MT when i enable PCQ Queue in Queue Tree.

omidkosari · September 19, 2009, 9:45am

My router packet loss is related to cpu usage . sometimes cpu usage grows without a logical reason and in that stage i have 5% to 10% packet loss and bandwidth could not be at its maximum .but after a long time suddenly cpu usage drops down and every thing works like a charm and i have not even 1 timeout ! (Please don’t say that the timeouts are from cabling or switch etc . i have tested them all)

I could not found the reason which makes cpu grow up or down but it is related to PCQ .

as you see in graphs when cpu usage is up bandwidth can not go maximum but when cpu usage drops down bandwidth goes at maximum.
Note : this kind of peak in cpu usage is not normal cpu usage and it caused by a problem or bug . because in normal cpu usage routeros works fine even if cpu usage is about 80%.

omidkosari · September 23, 2009, 7:21am

I have found that the problem is related to connection tracking . problem appears when total-entries is high (for example more than 100000) . but max-entries: 524288 . so what max-entries mean ?!

/ip firewall connection tracking> print
                   enabled: yes
      tcp-syn-sent-timeout: 1m
  tcp-syn-received-timeout: 1m
   tcp-established-timeout: 1d
      tcp-fin-wait-timeout: 10s
    tcp-close-wait-timeout: 10s
      tcp-last-ack-timeout: 20s
     tcp-time-wait-timeout: 10s
         tcp-close-timeout: 10s
               udp-timeout: 10s
        udp-stream-timeout: 3m
              icmp-timeout: 10s
           generic-timeout: 10m
             tcp-syncookie: no
               max-entries: 524288
             [b]total-entries: 127990[/b]

mikrotik guys said

Hello,

In this case you can reduce tcp-established-timeout. It should normalize
number of connections in your router.

Regards,
Janis Megis

i said

why i should reduce established timeout ? i want to use maximum potential of resources

and he says

Hello,

That is only number how many connections can be stored in the memory.

Regards,
Janis Megis

so still the main question is what does max-entries mean ?!
if it is only a number so what other resources we need to achieve the max-entries ?

omidkosari · September 30, 2009, 6:33am

Any idea ?!

NikolayN · September 30, 2009, 7:06am

tcp-established-timeout is how long established tcp session will be in mikrotik memory.
More entrys, more memory used, more CPU used.
I set this value to 6 hours on MT, and one MT to 1 hour.
After some tests i will post results here.

Ozelo · October 2, 2009, 11:17am

What if we set this tcp-established-timeout for 10 min? Ive playing around with this, but couldn’t notice any effects.

dssmiktik · October 3, 2009, 7:13pm

You can theoretically go up to the max-entries in connections, but based on bandwidth used by the connections and the sate of each connection could consume more cpu, and tax the pcq more heavily. More cpu horsepower should equal better pcq performance and overall better connection tracking performance.

I set the tcp-established-timeout to: 12h. This way if a tcp connection drops (say you are downloading a file and just unplug the power cord), that connection only stays tracked by the router for 12 hours.

omidkosari · October 4, 2009, 6:28am

I agree but cpu is about 15-20% and i have problems described above .

Really it is just useful for that situation ! for example if we have a telnet or ssh to a server it does not affected ? or an active download which takes for more than that time ?

dssmiktik · October 4, 2009, 7:36am

[quote=“dssmiktik”]
I set the tcp-established-timeout to: 12h. This way if a tcp connection drops (say you are downloading a file and just unplug the power cord), that connection only stays tracked by the router for 12 hours.
[/quote]

Really it is just useful for that situation ! for example if we have a telnet or ssh to a server it does not affected ? or an active download which takes for more than that time ?

From what I can tell, there haven’t been any problems so far.

I agree at 15-20 % CPU there should not be any problems regarding processing of the packets.

omidkosari · October 5, 2009, 6:07am

What is your opinion about the graphs i posted above ?

macgaiver · October 14, 2009, 10:51am

you are talking about CPU usage between 10%-30% - don’t you have something better to do than look at the numbers all the time - only CPU usage that matters is 100% and only if it for more that 1sek

10% CPU usage doesn’t mean it is faster than 30% CPU usage or 70% - 10% of CPU usage means you wasted your money cause your CPU is doing nothing 9/10 of the iteration (or cycle). And as results can be given to you only in the end of iteration from your point of view 1%-99% CPU load are equally fast, only when it is 100% CPU need to schedule something for later.

packet drops in this case are clearly from queues - queues limitation results in a packet drops - that is the way it works . But there is queue-size that you can increase to save some traffic from dropping.

Chupaka · October 14, 2009, 7:37pm

macgaiver, actually, Linux ‘load average’ shows more info than just CPU load, especially for time-critical applications (like routing )

NetworkPro · October 17, 2009, 9:53am

Replace router (entirely different hardware) - try with same config - if you do not have any packet loss with the different HW - problem is somewhere in the old router. If you still have packet loss - problem could be in config - packet queue, multiple queues in different places etc.