Community discussions

MikroTik App
 
ChipsAhoy
just joined
Topic Author
Posts: 5
Joined: Sat Jan 29, 2022 12:07 pm

RB3011 performance issues

Sat Jan 29, 2022 12:22 pm

Hi,

I'm having trouble receiving full gigabit on my RB3011UiAS in router mode and the speed fluctautes a lot, so results can be around 100-700Mbps less than expected.

On the other hand when I plug my laptop into my ISP router directly I get full gigabit wire speed.

Are there anything settings in my configuration that stands out wrong?

Kind regards,
[oats@MikroTik] > export
# jan/29/2022 05:13:57 by RouterOS 6.49.2
#
# model = RouterBOARD 3011UiAS
/interface bridge
add admin-mac=64:D1:54:F5:10:B9 auto-mac=no comment=LAN name=bridge
/interface ethernet
set [ find default-name=ether1 ] advertise=1000M-full comment=WAN1-ISP1 l2mtu=8156 loop-protect=off mac-address=64:D1:54:F5:10:B5 mtu=8156 rx-flow-control=auto speed=100Mbps tx-flow-control=auto
set [ find default-name=ether2 ] advertise=1000M-full comment=WAN2-ISP2 l2mtu=8156 loop-protect=off mtu=8156 rx-flow-control=auto speed=100Mbps tx-flow-control=auto
set [ find default-name=ether3 ] l2mtu=8156 loop-protect=on mtu=8156 rx-flow-control=auto speed=100Mbps tx-flow-control=auto
set [ find default-name=ether4 ] l2mtu=8156 mtu=8156 rx-flow-control=auto speed=100Mbps tx-flow-control=auto
set [ find default-name=ether5 ] l2mtu=8156 mtu=8156 rx-flow-control=auto speed=100Mbps tx-flow-control=auto
set [ find default-name=ether6 ] l2mtu=8156 mtu=8156 rx-flow-control=auto speed=100Mbps tx-flow-control=auto
set [ find default-name=ether7 ] l2mtu=8156 mtu=8156 rx-flow-control=auto speed=100Mbps tx-flow-control=auto
set [ find default-name=ether8 ] l2mtu=8156 mtu=8156 rx-flow-control=auto speed=100Mbps tx-flow-control=auto
set [ find default-name=ether9 ] l2mtu=8156 mtu=8156 rx-flow-control=auto speed=100Mbps tx-flow-control=auto
set [ find default-name=ether10 ] advertise=1000M-full l2mtu=8156 mtu=8156 poe-out=off rx-flow-control=auto speed=100Mbps tx-flow-control=auto
set [ find default-name=sfp1 ] advertise=1000M-full,2500M-full disabled=yes l2mtu=8158 mtu=8158 rx-flow-control=auto tx-flow-control=auto
/interface pppoe-client
add add-default-route=yes default-route-distance=2 disabled=no interface=ether2 max-mru=1508 max-mtu=1508 name=pppoe-out1 password=xxxxxxxx user=xxxxxxxx
/interface vlan
/interface ethernet switch
set 0 cpu-flow-control=no
set 1 cpu-flow-control=no
/interface list
add exclude=dynamic name=discover
add name=mactel
add name=mac-winbox
add name=WAN
add name=LAN
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip pool
add name=dhcp ranges=192.168.1.10-192.168.1.254
/ip dhcp-server
add address-pool=dhcp disabled=no interface=bridge lease-time=1d name=dhcp1
/ppp profile
set *FFFFFFFE local-address=192.168.1.80 remote-address=dhcp
/snmp community
set [ find default=yes ] addresses=0.0.0.0/0
/user group
set full policy=local,telnet,ssh,ftp,reboot,read,write,policy,test,winbox,password,web,sniff,sensitive,api,romon,dude,tikapp
/interface bridge port
add bridge=bridge disabled=yes interface=ether2
add bridge=bridge interface=ether6
add bridge=bridge hw=no interface=sfp1
add bridge=bridge interface=ether4
add bridge=bridge interface=ether5
add bridge=bridge interface=ether7
add bridge=bridge interface=ether8
add bridge=bridge interface=ether9
add bridge=bridge interface=ether10
add bridge=bridge disabled=yes interface=ether1
add bridge=bridge interface=ether3 tag-stacking=yes
/ip neighbor discovery-settings
set discover-interface-list=discover
/interface detect-internet
set detect-interface-list=WAN internet-interface-list=WAN lan-interface-list=LAN wan-interface-list=WAN
/interface list member
add interface=ether2 list=discover
add interface=ether3 list=discover
add interface=ether4 list=discover
add interface=ether5 list=discover
add interface=sfp1 list=discover
add interface=ether6 list=discover
add interface=ether7 list=discover
add interface=ether8 list=discover
add interface=ether9 list=discover
add interface=ether10 list=discover
add interface=bridge list=discover
add interface=pppoe-out1 list=discover
add interface=vlan1 list=discover
add interface=bridge list=mac-winbox
add interface=ether2 list=mactel
add interface=ether3 list=mactel
add interface=ether4 list=mactel
add interface=ether5 list=mactel
add interface=ether6 list=mactel
add interface=ether7 list=mactel
add interface=ether8 list=mactel
add interface=ether9 list=mactel
add interface=ether10 list=mactel
add interface=bridge list=mactel
add interface=pppoe-out1 list=WAN
add interface=ether1 list=WAN
add interface=bridge list=LAN
add list=WAN
/ip address
add address=192.168.1.1/24 interface=bridge network=192.168.1.0
/ip dhcp-client
# DHCP client can not run on slave interface!
add disabled=no interface=sfp1
add disabled=no interface=ether1
/ip dhcp-server lease
/ip dhcp-server network
add address=192.168.1.0/24 dns-server=192.168.1.1 gateway=192.168.1.1
/ip dns
set allow-remote-requests=yes servers=8.8.8.8,4.4.4.4 use-doh-server=https://dns.google/dns-query verify-doh-cert=yes
/ip dns static
/ip firewall address-list
add address=192.168.1.0/24 list=LAN
/ip firewall filter
add action=fasttrack-connection chain=forward comment=fasttrack connection-state=established,related
add action=accept chain=forward comment="allow related" connection-state=related
add action=drop chain=forward comment="drop invalid" connection-state=invalid protocol=tcp
add action=accept chain=input comment="accept established, related, untracked" connection-state=established,related,untracked
add action=drop chain=input comment="drop all not coming from LAN" in-interface-list=!LAN
add action=drop chain=forward comment="block outbound SMB traffic" dst-port=445 out-interface-list=WAN protocol=tcp
add action=drop chain=forward comment="block outbound SMB traffic" dst-port=445 out-interface-list=WAN protocol=udp
/ip firewall nat
add action=masquerade chain=srcnat comment="defconf: masquerade" out-interface-list=WAN
/ip route
add check-gateway=ping distance=1 gateway=ether1 routing-mark=WAN1-ISP1
add check-gateway=ping distance=2 gateway=pppoe-out1 routing-mark=WAN2-ISP2
/ip service
set telnet address=192.168.1.0/24
set ftp address=192.168.1.0/24
set www address=192.168.1.0/24 port=8080
set ssh address=192.168.1.0/24
set www-ssl address=192.168.1.0/24 certificate=webcert disabled=no port=8443
set api address=192.168.1.0/24
set winbox address=192.168.1.0/24 port=58291
set api-ssl address=192.168.1.0/24
/ip upnp interfaces
add interface=ether1 type=external
add interface=bridge type=internal
/lcd
set default-screen=stats-all time-interval=hour
/ppp secret
add name=aa password=aa profile=default-encryption
/system clock
set time-zone-name=America/Toronto
/system leds
set 0 disabled=yes
/system logging
set 0 topics=info,!dhcp
/system routerboard settings
set silent-boot=yes
/tool bandwidth-server
set enabled=no
/tool graphing resource
add
/tool mac-server
set allowed-interface-list=mactel
/tool mac-server mac-winbox
set allowed-interface-list=mac-winbox
 
User avatar
jbl42
Member Candidate
Member Candidate
Posts: 105
Joined: Sun Jun 21, 2020 12:58 pm

Re: RB3011 performance issues

Sat Jan 29, 2022 12:48 pm

What is the RB3011 CPU load while running speedtests to your ISP?
Is the fasttrack rule counting bytes, showing it works?

In general, RB3011 strugles with passing 1GBit with NAT and routing doing anything more than simple srcNAT and fasttrack.
The offcial RB3011 spec for small packets with NAT routing and fasttrack is 780MBit for 64k packets, getting better with larger packets.

If you need full GBit WAN bandwidth support without fasttrack and with some firewalling in place, you will need something like RB4011/RB5009.
Both having much more CPU power compared to RB3011.
 
ChipsAhoy
just joined
Topic Author
Posts: 5
Joined: Sat Jan 29, 2022 12:07 pm

Re: RB3011 performance issues

Sat Jan 29, 2022 7:59 pm

What is the RB3011 CPU load while running speedtests to your ISP?
Is the fasttrack rule counting bytes, showing it works?

Thanks for your reply! My speedtests start at 1Gbit and cpu0 jumps up to 100% usage, then the speed drops and CPU usage falls with it. cpu1 is barely used. The fasttrack counter is incrementing. I also switched all my interface queues to only-hardware-queue from ethernet-default with pfifo (how my router shipped) but never noticed a difference.

Are there any optimizations I can make to my filter rules?

I was hoping not having to buy another router but I am considering it now.
> system resource cpu print 
 # CPU                       LOAD         IRQ        DISK
 0 cpu0                      100%         95%          0%
 1 cpu1                        2%          0%          0%

> tool profile cpu=all
NAME                    CPU        USAGE
ethernet                  0          59%
firewall                  0         2.5%
networking                0          22%
winbox                    0           0%
management                0           0%
routing                   0           0%
ssl                       0           0%
bridging                  0         3.5%
unclassified              0           2%
cpu0                                 89%
lcd                       1         0.5%
ethernet                  1         5.5%
console                   1           0%
firewall                  1         0.5%
networking                1         2.5%
winbox                    1           0%
management                1         1.5%
ssl                       1           1%
internet-detect           1           0%
profiling                 1           0%
bridging                  1           0%
unclassified              1         1.5%
cpu1                                 13%

> ip settings print
              ip-forward: yes
          send-redirects: yes
     accept-source-route: no
        accept-redirects: no
        secure-redirects: yes
               rp-filter: no
          tcp-syncookies: no
    max-neighbor-entries: 8192
             arp-timeout: 30s
         icmp-rate-limit: 10
          icmp-rate-mask: 0x1818
             route-cache: yes
         allow-fast-path: yes
   ipv4-fast-path-active: no
  ipv4-fast-path-packets: 0
    ipv4-fast-path-bytes: 0
   ipv4-fasttrack-active: yes
  ipv4-fasttrack-packets: 878939818
    ipv4-fasttrack-bytes: 109371873347
 
mducharme
Trainer
Trainer
Posts: 1740
Joined: Tue Jul 19, 2016 6:45 pm
Location: Vancouver, BC, Canada

Re: RB3011 performance issues

Sat Jan 29, 2022 8:24 pm

Have you tried disabling flow control on all ports to see if there is any change? I don't know whether there would be or not, but I don't typically use flow control on these devices.
 
User avatar
jvanhambelgium
Long time Member
Long time Member
Posts: 677
Joined: Thu Jul 14, 2016 9:29 pm
Location: Belgium

Re: RB3011 performance issues

Sat Jan 29, 2022 8:24 pm

I think PPPoE stuff is (also) a single-core thing.
In ROS all packets of a single connection are handled by the same CPU-core.
So yeah, you might be hitting some limit here.

I'm running 3011 for years and very happy with the product, but I'm not near the 1Gbps WAN like you.
Its successor runs already side-by-side and is the RB5009.
 
tangent
Member
Member
Posts: 460
Joined: Thu Jul 01, 2021 3:15 pm

Re: RB3011 performance issues

Sat Jan 29, 2022 9:44 pm

Why are you hard-coding the MTU?

Are you certain it’s valid for your ISP? I’ve never seen one using anything beyond Ethernet specs, 1500 octets or so.

When your laptop is connected, what do you get from a path MTU discovery tool ?
 
ChipsAhoy
just joined
Topic Author
Posts: 5
Joined: Sat Jan 29, 2022 12:07 pm

Re: RB3011 performance issues

Sat Jan 29, 2022 9:50 pm

I tried toggling flow control on & off but there is no noticeable improvement. WAN1 (primary) uses standard ethernet. WAN2 (secondary) over PPPoE is mostly idle I just have it for failover. While forcing failover (i.e. disabling ether1 interface) the RB3011 shows similar performance over WAN2. Both ISPs support full duplex 1Gbit.

My MTUs were originally 1500 across the board. Everything in my network supports jumbo frames so I tried increasing the MTU. The WAN2 ISP supports baby jumbo frames (1500+8). No matter how the MTUs are set it's difficult to keep throughput above 900Mbit.
 
tangent
Member
Member
Posts: 460
Joined: Thu Jul 01, 2021 3:15 pm

Re: RB3011 performance issues

Sat Jan 29, 2022 10:01 pm

I tried toggling flow control on & off but there is no noticeable improvement.

Yes. Ethernet flow control is a crock. Use RouterOS's queuing features if TCP flow control doesn’t suffice.

That leaves UDP, but most uses of UDP don't benefit from queuing or flow control at all:

  • Small, brief protocols: DNS, DHCP, NTP, assorted "discovery" type protocols… Outside DDoS cases, they simply don't rise to the level to need it.
  • Real-time protocols: RTMP, RTP, SIP, IPTV… These can't be delayed, so queuing is either irrelevant or makes the problem worse, depending on whether there's enough bandwidth or not. The only option to handle congestion is to drop packets and wait for application-level elements in the stream to recover from the packet loss.
  • Tunneling protocols: most VPNs, plus things like GRE… Most of the payload data in these streams is likely to be encapsulated TCP, which includes its own flow control, so to put flow control on the "outside" as well is to invite the problems that lead to OpenVPN-over-TCP's famous performance problems. These streams should be passed without further flow control or queuing, since it should be done already on the "inside" instead.

WAN1 (primary) uses standard ethernet.

Huh? Quoting your first post:

/interface ethernet
set [ find default-name=ether1 ] … l2mtu=8156 … mtu=8156 …
 
ChipsAhoy
just joined
Topic Author
Posts: 5
Joined: Sat Jan 29, 2022 12:07 pm

Re: RB3011 performance issues

Sun Jan 30, 2022 1:16 am

I disabled flow control and reset all MTUs to 1500. No luck though
Below are fresh speedtests and my tracepath to both ISPs local servers:
$ speedtest --interface=enp1s3 --server-id=3025

   Speedtest by Ookla

     Server: Beanfield Metroconnect - Toronto, ON (id = 3025)
        ISP: Beanfield Technologies
    Latency:     0.81 ms   (0.18 ms jitter)
   Download:   846.97 Mbps (data used: 1.5 GB)
     Upload:   820.40 Mbps (data used: 390.7 MB)
Packet Loss:     0.0%
 Result URL: https://www.speedtest.net/result/c/338e472f-dc2c-4ef1-9544-89327e02871d

 $ speedtest --interface=enp1s3 --server-id=17394

   Speedtest by Ookla

     Server: Bell Canada - Toronto, ON (id = 17394)
        ISP: Bell Canada
    Latency:     2.03 ms   (0.40 ms jitter)
   Download:   825.63 Mbps (data used: 641.3 MB)
     Upload:   759.93 Mbps (data used: 1.1 GB)
Packet Loss:     0.0%
 Result URL: https://www.speedtest.net/result/c/966215b5-cb44-4441-8d73-ef80db6bc93a
 
 $ tracepath -b speedtest.beanfield.com
 1?: [LOCALHOST]                      pmtu 1500
 1:  192.168.1.1 (192.168.1.1)                             0.350ms
 1:  192.168.1.1 (192.168.1.1)                             0.559ms
 2:  199-68-125-129.beanfield.net (199.68.x.x)         1.630ms
 3:  be202.ppe01.18WynfordDr01.YYZ.beanfield.com (72.15.51.77)   1.919ms asymm 15
 4:  po203.agg01.18WynfordDr01.YYZ.beanfield.com (72.15.51.90)   1.671ms (This broken router returned corrupted payload) asymm 15
 5:  hu0-0-1-1.pe17.905KingStW01.YYZ.beanfield.com (199.167.152.68)   2.344ms asymm 13
 6:  hu0-0-1-1.pe10.905KingStW01.YYZ.beanfield.com (199.167.152.139)   1.989ms asymm 12
 7:  speedtest.beanfield.com (96.45.195.190)               1.546ms !H
     Resume: pmtu 1500
     
     $ tracepath -b tor47spd01.srvr.bell.ca
 1?: [LOCALHOST]                      pmtu 1500
 1:  192.168.1.1 (192.168.1.1)                             0.525ms
 1:  192.168.1.1 (192.168.1.1)                             0.555ms
 2:  10.50.x.x (10.50.x.x)                           4.050ms
 3:  no reply
 4:  64.230.59.188 (64.230.59.188)                         4.612ms asymm 10
 5:  cr02-toroon4772w-bundle-ether5.net.bell.ca (142.124.127.198)   4.358ms asymm  9
 6:  tcore3-toronto47-bundle-ether48.net.bell.ca (142.124.127.152)   3.691ms asymm  8
 7:  dis4-toronto47_7-0-0.net.bell.ca (64.230.107.3)       2.671ms
 8:  tor47spd01.srvr.bell.ca (184.150.167.230)             3.482ms !H
     Resume: pmtu 1500
Reference connection bypassing Mikro
$ speedtest --interface=ppp0 --server-id=17394

   Speedtest by Ookla

     Server: Bell Canada - Toronto, ON (id = 17394)
        ISP: Bell Canada
    Latency:     1.64 ms   (0.23 ms jitter)
   Download:   943.97 Mbps (data used: 426.2 MB)
     Upload:   943.32 Mbps (data used: 424.6 MB)
Packet Loss:     0.0%
 Result URL: https://www.speedtest.net/result/c/f2954dac-1ea7-4afc-bf66-699103bd8abf
 
mkx
Forum Guru
Forum Guru
Posts: 7672
Joined: Thu Mar 03, 2016 10:23 pm

Re: RB3011 performance issues

Sun Jan 30, 2022 3:56 pm

It's been mentioned before, I'll mention it again: official test results indicate that in real life this device is able to route at around 800Mbps give or take. With single TCP connection it's more of "take".
I'm affraid you're trying to achieve next to impossible result here.
BR,
Metod

Who is online

Users browsing this forum: Bing [Bot] and 9 guests