Bandwidth Issue

mohkhalifa · July 10, 2020, 2:04pm

Dear All,
I need your kind help regarding my configurations, I have 10x vDSL routers 100Mbps each with total bandwidth 921,600Mbps and the download speed must be around 115 MB/s BUT actually I’m downloading from 40-50 MB/s ?! and not even reaching near the 100 MB/s.
RouterOS CHR x86_64 6.47.1
8 Core CPU
4 GB Ram
Note: While downloading CPU around 50-60%

Please Advise!

/interface ethernet
set [ find default-name=ether1 ] comment="Internet Bonding" \
    disable-running-check=no name=Local
set [ find default-name=ether2 ] comment=03-3612794 disable-running-check=no \
    mtu=1492 name=WAN1
set [ find default-name=ether3 ] comment=03-3622138 disable-running-check=no \
    mtu=1492 name=WAN2
set [ find default-name=ether4 ] comment=03-3608053 disable-running-check=no \
    mtu=1492 name=WAN3
set [ find default-name=ether5 ] comment=03-3621162 disable-running-check=no \
    mtu=1492 name=WAN4
set [ find default-name=ether6 ] comment=03-3621163 disable-running-check=no \
    mtu=1492 name=WAN5
set [ find default-name=ether7 ] comment=03-3621164 disable-running-check=no \
    mtu=1492 name=WAN6
set [ find default-name=ether8 ] comment=03-3621165 disable-running-check=no \
    mtu=1492 name=WAN7
set [ find default-name=ether9 ] comment=03-3621177 disable-running-check=no \
    mtu=1492 name=WAN8
set [ find default-name=ether10 ] comment=03-3613636 disable-running-check=no \
    mtu=1492 name=WAN9
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip pool
add name=dhcp_pool1 ranges=192.168.3.3-192.168.3.254
add name=dhcp_pool2 ranges=192.168.3.2-192.168.3.254
add name=dhcp_pool3 ranges=192.168.0.1,192.168.0.3-192.168.0.254
/ip dhcp-server
add address-pool=dhcp_pool3 disabled=no interface=Local name=dhcp1
/user group
set full policy="local,telnet,ssh,ftp,reboot,read,write,policy,test,winbox,pas\
    sword,web,sniff,sensitive,api,romon,dude,tikapp"
/ip neighbor discovery-settings
set discover-interface-list=none
/ip address
add address=192.168.0.2/24 interface=Local network=192.168.0.0
add address=192.168.10.2/24 comment=Line1 interface=WAN1 network=192.168.10.0
add address=192.168.20.2/24 comment=Line2 interface=WAN2 network=192.168.20.0
add address=192.168.30.2/24 comment=Line3 interface=WAN3 network=192.168.30.0
add address=192.168.40.2/24 comment=Line4 interface=WAN4 network=192.168.40.0
add address=192.168.50.2/24 comment=Line5 interface=WAN5 network=192.168.50.0
add address=192.168.60.2/24 comment=Line6 interface=WAN6 network=192.168.60.0
add address=192.168.70.2/24 comment=Line7 interface=WAN7 network=192.168.70.0
add address=192.168.80.2/24 comment=Line8 interface=WAN8 network=192.168.80.0
add address=192.168.90.2/24 comment=Line9 interface=WAN9 network=192.168.90.0
/ip dhcp-server network
add address=192.168.0.0/24 dns-server=8.8.8.8,8.8.4.4 gateway=192.168.0.2
/ip dns
set servers=208.67.222.222,208.67.220.220
/ip firewall mangle
add action=accept chain=prerouting dst-address=10.0.0.0/24 in-interface=Local
add action=accept chain=prerouting dst-address=10.10.30.0/24 in-interface=\
    Local
add action=accept chain=prerouting dst-address=192.168.10.0/24 in-interface=\
    Local
add action=accept chain=prerouting dst-address=192.168.20.0/24 in-interface=\
    Local
add action=accept chain=prerouting dst-address=192.168.30.0/24 in-interface=\
    Local
add action=accept chain=prerouting dst-address=192.168.40.0/24 in-interface=\
    Local
add action=accept chain=prerouting dst-address=192.168.50.0/24 in-interface=\
    Local
add action=accept chain=prerouting dst-address=192.168.60.0/24 in-interface=\
    Local
add action=accept chain=prerouting dst-address=192.168.70.0/24 in-interface=\
    Local
add action=accept chain=prerouting dst-address=192.168.80.0/24 in-interface=\
    Local
add action=accept chain=prerouting dst-address=192.168.90.0/24 in-interface=\
    Local
add action=accept chain=prerouting in-interface=WAN1
add action=accept chain=prerouting in-interface=WAN2
add action=accept chain=prerouting in-interface=WAN3
add action=accept chain=prerouting in-interface=WAN4
add action=accept chain=prerouting in-interface=WAN5
add action=accept chain=prerouting in-interface=WAN6
add action=accept chain=prerouting in-interface=WAN7
add action=accept chain=prerouting in-interface=WAN8
add action=accept chain=prerouting in-interface=WAN9
add action=mark-connection chain=prerouting dst-address-type=!local \
    new-connection-mark=wan1_conn passthrough=yes per-connection-classifier=\
    both-addresses-and-ports:9/0
add action=mark-connection chain=prerouting dst-address-type=!local \
    new-connection-mark=wan2_conn passthrough=yes per-connection-classifier=\
    both-addresses-and-ports:9/1
add action=mark-connection chain=prerouting dst-address-type=!local \
    new-connection-mark=wan3_conn passthrough=yes per-connection-classifier=\
    both-addresses-and-ports:9/2
add action=mark-connection chain=prerouting dst-address-type=!local \
    new-connection-mark=wan4_conn passthrough=yes per-connection-classifier=\
    both-addresses-and-ports:9/3
add action=mark-connection chain=prerouting dst-address-type=!local \
    new-connection-mark=wan5_conn passthrough=yes per-connection-classifier=\
    both-addresses-and-ports:9/4
add action=mark-connection chain=prerouting dst-address-type=!local \
    new-connection-mark=wan6_conn passthrough=yes per-connection-classifier=\
    both-addresses-and-ports:9/5
add action=mark-connection chain=prerouting dst-address-type=!local \
    new-connection-mark=wan7_conn passthrough=yes per-connection-classifier=\
    both-addresses-and-ports:9/6
add action=mark-connection chain=prerouting dst-address-type=!local \
    new-connection-mark=wan8_conn passthrough=yes per-connection-classifier=\
    both-addresses-and-ports:9/7
add action=mark-connection chain=prerouting dst-address-type=!local \
    new-connection-mark=wan9_conn passthrough=yes per-connection-classifier=\
    both-addresses-and-ports:9/8
add action=mark-routing chain=prerouting connection-mark=wan1_conn \
    new-routing-mark=to_wan1 passthrough=yes
add action=mark-routing chain=prerouting connection-mark=wan2_conn \
    new-routing-mark=to_wan2 passthrough=yes
add action=mark-routing chain=prerouting connection-mark=wan3_conn \
    new-routing-mark=to_wan3 passthrough=yes
add action=mark-routing chain=prerouting connection-mark=wan4_conn \
    new-routing-mark=to_wan4 passthrough=yes
add action=mark-routing chain=prerouting connection-mark=wan5_conn \
    new-routing-mark=to_wan5 passthrough=yes
add action=mark-routing chain=prerouting connection-mark=wan6_conn \
    new-routing-mark=to_wan6 passthrough=yes
add action=mark-routing chain=prerouting connection-mark=wan7_conn \
    new-routing-mark=to_wan7 passthrough=yes
add action=mark-routing chain=prerouting connection-mark=wan8_conn \
    new-routing-mark=to_wan8 passthrough=yes
add action=mark-routing chain=prerouting connection-mark=wan9_conn \
    new-routing-mark=to_wan9 passthrough=yes
/ip firewall nat
add action=masquerade chain=srcnat out-interface=WAN1
add action=masquerade chain=srcnat out-interface=WAN2
add action=masquerade chain=srcnat out-interface=WAN3
add action=masquerade chain=srcnat out-interface=WAN4
add action=masquerade chain=srcnat out-interface=WAN5
add action=masquerade chain=srcnat out-interface=WAN6
add action=masquerade chain=srcnat out-interface=WAN7
add action=masquerade chain=srcnat out-interface=WAN8
add action=masquerade chain=srcnat out-interface=WAN9
add action=dst-nat chain=dstnat dst-port=11355 in-interface=all-ethernet \
    protocol=tcp to-addresses=192.168.0.252 to-ports=11355
/ip route
add check-gateway=ping distance=1 gateway=192.168.10.1 routing-mark=to_wan1
add check-gateway=ping distance=1 gateway=192.168.20.1 routing-mark=to_wan2
add check-gateway=ping distance=1 gateway=192.168.30.1 routing-mark=to_wan3
add check-gateway=ping distance=1 gateway=192.168.40.1 routing-mark=to_wan4
add check-gateway=ping distance=1 gateway=192.168.50.1 routing-mark=to_wan5
add check-gateway=ping distance=1 gateway=192.168.60.1 routing-mark=to_wan6
add check-gateway=ping distance=1 gateway=192.168.70.1 routing-mark=to_wan7
add check-gateway=ping distance=1 gateway=192.168.80.1 routing-mark=to_wan8
add check-gateway=ping distance=1 gateway=192.168.90.1 routing-mark=to_wan9
add check-gateway=ping distance=1 gateway=192.168.10.1
add check-gateway=ping distance=2 gateway=192.168.20.1
add check-gateway=ping distance=3 gateway=192.168.30.1
add check-gateway=ping distance=4 gateway=192.168.40.1
add check-gateway=ping distance=5 gateway=192.168.50.1
add check-gateway=ping distance=6 gateway=192.168.60.1
add check-gateway=ping distance=7 gateway=192.168.70.1
add check-gateway=ping distance=8 gateway=192.168.80.1
add check-gateway=ping distance=9 gateway=192.168.90.1
add distance=1 dst-address=10.0.0.0/8 gateway=192.168.0.1
/system clock
set time-zone-name=Africa/Cairo

mohkhalifa · July 11, 2020, 12:16am

Any Help ??

sindy · July 11, 2020, 9:58am

How do you test the download speed? Load distribution works per connection, so for each TCP session, only a single uplink is used. E.g. speedtest.net uses “only” four TCP sessions in parallel, so 400 Mbit/s, so about 40 MByte/s.

mohkhalifa · July 11, 2020, 1:15pm

@sindy I’m downloading large file using IDM with 32 connections and that’s the best way for bandwidth test.

sindy · July 11, 2020, 1:24pm

Can you check, while downloading that huge file, the actual distribution of the individual connections among the 10 uplinks?

/ip firewall connection {:foreach conn in=[find protocol~“tcp”] do={:put ("WAN address: “.[get $conn reply-dst-address].” download rate: ".[get $conn repl-rate])}} ; /

(supposing there is no other traffic, otherwise you have to filter the connections also by adding src-address~“ip.of.the.downloading.pc” to the find expression)

mohkhalifa · July 11, 2020, 3:21pm

that was the result

/ip firewall connection {:foreach conn in=[find protocol~"tcp"] do={:put ("                                                                                               WAN address: ".[get $conn reply-dst-address]." download rate: ".[get $conn repl-rate])}} ; /
WAN address: 192.168.80.2:9838 download rate: 0
WAN address: 192.168.80.2:48395 download rate: 31608264
WAN address: 192.168.10.2:59478 download rate: 0
WAN address: 192.168.30.2:44263 download rate: 0
WAN address: 192.168.70.2:25043 download rate: 0
WAN address: 192.168.70.2:19304 download rate: 10533472
WAN address: 192.168.50.2:63820 download rate: 0
WAN address: 192.168.40.2:36108 download rate: 0
WAN address: 192.168.10.2:61867 download rate: 22138520
WAN address: 192.168.10.2:23110 download rate: 7503816
WAN address: 192.168.80.2:62672 download rate: 27209776
WAN address: 192.168.50.2:24636 download rate: 127776
WAN address: 192.168.30.2:16114 download rate: 0
WAN address: 192.168.80.2:10516 download rate: 0
WAN address: 192.168.20.2:57741 download rate: 0
WAN address: 192.168.70.2:46327 download rate: 0
WAN address: 192.168.60.2:45441 download rate: 30107904
WAN address: 192.168.60.2:11818 download rate: 0
WAN address: 192.168.20.2:34320 download rate: 22610176
WAN address: 192.168.10.2:4330 download rate: 14847168
WAN address: 192.168.40.2:13738 download rate: 25253520
WAN address: 192.168.20.2:5726 download rate: 0
WAN address: 192.168.0.10:36885 download rate: 58280
WAN address: 192.168.50.2:11374 download rate: 8891328
WAN address: 192.168.80.2:3739 download rate: 0
WAN address: 192.168.40.2:14801 download rate: 38528024
WAN address: 192.168.70.2:59314 download rate: 0
WAN address: 192.168.20.2:16348 download rate: 0
WAN address: 192.168.70.2:31971 download rate: 680
WAN address: 192.168.10.2:61846 download rate: 18172840
WAN address: 192.168.70.2:38952 download rate: 25282200
WAN address: 192.168.60.2:38891 download rate: 22660912
WAN address: 192.168.70.2:13687 download rate: 0
WAN address: 192.168.80.2:1749 download rate: 0
WAN address: 192.168.40.2:6009 download rate: 0
WAN address: 192.168.70.2:19352 download rate: 9414912
WAN address: 192.168.80.2:27259 download rate: 23232
WAN address: 192.168.50.2:52071 download rate: 5946752
WAN address: 192.168.20.2:14073 download rate: 11616
WAN address: 192.168.50.2:60356 download rate: 27467064
WAN address: 192.168.80.2:37257 download rate: 0
WAN address: 192.168.30.2:59653 download rate: 21917296
WAN address: 192.168.60.2:10392 download rate: 0
WAN address: 192.168.50.2:12359 download rate: 545232
WAN address: 192.168.70.2:19244 download rate: 5868096
WAN address: 192.168.70.2:10777 download rate: 0
WAN address: 192.168.10.2:16775 download rate: 5991616
WAN address: 192.168.70.2:48371 download rate: 14658112
WAN address: 192.168.80.2:53911 download rate: 0
WAN address: 192.168.60.2:46146 download rate: 0
WAN address: 192.168.10.2:52226 download rate: 0
WAN address: 192.168.70.2:35493 download rate: 23089616
WAN address: 192.168.90.2:28172 download rate: 0
WAN address: 192.168.30.2:21049 download rate: 0
WAN address: 192.168.60.2:54937 download rate: 0
WAN address: 192.168.20.2:46498 download rate: 259584
WAN address: 192.168.0.10:61006 download rate: 5760
WAN address: 192.168.70.2:40503 download rate: 0
WAN address: 192.168.80.2:3016 download rate: 23906552
WAN address: 192.168.60.2:30226 download rate: 36660488
WAN address: 192.168.50.2:37724 download rate: 31275712
WAN address: 192.168.20.2:59660 download rate: 0

sindy · July 11, 2020, 4:14pm

OK. A little bit of postprocessing of that data shows the following:

WAN9 is not loaded with at least a single download thread (weird),
the summary flow across all WANs is those 525 Mbit/s as you’ve experienced,
the distribution of the load is not even, not only in terms of sessions per WAN (which is nothing unexpected) but also in terms of throughput per WAN (which is surprising): the most loaded WANs carry almost 90 Mbit/s, the least loaded ones carry just a bit above 20 Mbit/s.

So there is a bottleneck somewhere between the server and the client; the question is whether the bottleneck is the CHR itself, the virtualisation platform on which it is running, the test client PC, or something at the other end of the VDSL lines.

To determine this, you have to create a local setup with two CHRs, a file server and a file client, and try the same test. The CPU is not the only limiting factor, the networking can be slow on the virtualisation platform. Another test could be to use two CHRs, each serving four WANs, and two clients, one connected via each CHR, to see whether the issue is at your end or at the remote end.