Copy speed via trunk simply too slow

RouterOS General
1

Hello to the round,
I have to turn to you now because I’ve been looking for a suitable solution for a few days.
My problem is that I only get a data rate of about 60 MB/s when copying a 5GB file over a trunk.
A trunk should ideally even double the bandwidth, but that is not so important to me.
Failsafe and the 100 MB/s would be enough for my 1 Gibt Lan.

Here is a picture of copying a file over the trunk.
5Gb-Test-Kopieren-Reihe-ein-Kabel.PNG
On this picture you can see the copy speed,
if I use both computers only at one router. Here I then get to the 113 MB/s.
Aufbau.JPG
The last thing I did was to connect the two routers in series, without trunk only with one cable.
Here, too, I then get the 113 MB/s.
5Gb-Test-Kopieren-Trunk.PNG
What is wrong in my “config” for the trunk or bonding? Something is throttling the speed here.

Here is my config for the two routers:

Router1:

[admin@MikroTik] > export compact
# apr/05/2022 14:42:32 by RouterOS 7.1.5
# software id = N++++++++++
#
# model = CRS125-24G-1S-2HnD
# serial number = +++++++++++++++
/interface bridge
add name="bridge1 -Master-Netzwerk"
add name=bridge4-VLAN99
/interface ethernet
set [ find default-name=ether1 ] comment="LAN-Netzwerk 1-10" name=ether1-master-DHCP-192.168.1.0/24
set [ find default-name=ether8 ] advertise=1000M-full comment="Trunk Test" l2mtu=1598
set [ find default-name=ether9 ] advertise=1000M-full l2mtu=1598
set [ find default-name=ether10 ] comment=free
set [ find default-name=ether24 ] comment="INTERNET over LTE" name=ether24-master-10.10.10.0/24
set [ find default-name=sfp1 ] disabled=yes
/interface wireless
set [ find default-name=wlan1 ] antenna-gain=0 country=no_country_set frequency-mode=manual-txpower ssid=MikroTik station-roaming=enabled
/interface vlan
add interface=bridge4-VLAN99 name=VLAN99 vlan-id=99
/interface ethernet switch trunk
add member-ports=ether8,ether9 name=trunk-test
/interface lte apn
set [ find default=yes ] ip-type=ipv4 use-network-apn=no
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip pool
add name=192.168.1.0/24 ranges=192.168.1.100-192.168.1.200
add name=10.10.10.0/24 ranges=10.10.10.10-10.10.10.100
/ip dhcp-server
add address-pool=192.168.1.0/24 interface="bridge1 -Master-Netzwerk" name=192.168.1.0/24
add address-pool=10.10.10.0/24 interface=ether24-master-10.10.10.0/24 name=10.10.10.0/24
/port
set 0 name=serial0
/routing bgp template
set default as=65530 disabled=no name=default output.network=bgp-networks
/routing ospf instance
add name=default-v2
/routing ospf area
add disabled=yes instance=default-v2 name=backbone-v2
/user group
set full policy=local,telnet,ssh,ftp,reboot,read,write,policy,test,winbox,password,web,sniff,sensitive,api,romon,dude,tikapp,rest-api
/interface bridge port
add bridge="bridge1 -Master-Netzwerk" ingress-filtering=no interface=ether1-master-DHCP-192.168.1.0/24
add bridge="bridge1 -Master-Netzwerk" ingress-filtering=no interface=ether2
add bridge="bridge1 -Master-Netzwerk" ingress-filtering=no interface=ether3
add bridge="bridge1 -Master-Netzwerk" ingress-filtering=no interface=ether4
add bridge="bridge1 -Master-Netzwerk" ingress-filtering=no interface=ether5
add bridge="bridge1 -Master-Netzwerk" ingress-filtering=no interface=ether6
add bridge="bridge1 -Master-Netzwerk" ingress-filtering=no interface=ether7
add bridge=bridge4-VLAN99 interface=ether8
add bridge=bridge4-VLAN99 interface=ether9
add bridge="bridge1 -Master-Netzwerk" interface=VLAN99
/ip neighbor discovery-settings
set discover-interface-list=!dynamic
/ip settings
set max-neighbor-entries=8192
/ipv6 settings
set disable-ipv6=yes max-neighbor-entries=8192
/ip address
add address=10.10.10.11/24 interface=ether24-master-10.10.10.0/24 network=10.10.10.0
add address=20.20.20.1/24 interface=bridge4-VLAN99 network=20.20.20.0
add address=192.168.1.1/24 interface="bridge1 -Master-Netzwerk" network=192.168.1.0
/ip dhcp-server network
add address=10.10.10.0/24 dns-server=10.10.10.20 gateway=10.10.10.20 netmask=24 ntp-server=10.10.10.20
add address=192.168.1.0/24 dns-server=192.168.100.2,192.168.1.1 gateway=192.168.1.1 netmask=24 ntp-server=192.168.1.1
/ip dns
set allow-remote-requests=yes servers=1.1.1.1,1.0.0.1,8.8.8.8,192.168.2.1
/ip firewall filter
add action=accept chain=input connection-state=established,related,untracked
add action=drop chain=input connection-state=invalid
add action=accept chain=input protocol=icmp
add action=accept chain=forward ipsec-policy=in,ipsec
add action=accept chain=forward ipsec-policy=out,ipsec
add action=fasttrack-connection chain=forward connection-state=established,related hw-offload=yes
add action=accept chain=forward connection-state=established,related,untracked
add action=drop chain=forward connection-state=invalid
/ip firewall nat
add action=masquerade chain=srcnat out-interface="bridge1 -Master-Netzwerk"
add action=masquerade chain=srcnat out-interface=ether24-master-10.10.10.0/24
/ip route
add check-gateway=ping disabled=no dst-address=0.0.0.0/0 gateway=10.10.10.20
add disabled=no distance=1 dst-address=192.168.2.0/24 gateway=20.20.20.2 pref-src="" routing-table=main scope=30 suppress-hw-offload=no target-scope=10
/lcd interface pages
set 0 interfaces=wlan1
/system clock
set time-zone-name=Europe/Berlin
/system ntp client
set enabled=yes
/system ntp client servers
add address=de.pool.ntp.org
add address=ptbtime1.ptb.de
add address=ptbtime2.ptb.de
add address=ptbtime3.ptb.de
[admin@MikroTik] >

Router2:

[admin@MikroTik] > export compact
# apr/05/2022 14:43:25 by RouterOS 7.1.5
# software id = *********
#
# model = RBD53iG-5HacD2HnD
# serial number = ***********
/interface bridge
add name="bridge1 -Master-Netzwerk"
/interface ethernet
set [ find default-name=ether1 ] advertise=1000M-full comment=Bonding
set [ find default-name=ether2 ] advertise=1000M-full
set [ find default-name=ether3 ] comment=Netzwerk
/interface wireless
set [ find default-name=wlan1 ] ssid=MikroTik
set [ find default-name=wlan2 ] ssid=MikroTik
/interface bonding
add mode=balance-xor name=bonding1 slaves=ether1,ether2 transmit-hash-policy=layer-2-and-3
/interface vlan
add interface=bonding1 name=vlan99 vlan-id=99
/interface lte apn
set [ find default=yes ] ip-type=ipv4 use-network-apn=no
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/interface bridge port
add bridge="bridge1 -Master-Netzwerk" ingress-filtering=no interface=ether3
add bridge="bridge1 -Master-Netzwerk" ingress-filtering=no interface=ether4
add bridge="bridge1 -Master-Netzwerk" interface=vlan99
add bridge="bridge1 -Master-Netzwerk" interface=ether5
/ipv6 settings
set disable-ipv6=yes max-neighbor-entries=8192
/ip address
add address=20.20.20.2/24 interface=bonding1 network=20.20.20.0
add address=192.168.2.1/24 interface="bridge1 -Master-Netzwerk" network=192.168.2.0
/ip dns
set allow-remote-requests=yes servers=1.1.1.1,1.0.0.1,8.8.8.8,192.168.1.1
/ip firewall filter
add action=accept chain=input connection-state=established,related,untracked
add action=drop chain=input connection-state=invalid
add action=accept chain=input protocol=icmp
add action=accept chain=forward ipsec-policy=in,ipsec
add action=accept chain=forward ipsec-policy=out,ipsec
add action=fasttrack-connection chain=forward connection-state=established,related hw-offload=yes
add action=accept chain=forward connection-state=established,related,untracked
add action=drop chain=forward connection-state=invalid
/ip firewall nat
add action=masquerade chain=srcnat out-interface="bridge1 -Master-Netzwerk"
/ip route
add disabled=no dst-address=0.0.0.0/0 gateway=20.20.20.1 routing-table=main suppress-hw-offload=no
/system clock
set time-zone-name=Europe/Berlin
/system leds
set 0 interface=wlan1 leds=led1,led2,led3,led4,led5 type=wireless-signal-strength
set 1 leds=poe-led type=poe-out
/system ntp client
set enabled=yes
/system ntp client servers
add address=de.pool.ntp.org
add address=ptbtime1.ptb.de
add address=ptbtime2.ptb.de
add address=ptbtime3.ptb.de
/system routerboard settings
set cpu-frequency=auto
[admin@MikroTik] >

Structure:
5Gb-Test-Kopieren-Direkt.PNG
I would be very grateful for any tips.

Thanks a lot
Greetings
Tigger30926

2

20.20.20.x (microsoft.com) is not a free network, if you modify it for example is ok, but used real configuration is a dummy thing…

I’m expecting dhcp client on port 24 for LTE, not a DHCP server for provide the IP’s to LTE router…

On picture I see the Switch used as router and the Router used as switch… mah…

I do not know exactly if the bonding is done by hardware or by CPU, but on both case, why on layer 3?

3

There are two potential problems:


  1. the way bond distiributes traffic between physical links depends on mode and for some modes also on transmit-hash-policy. However, only bonds with mode=balance-rr actually use all (both) physical links for single TCP connection (copying a file to a SMB share is single TCP connection), the rest will use single physical link for the connection (and some modes will use single physical link for all connections between a pair of NICs). Bonds can be used to full potential only when there are multiple concurrent connections (either TCP or UDP or …) running over that bond.
  2. depending on exact devices used for bonding and exact bonding mode it might happen that some switch with slow CPU (e.g. CRSxxx) will actually perform bonding in software (as opposed to doing it in switch chip) which will make bond to perform very poorly. This is specially true for bond with mode=balance-rr which is linux-specific and none of Mikrotik devices can do it in hardware.

[edit] According to posted config it’s bullet #1 in the game, mode=balance-xor with trasmit-hash-policy=layer-2-and-3 will use same physical link for all packets towards same destination MAC and destination IP address (in case of normal machines, such as Windows, Linux, Mac, with single IP address, this is identical to transmit-hash-policy=layer2 as neither MAC nor IP address change).
Next thing would be to run /tool/profile cpu=all on both MT devices to see if any of them get CPU loaded by too much. Could be that your hAP ac3 might struggle, its CPU is powerful but not exactly without limits. I’m guesing CRS125 will be fine.

4

Thank you very much for the help.
I checked the CPU load when copying and unfortunately it is indeed the case that the CRS125 goes to the 90% load.
CRS125-24G-1S-2HnD.PNG
RBD531G-5Hac.JPG
Although I have followed Mikrotik’s instructions for the trunk and do not use bonding.

https://wiki.mikrotik.com/wiki/Manual:CRS1xx/2xx_VLANs_with_Trunks?msclkid=f5982d42b59211ecaecbc7389c16fb41

On the hAP I use a bonding since I can not set up a trunk there via the console.

If I understand all this correctly, a trunk or a bonding loads the CPU of the router very strongly.
This reduces the bandwidth and the copying speed. But how can I create a failsafe with this hardware and at the same time reach the 113 MB/s?
Or do I really have to buy other hardware?

Could I also do without a trunk or bonding and reach the same destination in a different way?
Maybe just use two ports and give each one a vlan and thus establish the transmission to the other router in parallel?
I also wonder why the devices can do all this, but then don’t have the necessary performance?

Yesterday I installed the latest firmware the 7.2 and since then I have only been able to achieve 25MB/s when copying, so I just installed the firmware 7.15 again.

5

The problem is not the trunk itself, rather that you are routing 192.168.1.x → 20.20.20.1 and 20.20.20.2 → 192.168.2.x. The CRS models are designed to be wire-speed switches with limited IP services such as routing and firewalling.

Use a single bridge on both Mikrotiks with the link between them added as a trunk on the CRS and bond on the hAP with addresses from the same subnet

6

Ok I implement very gladly so.
But I keep the Vlan99 config on both devices?
Do I add the two trunk ports to the bridge on the CRS123?
I give each trunk port its own IP address?
Which route do I specify then?

7

What surprised me was that you’ve configured ether8 and ether9 as member ports of the trunk, but you’ve made them individual ports of the bridge, rather than assigning the trunk port to the bridge. But the main issue is still the fact that you’ve got multiple bridges and/or that you ask CRS125 to route (either is enough to throttle the throughput).

8

The VLAN is not necessary for trunk operation.


Do I add the two trunk ports to the bridge on the CRS123?

On the CRS125, yes. The /interface ethernet switch trunk settings combine the ports within the switch chip to form a trunk.
On the hAP, no. The /interface bonding combines ports to form a trunk, the bonding1 interface is then added as a bridge port.


I give each trunk port its own IP address?
Which route do I specify then?

No, they are part of the bridges - the address should be set on the bridge, not bridge ports.
It depends on how you wish to set up the network, typically one device would be the router with an address 192.168.X.1, DHCP server, firewall rules, NAT, etc. and the other would be a simple layer 2 switch with an address such as 192.168.X.2 and gateway 192.168.X.1 but no DHCP server, etc.

As mentioned by others the hAP is a much more capable device when it comes to routing so it would be preferable to use that for the “router” role and the CRS for the “switch” role. The Fritz!BOX LTE “WAN” connection can be transported with a VLAN over the trunk, it requires a single VLAN-aware bridge on the hAP and the CRS configured as described in the CRS1xx/2xx switching documentation.

9

Thank you very much for the help.
I think I changed everything as suggested, but unfortunately the second router now no longer gets a connection
and the first router no longer reaches the second router. I currently do not know where my new error?

Router 1:

# apr/07/2022 08:26:26 by RouterOS 7.1.5
# software id = N9+++++++
#
# model = CRS125-24G-1S-2HnD
# serial number = 94++++++++++
/interface bridge
add name="bridge1 - Netzwerk"
/interface ethernet
set [ find default-name=ether1 ] comment="LAN-Netzwerk 1-2" name=ether1-master-DHCP-192.168.1.0/24
set [ find default-name=ether9 ] comment=Trunk
set [ find default-name=ether24 ] comment="INTERNET over LTE" name=ether24-master-10.10.10.0/24
set [ find default-name=sfp1 ] disabled=yes
/interface wireless
set [ find default-name=wlan1 ] antenna-gain=0 country=no_country_set frequency-mode=manual-txpower ssid=MikroTik station-roaming=enabled
/interface ethernet switch trunk
add member-ports=ether9,ether10 name=trunk-test
/interface lte apn
set [ find default=yes ] ip-type=ipv4 use-network-apn=no
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip pool
add name=192.168.1.0/24 ranges=192.168.1.100-192.168.1.200
add name=10.10.10.0/24 ranges=10.10.10.10-10.10.10.100
/ip dhcp-server
add address-pool=192.168.1.0/24 interface="bridge1 - Netzwerk" name=192.168.1.0/24
add address-pool=10.10.10.0/24 interface=ether24-master-10.10.10.0/24 name=10.10.10.0/24
/port
set 0 name=serial0
/routing bgp template
set default as=65530 disabled=no name=default output.network=bgp-networks
/routing ospf instance
add name=default-v2
/routing ospf area
add disabled=yes instance=default-v2 name=backbone-v2
/user group
set full policy=local,telnet,ssh,ftp,reboot,read,write,policy,test,winbox,password,web,sniff,sensitive,api,romon,dude,tikapp,rest-api
/interface bridge port
add bridge="bridge1 - Netzwerk" ingress-filtering=no interface=ether1-master-DHCP-192.168.1.0/24
add bridge="bridge1 - Netzwerk" ingress-filtering=no interface=ether2
add bridge="bridge1 - Netzwerk" interface=ether9
add bridge="bridge1 - Netzwerk" interface=ether10
/ip neighbor discovery-settings
set discover-interface-list=!dynamic
/ip settings
set max-neighbor-entries=8192
/ipv6 settings
set disable-ipv6=yes max-neighbor-entries=8192
/ip address
add address=192.168.1.1/24 interface="bridge1 - Netzwerk" network=192.168.1.0
add address=10.10.10.11/24 interface=ether24-master-10.10.10.0/24 network=10.10.10.0
/ip dhcp-server network
add address=10.10.10.0/24 dns-server=10.10.10.20 gateway=10.10.10.20 netmask=24 ntp-server=10.10.10.20
add address=192.168.1.0/24 dns-server=192.168.100.2,192.168.1.1,192.168.1.3 gateway=192.168.1.1 netmask=24 ntp-server=192.168.1.1
/ip dns
set allow-remote-requests=yes servers=1.1.1.1,1.0.0.1,8.8.8.8,192.168.1.3
/ip firewall filter
add action=accept chain=input connection-state=established,related,untracked
add action=drop chain=input connection-state=invalid
add action=accept chain=input protocol=icmp
add action=accept chain=forward ipsec-policy=in,ipsec
add action=accept chain=forward ipsec-policy=out,ipsec
add action=fasttrack-connection chain=forward connection-state=established,related hw-offload=yes
add action=accept chain=forward connection-state=established,related,untracked
add action=drop chain=forward connection-state=invalid
/ip firewall nat
add action=masquerade chain=srcnat out-interface="bridge1 - Netzwerk"
add action=masquerade chain=srcnat out-interface=ether24-master-10.10.10.0/24
/ip route
add check-gateway=ping disabled=no dst-address=0.0.0.0/0 gateway=10.10.10.20
/lcd interface pages
set 0 interfaces=wlan1
/system clock
set time-zone-name=Europe/Berlin
/system ntp client
set enabled=yes
/system ntp client servers
add address=de.pool.ntp.org
add address=ptbtime1.ptb.de
add address=ptbtime3.ptb.de
add address=ptbtime2.ptb.de
[admin@MikroTik] >

Router 2:

[admin@MikroTik] > export compact
# apr/06/2022 08:27:10 by RouterOS 7.1.5
# software id = ZZ+++++++++
#
# model = RBD53iG-5HacD2HnD
# serial number = E++++++++
/interface bridge
add name="bridge1 -Master-Netzwerk"
/interface ethernet
set [ find default-name=ether1 ] advertise=1000M-full comment=Bonding
set [ find default-name=ether2 ] advertise=1000M-full
set [ find default-name=ether3 ] comment=Netzwerk
/interface wireless
set [ find default-name=wlan1 ] ssid=MikroTik
set [ find default-name=wlan2 ] ssid=MikroTik
/interface bonding
add mode=802.3ad name=bonding1 slaves=ether1,ether2 transmit-hash-policy=layer-2-and-3
/interface lte apn
set [ find default=yes ] ip-type=ipv4 use-network-apn=no
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/interface bridge port
add bridge="bridge1 -Master-Netzwerk" ingress-filtering=no interface=ether3
add bridge="bridge1 -Master-Netzwerk" ingress-filtering=no interface=ether4
add bridge="bridge1 -Master-Netzwerk" interface=ether5
add bridge="bridge1 -Master-Netzwerk" interface=bonding1
/ipv6 settings
set disable-ipv6=yes max-neighbor-entries=8192
/ip address
add address=192.168.1.4/24 interface=bonding1 network=192.168.1.0
add address=192.168.1.3/24 interface="bridge1 -Master-Netzwerk" network=192.168.1.0
/ip dns
set allow-remote-requests=yes servers=1.1.1.1,1.0.0.1,8.8.8.8,192.168.1.1
/ip firewall filter
add action=accept chain=input connection-state=established,related,untracked
add action=drop chain=input connection-state=invalid
add action=accept chain=input protocol=icmp
add action=accept chain=forward ipsec-policy=in,ipsec
add action=accept chain=forward ipsec-policy=out,ipsec
add action=fasttrack-connection chain=forward connection-state=established,related hw-offload=yes
add action=accept chain=forward connection-state=established,related,untracked
add action=drop chain=forward connection-state=invalid
/ip firewall nat
add action=masquerade chain=srcnat out-interface="bridge1 -Master-Netzwerk"
/ip route
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway=192.168.1.1 pref-src=0.0.0.0 routing-table=main scope=30 suppress-hw-offload=no target-scope=10
/system clock
set time-zone-name=Europe/Berlin
/system leds
set 0 interface=wlan1 leds=led1,led2,led3,led4,led5 type=wireless-signal-strength
set 1 leds=poe-led type=poe-out
/system ntp client
set enabled=yes
/system ntp client servers
add address=de.pool.ntp.org
add address=ptbtime1.ptb.de
add address=ptbtime2.ptb.de
add address=ptbtime3.ptb.de
/system package update
set channel=long-term
/system routerboard settings
set cpu-frequency=auto
[admin@MikroTik] >