IPsec site-to-site low performance

reima · November 8, 2020, 11:45am

Hey guys,

I have set up a IPsec site-to-site VPN with a hap ac2 and a hexs trough the internet. Everything is working quite good, but with really really low performance.
Using iperf3 i get following results:

Connecting to host 192.168.21.23, port 5201
[  4] local 10.0.2.84 port 51820 connected to 192.168.21.23 port 5201
[ ID] Interval           Transfer     Bandwidth
[  4]   0.00-1.00   sec   512 KBytes  4.19 Mbits/sec
[  4]   1.00-2.01   sec   384 KBytes  3.13 Mbits/sec
[  4]   2.01-3.00   sec   256 KBytes  2.11 Mbits/sec
[  4]   3.00-4.00   sec   384 KBytes  3.13 Mbits/sec
[  4]   4.00-5.01   sec   256 KBytes  2.09 Mbits/sec
[  4]   5.01-6.01   sec   384 KBytes  3.16 Mbits/sec
[  4]   6.01-7.01   sec   256 KBytes  2.10 Mbits/sec
[  4]   7.01-8.00   sec   384 KBytes  3.16 Mbits/sec
[  4]   8.00-9.00   sec   256 KBytes  2.10 Mbits/sec
[  4]   9.00-10.00  sec   384 KBytes  3.15 Mbits/sec
- - - - - - - - - - - - - - - - - - - - - - - - -
[ ID] Interval           Transfer     Bandwidth
[  4]   0.00-10.00  sec  3.38 MBytes  2.83 Mbits/sec                  sender
[  4]   0.00-10.00  sec  3.17 MBytes  2.66 Mbits/sec                  receiver

iperf Done.

My interesting export, i reduced it to neccessary things:

# nov/08/2020 12:28:54 by RouterOS 6.47.4
# software id = FVNK-2IPL
#
# model = RBD52G-5HacD2HnD
# serial number = C6140C7A47AD

/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip hotspot profile
set [ find default=yes ] html-directory=flash/hotspot
/ip ipsec profile
set [ find default=yes ] enc-algorithm=aes-256,aes-128,3des
add dh-group=ecp256,modp2048 enc-algorithm=aes-256 hash-algorithm=sha256 \
    name=profile_site-to-site
/ip ipsec peer
add address=xxx.net name=landeck profile=\
    profile_site-to-site
add address=xxx.at name=vialli profile=profile_site-to-site
/ip ipsec proposal
set [ find default=yes ] auth-algorithms=sha256,sha1 enc-algorithms="aes-256-c\
    bc,aes-256-gcm,aes-192-cbc,aes-192-gcm,aes-128-cbc,aes-128-gcm,3des" \
    pfs-group=modp2048
add auth-algorithms=sha256 enc-algorithms=aes-256-cbc name=\
    proposal_site-to-site pfs-group=none
/ip ipsec mode-config
add address-pool=pool_vpn name=vpndhcp
/ppp profile
add local-address=10.0.254.254 name=l2tp_vpn remote-address=pool_vpn \
    use-encryption=required use-mpls=yes
/queue simple
add max-limit=1M/10M name=queue1 target=vlan_guest

/interface bridge port
add bridge=bridge1 interface=ether2 pvid=20
add bridge=bridge1 interface=ether3
add bridge=bridge1 interface=ether4
add bridge=bridge1 interface=ether5
add bridge=bridge1 interface=hap1_solo
add bridge=bridge1 interface=hap1_2
add bridge=bridge1 interface=hap1_5
add bridge=bridge1 interface=cap1_2
add bridge=bridge1 interface=cap1_5
add bridge=bridge1 interface=cap1_iot pvid=10
add bridge=bridge1 interface=hap1_iot pvid=10
add bridge=bridge1 interface=hap1_m2 pvid=20
add bridge=bridge1 interface=hap1_m5 pvid=20
add bridge=bridge1 interface=cap1_m2 pvid=20
add bridge=bridge1 interface=cap1_m5 pvid=20
add bridge=bridge1 interface=cap1_solo
add bridge=bridge1 interface=cap1_guest pvid=30
add bridge=bridge1 interface=hap1_guest pvid=30
/interface bridge vlan
add bridge=bridge1 tagged=bridge1,ether3,ether5,cap1_iot,hap1_iot vlan-ids=10
add bridge=bridge1 tagged=\
    bridge1,ether1,ether3,ether5,cap1_m2,cap1_m5,hap1_m2,hap1_m5 untagged=\
    ether2 vlan-ids=20
add bridge=bridge1 tagged=bridge1,ether3 vlan-ids=30
add bridge=bridge1 tagged=bridge1,ether3 vlan-ids=40
/interface l2tp-server server
set default-profile=l2tp_vpn enabled=yes max-mru=1480 max-mtu=1460 use-ipsec=\
    yes
/interface list member
add interface=ether1 list=WAN
add interface=bridge1 list=LAN
add interface=vlan_manuel list=LAN
add interface=vlan_iot list=LAN
add interface=vlan_guest list=LAN
/ip accounting
set account-local-traffic=yes enabled=yes
/ip accounting web-access
set accessible-via-web=yes
/ip cloud
set ddns-enabled=yes ddns-update-interval=30m
/ip firewall address-list
add address=10.0.0.0/24 list=Main
add address=10.0.1.0/24 list=Main
add address=10.0.2.0/24 list=Main
add address=10.0.3.0/24 list=Main
/ip firewall filter
add action=accept chain=input connection-state=established,related,untracked \
    log-prefix=input_accept
add action=drop chain=input connection-state=invalid
add action=accept chain=input protocol=icmp
add action=accept chain=input in-interface-list=LAN protocol=tcp src-port=53
add action=accept chain=input in-interface-list=LAN protocol=udp src-port=53
add action=accept chain=input src-address=127.0.0.1
add action=accept chain=input disabled=yes log-prefix=input_accept \
    src-address=192.168.10.0/24
add action=accept chain=input in-interface-list=LAN log-prefix=input_accept
add action=accept chain=input dst-port=500,1701,4500 in-interface=ether1 \
    protocol=udp
add action=accept chain=input in-interface=ether1 protocol=ipsec-esp
add action=drop chain=input log-prefix=input_drop
add action=accept chain=forward comment=vialli dst-port=8001,445,5201,443 \
    log-prefix=vialli protocol=tcp src-address=192.168.10.0/24
add action=accept chain=forward comment=landeck dst-port=443,445,5201,8080 \
    log-prefix=vialli protocol=tcp src-address=192.168.21.0/24
add action=fasttrack-connection chain=forward connection-state=\
    established,related disabled=yes in-interface=!vlan_guest out-interface=\
    !vlan_guest
add action=accept chain=forward connection-state=\
    established,related,untracked log-prefix=input_accept
add action=drop chain=forward comment="drop invalid" connection-state=invalid
add action=accept chain=forward comment="pass all outgoing" out-interface=\
    ether1
add action=accept chain=forward comment="Accept vlan_manuel -> AP" \
    dst-address=10.0.0.81 in-interface=vlan_manuel
add action=accept chain=forward comment="accept firetv-plex" dst-address=\
    10.0.2.200 dst-port=32400 protocol=tcp src-address=10.0.1.95 src-port=\
    48170
add action=accept chain=forward comment="allow vlan_iot->pihole udp" \
    dst-address=10.0.2.201 dst-port=53 in-interface=vlan_iot protocol=udp
add action=accept chain=forward comment="allow vlan_iot->pihole tcp" \
    dst-address=10.0.2.201 dst-port=53 in-interface=vlan_iot protocol=tcp
add action=drop chain=forward comment="drop all forward" log-prefix="dp fw"
/ip firewall mangle
add action=change-mss chain=forward dst-address=192.168.21.0/24 log=yes \
    log-prefix=mangle_la new-mss=1350 passthrough=yes protocol=tcp \
    src-address=192.168.0.1 tcp-flags=syn tcp-mss=!0-1350
add action=change-mss chain=forward dst-address=192.168.10.0/24 log-prefix=\
    mangle new-mss=1350 passthrough=yes protocol=tcp src-address=192.168.0.1 \
    tcp-flags=syn
/ip firewall nat
add action=accept chain=srcnat comment=vialli dst-address=192.168.10.0/24 \
    src-address=10.0.2.0/24
add action=accept chain=srcnat comment=ldk dst-address=192.168.21.0/24 \
    src-address=10.0.2.0/24
add action=masquerade chain=srcnat out-interface-list=WAN
/ip ipsec identity
add peer=vialli
add peer=landeck
/ip ipsec policy
add template=yes
add dst-address=192.168.10.0/24 peer=vialli sa-dst-address=XXXXXXXXX\
    sa-src-address=192.168.0.1 src-address=10.0.2.0/24 tunnel=yes
add dst-address=192.168.21.0/24 peer=landeck proposal=proposal_site-to-site \
    sa-dst-address=XXXXXXXXXXX sa-src-address=192.168.0.1 src-address=\
    10.0.2.0/24 tunnel=yes
/ip service
set www-ssl certificate=ssl disabled=no
/ppp secret
add name=reima profile=l2tp_vpn service=l2tp
/system clock
set time-zone-name=Europe/Vienna
/tool bandwidth-server
set authenticate=no

i also tried with disabling fasttrack, without results. On the second site there is the same configuration

EDIT: The speed from the Internet Uplink i on one site 150/20 and on the other 250/25

thanks
Manuel

sindy · November 8, 2020, 3:44pm

Fasttracking does interfere with IPsec handlin of forwarded packets.

Once a connection gets fasttracked, it remains fasttracked until it ends. Disabling the action=fasttrack-connection rule doesn’t affect already existing connections, only newly created ones.

By default, the connection tracking module considers UDP connections to be finished 3 minutes after the last packet seen. So if you run the iperf always between the same pair of UDP ports, even stopping iperf, disabling the action=fasttrack-connection rule, and running iperf again won’t show you the result without fasttracking. You have to manually remove the connection, or not restart the iperf sooner than 3 minutes after stopping the previous one.

And the action=fasttrack-connection rule must be disabled on both devices of course.

reima · November 8, 2020, 7:13pm

Is fasttrack actually not the same as an accept established, related, untracked rule? I wasn’t able to find a real big difference between them.

I had deactived the rule actually the whole day, and tested it again now: same results…

thanks
Manuel

sindy · November 8, 2020, 7:33pm

The difference is huge. “accept established,related,untracked” is just one of the key elements of a stateful firewall filter, where mid-connection packets are accepted by this single rule, and only the initial packet of each connection passes through more rules in the filter chain. So every mid-connection packet hits this rule.

If a connection gets fasttracked, a lot of firewall handling, queue handling except interface queues, and IPsec policy matching is skipped completely for most mid-connection packets. And the connection gets fasttracked once a single mid-connection packet of that connection hits this rule, most subsequent ones don’t ever get to the filter (some percentage does, as the traffic statistics for fasttracked connections is built this way).

What puzzles me most is that you say that the results are the same with and without fasttracking.

While testing, what does /ip firewall connection print where src-address~“10.0.2.84” dst-address~“192.168.21.23” show at both machines?

reima · November 8, 2020, 7:44pm

It’s on both machines the same:

[admin@MikroTik-HEXs] > /ip firewall connection print where src-address~"10.0.2.84
" dst-address~"192.168.21.23" 
Flags: E - expected, S - seen-reply, A - assured, C - confirmed, D - dying, 
F - fasttrack, s - srcnat, d - dstnat 
 #          PR.. SRC-ADDRESS           DST-ADDRESS           TCP-STATE  
 0  SAC     tcp  10.0.2.84:53688       192.168.21.23:5201    established
 1  SAC     tcp  10.0.2.84:53689       192.168.21.23:5201    established

sindy · November 8, 2020, 7:55pm

OK, so we are sure that fasttrack is out of the table. Now what is the ping response time if you ping from 10.0.2.84 to 192.168.21.23, and what is the bandwidth if you use UDP rather than TCP for testing?

Off topic, you may want to edit sa-dst-address in /ip ipsec policy rows in your configuration export.

reima · November 8, 2020, 8:36pm

Ping wird ausgeführt für 192.168.21.10 mit 32 Bytes Daten:
Antwort von 192.168.21.10: Bytes=32 Zeit=32ms TTL=62
Antwort von 192.168.21.10: Bytes=32 Zeit=32ms TTL=62
Antwort von 192.168.21.10: Bytes=32 Zeit=30ms TTL=62
Antwort von 192.168.21.10: Bytes=32 Zeit=31ms TTL=62

and

Ping wird ausgeführt für 10.0.2.84 mit 32 Bytes Daten:
Antwort von 10.0.2.84: Bytes=32 Zeit=33ms TTL=126
Antwort von 10.0.2.84: Bytes=32 Zeit=31ms TTL=126
Antwort von 10.0.2.84: Bytes=32 Zeit=35ms TTL=126
Antwort von 10.0.2.84: Bytes=32 Zeit=32ms TTL=126

Oh, thank you!

Thanks
Manuel

sindy · November 9, 2020, 7:16am

Are the UDP results same or better?

Regardless that, what does /ip ipsec installed-sa print show at both ends? It is enough to obfuscate the IPs, the encryption and authentication keys are ephemeral so it is sufficient to delay posting until they change - by default in 30 minutes at the latest.

reima · November 9, 2020, 1:37pm

The UDP Result with nmap (nmap -PU) for latency is 0.063s

First side:

admin@MikroTik] > /ip ipsec installed-sa print      
Flags: H - hw-aead, A - AH, E - ESP 
 0 HE spi=0xDE183E4 src-address=:4500 dst-address=192.168.0.1:4500 
      state=dying auth-algorithm=sha256 enc-algorithm=aes-cbc enc-key-size=256 
      auth-key="22b5fbf6b7758c1689506ef0680e8647beae21bcbd0d3686e1220d3137df2a49"
          
      
      enc-key="c93b3b061f5280d8351f10924e143a0fe765030d10a2dcb8baf67f113e6b98da" 
      addtime=nov/09/2020 10:30:37 expires-in=3m19s add-lifetime=24m/30m 
      current-bytes=27984 current-packets=144 replay=128 

 1 HE spi=0x410A330 src-address=192.168.0.1:4500 dst-address=:4500 
      state=dying auth-algorithm=sha256 enc-algorithm=aes-cbc enc-key-size=256 
      auth-key="db4f7a97355d2011d20bf19446978cd10128ed2a98f68ff52ec4ea18a8bec68d"
          
      
      enc-key="d9627c02dec92d873a6328373b7554288ef2402ee322a8c53ffd108afef69913" 
      addtime=nov/09/2020 10:30:37 expires-in=3m19s add-lifetime=24m/30m 
      current-bytes=25968 current-packets=192 replay=128 

 2 HE spi=0x90075C9 src-address=:4500 dst-address=192.168.0.1:4500 
      state=dying auth-algorithm=sha256 enc-algorithm=aes-cbc enc-key-size=256 
      auth-key="5e3d7cc256831f0fbc1b8184cc2ea8934f673d1caae90fee70ad04172d1e0452"

And second side:

admin@MikroTik-HEXs] > /ip ipsec installed-sa print
Flags: H - hw-aead, A - AH, E - ESP 
 0 HE spi=0x4C33213 src-address=:4500 dst-address=:450>
      state=dying auth-algorithm=sha256 enc-algorithm=aes-cbc enc-key-size=256 
      auth-key="d703efa00f6e4a777b7fce71fbafa10f19ef75ab0adbd6dc7cb055ce8d105407"
          
      
      enc-key="3fed518c9fc3acfb353b334c31d6c9c701b6d3f03330b3db6b32fe68cef5468c" 
      addtime=nov/09/2020 10:31:12 expires-in=2m44s add-lifetime=24m/30m 
      current-bytes=960 current-packets=24 replay=128 

 1 HE spi=0x90075C9 src-address=4500 dst-address=:450>
      state=dying auth-algorithm=sha256 enc-algorithm=aes-cbc enc-key-size=256 
      auth-key="5e3d7cc256831f0fbc1b8184cc2ea8934f673d1caae90fee70ad04172d1e0452"
          
      
      enc-key="697f89355fb1e35979d01e46fe3518e174769dd49d367db1aa4f0fa42ed7fae8" 
      addtime=nov/09/2020 10:31:12 expires-in=2m44s add-lifetime=24m/30m 
      current-bytes=960 current-packets=24 replay=128 

 2 HE spi=0x546B457 src-address= dst-address=450>
      state=mature auth-algorithm=sha256 enc-algorithm=aes-cbc enc-key-size=256 
      auth-key="e9da0390b069d618a831477e7bdf480ef673c74b10c073f18c43054900cc3e11"

I’m sorry for the bad text formatting, but i wasn’t able to do it better. I don’t now why my code block does not look like this sometimes…

thanks
Manuel

sindy · November 9, 2020, 2:19pm

I had in mind throughput testing using UDP, not latency testing. I assume the latency may impact the TCP throughput.

Other than that, I’ve seen ISPs to throttle IPsec, but it was on a long-distance path, not within a single bundesland.

It needs time to find out how to overcome some funny behaviour of the phpBB. But you’ve again let your public IPs slip through

The purpose of the question was to make sure that hardware encryption is activated at both ends. It is, so unless the 30-60 ms of round-trip delay cause the TCP test to show such a low bandwidth, I would suspect the ISP to impose some traffic shaping.

Once you test the UDP throughput via the IPsec tunnel, can you test it again directly to the public IP of the other peer, maybe with some port forwarding there?