L3HW on a switch

mbrmik · April 5, 2023, 7:17am

I am using a CRS317 as core switch with an OPNsense router. Some how I do not manage to get the CRS317 to do L3 VLAN routing.

# apr/05/2023 09:15:19 by RouterOS 7.8
# software id = R85S-8D6C
#
# model = CRS317-1G-16S+
# serial number = D7EB0DBE5D83
/interface bridge
add fast-forward=no frame-types=admit-only-vlan-tagged ingress-filtering=no \
    name=BR1 priority=0x3000 vlan-filtering=yes
/interface ethernet
set [ find default-name=ether1 ] l2mtu=1592
set [ find default-name=sfp-sfpplus1 ] l2mtu=1592
set [ find default-name=sfp-sfpplus2 ] l2mtu=1592
set [ find default-name=sfp-sfpplus3 ] l2mtu=1592
set [ find default-name=sfp-sfpplus4 ] l2mtu=1592
set [ find default-name=sfp-sfpplus5 ] l2mtu=1592
set [ find default-name=sfp-sfpplus6 ] l2mtu=1592
set [ find default-name=sfp-sfpplus7 ] l2mtu=1592
set [ find default-name=sfp-sfpplus8 ] l2mtu=1592
set [ find default-name=sfp-sfpplus9 ] l2mtu=1592
set [ find default-name=sfp-sfpplus10 ] l2mtu=1592
set [ find default-name=sfp-sfpplus11 ] l2mtu=1592
set [ find default-name=sfp-sfpplus12 ] l2mtu=1592
set [ find default-name=sfp-sfpplus13 ] l2mtu=1592
set [ find default-name=sfp-sfpplus14 ] l2mtu=1592
set [ find default-name=sfp-sfpplus15 ] l2mtu=1592
set [ find default-name=sfp-sfpplus16 ] l2mtu=1592
/interface vlan
add interface=BR1 name=GUEST_VLAN vlan-id=90
add interface=BR1 name=LAN_VLAN vlan-id=240
add interface=BR1 name=MANAGEMENT_VLAN vlan-id=99
add interface=BR1 name=SELIM_VLAN vlan-id=82
add interface=BR1 name=SERVER_VLAN vlan-id=820
add interface=BR1 name=TANJA_VLAN vlan-id=81
add interface=BR1 name=WORK_VLAN vlan-id=35
/interface ethernet switch
set 0 l3-hw-offloading=yes
/interface ethernet switch port
set 0 l3-hw-offloading=no
set 1 l3-hw-offloading=no
set 2 l3-hw-offloading=no
set 3 l3-hw-offloading=no
set 4 l3-hw-offloading=no
set 5 l3-hw-offloading=no
set 6 l3-hw-offloading=no
set 7 l3-hw-offloading=no
set 8 l3-hw-offloading=no
set 9 l3-hw-offloading=no
set 10 l3-hw-offloading=no
set 11 l3-hw-offloading=no
set 12 l3-hw-offloading=no
set 13 l3-hw-offloading=no
set 14 l3-hw-offloading=no
set 15 l3-hw-offloading=no
set 16 l3-hw-offloading=no
/interface list
add name=BASE
/interface lte apn
set [ find default=yes ] ip-type=ipv4 use-network-apn=no
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/port
set 0 name=serial0
/routing bgp template
set default disabled=no output.network=bgp-networks
/routing ospf instance
add disabled=no name=default-v2
/routing ospf area
add disabled=yes instance=default-v2 name=backbone-v2
/interface bridge port
add bridge=BR1 frame-types=admit-only-vlan-tagged interface=sfp-sfpplus1
add bridge=BR1 frame-types=admit-only-vlan-tagged interface=sfp-sfpplus2
add bridge=BR1 frame-types=admit-only-vlan-tagged interface=sfp-sfpplus3
add bridge=BR1 frame-types=admit-only-vlan-tagged interface=sfp-sfpplus4
add bridge=BR1 frame-types=admit-only-vlan-tagged interface=sfp-sfpplus5
add bridge=BR1 frame-types=admit-only-vlan-tagged interface=sfp-sfpplus6
add bridge=BR1 frame-types=admit-only-vlan-tagged interface=sfp-sfpplus7
add bridge=BR1 frame-types=admit-only-vlan-tagged interface=sfp-sfpplus8
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    sfp-sfpplus9 pvid=240
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    sfp-sfpplus10 pvid=820
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    sfp-sfpplus11 pvid=820
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    sfp-sfpplus12 pvid=820
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    sfp-sfpplus13 pvid=820
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    sfp-sfpplus14 pvid=820
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    sfp-sfpplus15 pvid=820
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    sfp-sfpplus16 pvid=820
add bridge=BR1 frame-types=admit-only-vlan-tagged ingress-filtering=no \
    interface=ether1
/ip neighbor discovery-settings
set discover-interface-list=BASE
/ip settings
set max-neighbor-entries=8192
/ipv6 settings
set disable-ipv6=yes max-neighbor-entries=8192
/interface bridge vlan
add bridge=BR1 tagged="BR1,sfp-sfpplus1,sfp-sfpplus2,sfp-sfpplus3,sfp-sfpplus4\
    ,sfp-sfpplus5,sfp-sfpplus6,sfp-sfpplus7,ether1" vlan-ids=99
add bridge=BR1 tagged="BR1,ether1,sfp-sfpplus1,sfp-sfpplus2,sfp-sfpplus3,sfp-s\
    fpplus4,sfp-sfpplus5,sfp-sfpplus6,sfp-sfpplus7" untagged="sfp-sfpplus10,sf\
    p-sfpplus11,sfp-sfpplus12,sfp-sfpplus13,sfp-sfpplus14,sfp-sfpplus15,sfp-sf\
    pplus16" vlan-ids=820
add bridge=BR1 tagged="BR1,sfp-sfpplus1,sfp-sfpplus2,sfp-sfpplus3,sfp-sfpplus4\
    ,sfp-sfpplus5,sfp-sfpplus6,sfp-sfpplus7,ether1" untagged=sfp-sfpplus9 \
    vlan-ids=240
add bridge=BR1 tagged="BR1,sfp-sfpplus1,sfp-sfpplus2,sfp-sfpplus3,sfp-sfpplus4\
    ,sfp-sfpplus5,sfp-sfpplus6,sfp-sfpplus7,ether1" vlan-ids=81
add bridge=BR1 tagged="BR1,sfp-sfpplus1,sfp-sfpplus2,sfp-sfpplus3,sfp-sfpplus4\
    ,sfp-sfpplus5,sfp-sfpplus6,sfp-sfpplus7,ether1" vlan-ids=82
add bridge=BR1 tagged="BR1,sfp-sfpplus1,sfp-sfpplus2,sfp-sfpplus3,sfp-sfpplus4\
    ,sfp-sfpplus5,sfp-sfpplus6,sfp-sfpplus7,ether1" vlan-ids=35
add bridge=BR1 tagged="BR1,sfp-sfpplus1,sfp-sfpplus2,sfp-sfpplus3,sfp-sfpplus4\
    ,sfp-sfpplus5,sfp-sfpplus6,sfp-sfpplus7,ether1" vlan-ids=90
/interface list member
add interface=MANAGEMENT_VLAN list=BASE
/interface ovpn-server server
set auth=sha1,md5
/ip address
add address=10.8.20.17/24 interface=SERVER_VLAN network=10.8.20.0
add address=192.168.90.17/24 interface=GUEST_VLAN network=192.168.90.0
add address=192.168.8.17/24 interface=LAN_VLAN network=192.168.8.0
add address=172.16.99.17/24 interface=MANAGEMENT_VLAN network=172.16.99.0
add address=172.16.82.17/24 interface=SELIM_VLAN network=172.16.82.0
add address=172.16.81.17/24 interface=TANJA_VLAN network=172.16.81.0
add address=172.16.35.17/24 interface=WORK_VLAN network=172.16.35.0
/ip dns
set servers=172.16.99.254
/ip firewall filter
add action=fasttrack-connection chain=forward connection-state=\
    established,related hw-offload=yes
add action=accept chain=forward connection-state=established,related
/ip route
add disabled=no dst-address=0.0.0.0/0 gateway=172.16.99.254 routing-table=\
    main suppress-hw-offload=no
/system clock
set time-zone-name=Europe/Zurich
/system identity
set name=crs317-c1
/system ntp client
set enabled=yes
/system ntp client servers
add address=172.16.99.254
/system routerboard settings
set boot-os=router-os
/tool mac-server
set allowed-interface-list=BASE
/tool mac-server mac-winbox
set allowed-interface-list=BASE
/tool romon
set enabled=yes

I also tried the following I read in some other forum post about L3HW routing.

/interface/bridge/settings
set use-ip-firewall=yes
set use-ip-firewall-for-vlan=no
set use-ip-firewall-for-pppoe=no
set use-ip-firewall=no

Do I have to handle the port where the router is connected specially that routing is not going via the router?

accarda · April 5, 2023, 7:45am

Have you checked this article from MikroTik about Inter VLAN routing with L3HW enabled ?

This is the example part, but it’s worth reading the whole article.
https://help.mikrotik.com/docs/display/ROS/L3+Hardware+Offloading#L3HardwareOffloading-ConfigurationExamples

blacksnow · April 5, 2023, 7:49am

Check this thread (http://forum.mikrotik.com/t/l3hw-firewall-offloading-doesnt-offload-inter-vlan-traffic/164575/1), in short, use-ip-firewall=yes needs to be set to use-ip-firewall=no for L3HW Vlan offload to work properly.

sirbryan · April 5, 2023, 8:42am

Actually, what’s missing is all the ports in the switch have L3HW turned OFF. The firewall settings only matter if you want L3HW-accelerated NAT (instead of interVLAN routing).

/interface ethernet switch port
set 0 l3-hw-offloading=no
set 1 l3-hw-offloading=no
set 2 l3-hw-offloading=no
set 3 l3-hw-offloading=no

All of that should be

/interface ethernet switch port
set 0 l3-hw-offloading=yes
set 1 l3-hw-offloading=yes
set 2 l3-hw-offloading=yes
set 3 l3-hw-offloading=yes

…

mbrmik · April 5, 2023, 9:11am

I did try to apply https://help.mikrotik.com/docs/display/ROS/L3+Hardware+Offloading#L3HardwareOffloading-Inter-VLANRouting to my case as good as possible but some how I did not get it yet working in a way that routing does not go via the router instead of staying on the switch.

I did now run

/interface/ethernet/switch/port set [find] l3-hw-offloading=yes

and rebooted the switch.

# apr/05/2023 11:08:11 by RouterOS 7.8
# software id = R85S-8D6C
#
# model = CRS317-1G-16S+
# serial number = D7EB0DBE5D83
/interface bridge
add fast-forward=no frame-types=admit-only-vlan-tagged ingress-filtering=no \
    name=BR1 priority=0x3000 vlan-filtering=yes
/interface ethernet
set [ find default-name=ether1 ] l2mtu=1592
set [ find default-name=sfp-sfpplus1 ] l2mtu=1592
set [ find default-name=sfp-sfpplus2 ] l2mtu=1592
set [ find default-name=sfp-sfpplus3 ] l2mtu=1592
set [ find default-name=sfp-sfpplus4 ] l2mtu=1592
set [ find default-name=sfp-sfpplus5 ] l2mtu=1592
set [ find default-name=sfp-sfpplus6 ] l2mtu=1592
set [ find default-name=sfp-sfpplus7 ] l2mtu=1592
set [ find default-name=sfp-sfpplus8 ] l2mtu=1592
set [ find default-name=sfp-sfpplus9 ] l2mtu=1592
set [ find default-name=sfp-sfpplus10 ] l2mtu=1592
set [ find default-name=sfp-sfpplus11 ] l2mtu=1592
set [ find default-name=sfp-sfpplus12 ] l2mtu=1592
set [ find default-name=sfp-sfpplus13 ] l2mtu=1592
set [ find default-name=sfp-sfpplus14 ] l2mtu=1592
set [ find default-name=sfp-sfpplus15 ] l2mtu=1592
set [ find default-name=sfp-sfpplus16 ] l2mtu=1592
/interface vlan
add interface=BR1 name=GUEST_VLAN vlan-id=90
add interface=BR1 name=LAN_VLAN vlan-id=240
add interface=BR1 name=MANAGEMENT_VLAN vlan-id=99
add interface=BR1 name=SELIM_VLAN vlan-id=82
add interface=BR1 name=SERVER_VLAN vlan-id=820
add interface=BR1 name=TANJA_VLAN vlan-id=81
add interface=BR1 name=WORK_VLAN vlan-id=35
/interface ethernet switch
set 0 l3-hw-offloading=yes
/interface list
add name=BASE
/interface lte apn
set [ find default=yes ] ip-type=ipv4 use-network-apn=no
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/port
set 0 name=serial0
/routing bgp template
set default disabled=no output.network=bgp-networks
/routing ospf instance
add disabled=no name=default-v2
/routing ospf area
add disabled=yes instance=default-v2 name=backbone-v2
/interface bridge port
add bridge=BR1 frame-types=admit-only-vlan-tagged interface=sfp-sfpplus1
add bridge=BR1 frame-types=admit-only-vlan-tagged interface=sfp-sfpplus2
add bridge=BR1 frame-types=admit-only-vlan-tagged interface=sfp-sfpplus3
add bridge=BR1 frame-types=admit-only-vlan-tagged interface=sfp-sfpplus4
add bridge=BR1 frame-types=admit-only-vlan-tagged interface=sfp-sfpplus5
add bridge=BR1 frame-types=admit-only-vlan-tagged interface=sfp-sfpplus6
add bridge=BR1 frame-types=admit-only-vlan-tagged interface=sfp-sfpplus7
add bridge=BR1 frame-types=admit-only-vlan-tagged interface=sfp-sfpplus8
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    sfp-sfpplus9 pvid=240
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    sfp-sfpplus10 pvid=820
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    sfp-sfpplus11 pvid=820
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    sfp-sfpplus12 pvid=820
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    sfp-sfpplus13 pvid=820
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    sfp-sfpplus14 pvid=820
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    sfp-sfpplus15 pvid=820
add bridge=BR1 frame-types=admit-only-untagged-and-priority-tagged interface=\
    sfp-sfpplus16 pvid=820
add bridge=BR1 frame-types=admit-only-vlan-tagged ingress-filtering=no \
    interface=ether1
/ip neighbor discovery-settings
set discover-interface-list=BASE
/ip settings
set max-neighbor-entries=8192
/ipv6 settings
set disable-ipv6=yes max-neighbor-entries=8192
/interface bridge vlan
add bridge=BR1 tagged="BR1,sfp-sfpplus1,sfp-sfpplus2,sfp-sfpplus3,sfp-sfpplus4\
    ,sfp-sfpplus5,sfp-sfpplus6,sfp-sfpplus7,ether1" vlan-ids=99
add bridge=BR1 tagged="BR1,ether1,sfp-sfpplus1,sfp-sfpplus2,sfp-sfpplus3,sfp-s\
    fpplus4,sfp-sfpplus5,sfp-sfpplus6,sfp-sfpplus7" untagged="sfp-sfpplus10,sf\
    p-sfpplus11,sfp-sfpplus12,sfp-sfpplus13,sfp-sfpplus14,sfp-sfpplus15,sfp-sf\
    pplus16" vlan-ids=820
add bridge=BR1 tagged="BR1,sfp-sfpplus1,sfp-sfpplus2,sfp-sfpplus3,sfp-sfpplus4\
    ,sfp-sfpplus5,sfp-sfpplus6,sfp-sfpplus7,ether1" untagged=sfp-sfpplus9 \
    vlan-ids=240
add bridge=BR1 tagged="BR1,sfp-sfpplus1,sfp-sfpplus2,sfp-sfpplus3,sfp-sfpplus4\
    ,sfp-sfpplus5,sfp-sfpplus6,sfp-sfpplus7,ether1" vlan-ids=81
add bridge=BR1 tagged="BR1,sfp-sfpplus1,sfp-sfpplus2,sfp-sfpplus3,sfp-sfpplus4\
    ,sfp-sfpplus5,sfp-sfpplus6,sfp-sfpplus7,ether1" vlan-ids=82
add bridge=BR1 tagged="BR1,sfp-sfpplus1,sfp-sfpplus2,sfp-sfpplus3,sfp-sfpplus4\
    ,sfp-sfpplus5,sfp-sfpplus6,sfp-sfpplus7,ether1" vlan-ids=35
add bridge=BR1 tagged="BR1,sfp-sfpplus1,sfp-sfpplus2,sfp-sfpplus3,sfp-sfpplus4\
    ,sfp-sfpplus5,sfp-sfpplus6,sfp-sfpplus7,ether1" vlan-ids=90
/interface list member
add interface=MANAGEMENT_VLAN list=BASE
/interface ovpn-server server
set auth=sha1,md5
/ip address
add address=10.8.20.17/24 interface=SERVER_VLAN network=10.8.20.0
add address=192.168.90.17/24 interface=GUEST_VLAN network=192.168.90.0
add address=192.168.8.17/24 interface=LAN_VLAN network=192.168.8.0
add address=172.16.99.17/24 interface=MANAGEMENT_VLAN network=172.16.99.0
add address=172.16.82.17/24 interface=SELIM_VLAN network=172.16.82.0
add address=172.16.81.17/24 interface=TANJA_VLAN network=172.16.81.0
add address=172.16.35.17/24 interface=WORK_VLAN network=172.16.35.0
/ip dns
set servers=172.16.99.254
/ip firewall filter
add action=fasttrack-connection chain=forward connection-state=\
    established,related hw-offload=yes
add action=accept chain=forward connection-state=established,related
/ip route
add disabled=no dst-address=0.0.0.0/0 gateway=172.16.99.254 routing-table=\
    main suppress-hw-offload=no
/system clock
set time-zone-name=Europe/Zurich
/system identity
set name=crs317-c1
/system ntp client
set enabled=yes
/system ntp client servers
add address=172.16.99.254
/system routerboard settings
set boot-os=router-os
/tool mac-server
set allowed-interface-list=BASE
/tool mac-server mac-winbox
set allowed-interface-list=BASE
/tool romon
set enabled=yes

I have no idea what I oversaw in the configuration, still inter vlan traffic goes via the router. Do I have to disable l3hw for the router port on the switch?

[admin@crs317-c1] > /interface/bridge/settings print
              use-ip-firewall: no
     use-ip-firewall-for-vlan: no
    use-ip-firewall-for-pppoe: no
              allow-fast-path: yes
      bridge-fast-path-active: yes
     bridge-fast-path-packets: 69
       bridge-fast-path-bytes: 30603
  bridge-fast-forward-packets: 0
    bridge-fast-forward-bytes: 0

accarda · April 5, 2023, 9:32am

If you read carefully that example that I indicated from MT, you will see that L3HW is disabled for the WAN port.
Try to match that example with your case, which is not so different from their case.

mbrmik · April 5, 2023, 9:59am

In the example the delta to my configuration seems to be firewall settings

/ip address
add address=192.168.88.1/24 interface=ether1
add address=10.0.0.17/24 interface=sfp-sfpplus16
 
/ip route
add gateway=10.0.0.1
 
/ip firewall filter
add action=fasttrack-connection chain=forward connection-state=established,related hw-offload=yes
add action=accept chain=forward connection-state=established,related
 
/ip firewall nat
add action=masquerade chain=srcnat out-interface-list=WAN

There is an IP for WAN and MGMT.

Also the route I use looks like this:

/ip route
add disabled=no dst-address=0.0.0.0/0 gateway=172.16.99.254 routing-table=\
    main suppress-hw-offload=no

Firewall settings are basically the same:

Mine

/ip firewall filter
add action=fasttrack-connection chain=forward connection-state=\
    established,related hw-offload=yes
add action=accept chain=forward connection-state=established,related

Sample:

/ip firewall filter
add action=fasttrack-connection chain=forward connection-state=established,related hw-offload=yes
add action=accept chain=forward connection-state=established,related

And I have not nat defined.

I wonder now what IP I would give my WAN port… for example 172.16.99.16 to match the route?

Adding:

/ip address
add address=172.16.99.16/24 interface=sfp-sfpplus3

Did not change anything. Why would I need nat for this? Since this seems to be the last difference I could spot…

sirbryan · April 5, 2023, 10:24am

If your OPNSense box is doing all the firewall/gateway work, then there should be no need for firewall filters in the 317 (except for input to the router itself, i.e. for trust/ACL rules), and there should be no NAT rules.

You can have L3HW-offloading for NAT/firewall or for interVLAN routing, but they don’t work well together (if at all).

mbrmik · April 5, 2023, 11:24am

I try to purely get interVLAN routing working. Since I have NAT covered. That is why I tried to stick to the VLAN Configuration Example:

/interface/ethernet/switch set 0 l3-hw-offloading=no
/interface/bridge/port add bridge=bridge interface=ether2
/interface/bridge/vlan add bridge=bridge tagged=bridge,ether2 vlan-ids=20
/interface/vlan add interface=bridge name=vlan20 vlan-id=20
/ip/address add address=192.0.2.1/24 interface=vlan20
/interface/bridge set bridge vlan-filtering=yes
/interface/ethernet/switch set 0 l3-hw-offloading=yes

But till now no success.

sirbryan · April 5, 2023, 3:47pm

Are you on 7.8? I had issues with L3HW offload on 7.8 on some 2116’s this morning, but it works on my CRS310’s. My 317 is on 7.7 and it works very well. I haven’t tried 7.8 on it yet.

tdw · April 5, 2023, 6:32pm

So you have static routes on your clients sending the inter-VLAN traffic to the switch, separate from the default route to your OPNsense router?

mbrmik · April 5, 2023, 6:44pm

Yes, 7.8

No, I thought the switch chip would register the destination… and then route the traffic directly automatically?

mbrmik · April 9, 2023, 10:17am

I finally got an example running based on the sampel from the L3HW page

# apr/09/2023 12:12:54 by RouterOS 7.8
# software id = 471Q-CR0D
#
# model = CRS309-1G-8S+
# serial number = BE150B46E3BA
/interface bridge
add name=bridge vlan-filtering=yes
/interface vlan
add interface=bridge name=vlan20 vlan-id=20
add interface=bridge name=vlan30 vlan-id=30
/interface ethernet switch
set 0 l3-hw-offloading=yes
/interface ethernet switch port
set 0 l3-hw-offloading=no
set 8 l3-hw-offloading=no
/interface list
add name=LAN
add name=WAN
add name=MGMT
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip hotspot profile
set [ find default=yes ] html-directory=hotspot
/port
set 0 name=serial0
/interface bridge port
add bridge=bridge interface=sfp-sfpplus2 pvid=20
add bridge=bridge interface=sfp-sfpplus3 pvid=20
add bridge=bridge interface=sfp-sfpplus4 pvid=20
add bridge=bridge interface=sfp-sfpplus5 pvid=30
add bridge=bridge interface=sfp-sfpplus6 pvid=30
add bridge=bridge interface=sfp-sfpplus7 pvid=30
add bridge=bridge interface=sfp-sfpplus8 pvid=30
/interface bridge vlan
add bridge=bridge tagged=bridge untagged=\
    sfp-sfpplus2,sfp-sfpplus3,sfp-sfpplus4 vlan-ids=20
add bridge=bridge tagged=bridge untagged=\
    sfp-sfpplus5,sfp-sfpplus6,sfp-sfpplus7,sfp-sfpplus8 vlan-ids=30
/interface list member
add interface=sfp-sfpplus1 list=MGMT
add interface=sfp-sfpplus2 list=LAN
add interface=sfp-sfpplus3 list=LAN
add interface=sfp-sfpplus4 list=LAN
add interface=sfp-sfpplus5 list=LAN
add interface=sfp-sfpplus6 list=LAN
add interface=sfp-sfpplus7 list=LAN
add interface=sfp-sfpplus8 list=LAN
add interface=ether1 list=WAN
/ip address
add address=192.168.20.17/24 interface=vlan20 network=192.168.20.0
add address=192.168.30.17/24 interface=vlan30 network=192.168.30.0
/ip dhcp-client
add interface=ether1
/ip firewall filter
add action=fasttrack-connection chain=forward connection-state=\
    established,related hw-offload=yes
add action=accept chain=forward connection-state=established,related
/ip firewall nat
add action=masquerade chain=srcnat out-interface-list=WAN
/system clock
set time-zone-name=Europe/Zurich
/system routerboard settings
set boot-os=router-os

I got a Windows 10 client with IP 192.168.30.30 and a Windows 11 client with ip 192.168.20.20

Ethernet adapter Ethernet 3:

   Connection-specific DNS Suffix  . :
   Link-local IPv6 Address . . . . . : fe80::5ba5:348c:9896:a70%4
   IPv4 Address. . . . . . . . . . . : 192.168.20.20
   Subnet Mask . . . . . . . . . . . : 255.255.255.0
   Default Gateway . . . . . . . . . : 192.168.20.17

I added the static routes as follows for my Windows 11 client (ip 192.168.20.20)

route ADD 192.168.30.0 MASK 255.255.255.0 192.168.20.17

and for my Windows 10 client (ip 192.168.30.30)

route ADD 192.168.20.0 MASK 255.255.255.0 192.168.30.17

Results with iperf3

PS C:\Users\bodo> iperf3 -s
-----------------------------------------------------------
Server listening on 5201 (test #1)
-----------------------------------------------------------
Accepted connection from 192.168.30.30, port 53987
[  5] local 192.168.20.20 port 5201 connected to 192.168.30.30 port 53988
[ ID] Interval           Transfer     Bitrate
[  5]   0.00-1.00   sec   919 MBytes  7.71 Gbits/sec
[  5]   1.00-2.00   sec   918 MBytes  7.70 Gbits/sec
[  5]   2.00-3.00   sec  1022 MBytes  8.57 Gbits/sec
[  5]   3.00-4.00   sec   978 MBytes  8.20 Gbits/sec
[  5]   4.00-5.00   sec  1000 MBytes  8.39 Gbits/sec
[  5]   5.00-6.00   sec   976 MBytes  8.19 Gbits/sec
[  5]   6.00-7.00   sec  1004 MBytes  8.42 Gbits/sec
[  5]   7.00-8.00   sec  1017 MBytes  8.53 Gbits/sec
[  5]   8.00-9.00   sec  1001 MBytes  8.40 Gbits/sec
[  5]   9.00-10.00  sec   994 MBytes  8.34 Gbits/sec
- - - - - - - - - - - - - - - - - - - - - - - - -
[ ID] Interval           Transfer     Bitrate
[  5]   0.00-10.00  sec  9.60 GBytes  8.24 Gbits/sec                  receiver
-----------------------------------------------------------
Server listening on 5201 (test #2)
-----------------------------------------------------------
Accepted connection from 192.168.30.30, port 53989
[  5] local 192.168.20.20 port 5201 connected to 192.168.30.30 port 53990
[ ID] Interval           Transfer     Bitrate
[  5]   0.00-1.00   sec  1.02 GBytes  8.73 Gbits/sec
[  5]   1.00-2.00   sec  1.06 GBytes  9.07 Gbits/sec
[  5]   2.00-3.00   sec  1.08 GBytes  9.29 Gbits/sec
[  5]   3.00-4.00   sec  1.07 GBytes  9.18 Gbits/sec
[  5]   4.00-5.00   sec  1.09 GBytes  9.35 Gbits/sec
[  5]   5.00-6.00   sec  1.08 GBytes  9.29 Gbits/sec
[  5]   6.00-7.00   sec  1.08 GBytes  9.29 Gbits/sec
[  5]   7.00-8.00   sec  1.09 GBytes  9.34 Gbits/sec
[  5]   8.00-9.00   sec  1.08 GBytes  9.24 Gbits/sec
[  5]   9.00-10.00  sec  1.08 GBytes  9.28 Gbits/sec
[  5]  10.00-10.00  sec   896 KBytes  10.5 Gbits/sec
- - - - - - - - - - - - - - - - - - - - - - - - -
[ ID] Interval           Transfer     Bitrate
[  5]   0.00-10.00  sec  10.7 GBytes  9.21 Gbits/sec                  sender
-----------------------------------------------------------
Server listening on 5201 (test #3)
-----------------------------------------------------------

Now I have to apply this to my setup.

Hominidae · April 9, 2023, 11:58am

same here…of course, I do have the individual routes on server side, but never thought that these would be needed on client side as well (which are connected to the switch…it’s called L3-offload after all, isn’t it … and with proper routing info for each VLAN in the switch itself, it should be possible to detect an inter-vlan comms … just sayin
Many thanks to both of your for reporting on how to solve that problem.