RB4011 Slow Inter-VLAN Routing

RouterOS General
1

I have my RGB4011 setup with all its VLANs on sfp-sfpplus1, it’s also the uplink to my switch. Internet comes in on ether1. For some reason routing between VLANs is super slow. Any ideas on what could cause this, or is it due to the fact that I do not have a bridge setup?

Config here.

# jan/18/2002 23:56:34 by RouterOS 7.1rc4
# software id = 
#
# model = RB4011iGS+
# serial number = 
/interface ethernet
set [ find default-name=ether2 ] disabled=yes
set [ find default-name=ether3 ] disabled=yes
set [ find default-name=ether4 ] disabled=yes
set [ find default-name=ether5 ] disabled=yes
set [ find default-name=ether6 ] disabled=yes
set [ find default-name=ether7 ] disabled=yes
set [ find default-name=ether8 ] disabled=yes
set [ find default-name=ether9 ] disabled=yes
set [ find default-name=ether10 ] disabled=yes poe-out=off
/interface vlan
add interface=sfp-sfpplus1 name="Guest Wifi" vlan-id=200
add interface=sfp-sfpplus1 name=IoT vlan-id=10
add interface=sfp-sfpplus1 name=VMs vlan-id=20
add interface=sfp-sfpplus1 name=Wifi vlan-id=7
/interface pppoe-client
add add-default-route=yes disabled=no interface=ether1 name=pppoe-out user=\
    no@no.no
/interface list
add name=WAN
add name=LAN
/interface lte apn
set [ find default=yes ] ip-type=ipv4
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip dhcp-server option
add code=42 name=NTPVMs value="'172.16.20.1'"
add code=42 name=NTPLAN value="'172.16.6.1'"
add code=42 name=NTPIoT value="'172.16.10.1'"
add code=42 name=NTPWifi value="'172.16.7.1'"
add code=42 name="NTPGuest Wifi" value="'172.16.200.1'"
/ip dhcp-server option sets
add name=Wifi options=NTPWifi
add name=LAN options=NTPLAN
add name=VMs options=NTPVMs
add name="Guest Wifi" options="NTPGuest Wifi"
add name=IoT options=NTPIoT
/ip ipsec profile
set [ find default=yes ] dh-group=modp2048 enc-algorithm=aes-128 \
    hash-algorithm=sha256
/ip ipsec proposal
set [ find default=yes ] auth-algorithms=sha256 enc-algorithms=\
    aes-256-gcm,aes-192-ctr,aes-192-gcm,aes-128-gcm lifetime=0s pfs-group=\
    modp2048
/ip pool
add name=IoT_pool ranges=172.16.10.100-172.16.10.254
add name=LAN_pool ranges=172.16.6.100-172.16.6.254
add name="Guest Wifi_pool" ranges=172.16.200.2-172.16.200.254
add name=VMs_pool ranges=172.16.20.100-172.16.20.254
add name=Wifi_pool ranges=172.16.7.100-172.16.7.254
/ip dhcp-server
add address-pool=IoT_pool dhcp-option-set=IoT interface=IoT lease-time=1w \
    name=IoT
add address-pool=LAN_pool dhcp-option-set=LAN interface=sfp-sfpplus1 \
    lease-time=1w name=LAN
add address-pool="Guest Wifi_pool" dhcp-option-set="Guest Wifi" interface=\
    "Guest Wifi" lease-time=1w name="Guest Wifi"
add address-pool=VMs_pool dhcp-option-set=VMs interface=VMs lease-time=1w \
    name=VMs
add address-pool=Wifi_pool dhcp-option-set=Wifi interface=Wifi lease-time=1w \
    name=Wifi
/queue simple
add burst-limit=2M/2M burst-threshold=2M/2M burst-time=10s/10s comment=\
    "Guest Wifi" limit-at=1M/1M max-limit=1M/1M name="Guest Wifi" priority=\
    6/6 queue=default/default target="Guest Wifi"
/routing bgp template
set default as=65530 disabled=no name=default output.network=bgp-networks
/routing table
add fib name=""
/system logging action
set 3 remote=172.16.6.2
add bsd-syslog=yes name=unRAID remote=172.16.6.2 src-address=172.16.6.1 \
    target=remote
/zerotier
set zt1 comment="ZeroTier Central controller - https://my.zerotier.com/" \
    identity="dd" name=zt1 \
    port=9993
/zerotier interface
add instance=zt1 mac-address=dd name=zerotier1 network=\
    dd
/ip neighbor discovery-settings
set discover-interface-list=none
/interface list member
add interface=ether1 list=WAN
add interface=sfp-sfpplus1 list=LAN
add interface="Guest Wifi" list=LAN
add interface=IoT list=LAN
add interface=VMs list=LAN
add interface=Wifi list=LAN
add interface=pppoe-out list=WAN
/ip address
add address=172.16.6.1/24 interface=sfp-sfpplus1 network=172.16.6.0
add address=172.16.7.1/24 interface=Wifi network=172.16.7.0
add address=172.16.10.1/24 interface=IoT network=172.16.10.0
add address=172.16.20.1/24 interface=VMs network=172.16.20.0
add address=172.16.200.1/24 interface="Guest Wifi" network=172.16.200.0
add address=192.168.254.253/24 interface=ether1 network=192.168.254.0
/ip cloud
set ddns-enabled=yes
/ip dhcp-server lease
add address=172.16.10.2 client-id=HeidiNightstand mac-address=\
    60:38:E0:F1:C8:71
add address=172.16.10.5 client-id=HueBridge mac-address=dd
add address=172.16.7.5 client-id=erx mac-address=04:18:D6:06:18:6F
add address=172.16.7.15 mac-address=70:2C:09:69:FF:88
add address=172.16.10.4 client-id=1:b0:be:76:46:b9:92 mac-address=\
    B0:BE:76:46:B9:92 server=IoT
add address=172.16.7.4 client-id=1:44:90:bb:5:c0:cd mac-address=\
    44:90:BB:05:C0:CD server=Wifi
add address=172.16.10.3 client-id=1:2c:aa:8e:d6:93:4c mac-address=\
    2C:AA:8E:D6:93:4C server=IoT
add address=172.16.7.3 client-id=1:dc:52:85:d4:15:9f mac-address=\
    DC:52:85:D4:15:9F server=Wifi
add address=172.16.20.3 client-id=1:52:54:0:c8:d0:49 mac-address=\
    52:54:00:C8:D0:49 server=VMs
add address=172.16.20.4 client-id=1:52:54:0:be:8c:1c mac-address=\
    52:54:00:BE:8C:1C server=VMs
/ip dhcp-server network
add address=172.16.6.0/24 dns-server=172.16.6.1 domain=mccloud.lan gateway=\
    172.16.6.1 netmask=24
add address=172.16.7.0/24 dns-server=172.16.7.1 domain=mccloud.lan gateway=\
    172.16.7.1
add address=172.16.10.0/24 dns-server=172.16.10.1 domain=mccloud.lan gateway=\
    172.16.10.1
add address=172.16.20.0/24 dns-server=172.16.20.1 domain=mccloud.lan gateway=\
    172.16.20.1
add address=172.16.200.0/24 dns-server=172.16.200.1 domain=mccloud.lan \
    gateway=172.16.200.1
/ip dns
set allow-remote-requests=yes servers=1.1.1.1,1.0.0.1
/ip dns static
add address=172.16.6.2 name=transmission.no.no
add address=172.16.6.2 name=unimus.no.no
add address=172.16.6.2 name=airsonic.no.no
add address=172.16.6.2 name=home.no.no
add address=172.16.6.2 name=jackett.no.no
add address=172.16.20.3 name=jenkins.no.no
add address=172.16.6.2 name=lidarr.no.no
add address=172.16.6.2 name=nzbget.no.no
add address=172.16.6.2 name=omada.no.no
add address=172.16.6.2 name=ombi.no.no
add address=172.16.6.2 name=paperless.no.no
add address=172.16.6.2 name=piwigo.no.no
add address=172.16.6.2 name=plex.no.no
add address=172.16.6.2 name=radarr.no.no
add address=172.16.6.2 name=sonarr.no.no
add address=172.16.6.2 name=speedtest.no.no
add address=172.16.6.2 name=subversion.no.no
add address=172.16.6.2 name=syncthing.no.no
add address=172.16.6.2 name=tautulli.no.no
add address=172.16.6.2 name=tdarr.no.no
add address=172.16.20.3 name=jumpbox
add address=172.16.6.2 name=bb-8
add address=172.16.20.3 name=jumpbox.mccloud.lan
add address=172.16.6.2 name=bb-8.mccloud.lan
/ip firewall filter
add action=accept chain=forward connection-state=\
    established,related,untracked
add action=accept chain=input connection-state=established,related,untracked
add action=accept chain=output connection-state=established,related,untracked
add action=accept chain=input connection-state=established,related
add action=drop chain=input connection-state=invalid
add action=accept chain=input in-interface=pppoe-out protocol=icmp
add action=drop chain=input in-interface=pppoe-out
add action=accept chain=forward comment=\
    "defconf: accept established,related, untracked" connection-state=\
    established,related,untracked
add action=drop chain=forward comment="defconf: drop invalid" \
    connection-state=invalid
add action=accept chain=forward comment=\
    "defconf: accept established,related, untracked" connection-state=\
    established,related,untracked
add action=drop chain=forward comment="defconf: drop invalid" \
    connection-state=invalid
add action=drop chain=forward comment="defconf: drop invalid" \
    connection-state=invalid
add action=drop chain=forward comment=\
    "defconf: drop all from WAN not DSTNATed" connection-nat-state=!dstnat \
    connection-state=new in-interface-list=WAN
add action=drop chain=forward in-interface="Guest Wifi" out-interface=IoT
add action=drop chain=forward in-interface="Guest Wifi" out-interface=VMs
add action=drop chain=forward in-interface="Guest Wifi" out-interface=Wifi
add action=drop chain=forward in-interface="Guest Wifi" out-interface=\
    sfp-sfpplus1
add action=drop chain=forward in-interface=IoT out-interface="Guest Wifi"
add action=drop chain=forward in-interface=VMs out-interface="Guest Wifi"
add action=drop chain=forward in-interface=Wifi out-interface="Guest Wifi"
add action=drop chain=forward in-interface=sfp-sfpplus1 out-interface=\
    "Guest Wifi"
add action=fasttrack-connection chain=forward connection-mark=!ipsec \
    connection-state=established,related hw-offload=yes
add action=fasttrack-connection chain=input connection-mark=!ipsec \
    connection-state=established,related hw-offload=yes
add action=fasttrack-connection chain=output connection-mark=!ipsec \
    connection-state=established,related hw-offload=yes
add action=fasttrack-connection chain=forward comment="defconf: fasttrack" \
    connection-mark=!ipsec connection-state=established,related hw-offload=\
    yes
add action=accept chain=forward comment="all from WAN DSTNATed" \
    connection-nat-state=dstnat connection-state=new in-interface-list=WAN
/ip firewall nat
add action=dst-nat chain=dstnat comment=SSH in-interface=pppoe-out port=no \
    protocol=tcp to-addresses=172.16.6.2 to-ports=no
add action=dst-nat chain=dstnat comment=HTTP in-interface=pppoe-out port=no \
    protocol=tcp to-addresses=172.16.6.2 to-ports=no
add action=dst-nat chain=dstnat comment=HTTPS in-interface=pppoe-out port=no \
    protocol=tcp to-addresses=172.16.6.2 to-ports=no
add action=dst-nat chain=dstnat comment=RDP in-interface=pppoe-out port=no \
    protocol=tcp to-addresses=172.16.20.3 to-ports=no
add action=dst-nat chain=dstnat comment=RDP in-interface=pppoe-out port=no \
    protocol=udp to-addresses=172.16.20.3 to-ports=no
add action=dst-nat chain=dstnat comment=Plex in-interface=pppoe-out port=\
    no protocol=tcp to-addresses=172.16.6.2 to-ports=no
add action=dst-nat chain=dstnat comment=Syncthing in-interface=pppoe-out \
    port=no protocol=tcp to-addresses=172.16.6.2 to-ports=no
add action=dst-nat chain=dstnat comment=Syncthing port=no protocol=udp \
    to-addresses=172.16.6.2 to-ports=no
add action=dst-nat chain=dstnat comment=Transmission dst-address=172.16.6.2 \
    in-interface=pppoe-out port=no protocol=tcp to-addresses=172.16.6.2 \
    to-ports=no
add action=dst-nat chain=dstnat comment=Transmission dst-address=172.16.6.2 \
    in-interface=pppoe-out port=no protocol=udp to-addresses=172.16.6.2 \
    to-ports=no
add action=dst-nat chain=dstnat comment="Resilio Sync" in-interface=pppoe-out \
    port=no protocol=tcp to-addresses=172.16.6.2 to-ports=no
add action=dst-nat chain=dstnat comment="Resilio Sync" in-interface=pppoe-out \
    port=no protocol=tcp to-addresses=172.16.6.2 to-ports=no
add action=dst-nat chain=dstnat comment="Resilio Sync" in-interface=pppoe-out \
    port=no protocol=udp to-addresses=172.16.6.2 to-ports=no
add action=masquerade chain=srcnat comment="nat to modem" dst-address=\
    192.168.254.254 out-interface=ether1
add action=masquerade chain=srcnat comment=Masquerade out-interface-list=WAN
/ip service
set telnet disabled=yes
set ftp disabled=yes
set www disabled=yes
set www-ssl certificate=router disabled=no tls-version=only-1.2
set api disabled=yes
set api-ssl certificate=router tls-version=only-1.2
/ip ssh
set strong-crypto=yes
/ipv6 nd
set [ find default=yes ] advertise-dns=no
/snmp
set contact=smccloud@no.no enabled=yes location="Mechanical  Room"
/system clock
set time-zone-name=America/Chicah
/system identity
set name=RB4011iGS+RM
/system logging
add action=remote topics=critical
add action=remote topics=error
add action=remote topics=info
add action=remote topics=warning
/system ntp client
set enabled=yes mode=multicast
/system ntp server
set enabled=yes manycast=yes multicast=yes
/system ntp client servers
add address=128.101.101.101
add address=134.84.84.84
/system package update
set channel=development
/system resource irq rps
set sfp-sfpplus1 disabled=no
/tool bandwidth-server
set authenticate=no enabled=no
/tool mac-server
set allowed-interface-list=none
/tool mac-server mac-winbox
set allowed-interface-list=none
/tool mac-server ping
set enabled=no
2

Firewall and NAT aren’t Perfect (Duplicates, sequence, etc..)
But shouldn’t make a big difference!


How Slow is your Routing ?
Copying a File from one PC to another ?
Less then 100 MBytes/s ?
During the Transfer, how high is you CPU-Usage on the Mirkotik ?

3

I get the same speed if I stay on the same VLAN or go between them in file transfers. Between VLANs I get around 25% CPU usage on the RB4011. The main slowdown I see is accessing services on my home server from Wifi across VLANs. i.e. my home-automation takes forever to connect when it doesn’t from my desktop.

What would you recommend to fix the Firewall & NAT?

4

I marked the duplicates in “Bold”

/ip firewall filter
add action=accept chain=forward connection-state=established,related,untracked
add action=accept chain=input connection-state=established,related,untracked
add action=accept chain=output connection-state=established,related,untracked
add action=accept chain=input connection-state=established,related
add action=drop chain=input connection-state=invalid
add action=accept chain=input in-interface=pppoe-out protocol=icmp
add action=drop chain=input in-interface=pppoe-out
add action=accept chain=forward comment=“defconf: accept established,related, untracked” connection-state=established,related,untracked
add action=drop chain=forward comment=“defconf: drop invalid” connection-state=invalid
add action=accept chain=forward comment=“defconf: accept established,related, untracked” connection-state=established,related,untracked
add action=drop chain=forward comment=“defconf: drop invalid” connection-state=invalid
add action=drop chain=forward comment=“defconf: drop invalid” connection-state=invalid
add action=drop chain=forward comment=“defconf: drop all from WAN not DSTNATed” connection-nat-state=!dstnat connection-state=new in-interface-list=WAN
add action=drop chain=forward in-interface=“Guest Wifi” out-interface=IoT
add action=drop chain=forward in-interface=“Guest Wifi” out-interface=VMs
add action=drop chain=forward in-interface=“Guest Wifi” out-interface=Wifi
add action=drop chain=forward in-interface=“Guest Wifi” out-interface=sfp-sfpplus1
add action=drop chain=forward in-interface=IoT out-interface=“Guest Wifi”
add action=drop chain=forward in-interface=VMs out-interface=“Guest Wifi”
add action=drop chain=forward in-interface=Wifi out-interface=“Guest Wifi”
add action=drop chain=forward in-interface=sfp-sfpplus1 out-interface=“Guest Wifi”
add action=fasttrack-connection chain=forward connection-mark=!ipsec connection-state=established,related hw-offload=yes
add action=fasttrack-connection chain=input connection-mark=!ipsec connection-state=established,related hw-offload=yes
add action=fasttrack-connection chain=output connection-mark=!ipsec connection-state=established,related hw-offload=yes
add action=fasttrack-connection chain=forward comment=“defconf: fasttrack” connection-mark=!ipsec connection-state=established,related hw-offload=yes
add action=accept chain=forward comment=“all from WAN DSTNATed” connection-nat-state=dstnat connection-state=new in-interface-list=WAN

5

So if i understand you correctly ,
Inter-VLAN Routing is only Slow,
when a Device connected to a Wireless-Access Point, communicated with another VLAN ?

Are your Wireless-AccessPoints Mikrotik ?

6

It’s between any VLAN, but wireless is what I notice the most. My access point is not a MikroTik, it’s a TP-Link Omada series. Worked fine on pfSense, so I’m 99% certain it’s a mistake I made.

7

I didnt look indepth but a shallow looks everything seems to be in order, for at least what I an understand…
Did you try changing this to the sfp+ interface…

/ip neighbor discovery-settings
set discover-interface-list=none

First time Ive ever seen this rule suggest you remove it
add action=accept chain=output connection-state=established,related,untracked

Your firewall rules can be severely improved in terms of efficiency but likely not the problem.


Finally I dont understand any of your DESTINATION NAT Rules as they have NO TO PORTS ???

8

They do, just don’t want everyone to know the from and to ports.

9

haha okay next time just put in sample ports, because its going to confuse the heck out of folks.

10

Given that this is a home setup, am I just making my life too difficult with VLANs? i.e. should I just flatten my network out and not care about it as much?

11

That’s a question, only you can answer ! =)
The Mikrotik RB4011 is more than capable to handel Multi-VLAN Routing, Firewall, QoS, etc…

If you don’t have many devices and don’t need “separation” between Devices and/ot Networks
There is nothing wrong not having VLAN’s in your Home-Network

12

Personally if you setup your router using this fine article you wouldnt be having any of your issues
Hint take the subnet and put it on a vlan like the other vlans
Put your ports on a bridge
and use the reference.

http://forum.mikrotik.com/t/using-routeros-to-vlan-your-network/126489/1

13

Does the SFP+ port on a RB4011 work correctly on a bridge?

14

Yes, no reason why it shouldnt, if it works now with your modules and connections it will work fine on the bridge.

15

Will a bridge help performance at all though?

16

Surprised no ones picked up on this yet

In order for you to go to the same VLAN, this means you either have a switch behind one of your ports, or if both devices were connected to the RB4011 you’d need a bridge setup
I’m going to assume the former, you have a switch you are sending traffic through. This means traffic doesn’t even go through the 4011 hence its not at fault (check your port speeds, maybe a port is running at 100mbit? or even 10mbit?)

17

My switch is a CSS326-24G-2S±RM, no routing in it. It has a single 10G trunk to the RB4011 with all the VLANs on it.

18

I’m not sure that your router is actually being slow at inter-VLAN routing. Have you actually done throughput tests with iPerf?

You say it is slow because accessing your server is slow from another VLAN, but maybe there is another reason for this. Perhaps it is a wrong firewall rule that is blocking traffic and it fails over.

Also, do not look at the overall CPU performance, look at the performance per core under System->Resources->CPU. You can have a bottleneck if one core is maxed out, and 25% CPU usage could be from one core maxed out. If this is the case you can use Tools->Profile to see what the cause is.

How much bandwidth are you sending over this? i.e. when you are doing things that cause the 25% CPU usage, what is the traffic going through the interfaces at that time?

19

The initial connect is super slow. Once it’s done, performance is ok but sometimes unstable.

20

Again, there are multiple reasons this could happen - from the scant info you have provided, this issue could be caused by anything. Please answer the other questions (and try the other suggestions) from my previous post.