RB3011 and CRS326 VLAN Trunk

petrz · July 31, 2023, 8:48pm

Hi.

I am trying to make work the following setup. RB3011 as the main router with CAPsMAN, CRS326 as the main switch + Dell 2824 for a second part + a few cAP ac, one connected to the RB3011, and others to the Dell switch. With some VLANs. I was able to correctly set up CAPsMAN with cAP connected to the RB3011 with 4 VLANs, I was able to use the VLAN trunk when my TrueNAS server was connected directly to the RB3011. But I can’t make it work with CRS326 and trunk. In this question, I would like to focus only on RB3011, CRS326, and the VLAN trunk between them, mainly VLAN 19 for a management. I tried to follow examples for both devices and this is the current config - not working, can’t ping each other. I checked many articles and watched many videos about different kinds of VLAN settings in ROS, but still don’t know where I have a mistake. Could someone please help me? I also tried to disable STP. Thanks in advance for any suggestions.

# 2023-07-31 21:55:16 by RouterOS 7.10.2
# software id = 6M6T-2GHR
#
# model = RB3011UiAS
# serial number = xxx

# Comments from https://wiki.mikrotik.com/wiki/Manual:Basic_VLAN_switching#Other_devices_with_built-in_switch_chip
 

# /interface bridge
# add name=bridge1

/interface bridge
add ingress-filtering=no name=br-sw2-vlan protocol-mode=none vlan-filtering=yes


# /interface bridge port
# add bridge=bridge1 interface=ether1 hw=yes
# add bridge=bridge1 interface=ether2 hw=yes
# add bridge=bridge1 interface=ether3 hw=yes

/interface bridge port
add bridge=br-sw2-vlan interface=ether8-dell-ladi-eth24
add bridge=br-sw2-vlan interface=ether7-crs-eth24
add bridge=br-sw2-vlan interface=ether10-lab-cap


# /interface ethernet switch vlan
# add ports=ether1,ether2 switch=switch1 vlan-id=20
# add ports=ether1,ether3 switch=switch1 vlan-id=30
# add ports=ether1,switch1-cpu switch=switch1 vlan-id=99

/interface ethernet switch vlan
add comment=crs-vlan10-lab-lan independent-learning=yes ports=ether7-crs-eth24 switch=sw2-vlan vlan-id=10
add comment=crs-vlan13-lab-srv independent-learning=yes ports=ether7-crs-eth24 switch=sw2-vlan vlan-id=13
add comment=crs-vlan14-lab-virt independent-learning=yes ports=ether7-crs-eth24 switch=sw2-vlan vlan-id=14
add comment=crs-vlan15-lab-jails independent-learning=yes ports=ether7-crs-eth24 switch=sw2-vlan vlan-id=15
add comment=crs-vlan16-lab-ent independent-learning=yes ports=ether7-crs-eth24 switch=sw2-vlan vlan-id=16
add comment=crs-vlan17-lab-dmz independent-learning=yes ports=ether7-crs-eth24 switch=sw2-vlan vlan-id=17
add comment=crs-vlan18-lab-fac independent-learning=yes ports=ether7-crs-eth24 switch=sw2-vlan vlan-id=18
add independent-learning=yes ports=ether7-crs-eth24,ether8-dell-eth24,ether10-lab-cap,sw2-vlan-cpu switch=sw2-vlan vlan-id=19


# /interface vlan
# add interface=bridge1 vlan-id=99 name=MGMT

/interface vlan
add interface=br-sw2-vlan name=vlan10-lab-lan vlan-id=10
add interface=br-sw2-vlan name=vlan11-lab-wifi vlan-id=11
add interface=br-sw2-vlan name=vlan12-lab-guest vlan-id=12
add interface=br-sw2-vlan name=vlan13-lab-srv vlan-id=13
add interface=br-sw2-vlan name=vlan14-lab-virt vlan-id=14
add interface=br-sw2-vlan name=vlan15-lab-jail vlan-id=15
add interface=br-sw2-vlan name=vlan16-lab-ent vlan-id=16
add interface=br-sw2-vlan name=vlan17-lab-dmz vlan-id=17
add interface=br-sw2-vlan name=vlan18-lab-fac vlan-id=18
add interface=br-sw2-vlan name=vlan19-lab-mngt vlan-id=19

# /ip address
# add address=192.168.99.1/24 interface=MGMT

/ip address
add address=192.168.10.1/24 interface=vlan10-lab-lan network=192.168.10.0
add address=192.168.11.1/24 interface=vlan11-lab-wifi network=192.168.11.0
add address=192.168.12.1/24 interface=vlan12-lab-guest network=192.168.12.0
add address=192.168.13.1/24 interface=vlan13-lab-srv network=192.168.13.0
add address=192.168.14.1/24 interface=vlan14-lab-virt network=192.168.14.0
add address=192.168.15.1/24 interface=vlan15-lab-jail network=192.168.15.0
add address=192.168.16.1/24 interface=vlan16-lab-ent network=192.168.16.0
add address=192.168.17.1/24 interface=vlan17-lab-dmz network=192.168.17.0
add address=192.168.18.1/24 interface=vlan18-lab-fac network=192.168.18.0
add address=192.168.19.1/24 interface=vlan19-lab-mngt network=192.168.19.0


# /interface ethernet switch port
# set ether1 vlan-mode=secure vlan-header=add-if-missing
# set ether2 vlan-mode=secure vlan-header=always-strip default-vlan-id=20
# set ether3 vlan-mode=secure vlan-header=always-strip default-vlan-id=30
# set switch1-cpu vlan-header=leave-as-is vlan-mode=secure

/interface ethernet switch port
;; eth7 - to the crs eth24
set 6 vlan-header=add-if-missing vlan-mode=secure
;; eth8 - to the dell 2824 eth24
set 7 vlan-header=add-if-missing vlan-mode=secure
;;eth 10 - to the cAP ac
set 9 vlan-header=add-if-missing vlan-mode=secure
;; sw2-vlan-cpu
set 11 vlan-mode=secure


/ip pool
add name=pool-lab-lan ranges=192.168.10.100-192.168.10.200
add name=pool-lab-wifi ranges=192.168.11.100-192.168.11.200
add name=pool-lab-guest ranges=192.168.12.100-192.168.12.200
add name=pool-lab-srv ranges=192.168.13.100-192.168.13.200
add name=pool-lab-virt ranges=192.168.14.100-192.168.14.200
add name=pool-lab-jail ranges=192.168.15.100-192.168.15.200
add name=pool-lab-ent ranges=192.168.16.100-192.168.16.200
add name=pool-lab-dmz ranges=192.168.17.100-192.168.17.200
add name=pool-lab-fac ranges=192.168.18.100-192.168.18.200
add name=pool-lab-mngt ranges=192.168.19.100-192.168.19.200

/ip dhcp-server
add address-pool=pool-lab-lan interface=vlan10-lab-lan name=dhcp-lab-lan
add address-pool=pool-lab-wifi interface=vlan11-lab-wifi name=dhcp-lab-wifi
add address-pool=pool-lab-guest interface=vlan12-lab-guest name=dhcp-lab-guest
add address-pool=pool-lab-srv interface=vlan13-lab-srv name=dhcp-lab-srv
add address-pool=pool-lab-virt interface=vlan14-lab-virt name=dhcp-lab-virt
add address-pool=pool-lab-jail interface=vlan15-lab-jail name=dhcp-lab-jail
add address-pool=pool-lab-ent interface=vlan16-lab-ent name=dhcp-lab-ent
add address-pool=pool-lab-dmz interface=vlan17-lab-dmz name=dhcp-lab-dmz
add address-pool=pool-lab-fac interface=vlan18-lab-fac name=dhcp-lab-fac
add address-pool=pool-lab-mngt interface=vlan19-lab-mngt name=dhcp-lab-mngt

/ip dhcp-server network
add address=192.168.10.0/24 comment=lab-lan dns-server=192.168.10.1 domain=my.tld gateway=192.168.10.1 ntp-server=192.168.10.1
add address=192.168.11.0/24 comment=lab-wifi dns-server=192.168.11.1 domain=my.tld gateway=192.168.11.1 ntp-server=192.168.11.1
add address=192.168.12.0/24 comment=lab-guest dns-server=8.8.8.8 domain=my.tld gateway=192.168.12.1 ntp-server=192.168.12.1
add address=192.168.13.0/24 comment=lab-srv dns-server=192.168.13.1 domain=my.tld gateway=192.168.13.1 ntp-server=192.168.13.1
add address=192.168.14.0/24 comment=lab-virt dns-server=192.168.14.1 domain=my.tld gateway=192.168.14.1 ntp-server=192.168.14.1
add address=192.168.15.0/24 comment=lab-jail dns-server=192.168.15.1 domain=my.tld gateway=192.168.15.1 ntp-server=192.168.15.1
add address=192.168.16.0/24 comment=lab-ent dns-server=192.168.16.1 domain=my.tld gateway=192.168.16.1 ntp-server=192.168.16.1
add address=192.168.17.0/24 comment=lab-dmz dns-server=192.168.17.1 domain=my.tld gateway=192.168.17.1 ntp-server=192.168.17.1
add address=192.168.18.0/24 comment=lab-fac dns-server=192.168.18.1 domain=my.tld gateway=192.168.18.1 ntp-server=192.168.18.1
add address=192.168.19.0/24 comment=lab-mngt dns-server=192.168.19.1 domain=my.tld gateway=192.168.19.1 ntp-server=192.168.19.1



# 2023-07-31 22:19:46 by RouterOS 7.10.2
# software id = HMM3-NKTK
#
# model = CRS326-24G-2S+
# serial number = xxx

# Comments from https://wiki.mikrotik.com/wiki/Manual:Basic_VLAN_switching#CRS3xx_series_switches

# /interface bridge
# set bridge1 vlan-filtering=yes

# /interface bridge
# add name=bridge1

/interface bridge
add name=br-crs protocol-mode=none vlan-filtering=yes

# /interface bridge port
# add bridge=bridge1 interface=ether1 hw=yes
# add bridge=bridge1 interface=ether2 hw=yes pvid=20
# add bridge=bridge1 interface=ether3 hw=yes pvid=30

/interface bridge port
add bridge=br-crs ingress-filtering=no interface=ether21-nas-trunk-bxe2
add bridge=br-crs ingress-filtering=no interface=ether22-nas-ilo pvid=19
add bridge=br-crs ingress-filtering=no interface=ether24-rb3011-eth7


# /interface bridge vlan
# add bridge=bridge1 tagged=ether1 untagged=ether2 vlan-ids=20
# add bridge=bridge1 tagged=ether1 untagged=ether3 vlan-ids=30
# add bridge=bridge1 tagged=ether1,bridge1 vlan-ids=99

/interface bridge vlan
add bridge=br-crs tagged=ether24-rb3011-eth7,br-crs untagged=ether22-nas-ilo vlan-ids=19


# /interface vlan
# add interface=bridge1 vlan-id=99 name=MGMT

/interface vlan
add interface=br-crs name=vlan-19-lab-mngt vlan-id=19


# /ip address
# add address=192.168.99.1/24 interface=MGMT

/ip address
add address=192.168.19.2/24 interface=vlan-19-lab-mngt network=192.168.19.0

petrz · August 1, 2023, 1:36am

Based on the config of somebody else I can ping each other using 192.168.19.0/24 when I do the following:
/interface/ethernet/switch/port - set header to leave-as-is
and

/interface/bridge/vlan/add bridge=br-sw2-vlan tagged=br-sw2-vlan,ether7-crs-eth24,vlan19-lab-mngt vlan-ids=19

The same way I made it work for vlan15-jails. RB3011 is running DHCP for VLANs, CRS is connected to RB3011, the TrueNAS server is connected to CRS and I can assign there a BSD jail to the vlan15-jails and it gets an IP correctly.

But I am afraid I am mixing different methods of ROS VLAN settings together. I would really appreciate some explanation, I am really lost in ROS VLAN methods.

petrz · August 9, 2023, 5:04pm

After some more study, I guess the main problem in the original config above was “vlan-filtering=yes” in the bridge, which enables the bridge VLAN filtering method, while the rest of the configuration used the switch chip VLAN filtering method. So the solution is disabling bridge VLAN filtering in this case.

Because of the wrong way mentioned in the post above (adding VLAN ID into the bridge) worked - it added a missing part for the bridge VLAN filtering method, while the switch chip method wasn’t used. So I was right that I was mixing two methods. Just to be sure - is that correct?

holvoetn · August 9, 2023, 5:10pm

Mixing bridge and switch VLAN settings is generally a bad idea if you are not experienced enough yet

Usually bridge approach is used by most.

anav · August 9, 2023, 7:59pm

IF you ditch capsman I will help.

petrz · August 10, 2023, 9:22pm

I am not experienced enough and exactly because of that I mixed it “accidentally”. Now it’s slightly better (still not enough) so that I know which configuration directives belong to which method.

I tried to follow the recommendations on Mikrotik’s WiFi. If I am not wrong, there were listed methods for each type of device - CRS1xx/2xx, CRS3xx, devices with the switch chip(s), and devices without.

petrz · August 10, 2023, 10:19pm

Thank you, @anav.
I was “fighting” with CAPsMAN and VLANs the last two evenings. I had a few SSIDs and VLANs configured, IPs, pools, networks, DHCP servers, VLANs, right bridges for local forwarding, etc. But only one SSID worked. Others established a connection, but no IP was assigned. I checked the config many times, they seemed OK. I exported them to the text files and I was comparing side by side by vimdiff parts for working SSID and not working. It looked exactly the same, just different IDs, subnets, and names. On the cAP there were correctly added interfaces in the bride by CAPsMAN, etc. I tried to remove all parts related to one not working SSID and added them again using the working one as a template. Nothing. I even removed almost all configurations for everything else, leaving just the bare minimum needed for that two SSIDs - working one and not working one. Nothing. Many hours, many cigarettes, lot of reading own config, lot of reading manuals, forums, etc. Nothing. I added on the cAP VLAN interfaces and DHCP clients for them to check if they will get IPs from the router. Again - only for that one VLAN. I had some parts configured as VLAN ID ranges. I asked myself if there could be a bug, so it gets only the last integer. Only VLAN with the highest ID worked. So I rewrote it using a separate line for each VLAN. I tried “everything”, but nothing worked. And then I realized that I have the DELL switch between RB3011 and cAP ac which I used for a test. And on the DELL switch, I didn’t have all VLANs configured, just a few I used for the previous test. I added missing VLANs to the DELL switch and it started to work. So I had all the configs in RB and the cAP the whole time correctly, but it didn’t work just because of this stupid mistake. I had those devices in another room, so during the remote work, it wasn’t so obvious. I guess I understand better now why Mikrotik is selling also the tabletop rack. I am almost sure I would realize that earlier if I would have it physically in my sight in front of me. But at least I will remember this experience quite well. Now I am going to re-add all the parts I removed yesterday and I am almost sure it will work.

Only one question, probably for somebody else as you are not the biggest fan of CAPsMAN, AFAIK. When CAPsMAN is used, it’s adding VLANs/wireless interfaces on the cAP only for bridged VLAN config, right? Or is CAPsMAN smart enough (or able to set) to determine that the switch chip VLANs method is used on the cAP and add VLANs/wireless interfaces into the switch chip mode config? I have it working using a bridged method, but I am curious.

And when I will finish this setup, I am going to buy a few more cAP XL ac and cAP ax to play a bit with them, together with RB5009 I already have, with the CAPsMAN v3 and wifiwave2. Is it still 802.11ac/ax only or Mikrotik guys were able to make b/g work also in wifiwave2?

Btw, I successfully used “the loop hack” for VLAN mismatch to put the DELL switch with VLAN 1 IP only into desired MGMT VLAN. So I hopefully solved all issues I had, with the help of the community. Thank you all. Now it will be just a boring part with adding all that VLANs, subnets, etc. As I am lazy, I used a bash script to prepare the configurations.

holvoetn · August 11, 2023, 5:37am

2 parts here:
Capsman controller: your choice how you set it up, switch / bridge, both can be used. For the normal functioning of the controller as a ROS-device using VLAN, that is.
caps: I never saw a config with switch chip method. Only bridge method. The whole point about capsman is that you actually don’t have to do anything on that caps apart from making sure in the config on the controller part, the datapath settings are set properly.
And then it will “automagically” work on the caps. Both for legacy and wifiwave2 it works like that.
Hybrid APs (devices having wifi and ethernet ports), I’m still figuring out myself how to tackle that most elegantly.

Careful there. Don’t mix legacy and wifiwave2 capsman/devices because it will NOT work.
If you choose to do so, that will be 2 separate environments with 2 different controllers (on possibly the same LAN but why would you want to do that ?).

re: b/g/n
From wifiwave2 wiki:

band (2ghz-g | 2ghz-n | 2ghz-ax | 5ghz-a | 5ghz-ac | 5ghz-an | 5ghz-ax)