Multiple bridges for multiple switch chips on CCR2004-16G-2S+?

RouterOS Beginner Basics
1

I read that since the CCR2004-16G-2S+ has multiple switch chips, for full hardware offloading, you need to create a separate bridge for each switch chip and then use an ethernet cable to join the bridges, so that’s what I did and things are working and the CPU never goes about 3%. But I have to do everything twice whenever I add a new VLAN and want to ensure it’s available on any port on the router. I wanted to check with the community if I’m making this harder on myself than it needs to be.

Relatedly, I recently tried to setup a firewall rule to isolate one VLAN from the other, and in the interface list, I saw both bridges and wasn’t sure if I needed to create an interface list for them, or if it was just enough to add a single bridge, or if I should just be using the subnet of the VLAN in the rule instead. Since I’m creating two bridges per VLAN, I’m going to end up with a log of bridges and a lot of manual work to keep them in sync. Out of curiosity, I moved one of the bridge ports (ether16 I believe) from bridge2 to bridge1, and saw the moved port kept its H indicator for hardware offloading, which doesn’t make sense since the port isn’t connected to bridge1’s switch chip, and so my understanding must be wrong somewhere.

# 2025-03-14 00:08:41 by RouterOS 7.18.1
# software id = KN49-Z4RX
#
# model = CCR2004-16G-2S+
/interface bridge
add name=bridge1 vlan-filtering=yes
add name=bridge2 vlan-filtering=yes
/interface ethernet
set [ find default-name=ether1 ] comment=U6-Mesh_Garage
set [ find default-name=ether2 ] comment=U6-Mesh_Basement rx-flow-control=on \
    tx-flow-control=on
set [ find default-name=ether3 ] comment=U6-Mesh_Backyard
set [ find default-name=ether4 ] comment=Lenovo
set [ find default-name=ether5 ] comment="Work Switch"
set [ find default-name=ether6 ] comment="U6-Enterprise Office"
set [ find default-name=ether7 ] comment="Unfi GCM"
set [ find default-name=ether8 ] comment=bridge1-to-bridge2 rx-flow-control=\
    on tx-flow-control=on
set [ find default-name=ether9 ] comment=bridge2-to-bridge1
set [ find default-name=ether10 ] comment="Floodlight Camera"
set [ find default-name=ether11 ] comment="Doorbell Camera"
set [ find default-name=ether13 ] comment=Lutron
set [ find default-name=ether15 ] comment=CCR2004-mgmt
set [ find default-name=sfp-sfpplus1 ] comment="Aggregation Switch"
set [ find default-name=sfp-sfpplus2 ] comment="Comcast Uplink" l2mtu=9192 \
    mtu=9192
/interface wireguard
add listen-port=13231 mtu=1420 name=wireguard1
/interface vlan
add interface=bridge1 name=vlan-IoT-bridge1 vlan-id=100
add interface=bridge2 name=vlan-IoT-bridge2 vlan-id=100
add interface=bridge1 name=vlan-Mgmt-bridge1 vlan-id=99
add interface=bridge2 name=vlan-Mgmt-bridge2 vlan-id=99
add interface=bridge1 name=vlan-Trusted-bridge1 vlan-id=86
add interface=bridge2 name=vlan-Trusted-bridge2 vlan-id=86
add interface=bridge1 name=vlan-Unifi-bridge1 vlan-id=98
add interface=bridge2 name=vlan-Unifi-bridge2 vlan-id=98
add interface=bridge1 name=vlan-Work-bridge1 vlan-id=200
add interface=bridge2 name=vlan-Work-bridge2 vlan-id=200
add interface=bridge1 name=vlan10-cgm vlan-id=10
/interface list
add name=WAN
add name=LAN
add name=TRUNK
add name=AP
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip dhcp-server option
add code=6 name="Google  DNS" value="'8.8.8.8'"
/ip pool
add name=dhcp_pool-Trusted ranges=192.168.86.10-192.168.86.254
add name=dhcp_pool-Work ranges=192.168.200.10-192.168.200.254
add name=dhcp_pool-IoT ranges=172.16.100.10-172.16.100.254
add name=dhcp_pool-Unifi ranges=192.168.98.10-192.168.98.254
add name=dhcp_pool9 ranges=192.168.10.2-192.168.10.254
/ip dhcp-server
add address-pool=dhcp_pool-Trusted interface=vlan-Trusted-bridge1 name=\
    dhcp-Trusted
add address-pool=dhcp_pool-Work interface=vlan-Work-bridge1 name=dhcp-Work
add address-pool=dhcp_pool-IoT interface=vlan-IoT-bridge1 name=dhcp-IoT
add address-pool=dhcp_pool-Unifi interface=vlan-Unifi-bridge1 name=dhcp-Unifi
add address-pool=dhcp_pool9 interface=vlan10-cgm name=dhcp1
/port
set 0 name=serial0
/snmp community
add addresses=::/0 name=public_v2
/interface bridge port
add bridge=bridge1 comment="U6-Mesh Garage" interface=ether1 pvid=98
add bridge=bridge1 comment="U6-Mesh Basement" interface=ether2 pvid=98
add bridge=bridge1 comment="U6-Mesh Backyard" interface=ether3 pvid=98
add bridge=bridge1 comment=Lenovo interface=ether4 pvid=86
add bridge=bridge1 comment="Work Switch" frame-types=\
    admit-only-untagged-and-priority-tagged interface=ether5 pvid=200
add bridge=bridge1 comment="U6-Enterprise Office" interface=ether6 pvid=99
add bridge=bridge1 comment="Unifi CGM" interface=ether7 pvid=10
add bridge=bridge1 comment="bridge1 to bridge 2" interface=ether8
add bridge=bridge2 comment="bridge2 to bridge 1" interface=ether9
add bridge=bridge2 comment="Floodlight Camera" frame-types=\
    admit-only-untagged-and-priority-tagged interface=ether10 pvid=100
add bridge=bridge2 comment="Doorbell Camera" frame-types=\
    admit-only-untagged-and-priority-tagged interface=ether11 pvid=100
add bridge=bridge2 frame-types=admit-only-untagged-and-priority-tagged \
    interface=ether12
add bridge=bridge2 comment=Lutron frame-types=\
    admit-only-untagged-and-priority-tagged interface=ether13 pvid=100
add bridge=bridge2 interface=ether14
add bridge=bridge2 interface=ether16 pvid=86
add bridge=bridge1 interface=sfp-sfpplus1
/ip neighbor discovery-settings
set discover-interface-list=all
/interface bridge vlan
add bridge=bridge1 tagged=bridge1,TRUNK,ether8,AP untagged=ether16 vlan-ids=\
    86
add bridge=bridge1 tagged=bridge1,TRUNK,ether8,ether9,AP untagged=\
    ether10,ether11 vlan-ids=100
add bridge=bridge1 tagged=ether6,ether1,bridge1 untagged=ether5 vlan-ids=200
add bridge=bridge1 tagged=bridge1,TRUNK,ether6 vlan-ids=99
add bridge=bridge2 tagged=bridge2,ether9 vlan-ids=99
add bridge=bridge2 tagged=bridge2,ether9,AP,ether6 untagged=ether16 vlan-ids=\
    86
add bridge=bridge2 tagged=bridge2,ether9,ether16 untagged=ether13 vlan-ids=\
    100
add bridge=bridge2 tagged=bridge2 vlan-ids=200
add bridge=bridge1 tagged=bridge1,TRUNK,ether8 untagged=AP,ether6 vlan-ids=98
add bridge=bridge2 tagged=bridge2,TRUNK,ether9 vlan-ids=98
add bridge=bridge1 tagged=bridge1 untagged=ether7 vlan-ids=10
/interface detect-internet
set detect-interface-list=WAN
/interface list member
add interface=bridge1 list=LAN
add interface=wireguard1 list=LAN
add interface=ether8 list=TRUNK
add interface=ether9 list=TRUNK
add interface=sfp-sfpplus1 list=TRUNK
add interface=sfp-sfpplus2 list=WAN
add interface=ether1 list=AP
add interface=ether2 list=AP
add interface=ether3 list=AP
add interface=ether6 list=AP
/ip address
add address=192.168.15.1/24 comment=wireguard interface=wireguard1 network=\
    192.168.15.0
add address=10.0.99.1/24 comment=management interface=ether15 network=\
    10.0.99.0
add address=192.168.86.1/24 interface=vlan-Trusted-bridge1 network=\
    192.168.86.0
add address=172.16.100.1/24 interface=vlan-IoT-bridge1 network=172.16.100.0
add address=192.168.200.1/24 interface=vlan-Work-bridge1 network=\
    192.168.200.0
add address=192.168.99.1/24 interface=vlan-Mgmt-bridge1 network=192.168.99.0
add address=192.168.98.1/24 interface=vlan-Unifi-bridge1 network=192.168.98.0
add address=192.168.10.1/24 interface=vlan10-cgm network=192.168.10.0
2

keep in mind with this switch chips we are talking about

only Layer 2 Bridge Hardware Offloading

https://help.mikrotik.com/docs/spaces/ROS/pages/328068/Bridging+and+Switching#BridgingandSwitching-BridgeHardwareOffloading

all L3 work will use CPU resources without any off-loading

3

Well, it depends.
If you can ensure that a group of vlans share the same bridge and then a group of vlans share the other bridge.

To ensure HW offloading, then yes two bridges one bridge for 1-5 and another bridge for ports 6-10.
Note that this supports traffic on the same vlan across ports.
So assess on a vlan by vlan basis
Ex.
So if VLAN5 as lots of cross traffic between users then attempt to ensure those users are handled by ports 1-5, or 6-10
DO the same for every vlan and come up with a plan.
To allow traffic between vlans, one would think firewall rules would suffice regardless of which bridge they are on.

4

If one configures single bridge and VLANs span both switch chips … then sure, CPU will work a bit to pass frames between both switch chips. Other than that this setup should not pose a bottleneck as both CPU-switch interconnects are 10Gbps (while each switch chip runs 8x 1Gbps port).
So yes, suggestion by @anav to try to keep entire VLANs on single switch chip is a fair suggestion … but device will survive of some VLANs cross over both switch chips.

5

Thanks for the replies. I did some experiments and learned that I don’t need a separate VLAN interface for each bridge (in the Interfaces window in winbox), so I deleted all of the VLAN interfaces for bridge2 and that cleaned things up a lot - and more importantly nothing broke. I also found that bridge2 didn’t need to be added to any of its bridge VLANs (on the Bridge → VLAN window). I expect this is because the bridge represents the cpu, and since I’m physically connecting the switch chips via port 8 and 9, there’s nothing for the cpu to do for the second switch chip.

I’m still a bit confused by the “Hardware Offload” setting on a bridge port. I changed the bridge setting of bridge port ether16 from bridge2 to bridge1, and saw that the bridge port still had the “H” label. But the PC connected to ether16 stopped receiving traffic until I unchecked “Hardware Offload”. I thought “offload” meant something like “try to run on the hardware if you can” but it seems it’s more like “only run on hardware”.

Since ether16 is not physically connected to switch1, it seems incorrect to indicate that the ether16 bridge port status is H - Hw. Offload. Especially since when I make this change, the “Hw Offload” status is cleared from ether9 - ether15. If I open the bridge port status tab for ether16 and ether11, for example, ether16 is listed as being in Hw Offload Group switch 2 and Hw Offload is checked. But for the bridge status tab for ether11 has Hw Offload unchecked and Hw Offload Group empty. It seems that either all ports on a switch would have Hw Offload status or none of them would – or that ether9 - ether15 would keep their Hw Offload status, and ether16 would lose it. I think there are still a lot of things about how hardware offloading works on the CCR2004 I don’t understand. If I had to guess, I think my confusion is thinking that a bridge is somehow connected to a switch chip - but there’s no setting I could find where you associate a bridge with a particular switch chip. I guess that happens behind the scenes based on which interfaces are ports of the bridge. Gotta go re-read the chapter on Bridging and Switching again.

6

Regarding HW offload: there are two places with similar setting … bridge and individual ports.

In case there are more bridges than switch chips or of port layout doesn’t follow physical layout, then ROS will decide which bridge will be offloaded (and which won’t). Sometimes such automatic decission is not the most optimal (e.g. it might decide to offload bridge with more ports but which handles way less traffic) and it is possible to mark bridges manually to offload or not offload.
And it seems that switch chip can only be controlled by single bridge (so it’s not possible to have some ports members of one bridge and other ports members of another bridge while having all of them offloaded).

Individual ports: if bridge as a whole is offloaded, but for some reason certain port must not be, then it’s possible to mark such port not to be offloaded. Often one wants to apply bridge filters to traffic of a particular port and for tgat traffic has to pass CPU, hence need to disable offload on that particular port.

The fact that you had to manually disable offload for bridge port, which is not served by switch chip to which corresponding bridge is offloaded … seems as a bug to me. ROS should be smart enough to do it automatically. OTOH single bridge can comfortably span multiple switch chips, so it’s normal to try to offload onto all involved switch chips. In your particular case bridge definition order might also play a role. You might try to have bridge, spanning single switch, “created” earlier and the one spanning both switches later. This way ROS might do the right thing regarding offload … but this is only my guess.

I still think that having single bridge on device with multiple switch chips is a completely valid option … specially do if there are multiple VLANs/subnets in the game and one can still set up a physical layout which minimises amount of traffic passing CPU en route between different switch chips.
Configuring multiple bridges seems to me an unnecessary complication leading to convoluted configuration (and it seems triggers a few obscure bugs).

7

This topic has some useful info:
http://forum.mikrotik.com/t/ccr2004-16g-2s-multiple-bridges-or-not/172985/1

8

It seems that either all ports on a switch would have Hw Offload status or none of them would – or that ether9 - ether15 would keep their Hw Offload status, and ether16 would lose it. I think there are still a lot of things about how hardware offloading works on the CCR2004 I don’t understand.

In my experience it is recommended to disable VLAN filtering on a bridge before changing ports and VLANs. If enabled again afterwards, there also should be a log entry showing for which ports HW forwarding was activated. And also port status will be updated accordingly. Changing port/VLAN configuration of bridges with running l2hw sometimes confused ROS.

For wiring ether8/9: This allows for HW offloading btw. switch1 and switch2, but also is limited to 1GB. The internal uplink to the CPU is 10Gb. As the CPU is quite fast, it is likely you get more than 1GB with CPU forwarding. But this only matters if there is a lot of horizontal traffic btw. ports on different switch chips.
https://cdn.mikrotik.com/web-assets/product_files/CCR2004-16G-2S_240151.png


Configuring multiple bridges seems to me an unnecessary complication leading to convoluted configuration (and it seems triggers a few obscure bugs).

In my experience it depends: On simple setups it is best to create a single bridge, configure all ports/VLANs/PVIDs (keeping ports grouped on chips as far as possible) and enable VLAN filtering. ROS will do the rest.
In more complicated scenarios with a heavy mix of different VLANs and trunked/hybrid/access ports, ROS gets confused with HW offloading. For such applications it works better to create one bridge per switch chip and avoid horizontal traffic btw them the as far as possible.

9

Uhh… it is officially recommended to disable L3HW when making L2 changes, yes. And if you don’t follow that recommendation, the pre-existing L3 flows may end up continuing down the “old” paths. If you really insist you can disable L2HW while editing L2 settings as well, although that never seemed necessary to me (and might have some pretty horrible performance impact, depending on the device / switchchip).

But disabling vlan filtering, uh… at best you get a security boundary violation, at worst you get that plus the overall network getting very confused as things which should remain separate get bridged together…?

10

or just reboot the router? often a good idea after many changes.