Bridge Vlan with QinQ

Rfulton · Thu Jul 27, 2023 10:45 pm

Like the subject says.

I have one cvlan Vlan20 and one svlan 2100(outer)

https://imgur.com/a/DBcuEu8

Vlan 20 gives out 192.168.20.x at both locations (no overlap)

The currently uploaded config works, but I'm seeing 1svlan and 2cvlans come across the provider network.

If I send tagged vlan 20 traffic down vlan2100-QinQ20 for vlan 20, it works but has 3 vlan headers (1svlan 2cvlan)

If i send untagged vlan 20 traffic down vlan2100-QinQ20 it doesn't work but the headers are correct (1svlan 1cvlan)

Also, if i add eth1-vlan2100 to the bridge and send tagged vlan 20 traffic down it, it has the correct headers but doesn't work.

The only config I can get working is if it double tags the cvlan by sending tagged 20 down vlan2100-QinQ20 (which tags it again) and then it gets outer service tagged by eth1-vlan2100

See the working config below that has the duplicate cvlan headers.

# 2023-07-26 17:19:15 by RouterOS 7.10.2
# software id = XFZ0-5LYE
#
# model = RB750Gr3
# serial number = HEC08V9AWMB
/interface bridge
add admin-mac=48:A9:8A:A9:85:6B auto-mac=no comment=defconf name=bridge \
    vlan-filtering=yes
/interface ethernet
set [ find default-name=ether1 ] name=eth1-provider
/interface vlan
add interface=bridge name=main-vlan20 vlan-id=20
add interface=bridge name=main-vlan21 vlan-id=21
add interface=eth1-provider name=vlan2100 use-service-tag=yes vlan-id=2100
add interface=vlan2100 name=vlan2100-QinQ20 vlan-id=20
/interface list
add comment=defconf name=WAN
add comment=defconf name=LAN
/interface lte apn
set [ find default=yes ] ip-type=ipv4 use-network-apn=no
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip hotspot profile
set [ find default=yes ] html-directory=hotspot
/ip pool
add name=Vlan20 ranges=192.168.20.10-192.168.20.20
add name=Vlan21 ranges=192.168.21.10-192.168.21.20
/ip dhcp-server
add address-pool=Vlan20 interface=main-vlan20 lease-time=10m name=Vlan20
add address-pool=Vlan21 interface=main-vlan21 lease-time=10m name=Vlan21
/port
set 0 name=serial0
/interface bridge port
add bridge=bridge comment=defconf interface=ether2 pvid=20
add bridge=bridge comment=defconf interface=ether3 pvid=21
add bridge=bridge comment=defconf ingress-filtering=no interface=ether4
add bridge=bridge interface=vlan2100-QinQ20
/ip neighbor discovery-settings
set discover-interface-list=LAN
/ipv6 settings
set disable-ipv6=yes max-neighbor-entries=8192
/interface bridge vlan
add bridge=bridge tagged=bridge,vlan2100-QinQ20 untagged=ether2 vlan-ids=20
add bridge=bridge tagged=bridge untagged=ether3 vlan-ids=21
/interface list member
add comment=defconf interface=bridge list=LAN
add comment=defconf interface=eth1-provider list=LAN
add interface=ether5 list=LAN
add interface=main-vlan20 list=LAN
add interface=main-vlan21 list=LAN
add interface=*9 list=LAN
/interface ovpn-server server
set auth=sha1,md5
/ip address
add address=192.168.1.1/24 comment=defconf interface=ether5 network=\
    192.168.1.0
add address=192.168.20.1/24 interface=main-vlan20 network=192.168.20.0
add address=192.168.21.1/24 interface=main-vlan21 network=192.168.21.0
/ip dhcp-client
add comment=defconf disabled=yes interface=eth1-provider
/ip dhcp-server network
add address=192.168.20.0/24 comment=defconf gateway=192.168.20.1
add address=192.168.21.0/24 comment=defconf gateway=192.168.21.1
/ip dns
set allow-remote-requests=yes
/ip dns static
add address=192.168.88.1 comment=defconf name=router.lan
/ip firewall filter
add action=accept chain=forward connection-state=\
    established,related,untracked
add action=accept chain=input connection-state=established,related,untracked
/routing bfd configuration
add disabled=no interfaces=all min-rx=200us min-tx=200us multiplier=5
/system clock
set time-zone-name=America/Detroit
/system identity
set name=Lab_A
/system note
set show-at-login=no
/tool mac-server
set allowed-interface-list=LAN
/tool mac-server mac-winbox
set allowed-interface-list=LAN
/tool sniffer
set file-name=pcapA.pcap filter-interface=eth1-provider filter-ip-protocol=\
    icmp


# 2023-07-26 19:29:08 by RouterOS 7.10.2
# software id = Y4LK-Y9I9
#
# model = RB750Gr3
# serial number = HEC08PP06KE
/interface bridge
add admin-mac=48:A9:8A:A9:88:76 auto-mac=no comment=defconf \
    ingress-filtering=no name=bridge vlan-filtering=yes
/interface ethernet
set [ find default-name=ether1 ] name=eth1-Provider
/interface vlan
add interface=eth1-Provider name=eth1-vlan2100 use-service-tag=yes vlan-id=\
    2100
add interface=bridge name=main-vlan20 vlan-id=20
add interface=bridge name=main-vlan21 vlan-id=21
add interface=eth1-vlan2100 name=vlan2100-QinQ20 vlan-id=20
/interface list
add comment=defconf name=WAN
add comment=defconf name=LAN
/interface lte apn
set [ find default=yes ] ip-type=ipv4 use-network-apn=no
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip hotspot profile
set [ find default=yes ] html-directory=hotspot
/ip pool
add name=Vlan20 ranges=192.168.20.30-192.168.20.40
add name=Vlan21 ranges=192.168.21.30-192.168.21.40
/ip dhcp-server
add address-pool=Vlan20 interface=main-vlan20 lease-time=10m name=Vlan20
add address-pool=Vlan21 interface=main-vlan21 lease-time=10m name=Vlan21
/port
set 0 name=serial0
/interface bridge port
add bridge=bridge comment=defconf interface=ether2 pvid=20
add bridge=bridge comment=defconf interface=ether3 pvid=21
add bridge=bridge comment=defconf ingress-filtering=no interface=ether4
add bridge=bridge interface=vlan2100-QinQ20 pvid=20
/ip neighbor discovery-settings
set discover-interface-list=LAN
/ipv6 settings
set disable-ipv6=yes max-neighbor-entries=8192
/interface bridge vlan
add bridge=bridge tagged=bridge,vlan2100-QinQ20 untagged=ether2 vlan-ids=20
add bridge=bridge tagged=bridge untagged=ether3 vlan-ids=21
/interface list member
add comment=defconf interface=bridge list=LAN
add comment=defconf interface=eth1-Provider list=LAN
add interface=ether5 list=LAN
add interface=main-vlan20 list=LAN
add interface=main-vlan21 list=LAN
add interface=*9 list=LAN
/interface ovpn-server server
set auth=sha1,md5
/ip address
add address=192.168.20.254/24 comment=defconf interface=main-vlan20 network=\
    192.168.20.0
add address=192.168.1.1/24 interface=ether5 network=192.168.1.0
add address=192.168.21.254/24 comment=defconf interface=main-vlan21 network=\
    192.168.21.0
/ip dhcp-client
add comment=defconf interface=eth1-Provider
/ip dhcp-server network
add address=192.168.20.0/24 comment=defconf gateway=192.168.20.254
add address=192.168.21.0/24 comment=defconf gateway=192.168.21.254
/ip dns
set allow-remote-requests=yes
/ip dns static
add address=192.168.88.1 comment=defconf name=router.lan
/ip firewall filter
add action=accept chain=input comment=\
    "defconf: accept established,related,untracked" connection-state=\
    established,related,untracked
add action=fasttrack-connection chain=forward comment="defconf: fasttrack" \
    connection-state=established,related hw-offload=yes
add action=accept chain=forward comment=\
    "defconf: accept established,related, untracked" connection-state=\
    established,related,untracked
/routing bfd configuration
add disabled=no interfaces=all min-rx=200us min-tx=200us multiplier=5 vrf=\
    main
/system clock
set time-zone-name=America/Detroit
/system identity
set name=LAB_B
/system note
set show-at-login=no
/tool mac-server
set allowed-interface-list=LAN
/tool mac-server mac-winbox
set allowed-interface-list=LAN
/tool sniffer
set file-name=pcap.pcap filter-interface=eth1-Provider filter-ip-protocol=\
    icmp

mkx · Thu Jul 27, 2023 11:15 pm

The error:

/interface bridge port
add bridge=bridge interface=vlan2100-QinQ20 pvid=20
/interface bridge vlan
add bridge=bridge tagged=bridge,vlan2100-QinQ20 untagged=ether2 vlan-ids=20

Bridge port (vlan2100-QinQ20 in this case) should not be untagged and tagged at the same time (setting pvid makes it untagged for that VLAN). In your particular case it has to be untagged because you're using VLAN interface which is untagged for that VLAN.
Essentially: pvid setting in bridge/port affects ingress packets (VLAN tag gets added) and type of vlan (tagged vs. untagged) setting in bridge/vlan affects egress packets (with setup above egress packets are tagged).

Alternatively you could avoid using vlan2100-QinQ20 (as untagged bridge port member of VLAN 20), instead you could use eth1-vlan2100 as bridge port, tagged with VID 20.

Rfulton · Thu Jul 27, 2023 11:54 pm

I did set pvid to 1 on vlan2100-QinQ20 which results in unchanged behavior.

"instead you could use eth1-vlan2100 as bridge port, tagged with VID 20."

I did do this, which gives you the correct headers but doesn't work.

Any ideas?

Rfulton · Fri Jul 28, 2023 12:28 am

Let me clarify, I also set pvid to 20 on vlan2100-QinQ20 and set it to untagged in bridge > vlan and the headers are correct but it doesn't work either.

only the tagged vlan2100-QinQ20 works, but gives me svlan 2100, cvlan 20, cvlan 20.

pants6000 · Fri Jul 28, 2023 3:45 am

If it works when it's triple-tagged like that, something in here must be stripping two layers of tags.

What is your provider like? Do they do anything with tags? Are you using 0x88a8 s-tags because they want that or is that of your own doing?

It might be helpful to post packet captures, it's easier to understand/explain I think.

Rfulton · Fri Jul 28, 2023 1:49 pm

Hello,

The provider does require an s-vlan across their network.

Attached is the packet captures when its working.

mkx · Fri Jul 28, 2023 2:16 pm

My only comment is this: when both ends are equally misconfigured (so that they work with two CVIDs), then everything works. I'm pretty sure that if they were both correctly configured (so that they would work with single CVID), then everything would work as well. However, if one end is misconfigured and the other is not, then things don't work (because one expects single CVID and the other expects double CVID).

Rfulton · Fri Jul 28, 2023 2:43 pm

"I'm pretty sure that if they were both correctly configured (so that they would work with single CVID), then everything would work as well."

Wow amazing analysis.

Turns out though no other configuration works.

Let me clarify for the third time.

TAGGED VLAN 20 down VLAN2100-QinQ works
Untagged Vlan 20 down Vlan 2100-QinQ DOESNT WORK
TAGGED Vlan 20 down Vlan2100 Doesn't work.

You are welcome to load my config and test like a couple others have done (and come to same conclusion as me)

Rfulton · Fri Jul 28, 2023 3:17 pm

I made the suggestion again either way to show pcaps and config.

tag20vlan2100.7z

mkx · Fri Jul 28, 2023 3:19 pm

You posted the pair of wireshark dumps which show the working double-CVID case. Do you have similar pair of dumps but for one of non-working cases? Analyzing those should give at least some insight as to what goes wrong ... the pcap files from latest post don't seem to show similar case as the previous ones.

Rfulton · Fri Jul 28, 2023 3:21 pm

Look above. I must have uploaded during your reply.

Rfulton · Fri Jul 28, 2023 3:27 pm

Here's some more though.

Pcap2.7z

mkx · Fri Jul 28, 2023 3:27 pm

Hmmm ... pcapA shows frames, egressing through ether1, as properly QinQ tagged (outer STAG and inner CTAG) ... pcapB doesn't show corresponding frames to ingress from ether1. Which indicates that these frames get dropped in transit ...
I'm referring to second set of files ...

mkx · Fri Jul 28, 2023 3:32 pm

The last set shows that ARP who has comes through ... sometimes.

Hmmm ... do pcapA and pcapB span same time period? Timestamps inside files say this is not the case (or so says my Wireshark).

Rfulton · Fri Jul 28, 2023 3:43 pm

These are lined up, check this.

Pcap3.7z

Rfulton · Fri Jul 28, 2023 4:49 pm

Anyone else want to try and load the config and take a look?

I swear this should work with vlan2100 tagged as 20 in bridge>vlan.

It works when you tag vlan2100-QinQ20 as 20 in bridge>vlan, but this results in a double stack of cvlan.

mkx · Fri Jul 28, 2023 5:49 pm

These are lined up, check this.

Dunno. pcapA.pcap file shows times between 21:51:19 and 21:51:46 (time span of 27 seconds) while pcapB.pcap fiel shows times between 22:13:40 and 22:14:18 (time span of 38 seconds).

But: let's say that message no. 63 from pcapA (ARP Who has) corresponds with message no. 18 from pcapB. And immediately follwos answer ... also seen in both pcap files. Meaning that VLAN works somehow end2end ...

Rfulton · Fri Jul 28, 2023 6:00 pm

I'm still not getting network connectivity though. something is not working as intended i believe.

I can get more pcaps for you, but they were running at the same time perhaps not the same length of time.

Rfulton · Fri Jul 28, 2023 6:14 pm

" 21:51:19 and 21:51:46 (time span of 27 seconds) while pcapB.pcap fiel shows times between 22:13:40 and 22:14:18 (time span of 38 seconds)."

NTP is off on the devices since they don't have network connectivity. good find at least.

pants6000 · Fri Jul 28, 2023 6:38 pm

Since ARP seems to work, can you mac-ping or mac-telnet between the two boxes? Do you see sensible things in the ARP table?

I wonder what the deal is with all the "bogus IPv4 version" stuff seen in the pcaps.

Rfulton · Fri Jul 28, 2023 6:41 pm

Mac-telnet and Mac-ping do not work.

Very weird.

mkx · Fri Jul 28, 2023 6:59 pm

If you connect a simple computer to untagged ether port, member of vlan20 (e.g. ether2 on Lab_A), can it communicate with local router? Both IP communication and MAC (e.g. winbox)? This should rule out possible higher layer configuration errors as this communication doesn't touch the QinQ link.

Rfulton · Fri Jul 28, 2023 7:07 pm

A computer hooked to either Tik on ethernet2 can reach the default gateway (main-vlan20) and login to winbox

pants6000 · Fri Jul 28, 2023 7:26 pm

Very weird indeed.

Do you know if your provider is using MPLS to transport this? It might be worth changing the MAC that starts with 4 to something else... I'm not (entirely) insane, read this:

https://mailman.nanog.org/pipermail/nan ... 89395.html

Rfulton · Fri Jul 28, 2023 8:34 pm

Right now I'm just pretending eth1 is the provider networking and doing svlan2100 (outer) and cvlan20(inner) and just trying to make that work first.

They are directly connected.

Rfulton · Fri Jul 28, 2023 9:59 pm

Where is Janis? Hes on lunch maybe?

Rfulton · Tue Aug 01, 2023 6:22 pm

Rfulton · Wed Aug 16, 2023 4:09 am

Bridge Vlan with QinQ

Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Re: Bridge Vlan with QinQ

Who is online