Hi there!
I’m trying to solve one last thorny issue with my new network after a week of fighting for the right to be called “not great network engineer, but technically a network engineer”.
I have a WAN coming in on eth1 on my MikroTik CCR2216-1G-12XS-2XQ. We’ll call this WAN oob-uplink (as that’s the interface name).
This is with the IP 123.123.123.62/30 and gateway 123.123.123.61. The first three octets have been censored for privacy.
I have a second WAN coming in via an LACP bond to a core switch, over VLAN2. We’ll call this WAN main-uplink (although it’s not referred to in the configs as such - just “vlan2”).
This is with the IP 23.227.172.66/27 and gateway 23.227.172.65. The IP address is a normal one.
What I am trying to solve specifically is the following:
- I want it so that Wireguard listens only on WAN oob-uplink, but does work
- (Optional) Router should only listen on WAN oob-uplink
- All traffic from or back to the BMC VLAN - VLAN50 - goes to or from the WAN oob-uplink
- All other traffic goes through the normal WAN main-uplink
I’ve attached my slightly redacted config below.
I have tried doing some reading - particularly http://forum.mikrotik.com/t/dual-wan-dual-subnet-multiple-vlans/148704/1 - and got close I think, to a working solution, but then Wireguard could no longer access the internet through the tunnel, and both switches could not ping 8.8.8.8 any more or really access just about anything.
I also asked ChatGPT (both 3.5 and 4) to no avail, although I carefully tried a bunch of things. The main thing I’ve tried so far is creating a second routing table and setting mangle rules to route BMC traffic appropriately.
Tearing my hair out and so close to a working network! Any help is appreciated. Thank you!!!
[notmyrealuser@DAL1-R001-ROUTER-01] > export
# apr/02/2023 20:17:24 by RouterOS 7.8
# software id = 8XEV-WHA0
#
# model = CCR2216-1G-12XS-2XQ
# serial number = notmyrealserial
/interface ethernet
set [ find default-name=ether1 ] comment="This interface provides an internet uplink on a special 1G connec\
tion that is to be used solely for management access and Wireguard. It allows entry into the network, j\
ust enough to Wireguard in and manage the BMCs." name=oob-uplink
set [ find default-name=qsfp28-1-1 ] comment=\
"These QSFP ports are all unused for now but may be used in the future."
set [ find default-name=sfp28-1 ] advertise=\
10M-half,10M-full,100M-half,100M-full,1000M-half,1000M-full,10000M-full comment="These first two SFP28 \
ports are used by the LACP link to the BMC/OOB switch DAL1-R001-BMC-SW-01. The SFP28 ports will be the \
primary thing we connect devices with." speed=10Gbps
set [ find default-name=sfp28-2 ] advertise=10M-half,10M-full,100M-half,100M-full,1000M-half,10000M-full \
speed=10Gbps
set [ find default-name=sfp28-9 ] comment="The following four SFP28 interfaces are the LACP quadruplet that\
\_is used in the main router <-> core switch bridge. They intentionally have FEC turned off on the swit\
ch side to prevent link failure." l2mtu=9084 mtu=9020
set [ find default-name=sfp28-10 ] l2mtu=9084 mtu=9020
set [ find default-name=sfp28-11 ] l2mtu=9084 mtu=9020
set [ find default-name=sfp28-12 ] l2mtu=9084 mtu=9020
/interface wireguard
add comment="A special Wireguard network solely for access to the Out of Band Management systems for EHI. D\
O NOT use for regular interaction." listen-port=13231 mtu=1420 name=wg-outofband
/interface vlan
add comment="Used by oob-uplink, since we needed a VLAN for access to the router (and Wireguard)" \
interface=oob-uplink name=vlan1 vlan-id=1
/interface bonding
add comment="This bond is used for connectivity to the core BMC/out of band switch." mode=802.3ad name=\
bmc-switch-bond slaves=sfp28-1,sfp28-2
add comment="This bond is used to connect the core 25G switch with the router." mode=802.3ad mtu=9020 \
name=core-switch-bond slaves=sfp28-9,sfp28-10,sfp28-11,sfp28-12
/interface vlan
add comment="Used to enable internet access from Hivelocity to the rest of EHI. The internet uplinks physic\
ally exist on the core switch, not here." interface=core-switch-bond loop-protect=on name=vlan2 \
vlan-id=2
add comment="The main VLAN for all internal EHI traffic." interface=core-switch-bond mtu=9000 name=vlan10 \
vlan-id=10
add interface=bmc-switch-bond name=vlan50 vlan-id=50
/interface list
add name=public
add name=local
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/port
set 0 name=serial0
/routing table
add disabled=no fib name=bmc-route
/interface list member
add interface=vlan2 list=public
add interface=oob-uplink list=public
add interface=vlan10 list=local
add interface=bmc-switch-bond list=local
add interface=vlan1 list=public
add interface=vlan50 list=local
/interface wireguard peers
add allowed-address=0.0.0.0/0 interface=wg-outofband public-key=\
"REDACTED"
add allowed-address=0.0.0.0/0 interface=wg-outofband public-key=\
"REDACTED"
/ip address
add address=123.123.123.62/30 comment="Address for the out of band internet connection" interface=oob-uplink \
network=123.123.123.60
add address=23.227.172.66/27 comment="Main internet uplink" interface=vlan2 network=23.227.172.64
add address=10.24.0.1/16 comment="Main address for the MikroTik from the EHI network." interface=\
core-switch-bond network=10.24.0.0
add address=10.25.2.1/24 comment="Wireguard native IP" interface=wg-outofband network=10.25.2.0
add address=10.25.1.1/24 comment="Main IP for the MikroTik on the BMC network" interface=bmc-switch-bond \
network=10.25.1.0
/ip dns
set servers=8.8.8.8,8.8.4.4
/ip firewall address-list
add address=REDACTED.81.REDACTED comment="Wings Home" list=allowed_to_router
add address=REDACTED.81.REDACTED comment="Wings Home" list=allowed_to_wg-outofband
add address=REDACTED.192.REDACTED comment="Wings PCHQ" list=allowed_to_router
add address=REDACTED.192.REDACTED comment="Wings PCHQ" list=allowed_to_wg-outofband
add address=0.0.0.0/8 comment=RFC6890 list=not_in_internet
add address=172.16.0.0/12 comment=RFC6890 list=not_in_internet
add address=192.168.0.0/16 comment=RFC6890 list=not_in_internet
add address=10.0.0.0/8 comment=RFC6890 list=not_in_internet
add address=169.254.0.0/16 comment=RFC6890 list=not_in_internet
add address=127.0.0.0/8 comment=RFC6890 list=not_in_internet
add address=224.0.0.0/4 comment=Multicast list=not_in_internet
add address=198.18.0.0/15 comment=RFC6890 list=not_in_internet
add address=192.0.0.0/24 comment=RFC6890 list=not_in_internet
add address=192.0.2.0/24 comment=RFC6890 list=not_in_internet
add address=198.51.100.0/24 comment=RFC6890 list=not_in_internet
add address=203.0.113.0/24 comment=RFC6890 list=not_in_internet
add address=100.64.0.0/10 comment=RFC6890 list=not_in_internet
add address=240.0.0.0/4 comment=RFC6890 list=not_in_internet
add address=192.88.99.0/24 comment="6to4 relay Anycast [RFC 3068]" list=not_in_internet
add address=10.24.0.0/16 comment="Allow VLAN 10 (main network) to send traffic from the LAN" list=\
allowed_LAN_IPs
add address=10.25.2.137 comment="Special Wireguard IP" list=allowed_to_router
add address=10.25.1.0/24 comment="Allow BMC switch to send traffic" list=allowed_LAN_IPs
add address=10.25.2.0/24 comment="Allow Wireguard machines to send traffic to the LAN and Internet" list=\
allowed_LAN_IPs
/ip firewall filter
add action=accept chain=input comment="Allow established,related" connection-state=established,related
add action=accept chain=input comment="Allow addresses on the allowed_to_router list to access the router" \
src-address-list=allowed_to_router
add action=accept chain=input comment="Allow ICMP (ping) from anyone, anywhere" protocol=icmp
add action=drop chain=input comment="Drop all traffic that doesn't match any other rules"
add action=fasttrack-connection chain=forward comment=FastTrack connection-state=established,related \
hw-offload=yes
add action=accept chain=forward comment="Allow established,related traffic (again)" connection-state=\
established,related
add action=drop chain=forward comment="Drop invalid" connection-state=invalid log=yes log-prefix=invalid
add action=drop chain=forward comment="Drop tries to reach not public addresses from LAN" \
dst-address-list=not_in_internet in-interface-list=local log=yes log-prefix=!public_from_LAN \
out-interface-list=!local
add action=drop chain=forward comment="Drop incoming not public IP from internet" in-interface-list=public \
log=yes log-prefix=!public src-address-list=not_in_internet
add action=drop chain=forward comment="Drop incoming packets that are not NAT'ed" connection-nat-state=\
!dstnat connection-state=new in-interface-list=public log=yes log-prefix=!NAT
add action=jump chain=forward comment="Jump to ICMP filters" jump-target=icmp protocol=icmp
add action=drop chain=forward comment="Drop incoming from internet which is not public IP" \
in-interface-list=public log=yes log-prefix=!public src-address-list=not_in_internet
add action=drop chain=forward comment="Drop packets from LAN that do not have LAN IP. (NOTE: This might not\
\_do anything\? Wait, yes it does!!)" in-interface-list=local log=yes log-prefix=LAN_!LAN \
src-address-list=!allowed_LAN_IPs
add action=drop chain=forward comment="Drop not public addresses from LAN" dst-address-list=\
not_in_internet in-interface-list=local log=yes log-prefix=!public_from_LAN out-interface-list=!local
add action=drop chain=forward comment="Drop non-wg-outofband traffic to vlan50" in-interface=!wg-outofband \
out-interface=bmc-switch-bond
add action=accept chain=forward comment="Allow router access to the BMC network" disabled=yes \
in-interface=all-ethernet out-interface=bmc-switch-bond
add action=accept chain=input comment="Allow WireGuard Out of Band access from specific IPs" dst-port=\
13231 in-interface=vlan2 protocol=udp src-address-list=allowed_to_wg-outofband
add action=accept chain=forward comment="Allow traffic from wg-outofband to anywhere" in-interface=\
wg-outofband
add action=drop chain=input
/ip firewall nat
add action=src-nat chain=srcnat comment=\
"Enable source NAT from VLAN10 main network (10.24.0.0/16) out to the internet" out-interface=\
all-ethernet src-address=10.24.0.0/16 to-addresses=23.227.172.66
add action=src-nat chain=srcnat comment=\
"Enable source NAT from VLAN50 BMC network (10.25.1.0/24) out to the internet" out-interface=\
all-ethernet src-address=10.25.1.0/24 to-addresses=123.123.123.62
add action=src-nat chain=srcnat comment="Allow Wireguard machines to access the internet" out-interface=\
all-ethernet src-address=10.25.2.0/24 to-addresses=23.227.172.66
/ip route
add comment="Main route for Out of Band Uplink" disabled=yes distance=1 dst-address=0.0.0.0/0 gateway=\
123.123.123.61 pref-src="" routing-table=main scope=30 suppress-hw-offload=no target-scope=10
add comment="Main (lower priority) route for All Normal Traffic" disabled=no distance=2 dst-address=\
0.0.0.0/0 gateway=23.227.172.65 pref-src="" routing-table=main scope=30 suppress-hw-offload=no \
target-scope=10
add comment="Main route for BMC IP" disabled=no distance=1 dst-address=0.0.0.0/0 gateway=123.123.123.61 \
pref-src="" routing-table=main scope=30 suppress-hw-offload=no target-scope=10
/ip service
set www-ssl certificate=letsencrypt-autogen_1970-01-02T04:07:29Z disabled=no
/system clock
set time-zone-name=America/Chicago
/system identity
set name=DAL1-R001-ROUTER-01
/system routerboard settings
set enter-setup-on=delete-key