We deployed a MikroTik CCR2216-1G-12XS-2XQ a few months ago, paired with an FS.com N8560-48BC (48x 25G, 4x 100G) switch. Things are going well so far, but it's becoming clear our setup is far from optimal - it was designed by me and I'm very new to MikroTiks and networking in general.
The network setup is a "router on a stick" (? maybe?) setup, as follows:
2-way LACP bond to internet
| |
MikroTik ================= FS.com switch
^ 4-way LACP bond
I've been noticing congestion, where a file transfer goes very quickly at first but then jumps around speed-wise really rapidly, and possibly packet loss too (still trying to verify). I was hoping to get some more experienced MikroTik users to review my configuration and see where I've gone wrong.
There was previously a second link on the MikroTik used for OOB access, we stopped using it as it was a little tricky to try to manage two uplinks correctly.
Code: Select all
[MikroTik] > export
# may/24/2023 21:57:37 by RouterOS 7.8
# software id = 8XEV-WHA0
#
# model = CCR2216-1G-12XS-2XQ
# serial number = REDACTED
/interface ethernet
set [ find default-name=ether1 ] comment="This interface provides an internet uplink on a special 1G connection that is to be used solely for management access and Wireguard. It allows entry into the n\
etwork, just enough to Wireguard in and manage the BMCs." name=oob-uplink
set [ find default-name=qsfp28-1-1 ] comment="These QSFP ports are all unused for now but may be used in the future."
set [ find default-name=sfp28-1 ] advertise=10M-half,10M-full,100M-half,100M-full,1000M-half,1000M-full,10000M-full comment=\
"These first two SFP28 ports are used by the LACP link to the BMC/OOB switch DAL1-R001-BMC-SW-01. The SFP28 ports will be the primary thing we connect devices with." speed=10Gbps
set [ find default-name=sfp28-2 ] advertise=10M-half,10M-full,100M-half,100M-full,1000M-half,10000M-full speed=10Gbps
set [ find default-name=sfp28-9 ] comment="The following four SFP28 interfaces are the LACP quadruplet that is used in the main router <-> core switch bridge. They intentionally have FEC turned off on \
the switch side to prevent link failure." l2mtu=9084 mtu=9020
set [ find default-name=sfp28-10 ] l2mtu=9084 mtu=9020
set [ find default-name=sfp28-11 ] l2mtu=9084 mtu=9020
set [ find default-name=sfp28-12 ] l2mtu=9084 mtu=9020
/interface wireguard
add comment="A special Wireguard network solely for access to the Out of Band Management systems for EHI. DO NOT use for regular interaction." listen-port=13231 mtu=1420 name=wg-outofband
add listen-port=13337 mtu=1420 name=wireguard-panicswitch
/interface bonding
add comment="This bond is used for connectivity to the core BMC/out of band switch." mode=802.3ad name=bmc-switch-bond slaves=sfp28-1,sfp28-2
add comment="This bond is used to connect the core 25G switch with the router." mode=802.3ad mtu=9020 name=core-switch-bond slaves=sfp28-9,sfp28-10,sfp28-11,sfp28-12
/interface vlan
add comment="Used by oob-uplink, since we needed a VLAN for access to the router (and Wireguard)" disabled=yes interface=core-switch-bond name=vlan1 vlan-id=1
add comment="Used to enable internet access from Hivelocity to the rest of EHI. The internet uplinks physically exist on the core switch, not here." disabled=yes interface=core-switch-bond \
loop-protect=on name=vlan2 vlan-id=2
add comment="The main VLAN for all internal EHI traffic." interface=core-switch-bond mtu=9020 name=vlan10 vlan-id=10
add comment="VLAN for the BMCs" interface=bmc-switch-bond name=vlan50 vlan-id=50
add comment="Temporary VLAN for JBODs" interface=bmc-switch-bond name=vlan51 vlan-id=51
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip pool
add comment="DHCP pool for BMC devices" name=dhcp_pool0 ranges=10.25.1.2-10.25.1.253
add comment="DHCP pool for main EHI" name=dhcp_pool1 ranges=10.24.2.20-10.24.6.254
/ip dhcp-server
add address-pool=dhcp_pool0 interface=bmc-switch-bond lease-time=2h name=dhcp1
add address-pool=dhcp_pool1 disabled=yes interface=vlan10 lease-time=2h name=dhcp2
/port
set 0 name=serial0
/interface wireguard peers
add allowed-address=10.25.2.2/32 comment="redacted" interface=wg-outofband public-key="JwS5t9+nAlIHCLAYTKm3p6AzmstdXXiJCEP8VZ2FrG4="
add allowed-address=10.25.2.141/32 comment="redacted" interface=wg-outofband public-key="liwOXXE83U2KKAq1m3Nj2zesCM4eS4UURcuPq00pNGE="
add allowed-address=10.25.2.201/32 comment="redacted" interface=wg-outofband public-key="27iugj71sIj3qwACHc3yU1aF/GVbG+luyOnK3R9OelM="
add allowed-address=10.25.2.142/32 comment="redacted" interface=wg-outofband public-key="O9zY687CxAHk0qc97fMaXpwjYK7oSyd7yIihmA2vX0g="
add allowed-address=10.25.2.2/32 comment="Wings JBOD hack" interface=wg-outofband public-key="WqV8Q4zah+PusOq1dMpCtBiqF1dQ8lNL7XoS4T1JqzE="
/ip address
add address=23.92.74.62/30 comment="Address for the out of band internet connection" interface=oob-uplink network=23.92.74.60
add address=23.227.172.66/27 comment="Main internet uplink" interface=core-switch-bond network=23.227.172.64
add address=10.24.0.1/16 comment="Main address for the MikroTik from the EHI network." interface=vlan10 network=10.24.0.0
add address=10.25.2.1/24 comment="Wireguard native IP" interface=wg-outofband network=10.25.2.0
add address=10.25.1.1/24 comment="Main IP for the MikroTik on the BMC network" interface=bmc-switch-bond network=10.25.1.0
add address=23.227.172.95/27 comment="ACME-DNS public IP" interface=core-switch-bond network=23.227.172.64
add address=23.227.172.73/27 comment=prod-bastion01 interface=core-switch-bond network=23.227.172.64
add address=23.227.172.74/27 comment=prod-bastion02 interface=core-switch-bond network=23.227.172.64
add address=23.227.172.75/27 comment=prod-bastion03 interface=core-switch-bond network=23.227.172.64
add address=23.227.172.67/27 comment=prod-haproxy-vip01 interface=core-switch-bond network=23.227.172.64
add address=192.168.11.1/24 comment="initial OOB for the JBODs" interface=vlan51 network=192.168.11.0
add address=23.227.172.68/27 comment=prod-haproxy-vip02 interface=core-switch-bond network=23.227.172.64
add address=23.227.172.69/27 comment=prod-haproxy-vip03 interface=core-switch-bond network=23.227.172.64
add address=23.227.172.93/27 comment="apollo shuttle-11 sctp" interface=core-switch-bond network=23.227.172.64
add address=23.227.172.78/27 comment="Shuttle 12 on EHI" interface=core-switch-bond network=23.227.172.64
add address=23.227.172.80/27 comment="JSON experiment" interface=core-switch-bond network=23.227.172.64
add address=23.227.172.92/27 comment="apollo shuttle-10 sctp" interface=core-switch-bond network=23.227.172.64
add address=23.227.172.89/27 comment="apollo shuttle-9 sctp" interface=core-switch-bond network=23.227.172.64
/ip dhcp-server lease
add address=10.24.6.232 client-id=ff:ca:53:9:5a:0:2:0:0:ab:11:2:99:ee:e6:26:a:c0:ee mac-address=FE:C1:46:3A:B1:14 server=dhcp2
add address=10.24.6.231 client-id=ff:ca:53:9:5a:0:2:0:0:ab:11:eb:b7:96:3c:d:2f:95:33 mac-address=52:2E:37:98:EC:1F server=dhcp2
add address=10.24.6.20 client-id=1:de:83:1a:f6:de:3f mac-address=DE:83:1A:F6:DE:3F server=dhcp2
add address=10.24.6.215 client-id=ff:ca:53:9:5a:0:2:0:0:ab:11:43:f7:36:a7:12:82:7a:e7 mac-address=76:63:8F:B1:A3:D4 server=dhcp2
add address=10.24.0.61 comment=prod-ebi-db01 mac-address=9A:01:65:B0:20:EF server=dhcp2
add address=10.24.0.62 comment=prod-ebi-db02 mac-address=72:4A:F3:8D:52:AA
add address=10.24.0.63 comment=prod-ebi-db03 mac-address=B2:B3:70:45:9F:FE
add address=10.24.6.217 client-id=ff:ca:53:9:5a:0:2:0:0:ab:11:10:df:4c:6b:ad:be:51:c5 mac-address=AE:D4:8B:4D:A2:E4 server=dhcp2
add address=10.24.6.221 client-id=ff:ca:53:9:5a:0:2:0:0:ab:11:9d:6a:d5:d3:43:29:38:1a mac-address=2A:37:64:13:B6:7C server=dhcp2
add address=10.24.6.211 client-id=ff:ca:53:9:5a:0:2:0:0:ab:11:60:e0:d1:52:b3:3c:1f:cb mac-address=2E:FA:2E:42:95:98 server=dhcp2
add address=10.24.6.210 client-id=ff:ca:53:9:5a:0:2:0:0:ab:11:fc:10:69:fc:45:9a:ce:5 mac-address=52:AA:82:0A:9A:FA server=dhcp2
add address=10.24.6.212 client-id=ff:ca:53:9:5a:0:2:0:0:ab:11:72:49:f8:c1:34:86:25:c4 mac-address=3A:39:A6:82:B2:2B server=dhcp2
add address=10.24.6.216 client-id=ff:ca:53:9:5a:0:2:0:0:ab:11:ab:6b:d9:b:9c:a3:1a:d0 mac-address=2A:98:80:C8:CD:E2 server=dhcp2
add address=10.24.6.214 client-id=ff:ca:53:9:5a:0:2:0:0:ab:11:84:2e:6d:76:34:e9:43:bc mac-address=4A:43:1C:C4:7E:60 server=dhcp2
add address=10.25.1.20 mac-address=74:56:3C:01:7F:71 server=dhcp1
/ip dhcp-server network
add address=10.24.0.0/16 gateway=10.24.0.1
add address=10.25.1.0/24 gateway=10.25.1.1
/ip dns
set servers=8.8.8.8,8.8.4.4
/ip firewall address-list
add address=redacted comment="redacted" list=allowed_to_router
add address=redacted comment="redacted" list=allowed_to_wg-outofband
add address=redacted comment="redacted" list=allowed_to_router
add address=redacted comment="redacted" list=allowed_to_wg-outofband
add address=0.0.0.0/8 comment=RFC6890 list=not_in_internet
add address=172.16.0.0/12 comment=RFC6890 list=not_in_internet
add address=192.168.0.0/16 comment=RFC6890 list=not_in_internet
add address=10.0.0.0/8 comment=RFC6890 list=not_in_internet
add address=169.254.0.0/16 comment=RFC6890 list=not_in_internet
add address=127.0.0.0/8 comment=RFC6890 list=not_in_internet
add address=224.0.0.0/4 comment=Multicast list=not_in_internet
add address=198.18.0.0/15 comment=RFC6890 list=not_in_internet
add address=192.0.0.0/24 comment=RFC6890 list=not_in_internet
add address=192.0.2.0/24 comment=RFC6890 list=not_in_internet
add address=198.51.100.0/24 comment=RFC6890 list=not_in_internet
add address=203.0.113.0/24 comment=RFC6890 list=not_in_internet
add address=100.64.0.0/10 comment=RFC6890 list=not_in_internet
add address=240.0.0.0/4 comment=RFC6890 list=not_in_internet
add address=192.88.99.0/24 comment="6to4 relay Anycast [RFC 3068]" list=not_in_internet
add address=10.24.0.0/16 comment="Allow VLAN 10 (main network) to send traffic from the LAN" list=allowed_LAN_IPs
add address=10.25.2.0/24 comment="Allow Wireguard machines to send traffic to the LAN and Internet" list=allowed_LAN_IPs
add address=1.146.202.158 comment="Wings temporary hotspot" list=allowed_to_wg-outofband
add address=110.174.59.255 comment="CCC router access" disabled=yes list=allowed_to_router
add address=110.174.59.255 list=allowed_to_wg-outofband
add address=192.168.11.0/24 list=allowed_LAN_IPs
add address=0.0.0.0/0 comment="TEMPORARY wireguard override" list=allowed_to_wg-outofband
/ip firewall filter
add action=accept chain=input comment="Allow WireGuard Out of Band access from specific IPs" dst-port=13231 in-interface=all-ethernet protocol=udp src-address-list=allowed_to_wg-outofband
add action=accept chain=forward comment="Accept dstnat packets" connection-nat-state=dstnat
add action=accept chain=forward comment="Route traffic from Wireguard peers to all Ethernet interfaces" in-interface=wg-outofband out-interface=all-ethernet
add action=accept chain=forward comment="Route traffic from Wireguard peers to Core Switch specifically. " in-interface=wg-outofband out-interface=core-switch-bond
add action=accept chain=input comment=\
"Accept traffic unconditionally from Wireguard peers. TODO (tidy): Likely redundant thanks to the rule above it. If the packet counter never goes above 2,074,236 (2023-04-05) delete it later" \
in-interface=wg-outofband
add action=accept chain=input comment="Allow traffic for connection states \"established\" and \"related\". Lowers load." connection-state=established,related
add action=fasttrack-connection chain=forward comment="FastTrack for QUIC" connection-nat-state=dstnat connection-state=established,related,new,untracked disabled=yes dst-port=51101-51250 hw-offload=\
yes protocol=udp
add action=fasttrack-connection chain=forward comment=FastTrack connection-state=established,related hw-offload=yes
add action=accept chain=forward comment="Forward traffic in established, related state" connection-state=established,related
add action=accept chain=input comment="Accept all traffic if in the allow_to_router address list" src-address-list=allowed_to_router
add action=accept chain=input comment="Accept pings from anyone" protocol=icmp
add action=accept chain=input comment="default configuration" connection-state=established,related
add action=drop chain=input comment="Drop all traffic that doesn't match any other rules"
add action=accept chain=forward comment="Netbird access from outside" in-interface=all-ethernet protocol=tcp src-port=51820
add action=drop chain=forward comment="Drop non-wg-outofband traffic to BMC network" in-interface=!wg-outofband out-interface=bmc-switch-bond src-address=!10.24.0.254
add action=accept chain=forward in-interface=vlan51 out-interface=all-ethernet
add action=accept chain=forward in-interface=wireguard-panicswitch out-interface=all-ethernet
add action=accept chain=forward in-interface=wireguard-panicswitch
add action=accept chain=input in-interface=wireguard-panicswitch
/ip firewall nat
add action=netmap chain=dstnat dst-address=23.227.172.80 to-addresses=10.24.3.200
add action=dst-nat chain=dstnat dst-address=23.227.172.67 to-addresses=10.24.0.2
add action=dst-nat chain=dstnat dst-address=23.227.172.68 to-addresses=10.24.0.3
add action=dst-nat chain=dstnat dst-address=23.227.172.69 to-addresses=10.24.0.4
add action=dst-nat chain=dstnat dst-port=5055 protocol=tcp to-addresses=10.24.0.137 to-ports=22
add action=src-nat chain=srcnat dst-address=!10.24.0.0/16 out-interface=core-switch-bond src-address=10.24.0.0/16 to-addresses=23.227.172.66
add action=src-nat chain=srcnat dst-address=!10.25.2.0/24 out-interface=wg-outofband src-address=10.25.2.0/24 to-addresses=23.227.172.66
add action=src-nat chain=srcnat dst-address=!10.25.2.0/24 out-interface=core-switch-bond src-address=10.25.2.0/24 to-addresses=23.227.172.66
add action=netmap chain=srcnat dst-address=!10.24.3.200 out-interface=core-switch-bond src-address=10.24.3.200 to-addresses=23.227.172.80
add action=dst-nat chain=dstnat comment=prod-acme-dns dst-address=23.227.172.95 to-addresses=10.24.6.232
add action=dst-nat chain=dstnat dst-address=23.227.172.73 to-addresses=10.24.0.8
add action=dst-nat chain=dstnat dst-address=23.227.172.74 to-addresses=10.24.0.9
add action=dst-nat chain=dstnat dst-address=23.227.172.75 to-addresses=10.24.0.10
add action=dst-nat chain=dstnat dst-address=23.227.172.94 to-addresses=192.168.11.11
add action=dst-nat chain=dstnat comment="pioneer SSH tar experiment" dst-address=23.227.172.93 dst-port=1414 protocol=tcp to-addresses=10.24.137.137 to-ports=1414
add action=dst-nat chain=dstnat dst-address=23.227.172.78 to-addresses=10.24.3.60
add action=dst-nat chain=dstnat comment="shuttle-2 netcat transfer experiment" dst-address=23.227.172.93 dst-port=24782 protocol=tcp src-port="" to-addresses=10.24.2.160 to-ports=24782
add action=dst-nat chain=dstnat dst-address=23.227.172.93 dst-port=24783 protocol=udp to-addresses=10.24.2.160 to-ports=24783
add action=dst-nat chain=dstnat comment="ZFS shuttle drain experiment" dst-address=23.227.172.93 dst-port=24799-24802 protocol=tcp to-addresses=10.24.0.207 to-ports=24799-24802
add action=dst-nat chain=dstnat dst-port=46224 protocol=udp to-addresses=10.24.0.207 to-ports=46224
add action=dst-nat chain=dstnat comment=Piper! dst-address=23.227.172.93 dst-port=51101-51250 protocol=udp to-addresses=10.24.0.207 to-ports=51101-51250
add action=dst-nat chain=dstnat comment="Shuttle-10 UDP" dst-address=23.227.172.90 dst-port=51117 protocol=udp to-addresses=10.24.0.207 to-ports=51117
add action=dst-nat chain=dstnat comment="socat sctp shuttle-9" dst-address=23.227.172.89 protocol=sctp to-addresses=10.24.0.207 to-ports=51111
add action=dst-nat chain=dstnat comment="socat sctp shuttle 10" dst-address=23.227.172.92 protocol=sctp to-addresses=10.24.0.207 to-ports=51116
add action=dst-nat chain=dstnat comment="socat sctp shuttle 11" dst-address=23.227.172.93 protocol=sctp to-addresses=10.24.0.207 to-ports=51117
add action=dst-nat chain=dstnat comment="socat tcp shuttle-9" dst-address=23.227.172.93 dst-port=51115 protocol=tcp to-addresses=10.24.0.207 to-ports=22
/ip route
add disabled=no distance=2 dst-address=0.0.0.0/0 gateway=23.92.74.61 pref-src="" routing-table=main suppress-hw-offload=no
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway=23.227.172.65 pref-src="" routing-table=main suppress-hw-offload=no
add disabled=yes distance=1 dst-address=192.168.11.0/24 gateway=192.168.11.1 pref-src="" routing-table=main scope=30 suppress-hw-offload=no target-scope=10
/ip service
set www-ssl certificate=letsencrypt-autogen_1970-01-02T04:07:29Z disabled=no
/system clock
set time-zone-name=America/Chicago
/system routerboard settings
set enter-setup-on=delete-key