Problem with WG Connections and Port Forwarding with ECMP Balancing

RouterOS General
1

Hi all!

Short and to the point, I have a gigabit fiber connection that drops occasionally. I setup a 5G Cell WAN using TMobile Home Internet for load balancing and to act as failover when Fiber goes out. Fiber has a static public IPV4 delivered to the MikroTik router (PUBLIC_IP in config below). 5G gateway can’t bridge and isn’t customizable, so MikroTik gets a DHCP address from the 5G Gateway in the 192.168.12.0/24 range, and gateway is 192.168.12.1. I’ve static configured the gateway in RouterOS, then use DHCP Client for IP.

ECMP load balancing works perfect. I’ve pushed around 1.6 gigabit total through the gigabit fiber and 5G combined. My issue is with port forwards and Wireguard VPN. The port forwards work occasionally, despite the mangle rules to mark the connections. Seemingly ignoring the mangle mark-routing rules and picking a load balanced gateway. I have seen in packet sniffer traffic coming in ether1-WAN(Fiber), dst-nat to server, server reply back out, then we send reply out ether2-CellWAN. I also am totally unable to connect to the Wireguard server on the Router, sending handshake initiations, but not finishing handshake. Mobile device doesn’t appear to receive anything at all. Disabling ECMP, port forwards and Wireguard work perfect.

I am fairly new to RouterOS, so still learning as I go. I’ve followed countless guides and tutorials, tried building rules 100 different ways and have never succeeded. Hoping you smart people can help! Thanks!

Configuration (Removed static LAN DHCP Leases as it was a giant block and Wireguard Peers):

[admin@MikroTik] > /export hide-sensitive
# dec/11/2022 11:16:57 by RouterOS 7.7beta9
# model = CCR2004-16G-2S+
/interface bridge
add name=dockers
/interface ethernet
set [ find default-name=ether1 ] advertise=\
    10M-half,10M-full,100M-half,100M-full,1000M-full mac-address=\
    2A:99:D3:A5:1D:06 name=ether1-WAN
set [ find default-name=ether2 ] advertise=\
    10M-half,10M-full,100M-half,100M-full,1000M-full name=ether2-CellWAN
set [ find default-name=sfp-sfpplus2 ] name=sfp2-LAN
/interface veth
add address=172.17.0.2/24 gateway=172.17.0.1 name=veth1-PiHole
/interface wireguard
add listen-port=51820 mtu=1420 name=wg1-Trusted
add listen-port=51825 mtu=1420 name=wg2-Untrusted
/interface vlan
add interface=sfp2-LAN name=vlan10-CAMS vlan-id=10
add interface=sfp2-LAN name=vlan20-ISO vlan-id=20
add interface=sfp2-LAN name=vlan30-NoInt vlan-id=30
/container mounts
add dst=/etc/pihole name=etc_pihole src=/disk1/pihole-mnt/etc
add dst=/etc/dnsmasq.d name=dnsmasq_pihole src=/disk1/pihole-mnt/etc-dnsmasq.d
/disk
set pcie1-usb1-part1 parent=pcie1-usb1 partition-offset=512 partition-size=\
    "32 080 199 680"
/interface list
add name=WAN
add name=LAN
add name=VLANS
add name=WG1
add name=VLAN20andUWG
add name=Untrusted
add name=Trusted
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip ipsec policy group
add name=MobileClients
/ip ipsec profile
add enc-algorithm=aes-128 name=MobileClients
/ip ipsec peer
add exchange-mode=ike2 name=MobileDevices passive=yes profile=MobileClients
/ip ipsec proposal
add name=MobileClient
/ip pool
add name=LAN ranges=10.0.0.100-10.0.0.200
add name=CAMS ranges=10.0.10.100-10.0.10.200
add name=ISO ranges=10.0.20.100-10.0.20.200
add name=NOINT ranges=192.168.146.100-192.168.146.200
/ip dhcp-server
add address-pool=LAN interface=sfp2-LAN name=dhcp1
add address-pool=CAMS interface=vlan10-CAMS name=dhcp2
add address-pool=ISO interface=vlan20-ISO name=dhcp3
add address-pool=NOINT interface=vlan30-NoInt name=dhcp4
/port
set 0 name=serial0
set 1 name=serial1
/routing table
add disabled=no fib name=Fiber_WAN
add disabled=no fib name=Cell_WAN
/container
add envlist=pihole_envs interface=veth1-PiHole mounts=etc_pihole,dnsmasq_pihole \
    root-dir=disk1/pihole
/container config
set registry-url=https://registry-1.docker.io tmpdir=disk1/pull
/container envs
add key=TZ name=pihole_envs value=America/Chicago
add key=WEBPASSWORD name=pihole_envs value=PASSWORDREMOVED
/interface bridge port
add bridge=dockers interface=veth1-PiHole
/ip settings
set max-neighbor-entries=8192
/ipv6 settings
set max-neighbor-entries=8192
/interface list member
add interface=ether1-WAN list=WAN
add interface=sfp2-LAN list=LAN
add interface=vlan10-CAMS list=VLANS
add interface=vlan20-ISO list=VLANS
add interface=wg1-Trusted list=WG1
add interface=vlan20-ISO list=VLAN20andUWG
add interface=wg2-Untrusted list=VLAN20andUWG
add interface=vlan10-CAMS list=Untrusted
add interface=vlan20-ISO list=Untrusted
add interface=vlan30-NoInt list=Untrusted
add interface=wg2-Untrusted list=Untrusted
add interface=sfp2-LAN list=Trusted
add interface=wg1-Trusted list=Trusted
add interface=vlan30-NoInt list=VLANS
add interface=vlan10-CAMS list=LAN
add interface=vlan30-NoInt list=LAN
add interface=dockers list=LAN
add interface=vlan20-ISO list=LAN
add interface=wg1-Trusted list=LAN
add interface=wg2-Untrusted list=LAN
add interface=ether2-CellWAN list=WAN
/interface ovpn-server server
set auth=sha1,md5
/ip address
add address=PUBLIC_IP/21 interface=ether1-WAN network=WAN_NETWORK
add address=10.0.0.1/24 interface=sfp2-LAN network=10.0.0.0
add address=10.0.10.1/24 interface=vlan10-CAMS network=10.0.10.0
add address=10.0.20.1/24 interface=vlan20-ISO network=10.0.20.0
add address=172.30.0.1/24 interface=wg1-Trusted network=172.30.0.0
add address=172.31.0.1/24 interface=wg2-Untrusted network=172.31.0.0
add address=192.168.146.1/24 interface=vlan30-NoInt network=192.168.146.0
add address=172.17.0.1/24 interface=dockers network=172.17.0.0
/ip dhcp-client
add add-default-route=no interface=ether2-CellWAN use-peer-dns=no use-peer-ntp=\
    no
/ip dhcp-server network
add address=10.0.0.0/24 dns-server=10.0.0.254,172.17.0.2 domain=dill gateway=\
    10.0.0.1 netmask=24
add address=10.0.10.0/24 dns-server=1.1.1.1,1.0.0.1 gateway=10.0.10.1
add address=10.0.20.0/24 dns-server=1.1.1.1,1.0.0.1 domain=dilliso gateway=\
    10.0.20.1
add address=192.168.146.0/24 dns-server=192.168.146.1 gateway=192.168.146.1
/ip dns
set allow-remote-requests=yes servers=1.1.1.1,1.0.0.1
/ip firewall address-list
add address=10.0.0.254 list=LocalDNS
add address=172.17.0.2 list=LocalDNS
add address=10.0.0.251 list=RemoteADInternal
add address=10.0.20.80 list=RemoteADInternal
add address=172.31.0.10 list=RDPFriends
add address=172.31.0.20 list=RDPFriends
add address=172.31.0.30 list=RDPFriends
add address=172.31.0.40 list=RDPFriends
add address=172.31.0.41 list=RDPFriends
add address=172.31.0.150 list=RDPFriends
add address=10.0.20.5 list=RDPFriends-Devices
add address=10.0.20.6 list=RDPFriends-Devices
add address=10.0.0.0/8 list=RFC1918
add address=172.16.0.0/12 list=RFC1918
add address=192.168.0.0/16 list=RFC1918
add address=0.0.0.0/8 list=RFC6890
add address=172.16.0.0/12 list=RFC6890
add address=192.168.0.0/16 list=RFC6890
add address=10.0.0.0/8 list=RFC6890
add address=169.254.0.0/16 list=RFC6890
add address=127.0.0.0/8 list=RFC6890
add address=224.0.0.0/4 list=RFC6890
add address=198.18.0.0/15 list=RFC6890
add address=192.0.0.0/24 list=RFC6890
add address=192.0.2.0/24 list=RFC6890
add address=198.51.100.0/24 list=RFC6890
add address=203.0.113.0/24 list=RFC6890
add address=100.64.0.0/10 list=RFC6890
add address=240.0.0.0/4 list=RFC6890
add address=192.88.99.0/24 list=RFC6890
/ip firewall filter
add action=accept chain=forward connection-state=established,related
add action=accept chain=input connection-state=established,related
add action=drop chain=input comment="Drop Invalid Connections" \
    connection-state=invalid
add action=accept chain=forward comment="Accept DST-NAT Packets" \
    connection-nat-state=dstnat
add action=accept chain=output comment="Allow Router Output" log-prefix=\
    ALLOWOUT
add action=accept chain=input comment="Allow Connections to WireGuard1" \
    dst-port=51820 protocol=udp
add action=accept chain=input comment="Allow Connections to WireGuard2" \
    dst-port=51825 protocol=udp
add action=accept chain=forward comment="Allow MikroTik PiHole to WAN" \
    out-interface-list=WAN src-address=172.17.0.2
add action=accept chain=forward comment="Allow Trusted to Everything" \
    in-interface-list=Trusted
add action=accept chain=input in-interface-list=Trusted
add action=accept chain=forward comment="Allow AD Server to DH Network IPs" \
    dst-address-list=RemoteADInternal in-interface=wg2-Untrusted src-address=\
    172.31.0.200
add action=accept chain=forward comment="Allow VLAN20 and UWG to Access PiHole" \
    dst-address-list=LocalDNS dst-port=53 in-interface-list=VLAN20andUWG \
    protocol=udp
add action=accept chain=forward dst-address-list=LocalDNS dst-port=53 \
    in-interface-list=VLAN20andUWG protocol=tcp
add action=accept chain=forward comment="Allow VLAN20 to Access Plex" \
    dst-address=10.0.0.20 dst-port=32400 in-interface-list=VLAN20andUWG \
    out-interface=sfp2-LAN protocol=tcp
add action=accept chain=forward comment="Allow VLAN20 to WAN" in-interface=\
    vlan20-ISO out-interface-list=WAN
add action=accept chain=forward comment=\
    "Allow Friends to Udoo Bolt (20.5) and LuckyVM (20.6)" dst-address-list=\
    RDPFriends-Devices src-address-list=RDPFriends
add action=drop chain=forward comment="Drop Everything Else" log-prefix=DROP
add action=drop chain=input log-prefix=DROPINPUT
/ip firewall mangle
add action=accept chain=prerouting dst-address=!192.168.12.0/24 \
    dst-address-list=RFC1918 src-address=!192.168.12.0/24 src-address-list=\
    RFC1918
add action=mark-connection chain=prerouting connection-mark=no-mark \
    in-interface=ether1-WAN new-connection-mark=Fiber_Mark passthrough=yes
add action=mark-connection chain=prerouting connection-mark=no-mark \
    in-interface=ether2-CellWAN new-connection-mark=Cell_Mark passthrough=yes
add action=mark-routing chain=output connection-mark=Fiber_Mark \
    new-routing-mark=Fiber_WAN passthrough=yes
add action=mark-routing chain=output connection-mark=Cell_Mark \
    new-routing-mark=Cell_WAN passthrough=yes
/ip firewall nat
add action=masquerade chain=srcnat comment="Base WAN Masquerade" out-interface=\
    ether1-WAN
add action=masquerade chain=srcnat out-interface=ether2-CellWAN
add action=dst-nat chain=dstnat comment="NAT ADMASTER LAN IP thru Wireguard" \
    dst-address=10.10.0.4 to-addresses=172.31.0.200
add action=dst-nat chain=dstnat comment="Plex Server" dst-address=PUBLIC_IP \
    dst-port=32400 protocol=tcp to-addresses=10.0.0.20
add action=dst-nat chain=dstnat comment="Lucky's Game Server" dst-address=\
    PUBLIC_IP dst-port=25560-25575 protocol=tcp to-addresses=10.0.20.6 \
    to-ports=25560-25575
add action=dst-nat chain=dstnat dst-address=PUBLIC_IP dst-port=25560-25575 \
    protocol=udp to-addresses=10.0.20.6 to-ports=25560-25575
add action=dst-nat chain=dstnat dst-address=PUBLIC_IP dst-port=4379-4380 \
    protocol=udp to-addresses=10.0.20.6 to-ports=4379-4380
add action=dst-nat chain=dstnat dst-address=PUBLIC_IP dst-port=3478 \
    protocol=udp to-addresses=10.0.20.6 to-ports=3478
/ip firewall service-port
set ftp disabled=yes
set sip disabled=yes
/ip ipsec identity
add generate-policy=port-strict mode-config=MobileDevices peer=MobileDevices \
    policy-template-group=MobileClients
/ip ipsec mode-config
add address-pool=*5 address-prefix-length=32 name=MobileDevices
/ip ipsec policy
set 0 disabled=yes
add group=MobileClients proposal=MobileClient template=yes
/ip route
add check-gateway=ping disabled=no distance=1 dst-address=0.0.0.0/0 gateway=\
    WAN_GATEWAY pref-src="" routing-table=main scope=30 suppress-hw-offload=no \
    target-scope=10
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway=192.168.12.1 pref-src=\
    "" routing-table=Cell_WAN scope=30 suppress-hw-offload=no target-scope=10
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway=WAN_GATEWAY pref-src=\
    "" routing-table=Fiber_WAN scope=30 suppress-hw-offload=no target-scope=10
add check-gateway=ping disabled=no distance=1 dst-address=0.0.0.0/0 gateway=\
    192.168.12.1 pref-src="" routing-table=main scope=30 suppress-hw-offload=no \
    target-scope=10
/ip service
set telnet disabled=yes
set ftp disabled=yes
set www address=10.0.0.0/24,172.30.0.0/24
set ssh address=10.0.0.0/24 disabled=yes
set api address=10.0.0.20/32 disabled=yes
set winbox address=10.0.0.0/24,172.30.0.0/24
set api-ssl disabled=yes
/ip smb
set allow-guests=no
/ip smb shares
add directory=/disk1 name=disk1
/ip smb users
add name=john
/ip ssh
set strong-crypto=yes
/snmp
set enabled=yes
/system clock
set time-zone-name=America/Chicago
/system ntp client
set enabled=yes
/system ntp client servers
add address=0.us.pool.ntp.org
add address=1.us.pool.ntp.org
add address=2.us.pool.ntp.org
add address=3.us.pool.ntp.org
/system package update
set channel=testing
/tool bandwidth-server
set enabled=no
/tool mac-server
set allowed-interface-list=Trusted
/tool mac-server mac-winbox
set allowed-interface-list=Trusted
/tool mac-server ping
set enabled=no
/tool sniffer
set filter-ip-address=172.58.105.140/32 filter-port=32400
/user aaa
set default-group=full use-radius=yes
2

Found another guide on Bandwidth Based Load Balancing that handled the mangle rules a bit different here: https://mum.mikrotik.com/presentations/US12/tomas.pdf

Adapted the mangle rules to ECMP and came up with this config (Static DHCP Leases and Wireguard Peers Removed):

[admin@MikroTik] > /export hide-sensitive
# dec/11/2022 17:40:57 by RouterOS 7.7beta9
# model = CCR2004-16G-2S+
/interface bridge
add name=dockers
/interface ethernet
set [ find default-name=ether1 ] advertise=10M-half,10M-full,100M-half,100M-full,1000M-full mac-address=2A:99:D3:A5:1D:06 name=ether1-WAN
set [ find default-name=ether2 ] advertise=10M-half,10M-full,100M-half,100M-full,1000M-full name=ether2-CellWAN
set [ find default-name=sfp-sfpplus2 ] name=sfp2-LAN
/interface veth
add address=172.17.0.2/24 gateway=172.17.0.1 name=veth1-PiHole
/interface wireguard
add listen-port=51820 mtu=1420 name=wg1-Trusted
add listen-port=51825 mtu=1420 name=wg2-Untrusted
/interface vlan
add interface=sfp2-LAN name=vlan10-CAMS vlan-id=10
add interface=sfp2-LAN name=vlan20-ISO vlan-id=20
add interface=sfp2-LAN name=vlan30-NoInt vlan-id=30
/container mounts
add dst=/etc/pihole name=etc_pihole src=/pcie1-usb1-part1/pihole-mnt/etc
add dst=/etc/dnsmasq.d name=dnsmasq_pihole src=/pcie1-usb1-part1/pihole-mnt/etc-dnsmasq.d
/disk
set pcie1-usb1-part1 parent=pcie1-usb1 partition-offset=512 partition-size="32 080 199 680"
/interface list
add name=WAN
add name=LAN
add name=VLANS
add name=WG1
add name=VLAN20andUWG
add name=Untrusted
add name=Trusted
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip ipsec policy group
add name=MobileClients
/ip ipsec profile
add enc-algorithm=aes-128 name=MobileClients
/ip ipsec peer
add exchange-mode=ike2 name=MobileDevices passive=yes profile=MobileClients
/ip ipsec proposal
add name=MobileClient
/ip pool
add name=LAN ranges=10.0.0.100-10.0.0.200
add name=CAMS ranges=10.0.10.100-10.0.10.200
add name=ISO ranges=10.0.20.100-10.0.20.200
add name=NOINT ranges=192.168.146.100-192.168.146.200
/ip dhcp-server
add address-pool=LAN interface=sfp2-LAN name=dhcp1
add address-pool=CAMS interface=vlan10-CAMS name=dhcp2
add address-pool=ISO interface=vlan20-ISO name=dhcp3
add address-pool=NOINT interface=vlan30-NoInt name=dhcp4
/port
set 0 name=serial0
set 1 name=serial1
/routing table
add disabled=no fib name=Fiber_WAN
add disabled=no fib name=Cell_WAN
/container
add envlist=pihole_envs interface=veth1-PiHole mounts=etc_pihole,dnsmasq_pihole root-dir=pcie1-usb1-part1/pihole
/container config
set registry-url=https://registry-1.docker.io tmpdir=/pcie1-usb1-part1/pull
/container envs
add key=TZ name=pihole_envs value=America/Chicago
add key=WEBPASSWORD name=pihole_envs value=PASS_REMOVED
/interface bridge port
add bridge=dockers interface=veth1-PiHole
/ip settings
set max-neighbor-entries=8192
/ipv6 settings
set max-neighbor-entries=8192
/interface list member
add interface=ether1-WAN list=WAN
add interface=sfp2-LAN list=LAN
add interface=vlan10-CAMS list=VLANS
add interface=vlan20-ISO list=VLANS
add interface=wg1-Trusted list=WG1
add interface=vlan20-ISO list=VLAN20andUWG
add interface=wg2-Untrusted list=VLAN20andUWG
add interface=vlan10-CAMS list=Untrusted
add interface=vlan20-ISO list=Untrusted
add interface=vlan30-NoInt list=Untrusted
add interface=wg2-Untrusted list=Untrusted
add interface=sfp2-LAN list=Trusted
add interface=wg1-Trusted list=Trusted
add interface=vlan30-NoInt list=VLANS
add interface=vlan10-CAMS list=LAN
add interface=vlan30-NoInt list=LAN
add interface=dockers list=LAN
add interface=vlan20-ISO list=LAN
add interface=wg1-Trusted list=LAN
add interface=wg2-Untrusted list=LAN
add interface=ether2-CellWAN list=WAN
/interface ovpn-server server
set auth=sha1,md5
/ip address
add address=192.168.88.1/24 comment=defconf interface=ether15 network=192.168.88.0
add address=PUBLIC_IP/21 interface=ether1-WAN network=FIBER_NETWORK
add address=10.0.0.1/24 interface=sfp2-LAN network=10.0.0.0
add address=10.0.10.1/24 interface=vlan10-CAMS network=10.0.10.0
add address=10.0.20.1/24 interface=vlan20-ISO network=10.0.20.0
add address=172.30.0.1/24 interface=wg1-Trusted network=172.30.0.0
add address=172.31.0.1/24 interface=wg2-Untrusted network=172.31.0.0
add address=192.168.146.1/24 interface=vlan30-NoInt network=192.168.146.0
add address=172.17.0.1/24 interface=dockers network=172.17.0.0
/ip dhcp-client
add add-default-route=no interface=ether2-CellWAN use-peer-dns=no use-peer-ntp=no
/ip dhcp-server network
add address=10.0.0.0/24 dns-server=10.0.0.254,172.17.0.2 domain=dill gateway=10.0.0.1 netmask=24
add address=10.0.10.0/24 dns-server=1.1.1.1,1.0.0.1 gateway=10.0.10.1
add address=10.0.20.0/24 dns-server=1.1.1.1,1.0.0.1 domain=dilliso gateway=10.0.20.1
add address=192.168.146.0/24 dns-server=192.168.146.1 gateway=192.168.146.1
/ip dns
set allow-remote-requests=yes servers=1.1.1.1,1.0.0.1
/ip firewall address-list
add address=10.0.0.254 list=LocalDNS
add address=172.17.0.2 list=LocalDNS
add address=10.0.0.251 list=RemoteADInternal
add address=10.0.20.80 list=RemoteADInternal
add address=172.31.0.10 list=RDPFriends
add address=172.31.0.20 list=RDPFriends
add address=172.31.0.30 list=RDPFriends
add address=172.31.0.40 list=RDPFriends
add address=172.31.0.41 list=RDPFriends
add address=172.31.0.150 list=RDPFriends
add address=10.0.20.5 list=RDPFriends-Devices
add address=10.0.20.6 list=RDPFriends-Devices
add address=10.0.0.0/8 list=RFC1918
add address=172.16.0.0/12 list=RFC1918
add address=192.168.0.0/16 list=RFC1918
add address=10.0.0.0/24 list=0HopNets
add address=10.0.10.0/24 list=0HopNets
add address=10.0.20.0/24 list=0HopNets
add address=172.17.0.0/24 list=0HopNets
add address=172.31.0.0/24 list=0HopNets
add address=172.30.0.0/24 list=0HopNets
add address=192.168.12.0/24 list=0HopNets
add address=192.168.88.0/24 list=0HopNets
add address=192.168.146.0/24 list=0HopNets
add address=FIBER_NETWORK/21 list=0HopNets
add address=10.0.0.0/24 list=LAN
add address=10.0.10.0/24 list=LAN
add address=10.0.20.0/24 list=LAN
add address=172.17.0.0/24 list=LAN
add address=172.31.0.0/24 list=LAN
add address=172.30.0.0/24 list=LAN
/ip firewall filter
add action=accept chain=forward connection-state=established,related
add action=accept chain=input connection-state=established,related
add action=drop chain=input comment="Drop Invalid Connections" connection-state=invalid
add action=accept chain=forward comment="Accept DST-NAT Packets" connection-nat-state=dstnat
add action=accept chain=output comment="Allow Router Output" log-prefix=ALLOWOUT
add action=accept chain=input comment="Allow Connections to WireGuard1" dst-port=51820 protocol=udp
add action=accept chain=input comment="Allow Connections to WireGuard2" dst-port=51825 protocol=udp
add action=accept chain=forward comment="Allow MikroTik PiHole to WAN" out-interface-list=WAN src-address=172.17.0.2
add action=accept chain=forward comment="Allow Trusted to Everything" in-interface-list=Trusted
add action=accept chain=input in-interface-list=Trusted
add action=accept chain=forward comment="Allow AD Server to DH Network IPs" dst-address-list=RemoteADInternal in-interface=wg2-Untrusted src-address=172.31.0.200
add action=accept chain=forward comment="Allow VLAN20 and UWG to Access PiHole" dst-address-list=LocalDNS dst-port=53 in-interface-list=VLAN20andUWG protocol=udp
add action=accept chain=forward dst-address-list=LocalDNS dst-port=53 in-interface-list=VLAN20andUWG protocol=tcp
add action=accept chain=forward comment="Allow VLAN20 to Access Plex" dst-address=10.0.0.20 dst-port=32400 in-interface-list=VLAN20andUWG out-interface=sfp2-LAN protocol=tcp
add action=accept chain=forward comment="Allow VLAN20 to WAN" in-interface=vlan20-ISO out-interface-list=WAN
add action=accept chain=forward comment="Allow Friends to Udoo Bolt (20.5) and LuckyVM (20.6)" dst-address-list=RDPFriends-Devices src-address-list=RDPFriends
add action=drop chain=forward comment="Drop Everything Else" log-prefix=DROP
add action=drop chain=input log-prefix=DROPINPUT
/ip firewall mangle
add action=accept chain=prerouting comment="Connect Accepted Networks" dst-address-list=0HopNets src-address-list=0HopNets
add action=mark-connection chain=input comment="WAN -> ROS" connection-mark=no-mark in-interface=ether1-WAN new-connection-mark=Fiber_Mark_ROS passthrough=yes
add action=mark-connection chain=input connection-mark=no-mark in-interface=ether2-CellWAN new-connection-mark=Cell_Mark_ROS passthrough=yes
add action=mark-routing chain=output connection-mark=Fiber_Mark_ROS log-prefix=FIBEROUT new-routing-mark=Fiber_WAN passthrough=yes
add action=mark-routing chain=output connection-mark=Cell_Mark_ROS new-routing-mark=Cell_WAN passthrough=yes
add action=mark-connection chain=forward comment="WAN -> LAN" connection-mark=no-mark in-interface=ether1-WAN new-connection-mark=Fiber_Mark_LAN passthrough=yes
add action=mark-connection chain=forward connection-mark=no-mark in-interface=ether2-CellWAN new-connection-mark=Cell_Mark_LAN passthrough=yes
add action=mark-routing chain=prerouting connection-mark=Fiber_Mark_LAN new-routing-mark=Fiber_WAN passthrough=yes src-address-list=LAN
add action=mark-routing chain=prerouting connection-mark=Cell_Mark_LAN new-routing-mark=Cell_WAN passthrough=yes src-address-list=LAN
/ip firewall nat
add action=masquerade chain=srcnat comment="Base WAN Masquerade" out-interface=ether1-WAN
add action=masquerade chain=srcnat out-interface=ether2-CellWAN
add action=dst-nat chain=dstnat comment="NAT ADMASTER LAN IP thru Wireguard" dst-address=10.10.0.4 to-addresses=172.31.0.200
add action=dst-nat chain=dstnat comment="Plex Server" dst-address=PUBLIC_IP dst-port=32400 protocol=tcp to-addresses=10.0.0.20
add action=dst-nat chain=dstnat comment="Lucky's Game Server" dst-address=PUBLIC_IP dst-port=25560-25575 protocol=tcp to-addresses=10.0.20.6 to-ports=25560-25575
add action=dst-nat chain=dstnat dst-address=PUBLIC_IP dst-port=25560-25575 protocol=udp to-addresses=10.0.20.6 to-ports=25560-25575
add action=dst-nat chain=dstnat dst-address=PUBLIC_IP dst-port=4379-4380 protocol=udp to-addresses=10.0.20.6 to-ports=4379-4380
add action=dst-nat chain=dstnat dst-address=PUBLIC_IP dst-port=3478 protocol=udp to-addresses=10.0.20.6 to-ports=3478
/ip firewall service-port
set ftp disabled=yes
set sip disabled=yes
/ip ipsec identity
add generate-policy=port-strict mode-config=MobileDevices peer=MobileDevices policy-template-group=MobileClients
/ip ipsec mode-config
add address-pool=*5 address-prefix-length=32 name=MobileDevices
/ip ipsec policy
set 0 disabled=yes
add group=MobileClients proposal=MobileClient template=yes
/ip route
add check-gateway=ping disabled=no distance=1 dst-address=0.0.0.0/0 gateway=FIBER_GATEWAY pref-src="" routing-table=main scope=30 suppress-hw-offload=no target-scope=10
add check-gateway=ping disabled=no distance=1 dst-address=0.0.0.0/0 gateway=192.168.12.1 pref-src="" routing-table=main scope=30 suppress-hw-offload=no target-scope=10
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway=FIBER_GATEWAY pref-src="" routing-table=Fiber_WAN scope=30 suppress-hw-offload=no target-scope=10
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway=192.168.12.1 pref-src="" routing-table=Cell_WAN scope=30 suppress-hw-offload=no target-scope=10
/ip service
set telnet disabled=yes
set ftp disabled=yes
set www address=10.0.0.0/24,172.30.0.0/24
set ssh address=10.0.0.0/24 disabled=yes
set api address=10.0.0.20/32 disabled=yes
set winbox address=10.0.0.0/24,172.30.0.0/24
set api-ssl disabled=yes
/ip smb
set allow-guests=no
/ip smb shares
add directory=/disk1 name=disk1
/ip smb users
add name=john
/ip ssh
set strong-crypto=yes
/snmp
set enabled=yes
/system clock
set time-zone-name=America/Chicago
/system ntp client
set enabled=yes
/system ntp client servers
add address=0.us.pool.ntp.org
add address=1.us.pool.ntp.org
add address=2.us.pool.ntp.org
add address=3.us.pool.ntp.org
/system package update
set channel=testing
/tool bandwidth-server
set enabled=no
/tool mac-server
set allowed-interface-list=Trusted
/tool mac-server mac-winbox
set allowed-interface-list=Trusted
/tool mac-server ping
set enabled=no
/tool sniffer
set filter-ip-address=98.183.98.101/32 filter-port=51825
/user aaa
set default-group=full use-radius=yes

Forward based routing, to servers behind LAN seems to be working fine now. No reports of issues or failed connections. Input for Wireguard still seems to be picking the Fiber or Cell gateways at random for replies. I have observed it receiving via Fiber and responding via Cell in the Packet Sniffer:
PacketSniff.png
98. is cell phone IP
192. is ROS’s IP on Cell Side.
158. is ROS’s IP on Fiber Side.

Any help is much appreciated.

EDIT: Forwarding is still experiencing issues, same as input.

3

Are you saying you expect the 5G WAN connection to be able to act as a wireguard server without getting a public IP address??

4

No, the WireGuard connections are directed to the Fiber connection which does have a static IP from the ISP. Without ECMP enabled, this works as expected. My issue is that RouterOS is receiving the packets from the WireGuard client over the Fiber gateway, as intended, then replying out the 5G gateway instead of replying out the fiber gateway, as configured in the mangle rules.

My apologies for the confusion, I appreciate your time!

EDIT: Forwarding is still have the same issue. Though it was resolved, but seems to still occasionally being going out cell gateway at random.

5

Well assuming your client is coming over for internet or to visit subnets or to config the router ( the admin that is ).
Therefore you have a route automatically pointing to the WG gateway for clients assuming .X/32 client.

Problem is I dont see any allowed IP settings. NO PEER settings???

Also not clear is why you are using dst-nat for wireguard, what is the purpose of this rule… Besides the fact that format wise its all wrong - but I could be wrong.
add action=dst-nat chain=dstnat comment=“NAT ADMASTER LAN IP thru Wireguard” dst-address=10.10.0.4 to-addresses=172.31.0.200

6

I didn’t know if they would be relevant to the routing issue. I have included them below:

/interface wireguard peers
add allowed-address=172.30.0.10/32 comment=*** interface=wg1-Trusted public-key=***
add allowed-address=172.31.0.50/32 comment=*** interface=wg2-Untrusted public-key=***
add allowed-address=172.31.0.201/32 comment=*** endpoint-address=ENDPOINT_IP endpoint-port=51820 interface=wg2-Untrusted public-key=***
add allowed-address=172.31.0.200/32 comment=*** endpoint-address=ENDPOINT_IP endpoint-port=51820 interface=wg2-Untrusted public-key=***
add allowed-address=172.31.0.150/32 comment=*** interface=wg2-Untrusted public-key=***
add allowed-address=172.30.0.30/32 comment=*** interface=wg1-Trusted public-key=***
add allowed-address=172.31.0.10/32 comment=*** interface=wg2-Untrusted public-key=***
add allowed-address=172.30.0.40/32 comment=*** interface=wg1-Trusted public-key=***
add allowed-address=172.31.0.20/32 comment=*** interface=wg2-Untrusted public-key=***
add allowed-address=172.31.0.30/32 comment=*** interface=wg2-Untrusted public-key=***
add allowed-address=172.31.0.40/32 comment=*** interface=wg2-Untrusted public-key=***
add allowed-address=172.31.0.41/32 comment=*** interface=wg2-Untrusted public-key=***



That’s a rule I had for a specific purpose. AD Server in the cloud had it’s main interface IP as 10.10.0.4 - It had Wireguard directly installed on it with a WG Interface IP of 172.31.0.200. AD based DNS would give clients the 10.10.0.4 on Azure end, which was correct, but also on our on-prem end. That rule essentially made calls on the on-prem end for 10.10.0.4 go to 172.31.0.200 and connect. Was a stupid hacky way to do it, but no longer used. Just deleted it.

The issue seems to be somewhere with the mangle rules. I edited above where I said the forward rules were working. Both forward and input have mangle rules to set a specific outgoing interface. Despite those rules, port forwards through the router and WG connections direct to the router are not obeying the mangle routing-mark rules. All connections come in ether1-WAN (Fiber Connection). Some replies go back out ether1-WAN and work fine, some go out ether2-CellWAN. It is at random which leads me to believe the routing-mark rules are not working correctly, and instead the replies are just hitting the ECMP load balance and going out whichever interface they pick at random. Hopefully that all makes sense.

Thanks again for your help!

7

Wanted to mention, I found anav’s Pathway to Config Success here: https://forum.mikrotik.com/viewtopic.php?p=906567

I removed all routing tables, routes, and mangle rules, and recreated them following section L in the above post. Everything worked as intended, port forwards and wireguard connected every time. I enabled ECMP, and it all went back to the randomly working, randomly not. I don’t know where I’ve messed it up, but I cannot seem to get replies to marked connections to follow the mark-routing rules.

I’m really not sure where to go from here.

8

Just as an aside…

Person A asks for help
Person B says, yes I can help but I need to see the config

Person A replies with partial config
Person B says I need to see full config

Person A, who HAS a problem, and doesn’t know what the problem is,
But very often states.

a. I didnt think the rest of the config was important
b. I didnt know if the rest would be relevant…

BUT miraculously KNOWS what NOT to send —> does not compute LOL.

9

So once again I will ask since I cannot respond to an emotional distress call… (ur last post),

for the complete config with all the new changes.
/export file=anynameyouwish ( minus serial number and any public WANIP info )

10

I understand, my apologies. I will ensure I provide complete information.

Full config is below:

# dec/12/2022 09:09:47 by RouterOS 7.7beta9
# model = CCR2004-16G-2S+
/interface bridge
add name=dockers
/interface ethernet
set [ find default-name=ether1 ] advertise=\
    10M-half,10M-full,100M-half,100M-full,1000M-full mac-address=\
    2A:99:D3:A5:1D:06 name=ether1-WAN
set [ find default-name=ether2 ] advertise=\
    10M-half,10M-full,100M-half,100M-full,1000M-full name=ether2-CellWAN
set [ find default-name=sfp-sfpplus2 ] name=sfp2-LAN
/interface veth
add address=172.17.0.2/24 gateway=172.17.0.1 name=veth1-PiHole
/interface wireguard
add listen-port=51820 mtu=1420 name=wg1-Trusted
add listen-port=51825 mtu=1420 name=wg2-Untrusted
/interface vlan
add interface=sfp2-LAN name=vlan10-CAMS vlan-id=10
add interface=sfp2-LAN name=vlan20-ISO vlan-id=20
add interface=sfp2-LAN name=vlan30-NoInt vlan-id=30
/container mounts
add dst=/etc/pihole name=etc_pihole src=/pcie1-usb1-part1/pihole-mnt/etc
add dst=/etc/dnsmasq.d name=dnsmasq_pihole src=\
    /pcie1-usb1-part1/pihole-mnt/etc-dnsmasq.d
/disk
set pcie1-usb1-part1 parent=pcie1-usb1 partition-offset=512 partition-size=\
    "32 080 199 680"
/interface list
add name=WAN
add name=LAN
add name=VLANS
add name=WG1
add name=VLAN20andUWG
add name=Untrusted
add name=Trusted
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip ipsec policy group
add name=MobileClients
/ip ipsec profile
add enc-algorithm=aes-128 name=MobileClients
/ip ipsec peer
add exchange-mode=ike2 name=MobileDevices passive=yes profile=MobileClients
/ip ipsec proposal
add name=MobileClient
/ip pool
add name=LAN ranges=10.0.0.100-10.0.0.200
add name=CAMS ranges=10.0.10.100-10.0.10.200
add name=ISO ranges=10.0.20.100-10.0.20.200
add name=NOINT ranges=192.168.146.100-192.168.146.200
/ip dhcp-server
add address-pool=LAN interface=sfp2-LAN name=dhcp1
add address-pool=CAMS interface=vlan10-CAMS name=dhcp2
add address-pool=ISO interface=vlan20-ISO name=dhcp3
add address-pool=NOINT interface=vlan30-NoInt name=dhcp4
/port
set 0 name=serial0
set 1 name=serial1
/routing table
add disabled=no fib name=Fiber_WAN
add disabled=no fib name=Cell_WAN
/container
add envlist=pihole_envs interface=veth1-PiHole mounts=\
    etc_pihole,dnsmasq_pihole root-dir=pcie1-usb1-part1/pihole
/container config
set registry-url=https://registry-1.docker.io tmpdir=/pcie1-usb1-part1/pull
/container envs
add key=TZ name=pihole_envs value=America/Chicago
add key=WEBPASSWORD name=pihole_envs value=PASSWORD_REMOVE
/interface bridge port
add bridge=dockers interface=veth1-PiHole
/ip settings
set max-neighbor-entries=8192
/ipv6 settings
set max-neighbor-entries=8192
/interface list member
add interface=ether1-WAN list=WAN
add interface=sfp2-LAN list=LAN
add interface=vlan10-CAMS list=VLANS
add interface=vlan20-ISO list=VLANS
add interface=wg1-Trusted list=WG1
add interface=vlan20-ISO list=VLAN20andUWG
add interface=wg2-Untrusted list=VLAN20andUWG
add interface=vlan10-CAMS list=Untrusted
add interface=vlan20-ISO list=Untrusted
add interface=vlan30-NoInt list=Untrusted
add interface=wg2-Untrusted list=Untrusted
add interface=sfp2-LAN list=Trusted
add interface=wg1-Trusted list=Trusted
add interface=vlan30-NoInt list=VLANS
add interface=vlan10-CAMS list=LAN
add interface=vlan30-NoInt list=LAN
add interface=dockers list=LAN
add interface=vlan20-ISO list=LAN
add interface=wg1-Trusted list=LAN
add interface=wg2-Untrusted list=LAN
add interface=ether2-CellWAN list=WAN
/interface ovpn-server server
set auth=sha1,md5
/interface wireguard peers
add allowed-address=172.30.0.10/32 comment=*** interface=wg1-Trusted public-key=***
add allowed-address=172.31.0.50/32 comment=*** interface=wg2-Untrusted public-key=***
add allowed-address=172.31.0.201/32 comment=*** endpoint-address=*** \
    endpoint-port=51820 interface=wg2-Untrusted public-key=***
add allowed-address=172.31.0.200/32 comment=*** \
    endpoint-address=20.124.88.159 endpoint-port=51820 interface=\
    wg2-Untrusted public-key=***
add allowed-address=172.31.0.150/32 comment=*** interface=\
    wg2-Untrusted public-key=***
add allowed-address=172.30.0.30/32 comment=*** interface=\
    wg1-Trusted public-key=***
add allowed-address=172.31.0.10/32 comment=*** interface=wg2-Untrusted \
    public-key=***
add allowed-address=172.30.0.40/32 comment=*** interface=wg1-Trusted \
    public-key=***
add allowed-address=172.31.0.20/32 comment=*** interface=wg2-Untrusted \
    public-key=***
add allowed-address=172.31.0.30/32 comment=*** interface=wg2-Untrusted \
    public-key=***
add allowed-address=172.31.0.40/32 comment=*** interface=\
    wg2-Untrusted public-key=***
add allowed-address=172.31.0.41/32 comment=*** interface=\
    wg2-Untrusted public-key=***
/ip address
add address=192.168.88.1/24 comment=defconf interface=ether15 network=\
    192.168.88.0
add address=FIBER_IP/21 interface=ether1-WAN network=FIBER_NETWORK
add address=10.0.0.1/24 interface=sfp2-LAN network=10.0.0.0
add address=10.0.10.1/24 interface=vlan10-CAMS network=10.0.10.0
add address=10.0.20.1/24 interface=vlan20-ISO network=10.0.20.0
add address=172.30.0.1/24 interface=wg1-Trusted network=172.30.0.0
add address=172.31.0.1/24 interface=wg2-Untrusted network=172.31.0.0
add address=192.168.146.1/24 interface=vlan30-NoInt network=192.168.146.0
add address=172.17.0.1/24 interface=dockers network=172.17.0.0
/ip dhcp-client
add add-default-route=no interface=ether2-CellWAN use-peer-dns=no \
    use-peer-ntp=no
/ip dhcp-server lease
add address=10.0.20.5 client-id=1:0:c0:8:9a:5e:a0 mac-address=\
    00:C0:08:9A:5E:A0 server=dhcp3
add address=10.0.0.2 client-id=1:2c:c8:1b:32:94:d4 mac-address=\
    2C:C8:1B:32:94:D4 server=dhcp1
add address=10.0.0.254 client-id=1:e4:5f:1:9:cb:ee mac-address=\
    E4:5F:01:09:CB:EE server=dhcp1
add address=10.0.10.106 client-id=1:9c:8e:cd:35:28:c9 mac-address=\
    9C:8E:CD:35:28:C9 server=dhcp2
add address=10.0.10.108 client-id=1:9c:8e:cd:38:6b:b mac-address=\
    9C:8E:CD:38:6B:0B server=dhcp2
add address=10.0.10.107 client-id=1:9c:8e:cd:34:56:15 mac-address=\
    9C:8E:CD:34:56:15 server=dhcp2
add address=10.0.0.50 client-id=1:52:54:0:74:75:8d mac-address=\
    52:54:00:74:75:8D server=dhcp1
add address=10.0.10.109 client-id=1:9c:8e:cd:32:19:7c mac-address=\
    9C:8E:CD:32:19:7C server=dhcp2
add address=10.0.10.110 client-id=1:9c:8e:cd:35:29:a1 mac-address=\
    9C:8E:CD:35:29:A1 server=dhcp2
add address=10.0.10.111 client-id=1:9c:8e:cd:35:28:fb mac-address=\
    9C:8E:CD:35:28:FB server=dhcp2
add address=10.0.20.6 client-id=1:52:54:0:5e:35:4a mac-address=\
    52:54:00:5E:35:4A server=dhcp3
add address=10.0.20.80 client-id=1:b4:22:0:6f:2a:db mac-address=\
    B4:22:00:6F:2A:DB server=dhcp3
add address=10.0.0.5 client-id=1:34:3a:20:c4:4:e6 mac-address=\
    34:3A:20:C4:04:E6 server=dhcp1
add address=10.0.0.21 client-id=1:4:42:1a:f0:f5:e0 mac-address=\
    04:42:1A:F0:F5:E0 server=dhcp1
add address=10.0.0.250 client-id=1:3c:ec:ef:c8:48:66 mac-address=\
    3C:EC:EF:C8:48:66 server=dhcp1
add address=10.0.0.3 client-id=1:18:fd:74:12:32:ed mac-address=\
    18:FD:74:12:32:ED server=dhcp1
add address=10.0.0.251 client-id=1:3c:ec:ef:af:9c:84 mac-address=\
    3C:EC:EF:AF:9C:84 server=dhcp1
add address=10.0.0.31 client-id=1:bc:9:1b:a:94:ca mac-address=\
    BC:09:1B:0A:94:CA server=dhcp1
add address=10.0.0.30 client-id=1:80:61:5f:15:48:e8 mac-address=\
    80:61:5F:15:48:E8 server=dhcp1
add address=10.0.0.20 client-id=1:78:2d:7e:12:54:d mac-address=\
    78:2D:7E:12:54:0D server=dhcp1
/ip dhcp-server network
add address=10.0.0.0/24 dns-server=10.0.0.254,172.17.0.2 domain=dill gateway=\
    10.0.0.1 netmask=24
add address=10.0.10.0/24 dns-server=1.1.1.1,1.0.0.1 gateway=10.0.10.1
add address=10.0.20.0/24 dns-server=1.1.1.1,1.0.0.1 domain=dilliso gateway=\
    10.0.20.1
add address=192.168.146.0/24 dns-server=192.168.146.1 gateway=192.168.146.1
/ip dns
set allow-remote-requests=yes servers=1.1.1.1,1.0.0.1
/ip firewall address-list
add address=10.0.0.254 list=LocalDNS
add address=172.17.0.2 list=LocalDNS
add address=10.0.0.251 list=RemoteADInternal
add address=10.0.20.80 list=RemoteADInternal
add address=172.31.0.10 list=RDPFriends
add address=172.31.0.20 list=RDPFriends
add address=172.31.0.30 list=RDPFriends
add address=172.31.0.40 list=RDPFriends
add address=172.31.0.41 list=RDPFriends
add address=172.31.0.150 list=RDPFriends
add address=10.0.20.5 list=RDPFriends-Devices
add address=10.0.20.6 list=RDPFriends-Devices
add address=10.0.0.0/8 list=RFC1918
add address=172.16.0.0/12 list=RFC1918
add address=192.168.0.0/16 list=RFC1918
add address=10.0.0.0/24 list=0HopNets
add address=10.0.10.0/24 list=0HopNets
add address=10.0.20.0/24 list=0HopNets
add address=172.17.0.0/24 list=0HopNets
add address=172.31.0.0/24 list=0HopNets
add address=172.30.0.0/24 list=0HopNets
add address=192.168.12.0/24 list=0HopNets
add address=192.168.88.0/24 list=0HopNets
add address=192.168.146.0/24 list=0HopNets
add address=FIBER_NETWORK/21 list=0HopNets
add address=10.0.0.0/24 list=LAN
add address=10.0.10.0/24 list=LAN
add address=10.0.20.0/24 list=LAN
add address=172.17.0.0/24 list=LAN
add address=172.31.0.0/24 list=LAN
add address=172.30.0.0/24 list=LAN
/ip firewall filter
add action=accept chain=forward connection-state=established,related
add action=accept chain=input connection-state=established,related
add action=drop chain=input comment="Drop Invalid Connections" \
    connection-state=invalid
add action=accept chain=forward comment="Accept DST-NAT Packets" \
    connection-nat-state=dstnat
add action=accept chain=output comment="Allow Router Output" log-prefix=\
    ALLOWOUT
add action=accept chain=input comment="Allow Connections to WireGuard1" \
    dst-port=51820 protocol=udp
add action=accept chain=input comment="Allow Connections to WireGuard2" \
    dst-port=51825 protocol=udp
add action=accept chain=forward comment="Allow MikroTik PiHole to WAN" \
    out-interface-list=WAN src-address=172.17.0.2
add action=accept chain=forward comment="Allow Trusted to Everything" \
    in-interface-list=Trusted
add action=accept chain=input in-interface-list=Trusted
add action=accept chain=forward comment="Allow AD Server to DH Network IPs" \
    dst-address-list=RemoteADInternal in-interface=wg2-Untrusted src-address=\
    172.31.0.200
add action=accept chain=forward comment=\
    "Allow VLAN20 and UWG to Access PiHole" dst-address-list=LocalDNS \
    dst-port=53 in-interface-list=VLAN20andUWG protocol=udp
add action=accept chain=forward dst-address-list=LocalDNS dst-port=53 \
    in-interface-list=VLAN20andUWG protocol=tcp
add action=accept chain=forward comment="Allow VLAN20 to Access Plex" \
    dst-address=10.0.0.20 dst-port=32400 in-interface-list=VLAN20andUWG \
    out-interface=sfp2-LAN protocol=tcp
add action=accept chain=forward comment="Allow VLAN20 to WAN" in-interface=\
    vlan20-ISO out-interface-list=WAN
add action=accept chain=forward comment=\
    "Allow Friends to Udoo Bolt (20.5) and LuckyVM (20.6)" dst-address-list=\
    RDPFriends-Devices src-address-list=RDPFriends
add action=drop chain=forward comment="Drop Everything Else" log-prefix=DROP
add action=drop chain=input log-prefix=DROPINPUT
/ip firewall mangle
add action=mark-connection chain=prerouting connection-state=new \
    in-interface=ether1-WAN new-connection-mark=Fiber_Mark passthrough=yes
add action=mark-routing chain=prerouting connection-mark=Fiber_Mark \
    in-interface-list=LAN new-routing-mark=Fiber_WAN passthrough=yes
add action=mark-connection chain=prerouting connection-state=new \
    in-interface=ether2-CellWAN new-connection-mark=Cell_Mark passthrough=yes
add action=mark-routing chain=prerouting connection-mark=Cell_Mark \
    in-interface-list=LAN new-routing-mark=Cell_WAN passthrough=yes
/ip firewall nat
add action=masquerade chain=srcnat comment="Base WAN Masquerade" \
    out-interface=ether1-WAN
add action=masquerade chain=srcnat out-interface=ether2-CellWAN
add action=dst-nat chain=dstnat comment="Plex Server" dst-address=\
    FIBER_IP dst-port=32400 protocol=tcp to-addresses=10.0.0.20
add action=dst-nat chain=dstnat comment="Lucky's Game Server" dst-address=\
    FIBER_IP dst-port=25560-25575 protocol=tcp to-addresses=10.0.20.6 \
    to-ports=25560-25575
add action=dst-nat chain=dstnat dst-address=FIBER_IP dst-port=\
    25560-25575 protocol=udp to-addresses=10.0.20.6 to-ports=25560-25575
add action=dst-nat chain=dstnat dst-address=FIBER_IP dst-port=4379-4380 \
    protocol=udp to-addresses=10.0.20.6 to-ports=4379-4380
add action=dst-nat chain=dstnat dst-address=FIBER_IP dst-port=3478 \
    protocol=udp to-addresses=10.0.20.6 to-ports=3478
/ip firewall service-port
set ftp disabled=yes
set sip disabled=yes
/ip ipsec identity
add generate-policy=port-strict mode-config=MobileDevices peer=MobileDevices \
    policy-template-group=MobileClients
/ip ipsec mode-config
add address-pool=*5 address-prefix-length=32 name=MobileDevices
/ip ipsec policy
set 0 disabled=yes
add group=MobileClients proposal=MobileClient template=yes
/ip route
add check-gateway=ping disabled=no distance=1 dst-address=0.0.0.0/0 gateway=\
    FIBER_GATEWAY pref-src="" routing-table=main scope=30 suppress-hw-offload=\
    no target-scope=10
add check-gateway=ping disabled=no distance=1 dst-address=0.0.0.0/0 gateway=\
    192.168.12.1 pref-src="" routing-table=main scope=30 suppress-hw-offload=\
    no target-scope=10
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway=FIBER_GATEWAY \
    pref-src="" routing-table=Fiber_WAN scope=30 suppress-hw-offload=no \
    target-scope=10
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway=192.168.12.1 \
    pref-src="" routing-table=Cell_WAN scope=30 suppress-hw-offload=no \
    target-scope=10
/ip service
set telnet disabled=yes
set ftp disabled=yes
set www address=10.0.0.0/24,172.30.0.0/24
set ssh address=10.0.0.0/24 disabled=yes
set api address=10.0.0.20/32 disabled=yes
set winbox address=10.0.0.0/24,172.30.0.0/24
set api-ssl disabled=yes
/ip smb
set allow-guests=no
/ip smb shares
add directory=/disk1 name=disk1
/ip smb users
add name=john
/ip ssh
set strong-crypto=yes
/snmp
set enabled=yes
/system clock
set time-zone-name=America/Chicago
/system ntp client
set enabled=yes
/system ntp client servers
add address=0.us.pool.ntp.org
add address=1.us.pool.ntp.org
add address=2.us.pool.ntp.org
add address=3.us.pool.ntp.org
/system package update
set channel=testing
/tool bandwidth-server
set enabled=no
/tool mac-server
set allowed-interface-list=Trusted
/tool mac-server mac-winbox
set allowed-interface-list=Trusted
/tool mac-server ping
set enabled=no
/tool sniffer
set filter-ip-address=98.183.98.101/32 filter-port=51825
/user aaa
set default-group=full use-radius=yes

Thank you.

11

(1) What is the purpose of this rule, never seen it before…

add action=accept chain=output comment=“Allow Router Output” log-prefix=
ALLOWOUT

12

Allow the router itself to connect to the internet. Without it, I couldn’t get the router to connect to MikroTik update servers to download packages, run it’s own DNS, etc.