Hello all. I have a CCR2004-16G-2S+ running with 10G ethernet to a Google Fiber modem with 8gbps service.
I noticed recently when troubleshooting some TCP streams that bursty download traffic (~4-5gbps) incremements the RX drop field on the WAN interface. I suspect this is causing some behavior I see where downloads start to slow down (as expected) when frames are dropped and window sizes reset.
Broadly, I am trying to confirm my suspicion that this is a hardware level limitation with the CCR2004 interfaces. I have increased the interface queue packet buffer to 500 and also applied some simple queues (may not be configured ideally), however it seems that this bursty traffic is simply overwhelming the interface because during these downloads the router CPU does not exceed ~35-40%.
I also tried implementing flow control on the WAN interface but Google fiber does not appear to respect the pause frames as the same rate of RX drops are observed with flow control enabled.
I have a CHR license, I am considering rebuilding an x86 box with CHR to validate this as I imagine with a powerful enough NIC (ConnectX 4 or better) these RX drops would be resolved.
I have tested with two different SFP+ 10G BaseT transceivers and see a similar rate of RX drops.
Any thoughts or advice would be appreciated!
Config below:
# 2025-05-14 13:38:19 by RouterOS 7.19rc2
#
# model = CCR2004-16G-2S+
/interface bridge
add admin-mac= auto-mac=no frame-types=\
admit-only-vlan-tagged name=RouterBridge priority=0x1000 vlan-filtering=\
yes
/interface ethernet
set [ find default-name=ether1 ] comment="FRACTAL IPMI"
set [ find default-name=ether2 ] comment="FRACTAL PROX MGMT"
set [ find default-name=ether3 ] comment="CRS310 MGMT" disabled=yes
set [ find default-name=ether4 ] comment="Jun Switch MGMT"
set [ find default-name=ether5 ] comment="Desktop Spare"
set [ find default-name=ether6 ] comment=PRINTER
set [ find default-name=ether7 ] disabled=yes
set [ find default-name=ether8 ] disabled=yes
set [ find default-name=ether9 ] disabled=yes
set [ find default-name=ether10 ] disabled=yes
set [ find default-name=ether11 ] disabled=yes
set [ find default-name=ether12 ] disabled=yes
set [ find default-name=ether13 ] disabled=yes
set [ find default-name=ether14 ] comment="NETGEAR FRONT ROOM UPLINK"
set [ find default-name=ether15 ] disabled=yes
set [ find default-name=ether16 ] comment="EX2200-C UPLINK"
set [ find default-name=sfp-sfpplus1 ] comment=WAN
set [ find default-name=sfp-sfpplus2 ] auto-negotiation=no comment=\
"CRS310 Switch" rx-flow-control=on tx-flow-control=on
/interface wireguard
add listen-port=13232 mtu=1420 name=WGDigitalOcean
/interface vlan
add interface=RouterBridge name=VLAN-10-CLIENT vlan-id=10
add interface=RouterBridge name=VLAN-20-BASTION vlan-id=20
add interface=RouterBridge name=VLAN-30-SERVER vlan-id=30
add interface=RouterBridge name=VLAN-40-DMZ vlan-id=40
add interface=RouterBridge name=VLAN-50-IOT vlan-id=50
add interface=RouterBridge name=VLAN-99-MGMT vlan-id=99
/interface list
add name=WAN
/ip pool
add name=MGMT-POOL ranges=10.218.99.15-10.218.99.254
add name=CLIENT-POOL ranges=10.218.10.2-10.218.10.254
add name=BASTION-POOL ranges=10.218.20.2-10.218.20.254
add name=SERVER-POOL ranges=10.218.30.20-10.218.30.254
add name=DMZ-POOL ranges=10.218.40.2-10.218.40.254
add name=IOT-POOL ranges=10.218.50.2-10.218.50.254
/ip dhcp-server
add address-pool=CLIENT-POOL interface=VLAN-10-CLIENT lease-time=10m name=\
CLIENT-DHCP
add address-pool=BASTION-POOL interface=VLAN-20-BASTION lease-time=10m name=\
BASTION-DHCP
add address-pool=SERVER-POOL interface=VLAN-30-SERVER lease-time=10m name=\
SERVER-DHCP
add address-pool=DMZ-POOL interface=VLAN-40-DMZ lease-time=10m name=DMZ-DHCP
add address-pool=MGMT-POOL interface=VLAN-99-MGMT lease-time=10m name=\
MGMT-DHCP
add address-pool=IOT-POOL interface=VLAN-50-IOT name=IOT-POOL
/port
set 0 name=serial0
set 1 name=serial1
/queue type
add cake-diffserv=diffserv4 cake-mpu=64 cake-overhead=38 kind=cake name=\
cake-default
set 9 mq-pfifo-limit=500
/queue interface
set sfp-sfpplus1 queue=multi-queue-ethernet-default
set sfp-sfpplus2 queue=multi-queue-ethernet-default
/queue simple
add max-limit=9500M/9500M name=Inter-VLAN queue=cake-default/cake-default \
target=VLAN-30-SERVER
add max-limit=9500M/9500M name=client-vlan queue=cake-default/cake-default \
target=VLAN-10-CLIENT
add dst=VLAN-30-SERVER max-limit=9500M/9500M name=WAN queue=\
cake-default/cake-default target=sfp-sfpplus1
add max-limit=9500M/9500M name=dmz-vlan queue=cake-default/cake-default \
target=VLAN-40-DMZ
/system logging action
set 3 remote=10.218.30.28 remote-port=12345
/interface bridge port
add bridge=RouterBridge frame-types=admit-only-vlan-tagged interface=\
sfp-sfpplus2
add bridge=RouterBridge frame-types=admit-only-untagged-and-priority-tagged \
interface=ether1 pvid=99
add bridge=RouterBridge frame-types=admit-only-untagged-and-priority-tagged \
interface=ether2 pvid=99
add bridge=RouterBridge frame-types=admit-only-untagged-and-priority-tagged \
interface=ether3 pvid=99
add bridge=RouterBridge frame-types=admit-only-untagged-and-priority-tagged \
interface=ether4 pvid=99
add bridge=RouterBridge frame-types=admit-only-untagged-and-priority-tagged \
interface=ether5 pvid=99
add bridge=RouterBridge frame-types=admit-only-untagged-and-priority-tagged \
interface=ether6 pvid=10
add bridge=RouterBridge frame-types=admit-only-vlan-tagged interface=ether16
add bridge=RouterBridge frame-types=admit-only-untagged-and-priority-tagged \
interface=ether14 pvid=10
/ip firewall connection tracking
set udp-timeout=10s
/ipv6 settings
set accept-redirects=no accept-router-advertisements=yes
/interface bridge vlan
add bridge=RouterBridge tagged=RouterBridge,sfp-sfpplus2,ether16 untagged=\
ether14 vlan-ids=10
add bridge=RouterBridge tagged=RouterBridge,sfp-sfpplus2,ether16 untagged=\
ether1,ether2,ether3,ether4,ether5 vlan-ids=99
add bridge=RouterBridge tagged=RouterBridge,sfp-sfpplus2,ether16 vlan-ids=20
add bridge=RouterBridge tagged=RouterBridge,sfp-sfpplus2 vlan-ids=30
add bridge=RouterBridge tagged=RouterBridge,sfp-sfpplus2 vlan-ids=40
add bridge=RouterBridge tagged=sfp-sfpplus2 vlan-ids=50
/interface list
add include=*2000013,*2000011,*2000012,*2000014 name=Internal
/interface list member
add interface=sfp-sfpplus1 list=WAN
add interface=VLAN-10-CLIENT list=Internal
add interface=VLAN-30-SERVER list=Internal
add interface=VLAN-40-DMZ list=Internal
add interface=VLAN-50-IOT list=Internal
add interface=VLAN-99-MGMT list=Internal
/ip address
add address=10.218.10.1/24 interface=VLAN-10-CLIENT network=10.218.10.0
add address=10.218.20.1/24 interface=VLAN-20-BASTION network=10.218.20.0
add address=10.218.30.1/24 interface=VLAN-30-SERVER network=10.218.30.0
add address=10.218.40.1/24 interface=VLAN-40-DMZ network=10.218.40.0
add address=10.218.99.1/24 interface=VLAN-99-MGMT network=10.218.99.0
add address=10.219.1.1/24 interface=WGDigitalOcean network=10.219.1.0
add address=10.218.50.1/24 interface=VLAN-50-IOT network=10.218.50.0
/ip cloud
set ddns-enabled=yes
/ip dhcp-client
add interface=sfp-sfpplus1
/ip dhcp-server network
add address=10.218.10.0/24 dns-server=10.218.30.20 gateway=10.218.10.1
add address=10.218.20.0/24 dns-server=10.218.30.20 gateway=10.218.20.1
add address=10.218.30.0/24 dns-server=10.218.30.20 gateway=10.218.30.1
add address=10.218.40.0/24 gateway=10.218.40.1
add address=10.218.50.0/24 dns-server=8.8.8.8,1.1.1.1 gateway=10.218.50.1
add address=10.218.99.0/24 dns-server=10.218.30.20 gateway=10.218.99.1
/ip dns
set allow-remote-requests=yes
/ip firewall filter
add action=accept chain=input comment="Accept established, related" \
connection-state=established,related
add action=drop chain=input comment="Drop invalid input" connection-state=\
invalid log=yes log-prefix=input_block
add action=accept chain=input comment="Allow input from MGMT" in-interface=\
VLAN-99-MGMT
add action=accept chain=input comment="Allow input from Desktop" disabled=yes \
in-interface=VLAN-10-CLIENT src-address=10.218.10.18
add action=accept chain=input comment="Allow Wireguard to Router admin" \
in-interface=VLAN-20-BASTION src-address=10.218.20.254
add action=accept chain=input comment="Accept ICMP from ALL" protocol=icmp
add action=accept chain=input comment="Allow DNS from all internal" dst-port=\
53 in-interface-list=Internal protocol=udp
add action=accept chain=input comment="Allow WireGuard from WAN" dst-port=\
13232 protocol=udp
add action=drop chain=input comment="Drop all other Input" log=yes \
log-prefix=input_block
add action=fasttrack-connection chain=forward comment="FAST TRACK FORWARD" \
connection-state=established,related hw-offload=yes
add action=accept chain=forward comment="Forward Established and Tracked" \
connection-state=established,related
add action=drop chain=forward comment="Drop invalid forward" \
connection-state=invalid
add action=accept chain=forward comment="Allow VLANs to Internet" \
in-interface-list=Internal out-interface-list=WAN
add action=accept chain=forward comment="Bastion to PiHole" dst-address=\
10.218.30.20 in-interface=VLAN-20-BASTION
add action=accept chain=forward comment="Bastion to PiHole" dst-address=\
10.218.30.28 src-address=10.218.40.3
add action=accept chain=forward comment="MGMT to Auth Hosts" dst-address=\
10.218.30.29 in-interface=VLAN-99-MGMT out-interface=VLAN-30-SERVER
add action=accept chain=forward dst-address=10.218.30.30 in-interface=\
VLAN-99-MGMT out-interface=VLAN-30-SERVER
add action=accept chain=forward comment="Proxmox to TrueNAS" dst-address=\
10.218.30.5 in-interface=VLAN-99-MGMT out-interface=VLAN-30-SERVER \
src-address=10.218.99.9
add action=accept chain=forward comment=\
"All clients to connect to WG Bastion" dst-address=10.218.20.254 \
dst-port=13214 in-interface=VLAN-10-CLIENT out-interface=VLAN-20-BASTION \
out-interface-list=all protocol=udp
add action=accept chain=forward comment="Allow WG host to ALL Internal" \
in-interface=VLAN-20-BASTION out-interface-list=all src-address=\
10.218.20.254
add action=accept chain=forward comment="Allow MGMT to ALL Internal" \
in-interface=VLAN-99-MGMT out-interface-list=all src-address=\
10.218.99.0/24
add action=accept chain=forward comment="PROXMOX Scraping" dst-address=\
10.218.99.9 dst-port=9199 in-interface=VLAN-30-SERVER out-interface=\
VLAN-99-MGMT protocol=tcp src-address=10.218.30.28
add action=accept chain=forward dst-address=10.218.99.9 dst-port=9290 \
in-interface=VLAN-30-SERVER out-interface=VLAN-99-MGMT protocol=tcp \
src-address=10.218.30.28
add action=accept chain=forward dst-address=10.218.99.9 dst-port=8006 \
in-interface=VLAN-30-SERVER out-interface=VLAN-99-MGMT protocol=tcp \
src-address=10.218.30.28
add action=accept chain=forward comment="Allow internal clients to Pihole" \
connection-state=new dst-address=10.218.30.20 dst-port=53 \
in-interface-list=Internal protocol=udp
add action=accept chain=forward comment="Allow DMZ to FreeIPA" \
connection-state=new dst-address=10.218.30.29 in-interface=VLAN-40-DMZ \
out-interface=VLAN-30-SERVER src-address=10.218.40.3
add action=accept chain=forward comment="Allow DMZ clients to Pihole" \
connection-state=new dst-address=10.218.30.20 dst-port=53 in-interface=\
VLAN-40-DMZ protocol=udp
add action=accept chain=forward comment="Allow phone to reach Asterisk" \
dst-address=10.218.30.23 in-interface=VLAN-10-CLIENT
add action=accept chain=forward comment="Remote WG to Bastion" dst-address=\
10.218.20.254 dst-port=13214 in-interface=sfp-sfpplus1 out-interface=\
VLAN-20-BASTION protocol=udp
add action=accept chain=forward comment="Allow Internal Clients to Plex DMZ" \
dst-address=10.218.40.3 in-interface=VLAN-10-CLIENT
add action=accept chain=forward comment="Allow DMZ Docker to access NFS" \
dst-address=10.218.30.5 in-interface=VLAN-40-DMZ out-interface=\
VLAN-30-SERVER src-address=10.218.40.3
add action=accept chain=forward comment="Allow NAS from internal Clients" \
dst-address=10.218.30.5 in-interface-list=Internal out-interface=\
VLAN-30-SERVER
add action=accept chain=forward comment="Allow WG Desktop to DO VPS" \
dst-address=10.219.1.0/24 src-address=10.219.0.0/24
add action=accept chain=forward comment="Allow WG Desktop to DO VPS" \
dst-port=32400 in-interface=WGDigitalOcean out-interface=VLAN-40-DMZ \
protocol=tcp
add action=accept chain=forward comment="Allow DO VPS to WG Desktop" \
dst-address=10.219.0.0/24 src-address=10.219.1.0/24
add action=accept chain=forward comment="Allow DO VPS to WG Desktop" \
in-interface=VLAN-99-MGMT out-interface-list=all
add action=drop chain=forward comment="Drop All Other Forward"
/ip firewall mangle
add action=change-mss chain=forward disabled=yes new-mss=clamp-to-pmtu \
protocol=tcp tcp-flags=syn
add action=mark-connection chain=forward disabled=yes ipsec-policy=out,ipsec \
new-connection-mark=ipsec passthrough=no
add action=mark-connection chain=forward disabled=yes ipsec-policy=in,ipsec \
new-connection-mark=ipsec passthrough=no
add action=change-mss chain=forward disabled=yes new-mss=1300 out-interface=\
*1A protocol=tcp tcp-flags=syn,ack
add action=change-mss chain=forward disabled=yes new-mss=1300 out-interface=\
VLAN-30-SERVER protocol=tcp tcp-flags=syn,ack
add action=change-mss chain=forward disabled=yes new-mss=1300 out-interface=\
VLAN-40-DMZ protocol=tcp tcp-flags=syn,ack
add action=mark-connection chain=forward disabled=yes in-interface=*1A \
new-connection-mark=wg passthrough=no
/ip firewall nat
add action=masquerade chain=srcnat dst-address=!10.219.0.0/24 \
out-interface-list=WAN
add action=masquerade chain=srcnat disabled=yes dst-address=10.218.30.21 \
src-address=10.218.30.21
add action=masquerade chain=srcnat disabled=yes out-interface=WGDigitalOcean
add action=dst-nat chain=dstnat dst-port=32400 in-interface=WGDigitalOcean \
protocol=tcp to-addresses=10.218.40.3 to-ports=32400
add action=dst-nat chain=dstnat disabled=yes dst-port=34198 in-interface=\
WGDigitalOcean protocol=udp to-addresses=10.218.40.3 to-ports=34198
add action=dst-nat chain=dstnat disabled=yes dst-port=34197 in-interface=\
sfp-sfpplus1 protocol=udp to-addresses=10.218.40.3 to-ports=34197
add action=dst-nat chain=dstnat disabled=yes dst-port=34198 in-interface=\
sfp-sfpplus1 protocol=udp to-addresses=10.218.40.3 to-ports=34198
add action=dst-nat chain=dstnat dst-port=13214 in-interface=sfp-sfpplus1 \
protocol=udp to-addresses=10.218.20.254 to-ports=13214
add action=masquerade chain=srcnat disabled=yes out-interface=sfp-sfpplus1 \
src-address=10.219.0.0/24
/ip firewall service-port
set ftp disabled=yes
set tftp disabled=yes
set h323 disabled=yes
set sip disabled=yes
set pptp disabled=yes
/ip ipsec profile
set [ find default=yes ] dpd-interval=2m dpd-maximum-failures=5
/ip route
add disabled=no distance=1 dst-address=10.219.0.0/24 gateway=10.218.20.254 \
routing-table=main scope=30 suppress-hw-offload=no target-scope=10
/ip service
set ftp disabled=yes
set ssh address=10.218.99.0/24,10.219.0.0/24,10.218.20.254/32
set telnet disabled=yes
set www disabled=yes
set winbox address=10.218.99.0/24,10.219.0.0/24,10.218.20.254/32
set api disabled=yes
set api-ssl disabled=yes
/ip ssh
set strong-crypto=yes
/ipv6 address
add address=::1 disabled=yes from-pool=google-fiber-ipv6-pool interface=\
VLAN-99-MGMT
add address=::1 from-pool=google-fiber-ipv6-pool interface=VLAN-10-CLIENT
add address=::1 from-pool=google-fiber-ipv6-pool interface=VLAN-30-SERVER
add address=::1 from-pool=google-fiber-ipv6-pool interface=VLAN-40-DMZ
/ipv6 dhcp-client
add add-default-route=yes default-route-tables=main interface=sfp-sfpplus1 \
pool-name=google-fiber-ipv6-pool request=address,prefix \
validate-server-duid=no
/ipv6 firewall address-list
add address=::/128 comment="defconf: unspecified address" list=bad_ipv6
add address=::1/128 comment="defconf: lo" list=bad_ipv6
add address=fec0::/10 comment="defconf: site-local" list=bad_ipv6
add address=::ffff:0.0.0.0/96 comment="defconf: ipv4-mapped" list=bad_ipv6
add address=::/96 comment="defconf: ipv4 compat" list=bad_ipv6
add address=100::/64 comment="defconf: discard only " list=bad_ipv6
add address=2001:db8::/32 comment="defconf: documentation" list=bad_ipv6
add address=2001:10::/28 comment="defconf: ORCHID" list=bad_ipv6
add address=3ffe::/16 comment="defconf: 6bone" list=bad_ipv6
/ipv6 firewall filter
add action=accept chain=input comment="accept established/related" \
connection-state=established,related
add action=drop chain=input comment="drop invalid" connection-state=invalid
add action=accept chain=input comment="allow ICMPv6" protocol=icmpv6
add action=accept chain=input comment="Allow DHCPv6 Advertise inbound" \
dst-port=546 in-interface=sfp-sfpplus1 protocol=udp
add action=accept chain=input comment="defconf: accept UDP traceroute" port=\
33434-33534 protocol=udp
add action=accept chain=input comment="defconf: accept IKE" dst-port=500,4500 \
protocol=udp
add action=accept chain=input comment="defconf: accept ipsec AH" protocol=\
ipsec-ah
add action=accept chain=input comment="defconf: accept ipsec ESP" protocol=\
ipsec-esp
add action=drop chain=input comment="drop everything else from WAN" \
connection-state="" log=yes
add action=fasttrack-connection chain=forward comment=\
"defconf: accept ICMPv6" connection-state=established,related
add action=accept chain=forward comment="allow established/related" \
connection-state=established,related,untracked
add action=drop chain=forward comment="drop invalid" connection-state=invalid
add action=drop chain=forward comment=\
"defconf: drop packets with bad src ipv6" src-address-list=bad_ipv6
add action=drop chain=forward comment=\
"defconf: drop packets with bad dst ipv6" dst-address-list=bad_ipv6
add action=drop chain=forward comment="defconf: rfc4890 drop hop-limit=1" \
hop-limit=equal:1 protocol=icmpv6
add action=accept chain=forward comment="allow established/related" \
connection-state=established,related,new in-interface-list=Internal \
out-interface=sfp-sfpplus1
add action=accept chain=forward comment="allow established/related" disabled=\
yes in-interface=VLAN-99-MGMT out-interface=VLAN-40-DMZ
add action=accept chain=forward comment="allow established/related" disabled=\
yes in-interface=VLAN-99-MGMT out-interface=VLAN-30-SERVER
add action=accept chain=forward comment="allow established/related" disabled=\
yes in-interface=VLAN-30-SERVER out-interface=VLAN-40-DMZ
add action=accept chain=forward comment="allow established/related" disabled=\
yes in-interface=VLAN-40-DMZ out-interface=VLAN-30-SERVER
add action=accept chain=forward comment="defconf: accept ICMPv6" protocol=\
icmpv6
add action=accept chain=forward comment="defconf: accept ICMPv6" protocol=\
ipv6-frag
add action=accept chain=forward comment="defconf: accept HIP" protocol=139
add action=accept chain=forward comment="defconf: accept IKE" dst-port=\
500,4500 protocol=udp
add action=accept chain=forward comment="defconf: accept ipsec AH" protocol=\
ipsec-ah
add action=accept chain=forward comment="defconf: accept ipsec ESP" protocol=\
ipsec-esp
add action=accept chain=forward comment=\
"defconf: accept all that matches ipsec policy" ipsec-policy=in,ipsec
add action=drop chain=forward comment="drop incoming WAN traffic" log=yes
/ipv6 nd
set [ find default=yes ] advertise-dns=no advertise-mac-address=no disabled=\
yes hop-limit=64 interface=sfp-sfpplus1 ra-interval=30s-2m
add disabled=yes interface=RouterBridge
add disabled=yes dns=2606:4700:4700::1111,2606:4700:4700::1001 interface=\
VLAN-99-MGMT
add dns=2605:a601:8014:c702:be24:11ff:fe1e:2198 interface=VLAN-10-CLIENT
add dns=2606:4700:4700::1001,2606:4700:4700::1111 interface=VLAN-30-SERVER
add dns=2606:4700:4700::1001,2606:4700:4700::1111 interface=VLAN-40-DMZ
/ipv6 nd prefix default
set preferred-lifetime=20m valid-lifetime=12h
/radius
add address=10.218.30.30 require-message-auth=no service=login src-address=\
10.218.99.1
/routing pimsm interface-template
add disabled=no instance=*1 interfaces=VLAN-20-BASTION,VLAN-40-DMZ \
source-addresses=10.218.20.6
/system clock
set time-zone-name=America/Denver
/system logging
add action=remote prefix=firewall
add disabled=yes topics=debug,dhcp
/system package update
set channel=long-term
/system routerboard settings
set enter-setup-on=delete-key
/tool bandwidth-server
set enabled=no
/tool mac-server
set allowed-interface-list=*2000012
/tool mac-server mac-winbox
set allowed-interface-list=*2000012
/tool mac-server ping
set enabled=no
/user aaa
set default-group=full use-radius=yes
