Internet Not Full-Speed

RouterOS Beginner Basics
1

So I have followed every tutorial I could find here and in other places and have hit a dead end. I have xfinity internet at 1Gbps speed and when I run speed tests connected directly to the modem my average download/upload is 850/40 which I can live with. However once I connect the Mikrotik router into the loop the best I can get is 450/40. Attached is the details from the /export hide-sensitive compact command on the interface further sanitized in a couple areas to not give away any personal details. Any help/insight you guys could provide would be awesome as I’ve hit the end of what I think I can figure out on my own.

The only odd ball thing that I have on my network that others may not is a Pi-Hole for internet ad blocking, it is at IP 192.168.1.15 and my DNS is pointed at it as the below config will reflect.


# may/04/2020 14:59:51 by RouterOS 6.46.6
# software id = CMVH-K794
#
# model = RB750Gr3
# serial number = CC220B49A972
/interface bridge
add admin-mac=C4:AD:34:98:38:6C auto-mac=no comment=defconf name=bridge
/caps-man configuration
add datapath.bridge=bridge datapath.client-to-client-forwarding=yes datapath.local-forwarding=yes mode=ap name=\
    "XXXXX Main" security.authentication-types=wpa2-psk ssid=XXXXX
add datapath.bridge=bridge name="XXXXX Guest" security.authentication-types=wpa2-psk ssid=XXXXX-Guest
/caps-man interface
add configuration="XXXXX Main" disabled=no l2mtu=1600 mac-address=C4:AD:34:D2:48:46 master-interface=none name=\
    "WAP1 W1" radio-mac=C4:AD:34:D2:48:46 radio-name=C4AD34D24846
add configuration="XXXXX Guest" disabled=no l2mtu=1600 mac-address=C4:AD:34:D2:48:47 master-interface=none name=\
    "WAP1 W2" radio-mac=C4:AD:34:D2:48:47 radio-name=C4AD34D24847
/interface list
add comment=defconf name=WAN
add comment=defconf name=LAN
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip hotspot profile
set [ find default=yes ] html-directory=flash/hotspot
/ip pool
add name=dhcp ranges=192.168.1.100-192.168.1.149
/ip dhcp-server
add add-arp=yes address-pool=dhcp disabled=no interface=bridge lease-time=20h name=defconf
/queue type
add kind=pcq name=PCQ_Download pcq-classifier=dst-address pcq-rate=64k
add kind=pcq name=PCQ_Upload pcq-classifier=src-address pcq-rate=32k
/queue interface
set ether1 queue=ethernet-default
set ether2 queue=ethernet-default
set ether3 queue=ethernet-default
/queue tree
add name="PCQ Down" packet-mark=client_download parent=global queue=PCQ_Download
add name="PCQ Up" packet-mark=client_upload parent=global queue=PCQ_Upload
/user group
set full policy=\
    local,telnet,ssh,ftp,reboot,read,write,policy,test,winbox,password,web,sniff,sensitive,api,romon,dude,tikapp
/caps-man manager
set ca-certificate=auto certificate=auto enabled=yes
/caps-man manager interface
set [ find default=yes ] forbid=yes
add disabled=no interface=bridge
/caps-man provisioning
add master-configuration="XXXXX Main" name-format=identity radio-mac=C4:AD:34:D2:48:44
/dude
set enabled=yes
/interface bridge port
add bridge=bridge comment=defconf interface=ether2 trusted=yes
add bridge=bridge comment=defconf interface=ether3 trusted=yes
add bridge=bridge comment=defconf interface=ether4 trusted=yes
add bridge=bridge comment=defconf interface=ether5 trusted=yes
/ip neighbor discovery-settings
set discover-interface-list=LAN
/interface detect-internet
set detect-interface-list=all
/interface list member
add comment=defconf interface=bridge list=LAN
add comment=defconf interface=ether1 list=WAN
/interface wireless cap
set certificate=request lock-to-caps-man=yes
/ip address
add address=192.168.1.1/24 comment=defconf interface=ether2 network=192.168.1.0
/ip cloud
set ddns-enabled=yes ddns-update-interval=15m
/ip dhcp-client
add comment=defconf disabled=no interface=ether1
/ip dhcp-server network
add address=192.168.1.0/24 comment=defconf dns-server=192.168.1.15 domain=XXXXX-Home gateway=192.168.1.1 netmask=24 \
    ntp-server=216.239.35.0
/ip dns
set allow-remote-requests=yes servers=192.168.1.15
/ip dns static
add address=192.168.1.1 name=router.lan
/ip firewall address-list
add address=XXXXXXXXXX.sn.mynetname.net list=WANIP
/ip firewall filter
add action=accept chain=forward comment="defconf: accept in ipsec policy" ipsec-policy=in,ipsec
add action=accept chain=forward comment="defconf: accept out ipsec policy" ipsec-policy=out,ipsec
add action=fasttrack-connection chain=forward comment="defconf: fasttrack" connection-state=established,related
add action=accept chain=input comment="defconf: accept established,related,untracked" connection-state=\
    established,related,untracked
add action=drop chain=input comment="defconf: drop invalid" connection-state=invalid
add action=accept chain=input comment="defconf: Allow ICMP" protocol=icmp
add action=accept chain=input comment="defconf: Allow Router Access from LAN" src-address=192.168.1.0/24
add action=drop chain=input comment="defconf: drop all not coming from LAN" in-interface-list=!LAN
add action=accept chain=forward comment="defconf: accept established,related, untracked" connection-state=\
    established,related,untracked
add action=drop chain=forward comment="defconf: drop invalid" connection-state=invalid
add action=drop chain=input comment="defconf: drop all from WAN not DSTNATed" connection-nat-state=!dstnat \
    connection-state=new in-interface=ether1
/ip firewall mangle
add action=mark-packet chain=prerouting comment="Packet Marking" in-interface=!ether1 new-packet-mark=client_upload \
    passthrough=yes
add action=mark-packet chain=prerouting in-interface=ether1 new-packet-mark=client_download passthrough=yes
/ip firewall nat
add action=masquerade chain=srcnat comment=HAIRPIN dst-address=192.168.1.0/24 src-address=192.168.1.0/24
add action=masquerade chain=srcnat comment="defconf: masquerade" ipsec-policy=out,none out-interface-list=WAN
add action=dst-nat chain=dstnat dst-address-list=WANIP dst-port=32400 protocol=tcp to-addresses=192.168.1.3
add action=dst-nat chain=dstnat dst-address-list=WANIP dst-port=443 protocol=tcp to-addresses=192.168.1.9
add action=dst-nat chain=dstnat dst-address-list=WANIP dst-port=21 protocol=tcp to-addresses=192.168.1.9
add action=dst-nat chain=dstnat disabled=yes dst-address-list=WANIP dst-port=22 protocol=tcp to-addresses=192.168.1.9
add action=dst-nat chain=dstnat comment="template port forward" disabled=yes dst-address-list=WANIP dst-port=8080 \
    protocol=tcp to-addresses=192.168.1.2
/ip service
set telnet disabled=yes
set ftp disabled=yes
set ssh port=2200
set api disabled=yes
set api-ssl disabled=yes
/ip upnp
set allow-disable-external-interface=yes enabled=yes
/ip upnp interfaces
add interface=ether1 type=external
add interface=ether2 type=internal
add interface=ether3 type=internal
/system clock
set time-zone-name=America/New_York
/system identity
set name=XXXXXX-Router
/system scheduler
add interval=1d name="Daily Reboot" on-event="/system reboot" policy=\
    ftp,reboot,read,write,policy,test,password,sniff,sensitive,romon start-date=may/01/2020 start-time=03:00:00
add interval=30m name=GoogleDNSUpdater on-event="/system  script  run  GoogleDNSUpdater\r \
    \n" policy=ftp,reboot,read,write,policy,test,password,sniff,sensitive,romon start-date=apr/30/2020 start-time=\
    09:57:53
/system script
add dont-require-permissions=no name=GoogleDNSUpdater owner=admin policy=\
    ftp,reboot,read,write,policy,test,password,sniff,sensitive,romon source="# credit to hhspiny\r\
    \n#\r\
    \n# Variables\r\
    \n# Username and password are long cryptic ones Google Domains generates for your dynDomain,\r\
    \n# IT IS NOT your regular Google account Username and Password\r\
    \n\r\
    \n:local GoogleDNSUsername \"XXXXXXXXXXXXX\"\r\
    \n:local GoogleDNSPassword \"XXXXXXXXXXXXXX\"\r\
    \n:local hostName \"XXXXXXXXXXXXXXXXXXX\"\r\
    \n:local currentIP \"\"\r\
    \n:local setResults \"\"\r\
    \n:local previousIP \"\"\r\
    \n\r\
    \n# Script\r\
    \n:set currentIP [/ip cloud get public-address]\r\
    \n:set previousIP [:resolve \"\$hostName\"]\r\
    \n:if (\$currentIP != \$previousIP) do={\r\
    \n:do {\r\
    \n/tool fetch url=\"https://\$GoogleDNSUsername:\$GoogleDNSPassword@domains.google.com/nic/update\?hostname=\$hostNam\
    e&myip=\$currentIP\" mode=https dst-path=GoogleDNS.txt\r\
    \n:set setResults [/file get GoogleDNS.txt contents];\r\
    \n:log info (\"GoogleDNS said this: \$setResults\")\r\
    \n} on-error={\r\
    \n:log error (\"GoogleDNS: script failed to set new IP address\") }}"
/system ups
add min-runtime=5m name="Main UPS" offline-time=5m port=usbhid1
/tool mac-server
set allowed-interface-list=LAN
/tool mac-server mac-winbox
set allowed-interface-list=LAN
2

I have the RB750Gr3 for testing out changes. I took a quick glance, and from my noob standpoint I can tell you this thing does not go full speed with any extra configs. I have seen 600Mb/s at home with default settings and fasttrack to the max. Any queues and packet marking will only slow it down.

3

Well poop. I was hoping since this thing was gigabit capable it would have the throughput the whole way downstream. I’ll get rid of the queues/packet marking, that was from an obscure link I found where one guy said that it fixed a similar issue he was having. When I implemented it I did not see any difference up or down in speed so I left them alone. But in the name of cleanliness I’ll probably remove them if they don’t affect anything. Thanks for the reply!

4

https://mikrotik.com/product/RB750Gr3#fndtn-testresults

5

I would suggest to test first entirely locally inside your LAN using a traffic server/client tool like iperf.
Only if the result of that test is ok should you do a WAN-to-LAN and vice-versa test. IMO.

Example (Linux): replace $PORT and $SERVER with their real values, ie. port number and IP address

On server machine:
iperf -s -p $PORT

On client machine (the client is the sender of the data; here it uses 3 threads and transfers 60s long, then prints the results):
iperf -c $SERVER -p $PORT -P 3 -t 60 -w 1M

See ‘man iperf’ or ‘iperf --help’ for details of the parameters.
Btw, iperf is available also in Cygwin (add-on Linux-like environment for Windows), but I haven’t tested it yet; https://cygwin.com/packages/summary/iperf.html

6

This is not a thread about linux iperf. Get a life!

THe Hex never got more that around 600ish for me with my 1gig connection.
Your 1 gig connection is lying to you if you are directly connected to the modem and only seeing 840, it should be over 900 easy.
That tells me that something is wrong at the connection to your house. Did the techs come and confirm 1 gig to the modem and connected to the modem during install??

In any case, I changed my router to the RB450Gx4 routerboard for my 1gig connection.
If you look at the specs for the Hex router - the most accurate line I have found that matches my real world results is somewhere between 25 ip filter rules and 25 simple queues and 512byte sized packets. In the case of the hex its 385-735Mbps. Since most of us have at least 15-20 filter rules bare minimum I would weight it more heavily on the ip filter rules. 2/3 - which means you should get around 500Mbps.

7

Thanks for the reply. I did look at this chart before purchasing the router and still picked it up because it’s tested upper limit was 1800mbps. In my mind that meant to me that even running a few firewall filter rules and some other routing config items I would still see around 1000mbps despite the overhead. Honestly of all the problems to have this is obviously a good one, I’m just one of those that like to see the full capability out of my equipment.

8

Thank you for the reply. However I don’t know how helpful this will be. I have attached a simple network diagram to the post to better show my network topology. In it you’ll see that for wireless devices I really only use internet and do very little file sharing between them and the wired machines. And I know my gigabit backbone is running full speed already because all machines that would need to utilize the speed have already been operating on it for the last couple of years since I implemented the infrastructure. I know my internet can operate at nearly the full speed I pay for because my previous router that this Mikrotik is replacing would regularly speed test in the upper 800mpbs range.
NetworkDiagram.png

9

Well as much as I would love to say Comcast is lying (because they probably are), taking into account the standard overhead inherent in cable based internet systems I figure that between 850 and 1000 is pretty decent. And no my cable system here at the house is solely used for internet, My connection comes in off the pole and basically goes straight to the modem. When the tech installed the line he ran his test tools on it and said that the connection is roughly 98% pure (his words) with little to no db drop on the run. And I purchase my own equipment (Renting is stupid) so when I got my new modem in I just had comcast register it to my account and then run their tests on it to verify its connectivity/viability.

I only have 12 IP filter rules which according to the data sheet https://mikrotik.com/product/RB750Gr3#fndtn-testresults should mean it’ll still operate at full speed. I guess I could get rid of a few of them but don’t know how that will impact the security of my system?

Is there a way to force the router to utilize 512byte packets? Seems like that may be the quickest and easiest fix, basically force it to use the packet size that it is most comfortable with.

10

You can always set the upper limit with the MTU size. A router will do packet split if the egress MTU size is too small for the packet.
The olld Internet over serial lines (SLIP protocol) was 576bytes remember. Old OS’es used 576byte packets when it was routed (not local to the ethernet).
In TCP the sender, after detecting the max MTU in the path, will set the appropriate MSS size. To force that process (for encapsulated protocols like PPPoE, pppoA, IPsec, PPTP …) we used to fiddle with the MSS in the packet.in transition.

I think setting MTU size will do. But this is experimental.

[IT advisors declared me crazy when I did set the ethernet MTU size to 1400bytes, to solve the 100ms ACK delay for 4Kbyte bursts resulting in an uneven number of packets. But it worked. We ran full speed, the others were stuck at 40kBps] Reducing the MTU size can be usefull, in some specific cases, if you know what you are doing..

11

When fast-track is in use (by default it’s enabled), then the most costly operation a firewall needs to perform is to track the connections. With fast track enabled, vast majority of packets (say 99.9%) bypass the whole firewall filter ruleset and it doesn’t matter much how many rules there are. Only packets not belonging to established or related connections have to traverse filter rules and if those are placed smartly, the effect on performance isn’t really high. So no need to delete too many rules if that might mean reduced security.


Well, packet size used is not up to router actually … it’s up to all of its clients (LAN clients included) and even then it’s maximum size. Clients are free to use (and quite often do so) smaller packets if that means reduced latency. One prime example is VoIP, where mouth-to-ear delay should be less than 200ms for both conversation parties to be comfortable. With voice codec that produces stream of say 64 kbps (ample enough for decent quality mono sound) and including IP overhead that would mean around 6 full-size packets per second. So already sender would introduce 166ms of delay. Add say 20-40 ms delay on the line and we reach the magical 200ms delay with no headroom for buffering on receiver’s side to deal with delay jitter. So rather VoIP uses smaller packets sent frequently (depending on implementation, but 4G VoLTE sends one VoIP data every 20ms), but then those packets are often quite small - 50 pps for 64kbps stream means 200-byte packets (including IP headers).

12

I mentioned iperf only because it’s a widely accepted standard professional tool for such network bandwidth/throughput tests.
Using such an own test server and client has the advantage that you can test any of your network segments to detect a bottleneck point.
See also the remarks at http://forum.mikrotik.com/t/mikrotik-bandwidth-test-software-using-iperf/6185/1

It seems that the routing throughput of the MT routers are unfortunately generally capped or limited, ie. wire-speed seems not possible
if there are some standard firewall rules configured according to the MT test pages of the few routers and also switch-routers I looked at, including yours.
OTOH there is another (advanced/low-level, but stateless) firewall in RouterOS that uses ACL filter rules if the device supports it, that promises wire-speed switching (I’m not sure yet about routing).

For comparison: from my 1000/50 Mbps WAN link I get about 950/50 Mpbs over 2 in-house routers and a switch, all in series: one is an AVM FritzBox Cable Router (WAN-router) and the other an Ubiquiti Edgerouter Lite-3. Attached to the ER is a MT CRS326 switch and to that the client PCs. The FritzBox does standard WAN/LAN NATting. On both routers I have currently 3 port-forwardings, and on the ER 12 firewall rules.

13

Sure. But apart from special cases (such as you described), reducing packet sizes means lower throughput, both due to higher load on any devices processing those packets (routers, firewalls, …) and due to higher share of packet headers overhead. If OP wants to reach high throughput and bottleneck is router/firewall processing power (limiting PPS), then using larger packets is the benefitial strategy. This is the very reason for inventing jumbo frames back when 100Mbps was “lightning fast” and FDDI gained some advantage over 100BaseTX also for this reason.

14

Ooops yes, sorry for the wording … “I think setting MTU size will do.” was the answer to “Is there a way to force the router to utilize 512byte packets?” only. Not to the idea that smaller packets will give better throughput. Jumbo packets indeed, also added to reduce the inter-packet gap loss that relatively increases with the speed of the connection. Standard packets on a 100Gbps link would be extremely short in time.

15

Did some work to calculate the optimal MTU for the Xfinity network and it looks like 1472(max packet size) was the magic number that didn’t require fragmentation. So accounting for the extra 28 bits the MTU of 1500 that the router auto detected was correct. I did play with this number a bit decreasing it by 10-25 bit blocks and running tests but the speed of the internet just gradually declined with each iteration. I put it back to the stock 1500, it was worth a shot though so thanks for the suggestion!

16

You can lead a piggy to water, but it just won’t drink. Upgrade, quest over, time saved.

17

Lukas 23,34
:open_mouth:

18

Alright guys I think I have it figured out. It did boil down to IP Filtering rules in the firewall area. I took a hard look at all the rules that were present and being used and trimmed out a couple of redundant ones and secured others that were being used as “shotgun” type rules such as dropping admin connections from !LAN. (Switched this to WAN as I had already set that up properly)
Below is what my ip filters look like now from the config dump:

/ip firewall address-list
add address=XXXXXXXXXXXXX.sn.mynetname.net list=WANIP
/ip firewall filter
add action=accept chain=forward comment="defconf: accept in ipsec policy" disabled=yes ipsec-policy=in,ipsec
add action=accept chain=forward comment="defconf: accept out ipsec policy" disabled=yes ipsec-policy=out,ipsec
add action=fasttrack-connection chain=forward comment="defconf: fasttrack" connection-state=established,related
add action=accept chain=input comment="defconf: accept established,related,untracked" connection-state=established,related,untracked
add action=drop chain=input comment="defconf: drop invalid" connection-state=invalid
add action=accept chain=input comment="defconf: Allow ICMP" protocol=icmp
add action=accept chain=input comment="defconf: Allow Router Access from LAN" disabled=yes src-address=192.168.1.0/24
add action=drop chain=input comment="defconf: drop all not coming from LAN" in-interface-list=WAN
add action=accept chain=forward comment="defconf: accept established,related, untracked" connection-state=established,related disabled=yes
add action=drop chain=forward comment="defconf: drop invalid" connection-state=invalid disabled=yes
add action=drop chain=input comment="defconf: drop all from WAN not DSTNATed" connection-nat-state=!dstnat connection-state=new disabled=yes in-interface=ether1
/ip firewall nat
add action=masquerade chain=srcnat comment=HAIRPIN dst-address=192.168.1.0/24 src-address=192.168.1.0/24
add action=masquerade chain=srcnat comment="defconf: masquerade" ipsec-policy=out,none out-interface-list=WAN
add action=dst-nat chain=dstnat dst-address-list=WANIP dst-port=32400 protocol=tcp to-addresses=192.168.1.3
add action=dst-nat chain=dstnat dst-address-list=WANIP dst-port=443 protocol=tcp to-addresses=192.168.1.9
add action=dst-nat chain=dstnat dst-address-list=WANIP dst-port=21 protocol=tcp to-addresses=192.168.1.9
add action=dst-nat chain=dstnat disabled=yes dst-address-list=WANIP dst-port=22 protocol=tcp to-addresses=192.168.1.9
add action=dst-nat chain=dstnat comment="template port forward" disabled=yes dst-address-list=WANIP dst-port=8080 protocol=tcp to-addresses=192.168.1.2

I left the rules in place just in case I need them at some point but you’ll see that they have been set to disable. My speed tests after completing this are now averaging between 750-850 regularly when I test with multiple different services ie. Google, Xfinity, and Ookla. Technically this isn’t my “full” speed but it is way better than what I was getting and I’ll continue to tweak this thing for optimal running as I continue to learn the operating system and it’s intricacies more. For now I’m marking this as solved and hopefully this helps out anyone experiencing a similar issue when they scroll through the boards here. Again thanks a bunch for all your help and insights while I was working to figure this thing out!

19

One of the things I have an issue with is your WANIP status.
Do you have a dynamic or static WANIP??

The reason I ask is your NAT rules show a mismatch.
The sourcenat rules are setup for a dynamic WANIP and your dstnat rules are setup for a static FIXED WANIP??

20

I know that there are users strongly against using action=masquerade for src-nat rules … But I constantly forget why their raid against it …
Humm, I seem to have found the source of the holly war: a MUM presentation, slides 25 and 26. Unlike other problems, tackled by that presentation, description of this “problem” doesn’t show huge Wrong! note. And the description of a problem implies that use of action=masquerade is a problem for routers with large number of PPPoE clients etc. (i.e. ISP router) with static WAN IP address. However, the “problematic” behaviour is completely correct in case where WAN IP does change. And doesn’t present a huge problem in a typical SOHO scenario with low to moderate number of active connections.