learn how to Shape Download Traffic only & leave Browsing

This is a way to isolate between download & Browsing and to give the user speed for example 64 Kb/sec for download but if he browsed the internet and surfing web sites for just browsing
he will browse the web pages with the entire network speed for example suppose that it’s
4 Mb/sec

and we took 64 kb for just an example
you can change it with what ever you want to

/ip firewall layer7-protocol
add comment=“” name=“Extension " .exe "” regexp=“\.(exe)”
add comment=“” name=“Extension " .rar "” regexp=“\.(rar)”
add comment=“” name=“Extension " .zip "” regexp=“\.(zip)”
add comment=“” name=“Extension " .7z "” regexp=“\.(7z)”
add comment=“” name=“Extension " .cab "” regexp=“\.(cab)”
add comment=“” name=“Extension " .asf "” regexp=“\.(asf)”
add comment=“” name=“Extension " .mov "” regexp=“\.(mov)”
add comment=“” name=“Extension " .wmv "” regexp=“\.(wmv)”
add comment=“” name=“Extension " .mpg "” regexp=“\.(mpg)”
add comment=“” name=“Extension " .mpeg "” regexp=“\.(mpeg)”
add comment=“” name=“Extension " .mkv "” regexp=“\.(mkv)”
add comment=“” name=“Extension " .avi "” regexp=“\.(avi)”
add comment=“” name=“Extension " .flv "” regexp=“\.(flv)”
add comment=“” name=“Extension " .pdf "” regexp=“\.(pdf)”
add comment=“” name=“Extension " .wav "” regexp=“\.(wav)”
add comment=“” name=“Extension " .rm "” regexp=“\.(rm)”
add comment=“” name=“Extension " .mp3 "” regexp=“\.(mp3)”
add comment=“” name=“Extension " .mp4 "” regexp=“\.(mp4)”
add comment=“” name=“Extension " .ram "” regexp=“\.(ram)”
add comment=“” name=“Extension " .rmvb "” regexp=“\.(rmvb)”
add comment=“” name=“Extension " .dat "” regexp=“\.(dat)”
add comment=“” name=“Extension " .daa "” regexp=“\.(daa)”
add comment=“” name=“Extension " .iso "” regexp=“\.(iso)”
add comment=“” name=“Extension " .nrg "” regexp=“\.(nrg)”
add comment=“” name=“Extension " .bin "” regexp=“\.(bin)”
add comment=“” name=“Extension " .vcd "” regexp=“\.(vcd)”

/ip firewall mangle
add action=mark-connection chain=prerouting comment=“7z DOWNS” disabled=no layer7-protocol=“Extension " .7z "” new-connection-mark=“7z DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“7z DOWNS” disabled=no new-packet-mark=7z passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“asf DOWNS” disabled=no layer7-protocol=“Extension " .asf "” new-connection-mark=“asf DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“asf DOWNS” disabled=no new-packet-mark=asf passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“avi DOWNS” disabled=no layer7-protocol=“Extension " .avi "” new-connection-mark=“avi DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“avi DOWNS” disabled=no new-packet-mark=avi passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“bin DOWNS” disabled=no layer7-protocol=“Extension " .bin "” new-connection-mark=“bin DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“bin DOWNS” disabled=no new-packet-mark=bin passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“flv DOWNS” disabled=no layer7-protocol=“Extension " .flv "” new-connection-mark=“flv DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“flv DOWNS” disabled=no new-packet-mark=flv passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“iso DOWNS” disabled=no layer7-protocol=“Extension " .iso "” new-connection-mark=“iso DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark= “iso DOWNS” disabled=no new-packet-mark=iso passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“mkv DOWNS” disabled=no layer7-protocol=“Extension " .mkv "” new-connection-mark=“mkv DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“mkv DOWNS” disabled=no new-packet-mark=mkv passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“exe DOWNS” disabled=no layer7-protocol=“Extension " .exe "” new-connection-mark=“exe DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“exe DOWNS” disabled=no new-packet-mark=exe passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“mov DOWNS” disabled=no layer7-protocol=“Extension " .mov "” new-connection-mark=“mov DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“mov DOWNS” disabled=no new-packet-mark=mov passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“mp3 DOWNS” disabled=no layer7-protocol=“Extension " .mp3 "” new-connection-mark=“mp3 DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“mp3 DOWNS” disabled=no new-packet-mark=mp3 passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“mp4 DOWNS” disabled=no layer7-protocol=“Extension " .mp4 "” new-connection-mark=“mp4 DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“mp4 DOWNS” disabled=no new-packet-mark=mp4 passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“mpeg DOWNS” disabled=no layer7-protocol=“Extension " .mpeg "” new-connection-mark=“mpeg DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“mpeg DOWNS” disabled=no new-packet-mark=mpeg passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“mpg DOWNS” disabled=no layer7-protocol=“Extension " .mpg "” new-connection-mark=“mpg DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“mpg DOWNS” disabled=no new-packet-mark=mpg passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“nrg DOWNS” disabled=no layer7-protocol=“Extension " .nrg "” new-connection-mark=“nrg DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“nrg DOWNS” disabled=no new-packet-mark=nrg passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“pdf DOWNS” disabled=no layer7-protocol=“Extension " .pdf "” new-connection-mark=“pdf DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“pdf DOWNS” disabled=no new-packet-mark=pdf passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“ram DOWNS” disabled=no layer7-protocol=“Extension " .ram "” new-connection-mark=“ram DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“ram DOWNS” disabled=no new-packet-mark=ram passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“rar DOWNS” disabled=no layer7-protocol=“Extension " .rar "” new-connection-mark=“rar DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“rar DOWNS” disabled=no new-packet-mark=rar passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“rm DOWNS” disabled=no layer7-protocol=“Extension " .rm "” new-connection-mark=“rm DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“rm DOWNS” disabled=no new-packet-mark=rm passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“rmvb DOWNS” disabled=no layer7-protocol=“Extension " .rmvb "” new-connection-mark=“rmvb DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“rmvb DOWNS” disabled=no new-packet-mark=rmvb passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“wav DOWNS” disabled=no layer7-protocol=“Extension " .wav "” new-connection-mark=“wav DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“wav DOWNS” disabled=no new-packet-mark=wav passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“wma DOWNS” disabled=no layer7-protocol=“Extension " .wma "” new-connection-mark=“wma DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“wma DOWNS” disabled=no new-packet-mark=wma passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“wmv DOWNS” disabled=no layer7-protocol=“Extension " .wmv "” new-connection-mark=“wmv DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“wmv DOWNS” disabled=no new-packet-mark=wmv passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“zip DOWNS” disabled=no layer7-protocol=“Extension " .zip "” new-connection-mark=“zip DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“zip DOWNS” disabled=no new-packet-mark=zip passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“youtube DOWNS” disabled=no layer7-protocol=“YouTube " new-connection-mark=“youtube DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=”" connection-mark=“youtube DOWNS” disabled=no new-packet-mark=youtube passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“daa DOWNS” disabled=no layer7-protocol=“Extension " .daa "” new-connection-mark=“daa DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“daa DOWNS” disabled=no new-packet-mark=daa passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“dat DOWNS” disabled=no layer7-protocol=“Extension " .dat "” new-connection-mark=“dat DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“dat DOWNS” disabled=no new-packet-mark=dat passthrough=no protocol=tcp


add action=mark-connection chain=prerouting comment=“vcd DOWNS” disabled=no layer7-protocol=“Extension " .vcd "” new-connection-mark=“vcd DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“vcd DOWNS” disabled=no new-packet-mark=vcd passthrough=no protocol=tcp

add action=mark-connection chain=prerouting comment=“cab DOWNS” disabled=no layer7-protocol=“Extension " .cab "” new-connection-mark=“cab DOWNS” passthrough=yes protocol=tcp
add action=mark-packet chain=postrouting comment=“” connection-mark=“cab DOWNS” disabled=no new-packet-mark=cab passthrough=no protocol=tcp





/queue type
set default-small kind=pcq name=default-small pcq-classifier=src-address,dst-address,src-port,dst-port pcq-limit=50 pcq-rate=0 pcq-total-limit=2000



/queue tree
add burst-limit=0 burst-threshold=0 burst-time=0s disabled=no max-limit=1M name=Download_queue packet-mark=“exe,flv,rar,zip,7z,bin,mp3,mp4,wav,pdf,asf,avi,mov,wmv,mpg,mpeg,rm,iso,nrg,daa,vcd,cab,3gp,ram,rmvb,mkv,wma,youtube” parent=global-out priority=8 queue=default-small





This shape download + upload with only 1 Mb/sec to the whole Network



if you want just to shape download you can use in layer7 regexp this rule for example

/ip firewall layer7-protocol
add comment=“” name=“Extension " .exe "” regexp=“^.get.+\.exe.$”

and apply it to the rest of the extensions


if you want to shape upload only use this code in layer7


/ip firewall layer7-protocol
add comment=“” name=“Extension " .exe "” regexp=“^.set.+\.exe.$”


and you can shape Download Traffic by 64 kb on one IP like 192.168.0.2 using simple Queue instead of Queue Tree like this


/queue simple
add burst-limit=0/0 burst-threshold=0/0 burst-time=0s/0s comment=“” direction=both disabled=no dst-address=0.0.0.0/0 interface=all limit-at=1M/1M max-limit=1M/1M name=“download max” packet-marks=exe,flv,rar,zip,7z,bin,mp3,mp4,wav,pdf,asf,avi,mov,wmv,mpg,mpeg,rm,iso,nrg,daa,vcd,cab,3gp,ram,rmvb,mkv,wma,youtube parent=none priority=8 queue=default-small/default-small total-queue=default-small

and then make 64 Kb a speed for the Specified user in this parent group and refer to it’s parent the first rule like this

add burst-limit=0/0 burst-threshold=0/0 burst-time=0s/0s comment=“” direction=both disabled=no dst-address=0.0.0.0/0 interface=all limit-at=64k/64k max-limit=64k/64k name=“User " 192.168.0.2 "” parent=“download max” priority=8 queue=default-small/default-small target-addresses=192.168.0.2/32 total-queue=default-small


using this we can Shape IP 192.168.0.2 Download Traffic By 64 Kb
or what ever the speed we want to do it for the client.



the only problem this code is not working with loadbalancing ADSL Lines
is there any way to make it work ???
any one know how to do both Load Balancing + Shape download Traffic please tell us .


Notice : All code above applied on 3.3 Mikrotik OS Version

any comment in this post :smiley:

very good job, what about the cpu resources? have you tested it with how many clients?

I’d be concerned about that, too.

The configuration can be significantly optimized:

  • There’s no need for passthrough=yes on all those mark-connection rules as a connection already identified as downloading a 7z file doesn’t need to undergo checks on whether it’s download an EXE file.
  • You should also put a rule that checks if there’s already a connection mark at the very top so that an already identified connection (from an earlier packet) never is checked again (connection-mark=!no-mark action=accept). Layer 7 scanning for a regex is pretty much the most expensive thing you could do, so you should do it as little as possible.
  • There’s also no need to do this for anything but HTTP traffic - it’ll be rare that that happens outside of tcp/80, tcp/3128, tcp/8080 and tcp/8888 and it’s probably worth missing a few connection but not scanning absolutely anything.
  • Lastly, start out with the rules in alphabetical order but ensure to watch packet counters and sort them so that the most often hit rules come first.

As an example for three file extensions:

/ip firewall mangle
add chain=prerouting action=jump jump-target=markL7packet connection-mark=!no-mark
add chain=prerouting action=jump jump-target=markL7conn protocol=tcp dst-port=80,3128,8080,8888

add action=mark-connection chain=markL7conn comment="mpeg DOWNS" disabled=no layer7-protocol="Extension \" .mpeg \"" new-connection-mark="mpeg DOWNS" passthrough=no
add action=mark-connection chain=markL7conn comment="mpg DOWNS" disabled=no layer7-protocol="Extension \" .mpg \"" new-connection-mark="mpg DOWNS" passthrough=no
add action=mark-connection chain=markL7conn comment="nrg DOWNS" disabled=no layer7-protocol="Extension \" .nrg \"" new-connection-mark="nrg DOWNS" passthrough=no

add action=mark-packet chain=markL7packet connection-mark="mpeg DOWNS" disabled=no new-packet-mark=mpeg passthrough=no 
add action=mark-packet chain=markL7packet connection-mark="mpg DOWNS" disabled=no new-packet-mark=mpeg passthrough=no 
add action=mark-packet chain=markL7packet connection-mark="nrg DOWNS" disabled=no new-packet-mark=mpeg passthrough=no

The packet on which the connection is identified won’t be marked and thus won’t be rate limited, but that’s one packet and for a download of a larger file that’s going to be insignificant. You’ll still want to order the rules with highest hit count first after running for a while.

add comment=“” name=“Extension " .exe "” regexp=“\.(exe)”
add comment=“” name=“Extension " .exe "” regexp=“\.(zip)”

question can we make it in one line
to add tow extension toghether

try

add name=“ExtensionZ” regexp=“\.(zip|exe)”

The packet on which the connection is identified won’t be marked and thus won’t be rate limited, but that’s one packet and for a download of a larger file that’s going to be insignificant. You’ll still want to order the rules with highest hit count first after running for a while.

Eng: fewi sorry correction has been made to the code try it again

doesn’t work … is there any way to merge the whole list of extensions in one line ??

Yeah, old topic, but - I have this question, too. What is an answer? It’s possible merge all extensions in one line? :confused:

Mr dreamrider , Samsoft08 and kazanova :
about your question if we can add all extensions in 1 line i say ;

yes it can be
and here it is


/ip firewall layer7-protocol
add name=test regexp=“^(.)(.exe|.rar|.zip|.7z|.iso|.nrg|.daa|.dat|.vcd|.flv|.mp4|.mov|.avi|.asf|.mpg|.mpeg|.wmv|. ra|.rar|.mp3|.wav|.ram|.rmvb|.mkv|.cab|.chm|.ctf|. flg)(.)\$|videoplayback”


test this and tell me , and you know of course what to do with mangle , it’s just only 1 mangle rule for all extensions , and i just added each extensions in one line because it was my first time to discover a L7 code that can detect downloadable files,

and this 1 line is from Mahmoud Abo Aldahab efforts,
and there is another code can make extensions in 3x seperated L7 Lines


.(exe)|.(rar)|.(zip)|.(7z)|.(iso)|.(nrg)|.(daa)|.(dat)|.(vcd)


.(flv)|.(mp4)|.(mov)|.(avi)|.(asf)|.(mpg)|.(mpeg)|.(wmv)


.(ra)|.(mp3)|.(wav)|.(ram)|.(rmvb)|.(mkv)|videoplayback


and of course if you use 3x lines you have to do 1 mangle rules for each L7 rule , good luck for all.

@hazemamer7

/ip firewall layer7-protocol
add name=test regexp="^(.*)(.exe|.rar|.zip|.7z|.iso|.nrg|.daa|.dat|.vcd|.flv|.mp4|.mov|.avi|.asf|.mpg|.mpeg|.wmv|. ra|.rar|.mp3|.wav|.ram|.rmvb|.mkv|.cab|.chm|.ctf|. flg)(.*)\\\$|videoplayback"

At first testing minutes - youtube watching is blocked, *.rar downloading is blocked, *.mp4 downloading from youtube is allowed, this forum - is blocked :slight_smile: Will continue to test. Thanks.

This will block everything, not just file extensions. It will block any page that mentions “.dat” in it’s text. Not just the downloads.

I began to understand, tests continue … But why downloading *.mp4 from youtube is allowed? In rule text - extension *.mp4 present.

Could be that youtube works over HTTPS, then you will not see the contents of the page, it’s encrypted

Test this and tell me


.(exe)|.(rar)|.(zip)|.(7z)|.(iso)|.(nrg)|.(daa)|.(dat)|.(vcd)


.(flv)|.(mp4)|.(mov)|.(avi)|.(asf)|.(mpg)|.(mpeg)|.(wmv)


.(ra)|.(mp3)|.(wav)|.(ram)|.(rmvb)|.(mkv)|videoplayback


and of course each layer7 code line need 1 mangle rule to mark the packet for the queues,
test and tell me and if still not work i can provide you with video of proving that it works.

@ hazemamer7
Can you write a complete code for at least one line? Begining, i mean. With regexp=. Or what else there need..
Thank you!

Dreamrider
here it is the code ,and if it works reply to me that it works




/ip firewall layer7-protocol
add name=ext1 regexp=“\.(exe)|\.(rar)|\.(zip)|\.(7z)|\.(iso)|\.(nrg)|\.
(daa)|\.(dat)|\.(vcd)”
add name=ext2 regexp=“\.(flv)|\.(mp4)|\.(mov)|\.(avi)|\.(asf)|\.(mpg)|\
.(mpeg)|\.(wmv)”
add name=ext3 regexp=
“\.(ra)|\.(mp3)|\.(wav)|\.(ram)|\.(rmvb)|\.(mkv)|videoplayback”


/ip firewall mangle
add action=mark-packet chain=prerouting layer7-protocol=ext1 new-packet-mark=
download1
add action=mark-packet chain=prerouting layer7-protocol=ext2 new-packet-mark=
download2
add action=mark-packet chain=prerouting layer7-protocol=ext3 new-packet-mark=
download3

/queue type
add kind=pcq name=pcqmanual pcq-classifier=src-address,dst-address,src-port
pcq-dst-address6-mask=64 pcq-rate=400k pcq-src-address6-mask=64
pcq-total-limit=5000

/queue tree
add max-limit=3M name=limitspeed packet-mark=download1,download2,download3
parent=global queue=pcqmanual

\


in queue tree (max-limit=3M) >>> this make the max download rate in the network = 3 Mb/sec
in queue type (pcq-rate=400k) >>> this make the speed for each user = 400 kb/sec = 50 KB/sec

you can edit those for get the optimum option for you,

try to download this link as a test

http://ardownload.adobe.com/pub/adobe/reader/win/11.x/11.0.04/en_US/AdbeRdr11004_en_US.exe

do not forget if it works to tell us.

@ hazemamer7
I do not need too complicated rules and I do not use mangles and queues for that. :slight_smile: There is part of my firewall rules that works perfectly for me. No more any downloads! Browsing works wherever I have tried. Thank you for working code!

/ip firewall layer7-protocol
add name=ext1 regexp="\\.(exe)|\\.(rar)|\\.(zip)|\\.(7z)|\\.(iso)|\\.(nrg)|\\.\
    (daa)|\\.(dat)|\\.(vcd)"
add name=ext2 regexp="\\.(flv)|\\.(mp4)|\\.(mov)|\\.(avi)|\\.(asf)|\\.(mpg)|\\\
    .(mpeg)|\\.(wmv)"
add name=ext3 regexp=\
    "\\.(ra)|\\.(mp3)|\\.(wav)|\\.(ram)|\\.(rmvb)|\\.(mkv)|videoplayback"
	
/ip firewall filter
add action=jump chain=forward jump-target=drop layer7-protocol=ext1
add action=jump chain=forward jump-target=drop layer7-protocol=ext2
add action=jump chain=forward jump-target=drop layer7-protocol=ext3
add action=drop chain=drop comment="Drop Everything that we drop"

P.S. But I can’t submit this answer… Works blocking rule ext3… Hm?

Curious..

It’s been a while since I’ve dug into the HTTP protocol, but aren’t there some headers that you could key in on?

IMO, the best, first test, would be Content-Length (if we could actually use the number from the string). Small files could be let through, with medium and large files being queued.

Next, would be the Content-Disposition, which should contain a filename. This filename would be where you would want to check for the type.

Content-Type might be another header that could be looked at, which could help identify specific mime types.

-Troy