@MTNick optimized logging, and also the displayed text when running in terminal.
Updated: shortend the time a list is unavailable on import by generating a temporary list and then after remove of the old one replacing it with temporary one containing the updated list. Advantage is that the old list is still active during reading the file and while it is being imported.
# Turris Import by Blacklister and edited by Optio
# 20210823 new version that directly downloads from the external server
# 20240331 rewritten to fetch the whole file and write it to a local file and then import it
# 20240401 avoiding perfect storm by reducing chunkSize when calculation the remainder
# 20240402 adding importing new address to temporary list and swap them out with the active list avoiding the list being not active for a short time as possible
# also save and display a count of static addresses present in a address-list
{
# import config - delay for slow routers
#:delay 1m
:log warning "IP-Blocker script started"
/ip firewall address-list
:local update do={
:if (heirule != null) do={:set $filtering ", filtering on: $heirule"}
:put "Start importing address-list: $listname$filtering"
:log warning "Start importing address-list: $listname$filtering"
/tool fetch url=$url dst-path="/$listname.txt" as-value
# delay to wait file flush after fetch
:delay 1
:local filesize [/file get "$listname.txt" size]
:local start 0
:local chunkSize 32767; # requested chunk size
:local partnumber ($filesize / $chunkSize); # how many chunk are chunkSize
:local remainder ($filesize % ($chunkSize-512)); # the last partly chunk and use reduced chunkSize
:if ($remainder > 0) do={ :set partnumber ($partnumber + 1) }; # total number of chunks
:local listCount [:len [find list=$listname dynamic]]
:put "Deleting $listCount entries (dynamic) from address-list: $listname"
:log warning "Deleting $listCount entries (dynamic) from address-list: $listname"
:if ($heirule = null) do={:set $heirule "."}
# remove the current dynamic entries completely
#:do {remove [find where list=$listname dynamic]} on-error={};
:set $listnameTemp ($listname."temp")
:for x from=1 to=$partnumber step=1 do={
:local data ([:file read offset=$start chunk-size=$chunkSize file="$listname.txt" as-value]->"data")
# Only remove the first line only if you are not at the start of list
:if ($start > 0) do={:set data [:pick $data ([:find $data "\n"]+1) [:len $data]]}
:while ([:len $data]!=0) do={
:local line [:pick $data 0 [:find $data "\n"]]; # create only once and checked twice as local variable
:if ($line~"^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}" && $line~heirule) do={
:local addr [:pick $data 0 [:find $data $delimiter]]
:do {add list=$listnameTemp address=$addr comment=$description timeout=$timeout} on-error={}; # on error avoids any panics
}; # if IP address && extra filter if present
:set data [:pick $data ([:find $data "\n"]+1) [:len $data]]; # removes the just added IP from the data array
# Cut of the end of the chunks by removing the last lines...very dirty but it works
:if (([:len $data] < 256) && (x < $partnumber)) do={:set data [:toarray ""]}
}; # while
#:set start ($start + $chunkSize)
:set start (($start-512) + $chunkSize); # shifts the subquential starts back with 512
}; #do for x
/file remove "$listname.txt"
:put "Deleted downloaded file: $listname.txt"
:log warning "Deleted downloaded file: $listname.txt"
# Swap out temp list and active list, shorten the time the list is empty
:do {set list=$listnameTemp [find list=$listname !dynamic]}; # backup any fixed IP addresses to the temporary list
:do {remove [find list=$listname]} on-error={}; # empty the complete list
:do {set list=$listname [find list=$listnameTemp]} on-error={
:put "Import failed: while swapping out the the old list with the temperorary list: $listname";
:log error "Import failed: while swapping out the the old list with the temperorary list: $listname"
}
:set $staticCount ""
:if ([:len [find list=$listname !dynamic]] > 0) do={:set $staticCount "of which $[:len [find list=$listname !dynamic]] are static addresses"}
:if ([:len [find list=$listnameTemp]] < 1) do={
:local listCount [:len [find list=$listname]]
:put "Completed updating address-list $listname with $listCount addresses $staticCount"
:log warning "Completed updating address-list $listname with $listCount addresses $staticCount"
}
}; # do
$update url=https://iplists.firehol.org/files/firehol_level2.netset delimiter=("\n") listname=z-blocklist-FireHOL-L2 timeout=3d
$update url=https://view.sentinel.turris.cz/greylist-data/greylist-latest.csv listname=z-blocklist-Sentinel delimiter=, timeout=8d heirule=http
$update url=https://www.spamhaus.org/drop/drop.txt delimiter=("\_") listname=z-blocklist-SpamHaus timeout=3d
$update url=https://www.spamhaus.org/drop/edrop.txt delimiter=("\_") listname=z-blocklist-SpamHaus-edrop timeout=3d
:log warning message="IP-Blocker script COMPLETED running"
}