Try with adding delimiter=(“\n”) to the $update line
Due to problem with “\n” have to be set manually I have adapted the script to do this for your this when no delimiter has been found:
{
/ip firewall address-list
:local update do={
:put "Starting import of address-list: $listname"
:if ($nolog = null) do={:log warning "Starting import of address-list: $listname"}
:local displayed true
:local maxretry 3
:local retrywaitingtime 120s
:local retryflag true
:for retry from=1 to=$maxretry step=1 do={
:if (retryflag) do={ :set $retryflag false; :set $sounter 0
:if (retry > 1) do={
:put "Source file changed. Retring after a $retrywaitingtime wait..."
:if ($nolog = null) do={:log warning "Source file changed. Retring after a $retrywaitingtime wait..."}
:delay $retrywaitingtime }
:local fetchResult [/tool fetch url=$url keep-result=no as-value]
:local filesize ($fetchResult->"total")
:local downsize ($fetchResult->"downloaded")
:if ($filesize = 0 && $downsize > 0) do={ :set $filesize $downsize}
:local start 0
:local maxsize 64000; # reqeusted chunk size
:local end ($maxsize - 1); # because start is zero the maxsize has to be reduced by one
:local partnumber ($filesize / ($maxsize / 1024)); # how many chunk are maxsize
:local remainder ($filesize % ($maxsize / 1024)); # the last partly chunk
:if ($remainder > 0) do={ :set $partnumber ($partnumber + 1) }; # total number of chunks
:if ($heirule != null) do={:put "Using as extra filtering: $heirule"} else={:set $heirule "."}
# remove the current list completely if "erase" is not present (default setting)
:if ($noerase = null) do={
:if ($timeout = null) do={:set $timeout 00:00:00; :do {:foreach i in=[/ip firewall address-list find list=$listname] do={/ip firewall address-list set list=("backup".$listname) $i }} on-error={} } else={
:do {:foreach i in=[/ip firewall address-list find list=$listname dynamic] do={/ip firewall address-list set list=("backup".$listname) $i }} on-error={} };
:put ("Conditional deleting all".$dynamic." entries in address-list: $listname")
:if ($nolog = null) do={:log warning ("Conditional deleting all".$dynamic." entries in address-list: $listname")}
} else={:put "Entries not conditional deleted in address-list: $listname"}; # ENDIF ERASE
:for x from=1 to=$partnumber step=1 do={
# get filesize to be compared to the orignal one and if changed then retry
:local comparesize ([/tool fetch url=$url keep-result=no as-value]->"total")
:if ($comparesize = 0 && $downsize > 0) do={ :set $comparesize $downsize}
# fetching the chunks from the webserver when the size of the source file has not changed
# empty array when the source file changed. No processing is done till the next complete retry
:if ($comparesize = $filesize) do={:set $data ([:tool fetch url=$url http-header-field="Range: bytes=$start-$end" output=user as-value]->"data")} else={:set $data [:toarray ""]; :set $retryflag true}
#:if ($ownposix = null) do={
# determining the used delimiter in the list, when not provided in the config
# this only run once and so the impact on the import time is low
:local ipv4Posix "^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}"
:local ipv4rangePosix "^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}/[0-9]{1,2}"
:local domainPosix "^.+\\.[a-z.]{2,7}"
:local sdata $data;
# removes any lines at the top of the file that could interfere with finding the correct posix. Setting remarksign is needed
:while ([:pick $sdata 0 1] = $remarksign) do={ :set $sdata [:pick $sdata ([:find $sdata "\n"]+1) [:len $sdata]] }
:while ([:len $sdata]!=0 && $delimiter = null) do={ # The check on length of $sdata is for if no delimiter is found.
:local sline [:pick $sdata 0 [:find $sdata "\n"]]; :local slen [:len $sline];
# set posix depending of type of data used in the list
:if ($sline ~ $ipv4Posix) do={:set $posix $ipv4Posix; :set $iden "List identified as a IPv4 list"}
:if ($sline ~ $ipv4rangePosix) do={:set $posix $ipv4rangePosix; :set $iden "List identified as a IPv4 with ranges list"}
:if ($sline ~ $domainPosix) do={:set $posix $domainPosix; :set $iden "List identified as a domain list"}
:if ($sline ~ $posix) do={:put $iden}
:if ($sline ~ $posix) do={ # only explore the line if there is a match at the start of the line.
:do {:if ([:pick $sline 0 ($slen-$send)] ~ ($posix."\$") || $send > $slen) do={
:set $delimiter [:pick $sline ($slen-$send) ($slen-($send-1))]; :set $result true} else={:set $send ($send+1)}
:if ($result) do={ :set $extra [:pick $sline ($slen-$send) ($slen-($send-1))]
:if ( $extra = " " ) do={ :set $delimiter [:pick $sline ($slen-$send) ($slen-($send-2))] }
:if ( $extra = " " ) do={ :set $delimiter [:pick $sline ($slen-$send) ($slen-($send-3))] }
:if ( $extra = " " ) do={ :set $delimiter [:pick $sline ($slen-$send) ($slen-($send-4))] }
}; # EndIf result
} while (!$result); # EndDoWhile
}; #IF sline posix
:set $sdata [:pick $sdata ([:find $sdata "\n"]+1) [:len $sdata]]; # cut off the already searched lines
:if ($delimiter != null) do={:local sdata [:toarray ""]} ; #Clearing sdata array ending the WhileDo loop
}; #WHILE END $sdata
:local sdata [:toarray ""]
:if ([:len $delimiter] = 0) do={ :set $delimiter "\n"; :set $delimiterShow "New Line" } else={ :set $delimiterShow $delimiter }; # when empty use NewLine 20220529
#} else={:put "User defind Posix: $ownposix"; :set $posix $ownposix } ; # ENDIF ownposix = null
:if ($delimiter != null && $displayed ) do={:set $displayed false; :put "Using config provided delimiter: \"$delimiterShow\""}
:if ($posix = null) do={:set $posix "."}; # Use a match all posix if nothing is defined or found
:if (!retryflag) do={:put "Reading Part: $x $start - $end"}
:if ($timeout = null) do={:local timeout 00:00:00}; # if no timeout is defined make it a static entry.
# Only remove the first line only if you are not at the start of list
:while ( [:pick $data 0 1] = $remarksign) do={ :set $data [:pick $data ([:find $data "\n"]+1) [:len $data]] }; # removes the invalid line (Spamhaus)
:if ($start > 0) do={:set $data [:pick $data ([:find $data "\n"]+1) [:len $data]]}
:while ([:len $data]!=0) do={
:local line [:pick $data 0 [:find $data "\n"]]; # create only once and checked twice as local variable
:if ( $line ~ $posix && $line~heirule) do={
:do {add list=$listname address=[:pick $data 0 [:find $data $delimiter]] comment=$comment timeout=$timeout; :set $counter ($counter + 1)} on-error={}; # on error avoids any panics
}; # if IP address && extra filter if present
:set $data [:pick $data ([:find $data "\n"]+1) [:len $data]]; # removes the just added IP from the data array
# Cut of the end of the chunks by removing the last lines...very dirty but it works
:if ([:len $data] < 256) do={:set $data [:toarray ""]}
}; # while
:set $start (($start-512) + $maxsize); # shifts the subquential start back by 512
:set $end (($end-512) + $maxsize); # shift the subquential ends back by 512 to keep the
}; # if retryflag
}; #do for x
}; # for retry
:if ($counter < 1) do={:set $resultline "Import was NOT successfull! Check if the list $listname is still being maintained."} else={:set $resultline "Completed reading $counter items into address-list $listname." }
:put $resultline
:if ($nolog = null) do={:log warning $resultline }
:if ($counter > 0) do={:do {/ip firewall address-list remove [find where list=("backup".$listname)]} on-error={} } else={
:do {:foreach i in=[/ip firewall address-list find list=("backup".$listname)] do={/ip firewall address-list set list=$listname $i }} on-error={}
:put "Restoring backup list: $listname"
:if ($nolog = null) do={:log warning "Restoring backup list: $listname"}
}; # if counter restore on failure and remove on success
}; # do
$update url=https://www.spamhaus.org/drop/drop.txt listname=spamhaus remarksign=";" timeout=1d nolog=1
$update url=https://lists.blocklist.de/lists/all.txt listname=blockDE timeout=1d nolog=1
}
# To be used configline settings:
# url= https://name.of.the.list
# listname= name of address-list
# Optinal settings
# timeout= the time the entry should be active. If omited then static entries are created.
# comment= puts this comment on every line in the choosen address-list (default: no comment)
# heirule= this will select on a word on each line if to import or not (default: no heirule)
# noerase= any value, then the current list is not erased (default: erase)
# ownPosix= allow to enter a onw regEX posix to be used (not ative at this moment)
# nolog= any value, then don't write to the log (default: writing to log)
I removed the previous versions of this script to avoid any confusion.
Updated the textual part of the script so it states when “\n” NewLine is enforced.
AWESOME @msatter !
It’s working with
delimiter=("\n")
seems to be mandatory also on your updated version of the script, correct?
No, just sometimes the correct delimiter isn’t detected properly.
Just like sometimes there is a URL about the list as the first line of the file and the script detects it as a list of FQDNs rather than IPs. My fix for this is to download it to my own server as a cron job and delete the first couple lines, then fetch it from my own server. I’m not advanced enough with my ROS scripting to have ROS do it, plus it keeps the scripts simplier.
Did you try the updated script that uses “\n” when no delimiter was detected?
If a list has matching posix in the header then the script could be pick a wrong delimiter. Then you can over write that setting it manual.
How did you get on with posix yourself after reading my earlier answer to you on that?
I only tested it on the first link you stated blocklist-DE. It is a very simple one, use NewLine when the length of the found delimiter is zero. Found delimiters, that are wrong, are not zero in length.
Update: updated later also the textual part of the script so it states when “\n” NewLine is enforced.
THANK YOU msatter for your AWESOME contribution!
Apologies for another dumb question, looking and found some great IP lists, Darklist.de, Greensnow.co and Snort are returning an error, any idea why?
Here are the ones found;
$update url=https://feodotracker.abuse.ch/downloads/ipblocklist.txt description/listname="Abuse.ch Feodo Tracker" delimiter=("\r")
$update url=https://sslbl.abuse.ch/blacklist/sslipblacklist.txt description/listname="Abuse.ch SSLBL" delimiter=("\r")
$update url=https://www.binarydefense.com/banlist.txt description/listname="Artillery Threat Intelligence Feed" delimiter=("\n")
$update url=https://lists.blocklist.de/lists/all.txt description/listname="BlockList.de - Fail2Ban" delimiter=("\n")
$update url=https://iplists.firehol.org/files/botscout.ipset description/listname="BotScout" delimiter=("\n")
$update url=https://cinsscore.com/list/ci-badguys.txt description/listname="CINS Army List" delimiter=("\n")
$update url=https://iplists.firehol.org/files/cleantalk.ipset description/listname="CleanTalk" delimiter=("\n")
$update url=https://iplists.firehol.org/files/cruzit_web_attacks.ipset description/listname="CruzIT" delimiter=("\n")
$update url=https://iplists.firehol.org/files/cybercrime.ipset description/listname="CyberCrime Tracker" delimiter=("\n")
$update url=https://iplists.firehol.org/files/darklist_de.netset description/listname="Darklist.de - Blacklisted" delimiter=("\n")
$update url=https://feeds.dshield.org/block.txt description/listname="DShield.org" delimiter=("\t") cidr=/24
$update url=https://iplists.firehol.org/files/greensnow.ipset description/listname="GreenSnow.co" delimiter=("\n")
$update url=https://myip.ms/files/blacklist/general/latest_blacklist.txt description/listname="MyIP.ms Blacklist" delimiter=("\n")
$update url=https://snort.org/downloads/ip-block-list description/listname="Snort - Talos IP Blacklist" delimiter=("\n")
$update url=https://www.spamhaus.org/drop/drop.txt description/listname="SpamHaus DROP" delimiter=("\_")
$update url=https://www.spamhaus.org/drop/edrop.txt description/listname="SpamHaus EDROP" delimiter=("\_")
$update url=https://stopforumspam.com/downloads/toxic_ip_cidr.txt description/listname="Stop Forum Spam" delimiter=("\n")
$update url=https://check.torproject.org/torbulkexitlist description/listname="Tor Exit List Service" delimiter=("\n")
$update url=https://iplists.firehol.org/files/voipbl.netset description/listname="VoIPBL.org" delimiter=("\n")
$update url=https://iplists.firehol.org/files/vxvault.ipset description/listname="VxVault" delimiter=("\n")
What error? You are missing some required variables.
Why are you adding so many IP lists? Especially ones that duplicate each other?
Have a look overhere: http://forum.mikrotik.com/t/download-list-from-https-similar-to-team-cymru/157710/4
@kevinds just an error on the log (for Darklist.de and Greensnow.co), there are more useful informations? I removed the other variables here but are present on the code 
Adding more lists just for learning purposes, which one is duplicated?
@msatter you had the answer for “Snort” list, it’s a redirect. You know an easy way to integrate the $url for a redirect?
No, following redirects is something that Mikrotik could implement in their code. Using JavaScript is something that is not feasible in RouterOS.
In the meantime you could an other device to download the files and the router is then downloading those from that device. That was the way I did it, before having this script.
I see is needed, I write the script.
Give me a few dozen minutes…
Most of your blocklist.de entries. Not sure of your other lists though.
*** REMOVED ***
check new version on next post
@kevinds you’re right… “all” Blocklist.de contains everything… 
@rextended is your solution easily implementable in msatter version? http://forum.mikrotik.com/t/address-lists-downloader-dshield-spamhaus-drop-edrop-etc/133640/1
Tried to integrate reading your post http://forum.mikrotik.com/t/download-list-from-https-similar-to-team-cymru/157710/4 without success.
search tag # rextended checkurl
A new advanced version of the function for check the URL
Check if the URL is valid and read server response and redirects
:global checkurl do={
/file remove [find where name~"checkurl.(txt|tmp)"]
{
:local jobid [:execute file=checkurl.txt \
script="/tool fetch http-header-field=\"Range: bytes=0-0\" dst-path=\"checkurl.tmp\" url=\"$1\""]
:local testsec 0
:while (([:len [/sys script job find where .id=$jobid]] = 1) && ($testsec < 20)) do={
:set testsec ($testsec + 1)
:delay 1s
}
:local error { cod="000.0" ; txt="NO CODE"}
:if ([:len [/file find where name="checkurl.txt"]] = 1) do={
:local check [/file get [/file find where name="checkurl.txt"] contents]
/file remove [find where name~"checkurl.(txt|tmp)"]
# 200 URL OK
:if ($check~"status: finished") do={
:set ($error->"cod") "200"
:set ($error->"txt") "OK"
:return $error
}
# 301 Permanent Redirect
:if ($check~" <301 Moved Permanently ") do={
:set ($error->"cod") "301"
:set ($error->"txt") [:pick $check ([:find $check " <301 Moved Permanently \"" -1] + 25) [:find $check "\"> " -1]]
:return $error
}
# 302 Redirect
:if ($check~" <302 Found ") do={
:set ($error->"cod") "302"
:set ($error->"txt") [:pick $check ([:find $check " <302 Found \"" -1] + 13) [:find $check "\"> " -1]]
:return $error
}
# other Codes (error or not)
:if ($check~" <.*> ") do={
:set ($error->"txt") [:pick $check ([:find $check " <" -1] + 2) [:find $check "> " -1]]
:set ($error->"cod") [:pick ($error->"txt") 0 [:find ($error->"txt") " " -1]]
:set ($error->"txt") [:pick ($error->"txt") ([:find ($error->"txt") " " -1] + 1) [:len ($error->"txt")]]
:return $error
}
# MikroTik fetch specific errors
:if ($check~"failure: ") do={
:set ($error->"cod") "666.1"
:set ($error->"txt") [:pick $check ([:find $check "failure: " -1] + 9) [:len $check]]
:return $error
}
# unexpected results
:set ($error->"cod") "000"
:set ($error->"txt") $check
:return $error
} else={
# :execute unsuccessful or timeout
:set error { cod="666.0" ; txt="TEMP FILE ERROR"}
/file remove [find where name~"checkurl.(txt|tmp)"]
:return $error
}
}
}
Syntax: call $checkurl and you have back one array with return code and text.
# 301 permanent redirect example
:put [$checkurl "http://forum.mikrotik.com"]
cod=301;txt=https://forum.mikrotik.com/
# 302 CDN redirect example:
:put [$checkurl "h ttps://snort.org/downloads/ip-block-list"]
cod=302;txt=https://snort-org-site.s3.amazonaws.com/.../ip_filter.blf?X-Amz-Algorithm=...
# 404 not found example
:put [$checkurl "https://forum.mikrotik.com/not-exist"]
cod=404;txt=Not Found
How to use on download lists:
{
:local url "https://snort.org/downloads/ip-block-list"
:local filename "ip_filter.blf"
:local testresult [$checkurl $url]
:local returncode ($testresult->"cod")
:local returntext ($testresult->"txt")
:if ($returncode = "200") do={
# can be downloaded directly
$update url=$url
} else={
:if ($returncode = "302") do={
# use redirected URL
$update url=$returntext
} else={
# some error happen
:log error "Error checking $url: $returncode $returntext"
}
}
}
To integrate checking for a redirect I tested it quick and dirty with these adaptations:
The import script changes:
{
/ip firewall address-list
:local update do={
:global checkurl
:if ($url ~ "invalid URL protocol") do={:log error "Could not import $listname due to a problem with the stated URL"} else={
:put "Starting import of address-list: $listname"
.
.
.
#$update url=https://view.sentinel.turris.cz/greylist-data/greylist-latest.csv delimiter=, listname=turris timeout=8d heirule=http nolog=1
$update url=[$checkurl https://view.sentinel.turris.cz/greylist-data/greylist-latest.csv] delimiter=, listname=turris timeout=8d heirule=http nolog=1
}
And slimmed the Global function by only returning one result:
:global checkurl do={
/file remove [find where name~"checkurl.(txt|tmp)"]
{
:local jobid [:execute file=checkurl.txt \
script="/tool fetch http-header-field=\"Range: bytes=0-0\" dst-path=\"checkurl.tmp\" url=\"$1\""]
:local testsec 0
:while (([:len [/sys script job find where .id=$jobid]] = 1) && ($testsec < 20)) do={
:set testsec ($testsec + 1)
:delay 1s
}
:local error { cod="000.0" ; txt="NO CODE"}
:if ([:len [/file find where name="checkurl.txt"]] = 1) do={
:local check [/file get [/file find where name="checkurl.txt"] contents]
/file remove [find where name~"checkurl.(txt|tmp)"]
# 200 URL OK
:if ($check~"status: finished") do={
:set $error $1
:return $error
}
# 301 Permanent Redirect
:if ($check~" <301 Moved Permanently ") do={
:set $error [:pick $check ([:find $check " <301 Moved Permanently \"" -1] + 25) [:find $check "\"> " -1]]
:return $error
}
# 302 Redirect
:if ($check~" <302 Found ") do={
:set $error [:pick $check ([:find $check " <302 Found \"" -1] + 13) [:find $check "\"> " -1]]
:return $error
}
# other Codes (error or not)
:if ($check~" <.*> ") do={
:set ($error->"txt") [:pick $check ([:find $check " <" -1] + 2) [:find $check "> " -1]]
:set ($error->"cod") [:pick ($error->"txt") 0 [:find ($error->"txt") " " -1]]
:set ($error->"txt") [:pick ($error->"txt") ([:find ($error->"txt") " " -1] + 1) [:len ($error->"txt")]]
:return $error
}
# MikroTik fetch specific errors
:if ($check~"failure: ") do={
:set $error [:pick $check ([:find $check "failure: " -1] + 9) [:len $check]]
:return $error
}
# unexpected results
:set $error $check
:return $error
} else={
# :execute unsuccessful or timeout
:set error { cod="666.0" ; txt="TEMP FILE ERROR"}
/file remove [find where name~"checkurl.(txt|tmp)"]
:return $error
}
}
}
This can be streamlined store the exact error to a Global variable that is read in the import script and then used in the log.
Thanks to rextended for the coding.
Grazie 