Code: Select all

Imported file length 64512 bytes

:global test "0123456789"
:for i from=1 to=22 do={:set test ($test . $test);:put [:len $test]}

refreshed old glory code

:global thefile ""
{
    :local url        "https://www.iwik.org/ipcountry/US.cidr"
    :local filesize   ([/tool fetch url=$url as-value output=none]->"downloaded")
    :local maxsize    64512 ; # is the maximum supported readable size of a block from a file
    :local start      0
    :local end        ($maxsize - 1)
    :local partnumber ($filesize / ($maxsize / 1024))
    :local reminder   ($filesize % ($maxsize / 1024))
    :if ($reminder > 0) do={ :set partnumber ($partnumber + 1) }
    :for x from=1 to=$partnumber step=1 do={
         :set thefile ($thefile . ([/tool fetch url=$url http-header-field="Range: bytes=$start-$end" as-value output=user]->"data"))
         :set start   ($start + $maxsize)
         :set end     ($end   + $maxsize)
    }
}
:put [:len $thefile]

:execute ":put \$thefile" file=newfile.txt

example with your url code

:global thefile ""
{
    :local url        ("https://" . "view.sentinel.turris.cz/greylist-data/greylist-latest.csv") ; # splitted just to not let the forum alter the url
    :local filesize   ([/tool fetch url=$url as-value output=none]->"downloaded")
    :local maxsize    64512 ; # is the maximum supported readable size of a block from a file
    :local start      0
    :local end        ($maxsize - 1)
    :local partnumber ($filesize / ($maxsize / 1024))
    :local reminder   ($filesize % ($maxsize / 1024))
    :if ($reminder > 0) do={ :set partnumber ($partnumber + 1) }
    :for x from=1 to=$partnumber step=1 do={
         :set thefile ($thefile . ([/tool fetch url=$url http-header-field="Range: bytes=$start-$end" as-value output=user]->"data"))
         :set start   ($start + $maxsize)
         :set end     ($end   + $maxsize)
    }
}
:put [:len $thefile]

Example script of reading a file from a webserver directly into a variable and generate a address-list from it.

Code: Select all

[…]
Starting import of address-list: testturris
Deleting all Dynamic entries in address-list: testturris
Using as extra filtering: http|smtp
Imported file length 64512 bytes
Completed importing testturris added 993 lines.

combined code

:global readfile do={
    :local url        $1
    :local thefile    ""
    :local filesize   ([/tool fetch url=$url as-value output=none]->"downloaded")
    :local maxsize    64512 ; # is the maximum supported readable size of a block from a file
    :local start      0
    :local end        ($maxsize - 1)
    :local partnumber ($filesize / ($maxsize / 1024))
    :local reminder   ($filesize % ($maxsize / 1024))
    :if ($reminder > 0) do={ :set partnumber ($partnumber + 1) }
    :for x from=1 to=$partnumber step=1 do={
         :set thefile ($thefile . ([/tool fetch url=$url http-header-field="Range: bytes=$start-$end" as-value output=user]->"data"))
         :set start   ($start + $maxsize)
         :set end     ($end   + $maxsize)
    }
    :return $thefile
}

{
/ip firewall address-list
:local update do={
 :global readfile
 :put "Starting import of address-list: $listname"
  :put "Deleting all Dynamic enties in address-list: $listname"
  :if (heirule != null) do={:put "Using as extra filtering: $heirule"}
  :if ($heirule = null) do={:set $heirule "."}
  :local n 0; # counter
  
 # remove the current list completely
 :do { /ip firewall address-list remove [find where list=$listname dynamic]} on-error={};
### line replaced ###  :local data ([:tool fetch url=$url output=user as-value]->"data")
   :local data [$readfile $url]
   :put "Imported file length $[:len $data] bytes"
     :while ([:len $data]!=0) do={ 
       :local line [:pick $data 0 [:find $data "\n"]]; # create only once and checked twice as local variable
       :if ($line~"^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}" && $line~heirule) do={
        :set $n ($n+1) 
        :do {add list=$listname address=[:pick $data 0 [:find $data $delimiter]] comment=$description timeout=$timeout} on-error={};
       }; # if IP address && extra filter if present
      :set data [:pick $data ([:find $data "\n"]+1) [:len $data]]; # removes the just added IP from the data array
     }; # while
 :put "Completed importing $listname added/replacing $n lines."
}; # do

$update url=("https://" . "view.sentinel.turris.cz/greylist-data/greylist-latest.csv") delimiter=, listname=turris timeout=8d heirule=http|smtp
}

Hi,
I know it’s an old post but I think both of your scripts are useful.
I use to download blacklist IP file from many sites (if you have some affordable site, it’ll be great!) and many times those files are very big!
I found this post but need some help because with the msatter’s script I can download .csv file indicate and it’s correctly imported, but with rextended’s combined code, the script exits with error “failure: Fetch failed with status 206”
During the script’s execution I see the log “Download from https://view.sentinel.turris.cz/greylis ... latest.csv to RAM FINISHED“ for 4 or 5 times and the last one is “failure: Fetch failed with status 206”.
How can I resolve?
With this script it’s possible also download .txt files? That is what I really matter
Thank you so much