Here is something that I created for POC for parsing quite large IP list and adding them into address list (shell script generates rsc for import), and it is fastest approach compared with file chunked read with pure RSC because it uses multiple cores for parsing by awk (controlled by maxParallelJobs variable):
:local listUrl "https://public-dns.info/nameservers-all.txt"
:local listName "PUBLIC_DNS"
:local listComment "[dnslist]"
:local listTimeout 35w
:local validRecordRegex "^[^#][0-9]{0,2}\\\\.[0-9]{1,3}\\\\.[0-9]{1,3}\\\\.[0-9]{1,3}"
:local maxParallelJobs 3
:local containerName "linux"
:local containerMount "linux_share"
:local containerWaitTimeoutSec 60
:local stopContainerWhenDone false
:local rand [:rndnum from=1000 to=9999]
:local shellScriptName "import_list_$rand.sh"
:local importRscName "import_list_$rand.rsc"
:local containerTmpDir "/tmp"
:local successMessagePrefix "RSC import file generated with number of records: "
:local shellScript "#!/bin/bash
LIST_URL='$listUrl'
WORK_DIR=\"import_list_$rand\"
DOWNLOAD_FILE='import_list.txt'
CHUNK_PREFIX='import_list_'
CWD=\"\$(pwd)\"
cleanup() {
cd \"\${CWD}\"
rm -rf \"\${WORK_DIR}\"
}
mkdir -p \"\${WORK_DIR}\"
if [ \$? -ne 0 ]; then
echo \"Unable to create work dir: \${WORK_DIR}\"
exit 1
fi
cd \"\${WORK_DIR}\"
curl -s -o \"\${DOWNLOAD_FILE}\" \"\${LIST_URL}\"
if [ \$? -ne 0 ]; then
echo \"Error downloading list from: \${LIST_URL}\"
cleanup
exit 1
fi
split -e -d -a 1 -n l/$maxParallelJobs --additional-suffix='.txt' \"\${DOWNLOAD_FILE}\" \"\${CHUNK_PREFIX}\"
if [ \$? -ne 0 ]; then
echo \"Error splitting list file: \${DOWNLOAD_FILE}\"
cleanup
exit 1
fi
chunk_count=0
for filename in \"\${CHUNK_PREFIX}\"*.txt; do
awk -v valid_record_regex='$validRecordRegex' '{
for (line=1; line<=NF; line++) {
if (\$line ~ valid_record_regex) {
print \":onerror e {/ip/firewall/address-list/add list=\\\"$listName\\\" address=\"\$line\" timeout=$listTimeout comment=\\\"$listComment\\\"} do={:log debug \\\"Unable to add address \"\$line\" to address list $listName, reason: \$e\\\"} \"
}
}
}' \"\${filename}\" > \"\${CHUNK_PREFIX}\${chunk_count}.rsc\" &
chunk_count=\$((chunk_count + 1))
done
wait
find . -type f -name \"\${CHUNK_PREFIX}*.rsc\" -exec cat {} + >> \"$importRscName\"
if [ \$? -ne 0 ]; then
echo \"Error concatenating parsed chunks\"
cleanup
exit 1
fi
RECORDS=\$(wc -l \"$importRscName\" | cut -d ' ' -f 1)
cp \"$importRscName\" \"\${CWD}/\"
cleanup
echo \"$successMessagePrefix\${RECORDS}\"
"
:log info "Public DNS servers list import started"
:local quitWIthError do={
:log error $1
:error $1
}
/container
:if ([:len [find name=$containerName mounts=[
:local findMount do={
:foreach m in=$2 do={ :if ($m = $1) do={ :return true } }
:return false
}
:if ([$findMount $containerMount $mounts]) do={ :return $mounts }
]]] = 0) do={ $quitWIthError ("Container: $containerName not found or mount: $containerMount is not assigned to container") }
:if (![get $containerName running]) do={
:if ([get $containerName stopping]) do={
:local sec 0
:local timeout ($containerWaitTimeoutSec * 2)
:while (![get $containerName stopped] && $sec < $timeout) do={
:delay .5
:set sec ($sec + 1)
}
:if ($sec >= $timeout) do={ $quitWIthError ("Timeout while waiting container: $containerName") }
}
:if (![get $containerName starting]) do={ start $containerName }
:local sec 0
:local timeout ($containerWaitTimeoutSec * 2)
:while (![get $containerName running] && $sec < $timeout) do={
:delay .5
:set sec ($sec + 1)
}
:if ($sec >= $timeout) do={ $quitWIthError ("Timeout while waiting container: $containerName") }
}
:local srcDir [mounts/get $containerMount src]
/file/add contents=$shellScript name="$srcDir/$shellScriptName"
:local dstDir [mounts/get $containerMount dst]
:local res [shell $containerName cmd="2>&1 mv -f \"$dstDir/$shellScriptName\" \"$containerTmpDir\" && 2>&1 cd \"$containerTmpDir\" && 2>&1 chmod 755 \"$shellScriptName\" && 2>&1 ./\"$shellScriptName\" && 2>&1 mv -f \"$importRscName\" \"$dstDir/\" && 2>&1 rm -f \"$shellScriptName\"" as-value]
:if (!($res ~ "^$successMessagePrefix[0-9]+")) do={ $quitWIthError ("Error while executing script in container: $containerName, reason: \"$res\"") }
:local records [:tonum [:pick $res [:len $successMessagePrefix] [:len $res]]]
:local importScriptPath "$srcDir/$importRscName"
:if ($records > 0) do={
/ip/firewall/address-list
remove [find dynamic list=$listName comment=$listComment]
:local jobId [:execute ":import file-name=\"$importScriptPath\" verbose=yes"]
/system/script/job
:while ([:len [find .id=$jobId]] != 0) do={ :delay .2 }
} else={
:log warn "Public DNS servers list was parsed empty. Check for script errors!"
}
:if ([:pick $importScriptPath 0 1] = "/") do={ :set importScriptPath [:pick $importScriptPath 1 [:len $importScriptPath]] }
/file/remove "$importScriptPath"
:if ($stopContainerWhenDone) do={ /container/stop $containerName }
:log info "Public DNS servers list import finished, records processed: $records"
Tested with linuxserver/openssh-server container which is Alpine based. For script to work container requires mount for in/out file sharing with container (variable containerMount).