From 3a35992397f3ac38bb3a77232715ce542913c23e Mon Sep 17 00:00:00 2001 From: onefang Date: Tue, 26 Nov 2019 00:59:20 +1000 Subject: Curl based downloader. Moving back to Lua-socket might be better, due to the crappy "modified" support in curl, but then I can't fork it. --- apt-panopticon.lua | 106 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 77 insertions(+), 29 deletions(-) diff --git a/apt-panopticon.lua b/apt-panopticon.lua index a3857b3..26ef4ac 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua @@ -70,8 +70,6 @@ options = } local defaultURL = {scheme = "http"} -local downloadLock = "flock -n results/wget-" -local download = "wget --timeout=300 -np -N -r -P results " -- Note wget has a default read timeout of 900 seconds (15 minutes). local releases = {"jessie", "ascii", "beowulf", "ceres"} local releaseFiles = { @@ -609,28 +607,66 @@ checkHost = function (orig, host, path, ip, file) end end -local downloads = function (cut, host, URL) - if 0 ~= cut then cd = " --cut-dirs=" .. cut .. " " else cd = "" end + +local addDownload = function(host, URL, f, r, k) + f:write('url "' .. 'http://' .. host .. URL .. '/merged/dists/' .. r .. k .. '"\n') + f:write('output "results/' .. host .. '/merged/dists/' .. r .. k .. '"\n') + -- Curls "check timestamp and overwrite file" stuff sucks. + -- Can only do ONE timestamp check per command. + -- Will DELETE the existing file if the timestamp fails to download a new one, unless we change directory first, which wont work here. + os.execute("if [ -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then mv" .. + " results/" .. host .. "/merged/dists/" .. r .. k .. + " results/" .. host .. "/merged/dists/" .. r .. k .. ".old; fi") +end + +local postDownload = function(host, r, k) + os.execute("if [ -f results/" .. host .. "/merged/dists/" .. r .. k .. ".old ]" .. + " && [ ! -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then cp" .. + " results/" .. host .. "/merged/dists/" .. r .. k .. ".old" .. + " results/" .. host .. "/merged/dists/" .. r .. k .. "; fi") +end + +local downloadLock = "flock -n results/curl-" +local download = "curl --connect-timeout " .. options.timeout.value .. " --create-dirs -L -z 'results/stamp.old' -v -R " +local downloads = function(host, URL, release, list) if nil == URL then URL = "/" end - local lock = "%s-" .. host .. ".log " - local log = " --rejected-log=results/wget-%s_REJECTS-" .. host .. ".log -a results/wget-%s-" .. host .. ".log " - I("starting file download commands for " .. host .. " " .. URL) - local cm = "ionice -c3 " .. downloadLock .. lock:format("debs") .. download .. log:format("debs", "debs") .. cd - for i, s in pairs(referenceDevs) do - cm = cm .. " https://" .. host .. URL .. "/" .. s - end - for i, s in pairs(referenceDebs) do - cm = cm .. " https://" .. host .. URL .. "/" .. s - end - for i, s in pairs(releases) do - fork(cm) - cm = "ionice -c3 " .. downloadLock .. lock:format(s) .. download .. log:format(s, s) .. cd - if repoExists(s .. k) then + local lock = "%s-" .. host .. ".lock" + local log = " --stderr results/curl-%s_" .. host .. ".log" + local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock:format("META") .. " " .. download .. log:format("META") .. " -K results/" .. host .. ".curl" + if testing("IPv4") and (not testing("IPv6")) then cm = cm .. ' -4' end + if (not testing("IPv4")) and testing("IPv6") then cm = cm .. ' -6' end + f, e = io.open("results/" .. host .. ".curl", "a+") + if nil == f then C("opening curl file - " .. e); return end + + if nil ~= list then + if "" ~= list then + for l in list:gmatch("\n*([^\n]+)\n*") do + addDownload(host, URL, f, release, "/" .. l) + end + f:close() + return + end + else + +--[[ + for i, s in pairs(referenceDevs) do + cm = cm .. " https://" .. host .. URL .. "/" .. s + end + for i, s in pairs(referenceDebs) do + cm = cm .. " https://" .. host .. URL .. "/" .. s + end + execute(cm) +]] + + for i, s in pairs(releases) do for j, k in pairs(releaseFiles) do - cm = cm .. " https://" .. host .. URL .. "/merged/dists/" .. s .. k + if repoExists(s .. k) then + addDownload(host, URL, f, s, k) + end end end end + f:close() fork(cm) end @@ -796,17 +832,13 @@ if 0 < #arg then results[v] = tests end end - if testing("Integrity") or testing("Updated") then - if nil == arg[3] then - if not keep then execute("rm -fr results/" .. pu.host) end - cut = 0 - for t in arg[1]:gmatch("(/)") do - cut = cut + 1 - end - downloads(cut, pu.host, pu.path) - checkExes("apt-panopticon.lua " .. sendArgs) - checkExes(downloadLock) + if nil == arg[2] then + I("Starting file downloads for " .. pu.host) +-- if not keep then execute("rm -fr results/" .. pu.host) end + downloads(pu.host, pu.path) +-- checkExes("apt-panopticon.lua " .. sendArgs) +-- checkExes(downloadLock) end end if origin then @@ -815,6 +847,21 @@ if 0 < #arg then checkHost(pu.host, pu.host, pu.path, arg[2], arg[3]) end + if testing("Integrity") or testing("Updated") then + if nil == arg[2] then + while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end + os.execute("rm -f results/" .. pu.host .. ".curl") + for i, n in pairs(releases) do + for l, o in pairs(releaseFiles) do + if repoExists(i .. o) then + postDownload(pu.host, n, o) + end + end + + end + end + end + local f = pu.host if "" ~= ip then f = f .. "_" .. ip end local rfile, e = io.open("results/" .. f .. ".lua", "w+") @@ -826,6 +873,7 @@ if 0 < #arg then logFile:close() else if not keep then + os.execute("rm -f results/*.curl") os.execute("rm -f results/*.log") os.execute("rm -f results/*.html") os.execute("rm -f results/*.txt") -- cgit v1.1