Curl based downloader.

Moving back to Lua-socket might be better, due to the crappy "modified"
support in curl, but then I can't fork it.

1 files changed, 77 insertions, 29 deletions

diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index a3857b3..26ef4ac 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -70,8 +70,6 @@ options =
 }
 
 local defaultURL = {scheme = "http"}
-local downloadLock = "flock -n results/wget-"
-local download = "wget --timeout=300 -np -N -r -P results "     -- Note wget has a default read timeout of 900 seconds (15 minutes).
 local releases = {"jessie", "ascii", "beowulf", "ceres"}
 local releaseFiles =
 {
@@ -609,28 +607,66 @@ checkHost = function (orig, host, path, ip, file)
     end
 end
 
-local downloads = function (cut, host, URL)
-    if 0 ~= cut then cd = " --cut-dirs=" .. cut .. " " else cd = "" end
+
+local addDownload = function(host, URL, f, r, k)
+    f:write('url "' .. 'http://' .. host .. URL .. '/merged/dists/' .. r .. k .. '"\n')
+    f:write('output "results/' .. host .. '/merged/dists/' .. r .. k .. '"\n')
+    -- Curls "check timestamp and overwrite file" stuff sucks.
+    -- Can only do ONE timestamp check per command.
+    -- Will DELETE the existing file if the timestamp fails to download a new one, unless we change directory first, which wont work here.
+    os.execute("if [ -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then mv" ..
+                    "   results/" .. host .. "/merged/dists/" .. r .. k .. 
+                    "   results/" .. host .. "/merged/dists/" .. r .. k .. ".old; fi")
+end
+
+local postDownload = function(host, r, k)
+    os.execute("if [ -f results/" .. host .. "/merged/dists/" .. r .. k .. ".old ]" ..
+            " && [ ! -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then cp" ..
+                    "   results/" .. host .. "/merged/dists/" .. r .. k .. ".old" ..
+                    "   results/" .. host .. "/merged/dists/" .. r .. k .. "; fi")
+end
+
+local downloadLock = "flock -n results/curl-"
+local download = "curl --connect-timeout " .. options.timeout.value .. " --create-dirs -L -z 'results/stamp.old' -v -R "
+local downloads = function(host, URL, release, list)
     if nil == URL then URL = "/" end
-    local lock = "%s-" .. host .. ".log "
-    local log = " --rejected-log=results/wget-%s_REJECTS-" .. host .. ".log -a results/wget-%s-" .. host .. ".log "
-    I("starting file download commands for " .. host .. " " .. URL)
-    local cm = "ionice -c3 " .. downloadLock .. lock:format("debs") .. download .. log:format("debs", "debs") .. cd
-    for i, s in pairs(referenceDevs) do
-        cm = cm .. " https://" .. host .. URL .. "/" .. s
-    end
-    for i, s in pairs(referenceDebs) do
-        cm = cm .. " https://" .. host .. URL .. "/" .. s
-    end
-    for i, s in pairs(releases) do
-        fork(cm)
-        cm = "ionice -c3 " .. downloadLock .. lock:format(s) .. download .. log:format(s, s) .. cd
-        if repoExists(s .. k) then
+    local lock = "%s-" .. host .. ".lock"
+    local log = " --stderr results/curl-%s_" .. host .. ".log"
+    local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock:format("META") .. " " .. download .. log:format("META") .. " -K results/" .. host .. ".curl"
+    if      testing("IPv4")  and (not testing("IPv6")) then cm = cm .. ' -4' end
+    if (not testing("IPv4")) and      testing("IPv6")  then cm = cm .. ' -6' end
+    f, e = io.open("results/" .. host .. ".curl", "a+")
+    if nil == f then C("opening curl file - " .. e); return end
+
+    if nil ~= list then
+        if "" ~= list then
+            for l in list:gmatch("\n*([^\n]+)\n*") do
+                addDownload(host, URL, f, release, "/" .. l)
+            end
+            f:close()
+            return
+        end
+    else
+
+--[[
+        for i, s in pairs(referenceDevs) do
+            cm = cm .. " https://" .. host .. URL .. "/" .. s
+        end
+        for i, s in pairs(referenceDebs) do
+            cm = cm .. " https://" .. host .. URL .. "/" .. s
+        end
+        execute(cm)
+]]
+
+        for i, s in pairs(releases) do
             for j, k in pairs(releaseFiles) do
-                cm = cm .. " https://" .. host .. URL .. "/merged/dists/" .. s .. k
+                if repoExists(s .. k) then
+                    addDownload(host, URL, f, s, k)
+                end
             end
         end
     end
+    f:close()
     fork(cm)
 end
 
@@ -796,17 +832,13 @@ if 0 < #arg then
             results[v] = tests
         end
     end
-
     if testing("Integrity") or testing("Updated") then
-        if nil == arg[3] then
-            if not keep then execute("rm -fr results/" .. pu.host) end
-            cut = 0
-            for t in arg[1]:gmatch("(/)") do
-                cut = cut + 1
-            end
-            downloads(cut, pu.host, pu.path)
-            checkExes("apt-panopticon.lua " .. sendArgs)
-            checkExes(downloadLock)
+        if nil == arg[2] then
+            I("Starting file downloads for " .. pu.host)
+--          if not keep then execute("rm -fr results/" .. pu.host) end
+            downloads(pu.host, pu.path)
+--          checkExes("apt-panopticon.lua " .. sendArgs)
+--          checkExes(downloadLock)
         end
     end
     if origin then
@@ -815,6 +847,21 @@ if 0 < #arg then
         checkHost(pu.host, pu.host, pu.path, arg[2], arg[3])
     end
 
+    if testing("Integrity") or testing("Updated") then
+        if nil == arg[2] then
+            while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
+            os.execute("rm -f results/" .. pu.host .. ".curl")
+            for i, n in pairs(releases) do
+                for l, o in pairs(releaseFiles) do
+                    if repoExists(i .. o) then
+                        postDownload(pu.host, n, o)
+                    end
+                end
+
+            end
+        end
+    end
+
     local f = pu.host
     if "" ~= ip then f = f .. "_" .. ip end
     local rfile, e = io.open("results/" .. f .. ".lua", "w+")
@@ -826,6 +873,7 @@ if 0 < #arg then
     logFile:close()
 else
     if not keep then
+        os.execute("rm -f results/*.curl")
         os.execute("rm -f results/*.log")
         os.execute("rm -f results/*.html")
         os.execute("rm -f results/*.txt")