Various changes to the download function.

Use flock to prevent duplicating downloads that are already in progress.
Change the command checker to include the flock command.
Add a timeout to the wget call.
Use --cut-dirs in wget to smooth out the extra path some mirrors have.
Remove the IP from the wget lof file names.

1 files changed, 18 insertions, 12 deletions

diff --git a/mirror-checker.lua b/mirror-checker.lua
index bba9d49..9f70bcd 100755
--- a/mirror-checker.lua
+++ b/mirror-checker.lua
@@ -36,7 +36,8 @@ options =
 }
 
 local defaultURL = {scheme = "http"}
-local download = "wget -np -N -r -P results "
+local downloadLock = "flock -n results/wget-"
+local download = "wget --timeout=300 -np -N -r -P results "     -- Note wget has a default read timeout of 900 seconds (15 minutes).
 local releases = {"jessie", "ascii", "beowulf", "ceres"}
 local releaseFiles =
 {
@@ -269,21 +270,22 @@ local checkHost = function (host, path, ip)
     checkIP(host, host, path, ip)
 end
 
-local downloads = function (host, URL, IP)
+local downloads = function (cut, host, URL)
+    if 0 ~= cut then cd = " --cut-dirs=" .. cut .. " " else cd = "" end
     if nil == URL then URL = "/" end
-    if nil == IP then IP = "" else IP = "-" .. IP end
-    local log = " --rejected-log=results/wget-%s_REJECTS-" .. host .. IP .. ".log -a results/wget-%s-" .. host .. IP ..".log "
+    local lock = "%s-" .. host .. ".log "
+    local log = " --rejected-log=results/wget-%s_REJECTS-" .. host .. ".log -a results/wget-%s-" .. host .. ".log "
     I("starting file download commands for " .. host .. " " .. URL)
-    local cm = "ionice -c3 " .. download .. log:format("debs", "debs")
+    local cm = "ionice -c3 " .. downloadLock .. lock:format("debs") .. download .. log:format("debs", "debs") .. cd
     for i, s in pairs(referenceDebs) do
         cm = cm .. " https://" .. host .. URL .. "/" .. s
     end
     for i, s in pairs(releases) do
         execute(cm .. " &")
-        cm = "ionice -c3 " .. download .. log:format(s, s)
-        for j, k in pairs(releaseFiles) do
-            cm = cm .. " https://" .. host .. URL .. "/merged/dists/" .. s .. k
+        cm = "ionice -c3 " .. downloadLock .. lock:format(s) .. download .. log:format(s, s) .. cd
         if repoExists(s .. k) then
+            for j, k in pairs(releaseFiles) do
+                cm = cm .. " https://" .. host .. URL .. "/merged/dists/" .. s .. k
             end
         end
     end
@@ -405,9 +407,13 @@ if 0 < #arg then
     if nil ~= arg[2] then I("  Using IP " .. arg[2]) end
     if testing("Integrity") or testing("Updated") then
         if not keep then execute("rm -fr results/" .. pu.host) end
-        downloads(pu.host, pu.path, arg[2])
+        cut = 0
+        for t in arg[1]:gmatch("(/)") do
+            cut = cut + 1
+        end
+        downloads(cut, pu.host, pu.path)
         checkExes("mirror-checker.lua " .. sendArgs)
-        checkExes(download)
+        checkExes(downloadLock)
     end
     checkHost(pu.host, pu.path, arg[2])
     logFile:close()
@@ -432,11 +438,11 @@ else
 --      checkHost(pu.host)
         forkIP(m.BaseURL)
         checkExes("mirror-checker.lua " .. sendArgs)
-        if testing("Integrity") or testing("Updated") then checkExes(download) end
+        if testing("Integrity") or testing("Updated") then checkExes(downloadLock) end
     end
     while 1 <= checkExes("mirror-checker.lua " .. sendArgs) do os.execute("sleep 30") end
     if testing("Integrity") or testing("Updated") then
-        while 1 < checkExes(download) do os.execute("sleep 30") end
+        while 0 < checkExes(downloadLock) do os.execute("sleep 30") end
     end
     logFile:close()
 end