From 39e2c8fa335aa8db8be1277d55f11aa57b9a402e Mon Sep 17 00:00:00 2001
From: onefang
Date: Wed, 26 Jun 2019 14:50:12 +1000
Subject: Various changes to the download function.

Use flock to prevent duplicating downloads that are already in progress.
Change the command checker to include the flock command.
Add a timeout to the wget call.
Use --cut-dirs in wget to smooth out the extra path some mirrors have.
Remove the IP from the wget lof file names.
---
 mirror-checker.lua | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/mirror-checker.lua b/mirror-checker.lua
index bba9d49..9f70bcd 100755
--- a/mirror-checker.lua
+++ b/mirror-checker.lua
@@ -36,7 +36,8 @@ options =
 }
 
 local defaultURL = {scheme = "http"}
-local download = "wget -np -N -r -P results "
+local downloadLock = "flock -n results/wget-"
+local download = "wget --timeout=300 -np -N -r -P results "	-- Note wget has a default read timeout of 900 seconds (15 minutes).
 local releases = {"jessie", "ascii", "beowulf", "ceres"}
 local releaseFiles =
 {
@@ -269,21 +270,22 @@ local checkHost = function (host, path, ip)
     checkIP(host, host, path, ip)
 end
 
-local downloads = function (host, URL, IP)
+local downloads = function (cut, host, URL)
+    if 0 ~= cut then cd = " --cut-dirs=" .. cut .. " " else cd = "" end
     if nil == URL then URL = "/" end
-    if nil == IP then IP = "" else IP = "-" .. IP end
-    local log = " --rejected-log=results/wget-%s_REJECTS-" .. host .. IP .. ".log -a results/wget-%s-" .. host .. IP ..".log "
+    local lock = "%s-" .. host .. ".log "
+    local log = " --rejected-log=results/wget-%s_REJECTS-" .. host .. ".log -a results/wget-%s-" .. host .. ".log "
     I("starting file download commands for " .. host .. " " .. URL)
-    local cm = "ionice -c3 " .. download .. log:format("debs", "debs")
+    local cm = "ionice -c3 " .. downloadLock .. lock:format("debs") .. download .. log:format("debs", "debs") .. cd
     for i, s in pairs(referenceDebs) do
 	cm = cm .. " https://" .. host .. URL .. "/" .. s
     end
     for i, s in pairs(releases) do
 	execute(cm .. " &")
-	cm = "ionice -c3 " .. download .. log:format(s, s)
-	for j, k in pairs(releaseFiles) do
-	    cm = cm .. " https://" .. host .. URL .. "/merged/dists/" .. s .. k
+	cm = "ionice -c3 " .. downloadLock .. lock:format(s) .. download .. log:format(s, s) .. cd
 	if repoExists(s .. k) then
+	    for j, k in pairs(releaseFiles) do
+		cm = cm .. " https://" .. host .. URL .. "/merged/dists/" .. s .. k
 	    end
 	end
     end
@@ -405,9 +407,13 @@ if 0 < #arg then
     if nil ~= arg[2] then I("  Using IP " .. arg[2]) end
     if testing("Integrity") or testing("Updated") then
 	if not keep then execute("rm -fr results/" .. pu.host) end
-	downloads(pu.host, pu.path, arg[2])
+	cut = 0
+	for t in arg[1]:gmatch("(/)") do
+	    cut = cut + 1
+	end
+	downloads(cut, pu.host, pu.path)
 	checkExes("mirror-checker.lua " .. sendArgs)
-	checkExes(download)
+	checkExes(downloadLock)
     end
     checkHost(pu.host, pu.path, arg[2])
     logFile:close()
@@ -432,11 +438,11 @@ else
 --	checkHost(pu.host)
 	forkIP(m.BaseURL)
         checkExes("mirror-checker.lua " .. sendArgs)
-        if testing("Integrity") or testing("Updated") then checkExes(download) end
+        if testing("Integrity") or testing("Updated") then checkExes(downloadLock) end
     end
     while 1 <= checkExes("mirror-checker.lua " .. sendArgs) do os.execute("sleep 30") end
     if testing("Integrity") or testing("Updated") then
-	while 1 < checkExes(download) do os.execute("sleep 30") end
+	while 0 < checkExes(downloadLock) do os.execute("sleep 30") end
     end
     logFile:close()
 end
-- 
cgit v1.1