From 5e8a68623c0241e3a6bdfefd35efdc8f4b16ed5f Mon Sep 17 00:00:00 2001
From: onefang
Date: Wed, 26 Jun 2019 14:52:32 +1000
Subject: Retry some of the HTTTP header downloads on certain errors, and
 report others.

---
 mirror-checker.lua | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/mirror-checker.lua b/mirror-checker.lua
index 9f70bcd..8b80e3f 100755
--- a/mirror-checker.lua
+++ b/mirror-checker.lua
@@ -173,23 +173,42 @@ local nlst = function (u)
     return r and table.concat(t), e
 end
 
-checkURL = function (host, URL, r)
+checkURL = function (host, URL, r, retry)
     if nil == r then r = 0 end
+    if nil == retry then retry = 0 end
     local check = "Checking file"
     if 0 < r then
 	check = "Redirecting to"
---        checkIP(host)
+--        checkHost(host, host)
+    end
+    if 0 < retry then
+	check = "Retry " .. retry .. " " .. check
     end
     if 10 < r then
 	E("too many redirects!  " .. check .. " " .. host .. " -> " .. URL)
 	return
     end
+    if 10 < retry then
+	E("too many retries!  " .. check .. " " .. host .. " -> " .. URL)
+	return
+    end
     local PU = url.parse(URL, defaultURL)
     D("   " .. PU.scheme .. " ://  " .. check .. " " .. host .. " -> " .. URL)
     if not testing(PU.scheme) then D("not testing " .. PU.scheme .. " " .. host .. " -> " .. URL);  return end
     local hd = {Host = host}
     local p, c, h, s = http.request{method = "HEAD", redirect = false, url = URL, headers = hd}
-    if nil == p then E(c .. "!   " .. check .. " " .. host .. " -> " .. URL) else
+    if nil == s then s = "" end
+    if nil == p then
+	E(c .. " " .. s .. "!   " .. check .. " " .. host .. " -> " .. URL)
+	-- So far the only errors are "timeout", "Network is unreachable", and "closed", and I suspect "closed" is due to saturating my bandwidth.
+	-- Might be worthwhile retrying those, some number of times.
+	if ("closed" == c) or ("Network is unreachable" == c) or ("timeout" == c) then checkURL(host, URL, r, retry + 1) end
+    else
+	if ("4" == tostring(c):sub(1, 1)) or ("5" == tostring(c):sub(1, 1)) then
+	    E(c .. " " .. s .. ".   " .. check .. " " .. host .. " -> " .. URL)
+	else
+	    I(c .. " " .. s .. ".   " .. check .. " " .. host .. " -> " .. URL)
+	end
 	l = h.location
 	if nil ~= l then
 	    local pu = url.parse(l, defaultURL)
@@ -203,7 +222,7 @@ checkURL = function (host, URL, r)
 		    if testing("Protocol") and pu.scheme ~= PU.scheme then
 			W("protocol changed during redirect!  " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l)
 		    end
-		    checkURL(pu.host, l, r + 1)
+		    checkURL(pu.host, l, r + 1, retry)
 		end
 	    end
 	end
-- 
cgit v1.1