From 58e293224ee9256937d49d3a83bf2f709f60660b Mon Sep 17 00:00:00 2001
From: onefang
Date: Mon, 4 Nov 2019 21:20:24 +1000
Subject: End early if the host keeps timing out, and random sleep for retries.

Otherwise it could take hours trying a timing out host over and over.
---
 mirror-checker.lua | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/mirror-checker.lua b/mirror-checker.lua
index 69dd9e8..e7de781 100755
--- a/mirror-checker.lua
+++ b/mirror-checker.lua
@@ -203,6 +203,7 @@ local nlst = function (u)
     return r and table.concat(t), e
 end
 
+local timeouts = 0;
 checkHEAD = function (host, URL, r, retry)
     if nil == r then r = 0 end
     if nil == retry then retry = 0 end
@@ -213,14 +214,18 @@ checkHEAD = function (host, URL, r, retry)
 	check = "Redirecting to"
     end
     if 0 < retry then
--- TODO - should have a random sleep here before retrying.
+	os.execute("sleep " .. math.random(1, 4))
 	check = "Retry " .. retry .. " " .. check
     end
+    if 3 < timeouts then
+	E("too many timeouts!  " .. check .. " " .. host .. " -> " .. URL, PU.scheme, host)
+	return
+    end
     if 20 < r then
 	E("too many redirects!  " .. check .. " " .. host .. " -> " .. URL, PU.scheme, host)
 	return
     end
-    if 10 < retry then
+    if 4 < retry then
 	E("too many retries!  " .. check .. " " .. host .. " -> " .. URL, PU.scheme, host)
 	return
     end
@@ -241,12 +246,14 @@ checkHEAD = function (host, URL, r, retry)
 	E("  " .. c .. " " .. s .. "!   " .. check .. " " .. host .. " -> " .. URL, PU.scheme, host)
 	-- So far the only errors are "timeout", "Network is unreachable", and "closed", and I suspect "closed" is due to saturating my bandwidth.
 	-- Might be worthwhile retrying those, some number of times.
-	if ("closed" == c) or ("Network is unreachable" == c) or ("timeout" == c) then checkHEAD(host, URL, r, retry + 1) end
+	if "timeout" == c then timeouts = timeouts + 1 end
+	if ("closed" == c) or ("Network is unreachable" == c) or ("timeout" == c) then checkHEAD(host, URL, r, retry + 1, timeouts) end
     else
 	if ("4" == tostring(c):sub(1, 1)) or ("5" == tostring(c):sub(1, 1)) then
 	    E("  " .. c .. " " .. s .. ".   " .. check .. " " .. host .. " -> " .. URL, PU.scheme, host)
 	else
 	    I("  " .. c .. " " .. s .. ".   " .. check .. " " .. host .. " -> " .. URL)
+	    timeouts = timeouts - 1	-- Backoff the timeouts count if we managed to get through.
 	end
 	l = h.location
 	if nil ~= l then
@@ -295,14 +302,18 @@ local checkFiles = function (host, ip, path, file)
 	I("  Checking IP " .. host .. " -> " .. ip .. " " .. path)
 	for i, s in pairs(referenceDevs) do
 	    if testing("http", host)  then checkHEAD(host, "http://"  .. ip .. path .. "/" .. s) end
+	    if 3 < timeouts then return end
 	    if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/" .. s) end
+	    if 3 < timeouts then return end
 	end
 
 	for i, s in pairs(releases) do
 	    for j, k in pairs(releaseFiles) do
 		if repoExists(s .. k) then
 		    if testing("http", host)  then checkHEAD(host, "http://"  .. ip .. path .. "/merged/dists/" .. s .. k) end
+	    if 3 < timeouts then return end
 		    if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/merged/dists/" .. s .. k) end
+	    if 3 < timeouts then return end
 		end
 	    end
 	end
-- 
cgit v1.1