Clean up the timeout code.

Make it a command line parameter.
Re-factor the code somewhat.
Comment out the timeout backoff, for now.
Reduce the retry sleep time.

1 files changed, 40 insertions, 16 deletions

diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index cd1bca0..ed08c66 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -48,6 +48,12 @@ options =
 --              "Updated",
             },
         },
+    timeout =
+        {
+            typ = "number",
+            help = "",
+            value = 15,
+        },
     reports =
         {
             typ = "table",
@@ -389,6 +395,7 @@ local nlst = function (u)
 end
 
 local timeouts = 0;
+local totalTimeouts = 0
 checkHEAD = function (host, URL, r, retry)
     if nil == r then r = 0 end
     if nil == retry then retry = 0 end
@@ -399,13 +406,17 @@ checkHEAD = function (host, URL, r, retry)
         check = "Redirecting to"
     end
     if 0 < retry then
-        os.execute("sleep " .. math.random(1, 4))
+        os.execute("sleep " .. math.random(1, 3))
         check = "Retry " .. retry .. " " .. check
     end
     if 2 <= timeouts then
         E("too many timeouts!  " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
         return
     end
+    if 4 <= (totalTimeouts) then
+        E("Way too many timeouts!", PU.scheme, "", host)
+        return
+    end
     if 20 <= r then
         E("too many redirects!  " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
         return
@@ -442,7 +453,7 @@ checkHEAD = function (host, URL, r, retry)
         end
         IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"'
     end
-    local cmd = 'curl -I --retry 0 -s --path-as-is --connect-timeout 15 --max-redirs 0 ' .. IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' ..
+    local cmd = 'ionice -c3 nice -n 19 curl -I --retry 0 -s --path-as-is --connect-timeout ' .. options.timeout.value .. ' --max-redirs 0 ' .. IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' ..
                 hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"'
     local status, result = execute(cmd)
     os.execute('cat results/"HEADERS_' .. fname .. '" >>results/"STATUS_' .. fname .. '" 2>/dev/null; rm results/"HEADERS_' .. fname .. '" 2>/dev/null')
@@ -479,7 +490,7 @@ checkHEAD = function (host, URL, r, retry)
         E("  " .. code .. " " .. cstr .. ".   " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
     else
         I("  " .. code .. " " .. cstr .. ".   " .. check .. " " .. host .. " -> " .. URL)
-        timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through.
+--      timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through.
         if nil ~= location then
             pu = url.parse(location, defaultURL)
             if (pu.scheme ~= PU.scheme) then
@@ -515,31 +526,44 @@ checkHEAD = function (host, URL, r, retry)
     end
 end
 
+local checkTimeouts = function(host, scheme, URL)
+    if testing(scheme) then
+        totalTimeouts = totalTimeouts + timeouts; timeouts = 0
+        checkHEAD(host, scheme .. "://" .. URL)
+        if 4 <= (totalTimeouts) then
+            E("Way too many timeouts!", scheme, "", host)
+            return true
+        end
+    end
+    return false
+end
+
 local checkFiles = function (host, ip, path, file)
+    timeouts = 0
     if nil == path then path = "" end
     if nil ~= file then
         if "redir" == ip then ip = host end
         I("  Checking IP for file " .. host .. " -> " .. ip .. " " .. path .. " " .. file)
-        if testing("http", host)  then checkHEAD(host, "http://"  .. ip .. path .. "/" .. file) end
-        if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/" .. file) end
+        if checkTimeouts(host, "http",  ip .. path .. "/" .. file) then return end
+        if checkTimeouts(host, "https", ip .. path .. "/" .. file) then return end
+--      if testing("http", host)  then checkHEAD(host, "http://"  .. ip .. path .. "/" .. file); if checkTimeouts(host, "http" ) then return end; end
+--      if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/" .. file); if checkTimeouts(host, "https") then return end; end
     else
         I("  Checking IP " .. host .. " -> " .. ip .. " " .. path)
         for i, s in pairs(referenceDevs) do
-            local t = timeouts; timeouts = 0
-            if testing("http", host)  then checkHEAD(host, "http://"  .. ip .. path .. "/" .. s) end
-            t = t + timeouts; timeouts = 0
-            if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/" .. s) end
-            if 4 <= (t + timeouts) then return end
+            if checkTimeouts(host, "http",  ip .. path .. "/" .. s) then return end
+            if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end
+--          if testing("http", host)  then checkHEAD(host, "http://"  .. ip .. path .. "/" .. s); if checkTimeouts(host, "http" ) then return end; end
+--          if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/" .. s); if checkTimeouts(host, "https") then return end; end
         end
 
         for i, s in pairs(releases) do
             for j, k in pairs(releaseFiles) do
                 if repoExists(s .. k) then
-                    local t = timeouts; timeouts = 0
-                    if testing("http", host)  then checkHEAD(host, "http://"  .. ip .. path .. "/merged/dists/" .. s .. k) end
-                    t = t + timeouts; timeouts = 0
-                    if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/merged/dists/" .. s .. k) end
-                    if 4 <= (t + timeouts) then return end
+                    if checkTimeouts(host, "http",  ip .. path .. "/merged/dists/" .. s .. k) then return end
+                    if checkTimeouts(host, "https", ip .. path .. "/merged/dists/" .. s .. k) then return end
+--                  if testing("http", host)  then checkHEAD(host, "http://"  .. ip .. path .. "/merged/dists/" .. s .. k); if checkTimeouts(host, "http" ) then return end; end
+--                  if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/merged/dists/" .. s .. k); if checkTimeouts(host, "https") then return end; end
                 end
             end
         end
@@ -726,7 +750,7 @@ if 0 ~= #args then
                         end
                     end
                 else
-                    options[option].value = a
+                    options[option].value = a:sub(e + 1, -1)
                 end
                 option = ""
             end