From 8290a02946e3e4f202b3377b132fabc57df80f63 Mon Sep 17 00:00:00 2001 From: onefang Date: Mon, 25 Nov 2019 02:31:15 +1000 Subject: Clean up the timeout code. Make it a command line parameter. Re-factor the code somewhat. Comment out the timeout backoff, for now. Reduce the retry sleep time. --- apt-panopticon.lua | 56 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/apt-panopticon.lua b/apt-panopticon.lua index cd1bca0..ed08c66 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua @@ -48,6 +48,12 @@ options = -- "Updated", }, }, + timeout = + { + typ = "number", + help = "", + value = 15, + }, reports = { typ = "table", @@ -389,6 +395,7 @@ local nlst = function (u) end local timeouts = 0; +local totalTimeouts = 0 checkHEAD = function (host, URL, r, retry) if nil == r then r = 0 end if nil == retry then retry = 0 end @@ -399,13 +406,17 @@ checkHEAD = function (host, URL, r, retry) check = "Redirecting to" end if 0 < retry then - os.execute("sleep " .. math.random(1, 4)) + os.execute("sleep " .. math.random(1, 3)) check = "Retry " .. retry .. " " .. check end if 2 <= timeouts then E("too many timeouts! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) return end + if 4 <= (totalTimeouts) then + E("Way too many timeouts!", PU.scheme, "", host) + return + end if 20 <= r then E("too many redirects! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) return @@ -442,7 +453,7 @@ checkHEAD = function (host, URL, r, retry) end IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"' end - local cmd = 'curl -I --retry 0 -s --path-as-is --connect-timeout 15 --max-redirs 0 ' .. IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' .. + local cmd = 'ionice -c3 nice -n 19 curl -I --retry 0 -s --path-as-is --connect-timeout ' .. options.timeout.value .. ' --max-redirs 0 ' .. IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' .. hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"' local status, result = execute(cmd) os.execute('cat results/"HEADERS_' .. fname .. '" >>results/"STATUS_' .. fname .. '" 2>/dev/null; rm results/"HEADERS_' .. fname .. '" 2>/dev/null') @@ -479,7 +490,7 @@ checkHEAD = function (host, URL, r, retry) E(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) else I(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL) - timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through. +-- timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through. if nil ~= location then pu = url.parse(location, defaultURL) if (pu.scheme ~= PU.scheme) then @@ -515,31 +526,44 @@ checkHEAD = function (host, URL, r, retry) end end +local checkTimeouts = function(host, scheme, URL) + if testing(scheme) then + totalTimeouts = totalTimeouts + timeouts; timeouts = 0 + checkHEAD(host, scheme .. "://" .. URL) + if 4 <= (totalTimeouts) then + E("Way too many timeouts!", scheme, "", host) + return true + end + end + return false +end + local checkFiles = function (host, ip, path, file) + timeouts = 0 if nil == path then path = "" end if nil ~= file then if "redir" == ip then ip = host end I(" Checking IP for file " .. host .. " -> " .. ip .. " " .. path .. " " .. file) - if testing("http", host) then checkHEAD(host, "http://" .. ip .. path .. "/" .. file) end - if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/" .. file) end + if checkTimeouts(host, "http", ip .. path .. "/" .. file) then return end + if checkTimeouts(host, "https", ip .. path .. "/" .. file) then return end +-- if testing("http", host) then checkHEAD(host, "http://" .. ip .. path .. "/" .. file); if checkTimeouts(host, "http" ) then return end; end +-- if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/" .. file); if checkTimeouts(host, "https") then return end; end else I(" Checking IP " .. host .. " -> " .. ip .. " " .. path) for i, s in pairs(referenceDevs) do - local t = timeouts; timeouts = 0 - if testing("http", host) then checkHEAD(host, "http://" .. ip .. path .. "/" .. s) end - t = t + timeouts; timeouts = 0 - if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/" .. s) end - if 4 <= (t + timeouts) then return end + if checkTimeouts(host, "http", ip .. path .. "/" .. s) then return end + if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end +-- if testing("http", host) then checkHEAD(host, "http://" .. ip .. path .. "/" .. s); if checkTimeouts(host, "http" ) then return end; end +-- if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/" .. s); if checkTimeouts(host, "https") then return end; end end for i, s in pairs(releases) do for j, k in pairs(releaseFiles) do if repoExists(s .. k) then - local t = timeouts; timeouts = 0 - if testing("http", host) then checkHEAD(host, "http://" .. ip .. path .. "/merged/dists/" .. s .. k) end - t = t + timeouts; timeouts = 0 - if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/merged/dists/" .. s .. k) end - if 4 <= (t + timeouts) then return end + if checkTimeouts(host, "http", ip .. path .. "/merged/dists/" .. s .. k) then return end + if checkTimeouts(host, "https", ip .. path .. "/merged/dists/" .. s .. k) then return end +-- if testing("http", host) then checkHEAD(host, "http://" .. ip .. path .. "/merged/dists/" .. s .. k); if checkTimeouts(host, "http" ) then return end; end +-- if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/merged/dists/" .. s .. k); if checkTimeouts(host, "https") then return end; end end end end @@ -726,7 +750,7 @@ if 0 ~= #args then end end else - options[option].value = a + options[option].value = a:sub(e + 1, -1) end option = "" end -- cgit v1.1