From f02b222005f7d656a6da8d25dca5a5fef56857c2 Mon Sep 17 00:00:00 2001 From: onefang Date: Mon, 30 Dec 2019 17:05:18 +1000 Subject: A lot of tweaking of the logging and reporting, making it easier to read. --- apt-panopticon.lua | 114 ++++++++++++++++++++++++----------------------------- 1 file changed, 51 insertions(+), 63 deletions(-) (limited to 'apt-panopticon.lua') diff --git a/apt-panopticon.lua b/apt-panopticon.lua index b140831..889058b 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua @@ -163,7 +163,7 @@ local ip = "" local cor = nil local downloadLock = "flock -n results/curl-" - +local arw = '   ->   ' local repoExists = function (r) r = r:match("([%a-]*)") @@ -215,14 +215,15 @@ end local timeouts = 0; local totalTimeouts = 0 +local spcd = '   ' checkHEAD = function (host, URL, r, retry, sanity) if nil == r then r = 0 end if nil == retry then retry = 0 end - if nil == sanity then sanity = false end - local check = "Checking file" + if true == sanity then sanity = 'URLSanity' else sanity = '' end + local check = "HEAD testing file" local PU = url.parse(URL, defaultURL) local pu = url.parse(PU.scheme .. "://" .. host, defaultURL) - if not APT.testing(PU.scheme, host) and APT.redir then I("Not supported, not tested " .. PU.scheme .. " " .. host .. " -> " .. URL, PU.scheme, "", host); return end + if not APT.testing(PU.scheme, host) and APT.redir then I(spcd .. string.upper(PU.scheme) .. " not supported, not tested.   " .. URL, host); return end if 0 < r then check = "Redirecting to" end @@ -231,23 +232,27 @@ checkHEAD = function (host, URL, r, retry, sanity) check = "Retry " .. retry .. " " .. check end if 2 <= timeouts then - E("too many timeouts! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) + E(spcd .. spcd .. "too many timeouts! " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host) return end if APT.options.timeouts.value <= (totalTimeouts) then - E("Way too many timeouts!", PU.scheme, "", host) + E(spcd .. spcd .. "Way too many timeouts!", PU.scheme, "", host) return end if 20 <= r then - E("too many redirects! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) + E(spcd .. spcd .. "too many redirects! " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host) return end if APT.options.retries.value <= retry then - E("too many retries! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) + E(spcd .. spcd .. "too many retries! " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host) + return + end + if "https" == PU.scheme and APT.options.roundRobin.value == host then + I(spcd .. "Not testing " .. APT.lnk(URL) .. " mirrors wont have the correct HTTPS certificate for the round robin.", host) return + else + I(spcd .. check .. " " .. APT.lnk(URL), host) end - D(PU.scheme .. " :// " .. check .. " " .. host .. " -> " .. URL) - if "https" == PU.scheme and APT.options.roundRobin.value == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert."); return end --[[ Using curl command line - -I - HEAD @@ -282,20 +287,13 @@ checkHEAD = function (host, URL, r, retry, sanity) local msg = curlStatus[status] if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end if (28 == status) or (7 == status) then - if sanity then - T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "URLSanity", host) - else - T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "", host) - end + T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host) timeouts = timeouts + 1 else - if sanity then - E(" The curl command return an error code of " .. status .. " - " .. msg, PU.scheme, "URLSanity", host) - else - E(" The curl command return an error code of " .. status .. " - " .. msg, PU.scheme, "", host) - end + E(spcd .. spcd .. "The curl command return an error code of " .. status .. " - " .. msg .. ' for '.. APT.lnk(URL), PU.scheme, sanity, host) end - checkHEAD(host, URL, r, retry + 1, sanity) + if 60 == status then return end -- Certificate is invalid, don't bother retrying. + checkHEAD(host, URL, r, retry + 1, '' ~= sanity) return end local code = "???" @@ -303,17 +301,13 @@ checkHEAD = function (host, URL, r, retry, sanity) local location = nil local tmot = 1 while not APT.checkFile('results/STATUS_' .. fname) do - I('Waiting for results/STATUS_' .. fname .. ' file.') + D(spcd .. spcd .. 'Waiting for results/STATUS_' .. fname .. ' file.') os.execute('sleep ' .. tmot) tmot = tmot * 2 if 8 < tmot then - if sanity then - T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "URLSanity", host) - else - T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "", host) - end + T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host) timeouts = timeouts + 1 - checkHEAD(host, URL, r, retry + 1, sanity) + checkHEAD(host, URL, r, retry + 1, '' ~= sanity) return end end @@ -323,10 +317,10 @@ checkHEAD = function (host, URL, r, retry, sanity) if "#" == line:sub(1, 1) then code = line:sub(2, 4) if ("https" == PU.scheme) and ("0" ~= line:sub(6, 6)) then - if sanity then - E(" The certificate is invalid.", PU.scheme, "URLSanity", host) + if '' ~= sanity then + E(spcd .. spcd .. "The certificate is invalid.", PU.scheme, sanity, host) else - E(" The certificate is invalid.", PU.scheme, "https", host) + E(spcd .. spcd .. "The certificate is invalid.", PU.scheme, "https", host) end end elseif "http" == line:sub(1, 4):lower() then @@ -339,39 +333,35 @@ checkHEAD = function (host, URL, r, retry, sanity) os.execute('rm -f results/STATUS_' .. fname .. ' 2>/dev/null') end if ("4" == tostring(code):sub(1, 1)) or ("5" == tostring(code):sub(1, 1)) then - if sanity then - E(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "URLSanity", host) - else - E(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) - end + E(spcd .. spcd .. code .. " " .. cstr .. ". " .. check .. " " .. APT.lnk(URL), PU.scheme, sanity, host) else if not APT.testing(PU.scheme, host) then - W("Not supported, but works " .. PU.scheme .. " " .. host .. " -> " .. URL, PU.scheme, "", host) + W(spcd .. spcd .. "Not supported, but works " .. PU.scheme .. " " .. APT.lnk(URL), PU.scheme, "", host) end - I(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL) + I(spcd .. spcd .. code .. " " .. cstr .. ". " .. check .. " " .. APT.lnk(URL), host) -- timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through. if (1 <= APT.options.bandwidth.value) and (nil ~= location) then pu = url.parse(location, defaultURL) if ('http' == location:sub(1, 4)) and (pu.scheme ~= PU.scheme) then -- Sometimes a location sans scheme is returned, this is not a protocol change. if APT.testing("Protocol") then if APT.options.roundRobin.value == host then -- Coz HTTPS shouldn't happen via the round robin. - E(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host) + E(spcd .. spcd .. "Protocol changed during redirect! " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "Protocol", host) end - W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host) + W(spcd .. spcd .. "Protocol changed during redirect! " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "Protocol", host) else end end if location == URL then - E(" redirect loop! " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location, PU.scheme, "", host) + E(spcd .. spcd .. "Redirect loop! " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "", host) elseif nil == pu.host then - I(" relative redirect. " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location) - checkHEAD(host, PU.scheme .. "://" .. PU.host .. location, r + 1, retry, sanity) + I(spcd .. spcd .. "Relative redirect. " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host) + checkHEAD(host, PU.scheme .. "://" .. PU.host .. location, r + 1, retry, '' ~= sanity) elseif (PU.host == pu.host) or (host == pu.host) then - I(" redirect to same host. " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location) - checkHEAD(pu.host, location, r + 1, retry, sanity) + I(spcd .. spcd .. "Redirect to same host. " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host) + checkHEAD(pu.host, location, r + 1, retry, '' ~= sanity) else - I(" redirect to different host. " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location) + I(spcd .. spcd .. "Redirect to different host. " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host) --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location. Not good for deb.debian.org, which we should only be throwing .debs at. What we do is loop through the DNS entries, and only test the specific protocol & file being tested here. @@ -384,10 +374,10 @@ checkHEAD = function (host, URL, r, retry, sanity) if 'https' == pu.scheme then extraArgs = extraArgs .. ' --tests=-http' end if 'http' == pu.scheme then extraArgs = extraArgs .. ' --tests=-https' end local pth = path:match('^(.*/pool/).*$') - if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E('Odd redirect path ' .. path) end - I(" Now checking redirected host " .. u) + if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E(spcd .. spcd .. 'Odd redirect path ' .. path) end + I(spcd .. spcd .. "Now checking redirected host " .. u .. '   for   ' .. APT.lnk(URL) .. arw .. APT.lnk(location), host) APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork() - D('logging to ' .. APT.logName(pu.host, nil, file)[2]) + D(spcd .. 'logging to ' .. APT.logName(pu.host, nil, file)[2]) end end end @@ -418,11 +408,10 @@ local checkFiles = function (host, ip, path, file) if nil == path then path = "" end if nil ~= file then if "redir" == ip then ip = host end - I(" Checking IP for file " .. host .. " -> " .. ip .. " " .. path .. " " .. file) if checkTimeouts(host, "http", ip .. path .. "/" .. file) then return end if checkTimeouts(host, "https", ip .. path .. "/" .. file) then return end else - I(" Checking IP " .. host .. " -> " .. ip .. " " .. path) + I(" HEAD testing files for " .. host .. arw .. ip .. " " .. path, host) if 1 <= APT.options.bandwidth.value then -- Do these first, coz they are likely to fork off a different server. for i, s in pairs(referenceDebs) do @@ -467,10 +456,10 @@ checkHost = function (orig, host, path, ip, file) end else if orig == host then - D("checkHost " .. orig .. "" .. file) + I("Testing mirror " .. orig .. "" .. file) APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork() D('logging to ' .. APT.logName(ph.host, nil, file)[2]) - else D("checkHost " .. orig .. " -> " .. host) end + else D("checkHost " .. orig .. arw .. host) end end end @@ -534,7 +523,7 @@ local downloads = function(host, URL, meta, release, list) addDownload(host, URL, f, release, "/" .. l) end else - D('Downloading http://' .. host .. URL .. '/merged/' .. list) + I('Downloading ' .. APT.lnk('http://' .. host .. URL .. '/merged/' .. list)) f:write('url "' .. 'http://' .. host .. URL .. '/merged/' .. list .. '"\n') f:write('output "results/' .. host .. '/merged/' .. list .. '"\n') end @@ -582,8 +571,8 @@ local getMirrors = function () local host = "" local m = {} local active = true - local URL = "http://" .. APT.options.referenceSite.value .. "/mirror_list.txt" - I("getting mirrors.") + local URL = 'http://' .. APT.options.referenceSite.value .. '/mirror_list.txt' + I('Downloadin gand parsing http://' .. APT.options.referenceSite.value .. '/mirror_list.txt') local p, c, h = http.request(URL) if nil == p then E(c .. " fetching " .. URL) else @@ -683,12 +672,12 @@ local parseDebs = function(host) end if APT.testing("Updated") then if sz ~= fsz:sub(2, -2) then - E('Package size mismatch - results/' .. host .. "/merged/" .. p, 'http', 'Updated', host) + E('Package size mismatch for ' .. host .. "/merged/" .. p, 'http', 'Updated', host) end end os.execute('rm -f results/' .. host .. "/merged/" .. p) else - E('Failed to download - results/' .. host .. "/merged/" .. p, 'http', 'Updated', host) + E('Failed to download ' .. host .. "/merged/" .. p, 'http', 'Updated', host) end end end @@ -803,7 +792,7 @@ local parseRelease = function(host) if APT.testing("Integrity") then local status = APT.exe( "gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status - if 0 ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end + if 0 ~= status then E("GPG check failed for " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end -- TODO - should check the PGP sig of InRelease as well. end os.execute('rm results/' .. host .. '/merged/dists/' .. n .. '/' .. o) @@ -992,12 +981,11 @@ if 0 < #arg then end end - if APT.origin or APT.redir then APT.results["IPs"] = gatherIPs(pu.host) end - if not APT.logOpen(pu.host, arg[2], arg[3]) then return end I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", ")) - if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end - if nil ~= arg[3] then I(" Using file " .. arg[3]); end + if APT.origin or APT.redir then APT.results["IPs"] = gatherIPs(pu.host) end + if nil ~= arg[2] then I("   Using IP " .. arg[2]); ip = arg[2] end + if nil ~= arg[3] then I("   Using file " .. arg[3]); end APT.mirrors = loadfile("results/mirrors.lua")() APT.results = APT.padResults(APT.results) -- cgit v1.1