From f02b222005f7d656a6da8d25dca5a5fef56857c2 Mon Sep 17 00:00:00 2001 From: onefang Date: Mon, 30 Dec 2019 17:05:18 +1000 Subject: A lot of tweaking of the logging and reporting, making it easier to read. --- apt-panopticommon.lua | 19 +++--- apt-panopticon-report-email-web.lua | 18 +++--- apt-panopticon.lua | 114 ++++++++++++++++-------------------- 3 files changed, 72 insertions(+), 79 deletions(-) diff --git a/apt-panopticommon.lua b/apt-panopticommon.lua index 4e75f3f..ca952d3 100644 --- a/apt-panopticommon.lua +++ b/apt-panopticommon.lua @@ -348,6 +348,11 @@ APT.allpairs = function(tbl, func) end end +APT.lnk = function(URL) + return '' .. URL .. '' +end + + APT.search = function(t, s) for i, v in pairs(t) do if v == s then return true end @@ -441,12 +446,12 @@ local log = function(v, t, s, prot, test, host) APT.logFile:flush() end end -APT.D = function(s) log(4, "DEBUG ", s) end -APT.I = function(s) log(3, "INFO ", s) end -APT.T = function(s, p, t, h) log(2, "TIMEOUT ", s, p, t, h) end -APT.W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end -APT.E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end -APT.C = function(s) log(-1, "CRITICAL", s) end +APT.D = function(s, h) log(4, "DEBUG ", s, nil, nil, h) end +APT.I = function(s, h) log(3, "INFO ", s, nil, nil, h) end +APT.T = function(s, p, t, h) log(2, "TIMEOUT ", s, p, t, h) end +APT.W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end +APT.E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end +APT.C = function(s) log(-1, "CRITICAL", s) end local D = APT.D local I = APT.I local T = APT.T @@ -463,7 +468,7 @@ APT.testing = function(t, host) if t == v then local h = APT.mirrors[host] if nil == h then return true end - if true == h["Protocols"][t] then return true else D("Skipping " .. t .. " checks for " .. host) end + if true == h["Protocols"][t] then return true else I("Skipping " .. t .. " checks for " .. host) end end end return false diff --git a/apt-panopticon-report-email-web.lua b/apt-panopticon-report-email-web.lua index 97f7cef..05699ee 100755 --- a/apt-panopticon-report-email-web.lua +++ b/apt-panopticon-report-email-web.lua @@ -213,7 +213,7 @@ end local makeTable = function(web, hosts) web:write("\n" .. "" .. - "" .. + "" .. "\n") local bg = '' for k, v in APT.orderedPairs(hosts) do @@ -235,13 +235,13 @@ local makeTable = function(web, hosts) local sanity = status(hosts, k, results, "URLSanity") local integrity = status(hosts, k, results, "Integrity") local updated = status(hosts, k, results, "Updated") - local rate = v.Rate - if nil ~= rate then updated = updated .. ' ' .. rate end + local rate = v.Rate local min = tonumber(results.speed.min) local max = tonumber(results.speed.max) local spd = '' - local week = '' + local week = '' local graph = 'graphs' + if nil == rate then rate = '' end -- DNS-RR test. if (APT.options.roundRobin.value ~= k) and (nil ~= hosts[APT.options.roundRobin.value]) then @@ -291,12 +291,12 @@ local makeTable = function(web, hosts) if '0.00' == percentUp then percentUpdated = '??' end -- We are counting errors, and you can't get an error if you can't check anything. -- TODO - try to account for this better, this is just a quick hack. end - week = '' + week = '' end web:write("\n") + " " .. ' ' .. spd .. " " .. week .." \n") if "" ~= active then web:write("\n") end @@ -482,7 +482,7 @@ if nil == web then C("opening mirrors file - " .. e) else " means the mirror had some timeouts, and tests where not yet aborted. The darker colour means unsupported by the mirror, but tested anyway.

" .. "

TIMEOUT or TIMEOUT" .. " means the mirror had too many timeouts, and tests where aborted, so there is no result for this test.

" .. - "

NOTE: timeouts may be due to a problem on the testing computer.

" .. + "

NOTE: timeouts may be due to a problem on the testing computer, it might be busy with other things, or be having it's own network problems..

" .. "

The DNS round robin (DNS-RR) column shows the IPs for that mirror, or no if it isn't part of the DNS-RR.   " .. "The IPs link to the testing log for that IP accessed via the DNS-RR.   " .. APT.options.roundRobin.value .. " is the DNS-RR itself, so it doesn't get tested directly.

\n" .. @@ -497,7 +497,7 @@ if nil == web then C("opening mirrors file - " .. e) else m = makeIPlist(APT.mirrors) web:write( "

This lists each mirror, and the DNS entries for that mirror.   " .. - "The links point to the testing log files for " .. logCount("apt-panopticon") .. " for each domain name / IP combination that was tested.   " .. + "The links point to the testing log files (" .. logCount("apt-panopticon") .. ") for each domain name / IP combination that was tested.   " .. "If a mirror has a CNAME, that CNAME is listed along with that CNAMEs DNS entries.   " .. APT.options.roundRobin.value .. " is the DNS round robin, which points to the mirrors that are part of the DNS-RR.   " .. "If an IP is part of the DNS-RR, it is marked with 'DNS-RR'   " .. diff --git a/apt-panopticon.lua b/apt-panopticon.lua index b140831..889058b 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua @@ -163,7 +163,7 @@ local ip = "" local cor = nil local downloadLock = "flock -n results/curl-" - +local arw = '   ->   ' local repoExists = function (r) r = r:match("([%a-]*)") @@ -215,14 +215,15 @@ end local timeouts = 0; local totalTimeouts = 0 +local spcd = '   ' checkHEAD = function (host, URL, r, retry, sanity) if nil == r then r = 0 end if nil == retry then retry = 0 end - if nil == sanity then sanity = false end - local check = "Checking file" + if true == sanity then sanity = 'URLSanity' else sanity = '' end + local check = "HEAD testing file" local PU = url.parse(URL, defaultURL) local pu = url.parse(PU.scheme .. "://" .. host, defaultURL) - if not APT.testing(PU.scheme, host) and APT.redir then I("Not supported, not tested " .. PU.scheme .. " " .. host .. " -> " .. URL, PU.scheme, "", host); return end + if not APT.testing(PU.scheme, host) and APT.redir then I(spcd .. string.upper(PU.scheme) .. " not supported, not tested.   " .. URL, host); return end if 0 < r then check = "Redirecting to" end @@ -231,23 +232,27 @@ checkHEAD = function (host, URL, r, retry, sanity) check = "Retry " .. retry .. " " .. check end if 2 <= timeouts then - E("too many timeouts! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) + E(spcd .. spcd .. "too many timeouts! " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host) return end if APT.options.timeouts.value <= (totalTimeouts) then - E("Way too many timeouts!", PU.scheme, "", host) + E(spcd .. spcd .. "Way too many timeouts!", PU.scheme, "", host) return end if 20 <= r then - E("too many redirects! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) + E(spcd .. spcd .. "too many redirects! " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host) return end if APT.options.retries.value <= retry then - E("too many retries! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) + E(spcd .. spcd .. "too many retries! " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host) + return + end + if "https" == PU.scheme and APT.options.roundRobin.value == host then + I(spcd .. "Not testing " .. APT.lnk(URL) .. " mirrors wont have the correct HTTPS certificate for the round robin.", host) return + else + I(spcd .. check .. " " .. APT.lnk(URL), host) end - D(PU.scheme .. " :// " .. check .. " " .. host .. " -> " .. URL) - if "https" == PU.scheme and APT.options.roundRobin.value == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert."); return end --[[ Using curl command line - -I - HEAD @@ -282,20 +287,13 @@ checkHEAD = function (host, URL, r, retry, sanity) local msg = curlStatus[status] if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end if (28 == status) or (7 == status) then - if sanity then - T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "URLSanity", host) - else - T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "", host) - end + T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host) timeouts = timeouts + 1 else - if sanity then - E(" The curl command return an error code of " .. status .. " - " .. msg, PU.scheme, "URLSanity", host) - else - E(" The curl command return an error code of " .. status .. " - " .. msg, PU.scheme, "", host) - end + E(spcd .. spcd .. "The curl command return an error code of " .. status .. " - " .. msg .. ' for '.. APT.lnk(URL), PU.scheme, sanity, host) end - checkHEAD(host, URL, r, retry + 1, sanity) + if 60 == status then return end -- Certificate is invalid, don't bother retrying. + checkHEAD(host, URL, r, retry + 1, '' ~= sanity) return end local code = "???" @@ -303,17 +301,13 @@ checkHEAD = function (host, URL, r, retry, sanity) local location = nil local tmot = 1 while not APT.checkFile('results/STATUS_' .. fname) do - I('Waiting for results/STATUS_' .. fname .. ' file.') + D(spcd .. spcd .. 'Waiting for results/STATUS_' .. fname .. ' file.') os.execute('sleep ' .. tmot) tmot = tmot * 2 if 8 < tmot then - if sanity then - T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "URLSanity", host) - else - T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "", host) - end + T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host) timeouts = timeouts + 1 - checkHEAD(host, URL, r, retry + 1, sanity) + checkHEAD(host, URL, r, retry + 1, '' ~= sanity) return end end @@ -323,10 +317,10 @@ checkHEAD = function (host, URL, r, retry, sanity) if "#" == line:sub(1, 1) then code = line:sub(2, 4) if ("https" == PU.scheme) and ("0" ~= line:sub(6, 6)) then - if sanity then - E(" The certificate is invalid.", PU.scheme, "URLSanity", host) + if '' ~= sanity then + E(spcd .. spcd .. "The certificate is invalid.", PU.scheme, sanity, host) else - E(" The certificate is invalid.", PU.scheme, "https", host) + E(spcd .. spcd .. "The certificate is invalid.", PU.scheme, "https", host) end end elseif "http" == line:sub(1, 4):lower() then @@ -339,39 +333,35 @@ checkHEAD = function (host, URL, r, retry, sanity) os.execute('rm -f results/STATUS_' .. fname .. ' 2>/dev/null') end if ("4" == tostring(code):sub(1, 1)) or ("5" == tostring(code):sub(1, 1)) then - if sanity then - E(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "URLSanity", host) - else - E(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) - end + E(spcd .. spcd .. code .. " " .. cstr .. ". " .. check .. " " .. APT.lnk(URL), PU.scheme, sanity, host) else if not APT.testing(PU.scheme, host) then - W("Not supported, but works " .. PU.scheme .. " " .. host .. " -> " .. URL, PU.scheme, "", host) + W(spcd .. spcd .. "Not supported, but works " .. PU.scheme .. " " .. APT.lnk(URL), PU.scheme, "", host) end - I(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL) + I(spcd .. spcd .. code .. " " .. cstr .. ". " .. check .. " " .. APT.lnk(URL), host) -- timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through. if (1 <= APT.options.bandwidth.value) and (nil ~= location) then pu = url.parse(location, defaultURL) if ('http' == location:sub(1, 4)) and (pu.scheme ~= PU.scheme) then -- Sometimes a location sans scheme is returned, this is not a protocol change. if APT.testing("Protocol") then if APT.options.roundRobin.value == host then -- Coz HTTPS shouldn't happen via the round robin. - E(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host) + E(spcd .. spcd .. "Protocol changed during redirect! " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "Protocol", host) end - W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host) + W(spcd .. spcd .. "Protocol changed during redirect! " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "Protocol", host) else end end if location == URL then - E(" redirect loop! " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location, PU.scheme, "", host) + E(spcd .. spcd .. "Redirect loop! " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "", host) elseif nil == pu.host then - I(" relative redirect. " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location) - checkHEAD(host, PU.scheme .. "://" .. PU.host .. location, r + 1, retry, sanity) + I(spcd .. spcd .. "Relative redirect. " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host) + checkHEAD(host, PU.scheme .. "://" .. PU.host .. location, r + 1, retry, '' ~= sanity) elseif (PU.host == pu.host) or (host == pu.host) then - I(" redirect to same host. " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location) - checkHEAD(pu.host, location, r + 1, retry, sanity) + I(spcd .. spcd .. "Redirect to same host. " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host) + checkHEAD(pu.host, location, r + 1, retry, '' ~= sanity) else - I(" redirect to different host. " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location) + I(spcd .. spcd .. "Redirect to different host. " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host) --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location. Not good for deb.debian.org, which we should only be throwing .debs at. What we do is loop through the DNS entries, and only test the specific protocol & file being tested here. @@ -384,10 +374,10 @@ checkHEAD = function (host, URL, r, retry, sanity) if 'https' == pu.scheme then extraArgs = extraArgs .. ' --tests=-http' end if 'http' == pu.scheme then extraArgs = extraArgs .. ' --tests=-https' end local pth = path:match('^(.*/pool/).*$') - if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E('Odd redirect path ' .. path) end - I(" Now checking redirected host " .. u) + if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E(spcd .. spcd .. 'Odd redirect path ' .. path) end + I(spcd .. spcd .. "Now checking redirected host " .. u .. '   for   ' .. APT.lnk(URL) .. arw .. APT.lnk(location), host) APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork() - D('logging to ' .. APT.logName(pu.host, nil, file)[2]) + D(spcd .. 'logging to ' .. APT.logName(pu.host, nil, file)[2]) end end end @@ -418,11 +408,10 @@ local checkFiles = function (host, ip, path, file) if nil == path then path = "" end if nil ~= file then if "redir" == ip then ip = host end - I(" Checking IP for file " .. host .. " -> " .. ip .. " " .. path .. " " .. file) if checkTimeouts(host, "http", ip .. path .. "/" .. file) then return end if checkTimeouts(host, "https", ip .. path .. "/" .. file) then return end else - I(" Checking IP " .. host .. " -> " .. ip .. " " .. path) + I(" HEAD testing files for " .. host .. arw .. ip .. " " .. path, host) if 1 <= APT.options.bandwidth.value then -- Do these first, coz they are likely to fork off a different server. for i, s in pairs(referenceDebs) do @@ -467,10 +456,10 @@ checkHost = function (orig, host, path, ip, file) end else if orig == host then - D("checkHost " .. orig .. "" .. file) + I("Testing mirror " .. orig .. "" .. file) APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork() D('logging to ' .. APT.logName(ph.host, nil, file)[2]) - else D("checkHost " .. orig .. " -> " .. host) end + else D("checkHost " .. orig .. arw .. host) end end end @@ -534,7 +523,7 @@ local downloads = function(host, URL, meta, release, list) addDownload(host, URL, f, release, "/" .. l) end else - D('Downloading http://' .. host .. URL .. '/merged/' .. list) + I('Downloading ' .. APT.lnk('http://' .. host .. URL .. '/merged/' .. list)) f:write('url "' .. 'http://' .. host .. URL .. '/merged/' .. list .. '"\n') f:write('output "results/' .. host .. '/merged/' .. list .. '"\n') end @@ -582,8 +571,8 @@ local getMirrors = function () local host = "" local m = {} local active = true - local URL = "http://" .. APT.options.referenceSite.value .. "/mirror_list.txt" - I("getting mirrors.") + local URL = 'http://' .. APT.options.referenceSite.value .. '/mirror_list.txt' + I('Downloadin gand parsing http://' .. APT.options.referenceSite.value .. '/mirror_list.txt') local p, c, h = http.request(URL) if nil == p then E(c .. " fetching " .. URL) else @@ -683,12 +672,12 @@ local parseDebs = function(host) end if APT.testing("Updated") then if sz ~= fsz:sub(2, -2) then - E('Package size mismatch - results/' .. host .. "/merged/" .. p, 'http', 'Updated', host) + E('Package size mismatch for ' .. host .. "/merged/" .. p, 'http', 'Updated', host) end end os.execute('rm -f results/' .. host .. "/merged/" .. p) else - E('Failed to download - results/' .. host .. "/merged/" .. p, 'http', 'Updated', host) + E('Failed to download ' .. host .. "/merged/" .. p, 'http', 'Updated', host) end end end @@ -803,7 +792,7 @@ local parseRelease = function(host) if APT.testing("Integrity") then local status = APT.exe( "gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status - if 0 ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end + if 0 ~= status then E("GPG check failed for " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end -- TODO - should check the PGP sig of InRelease as well. end os.execute('rm results/' .. host .. '/merged/dists/' .. n .. '/' .. o) @@ -992,12 +981,11 @@ if 0 < #arg then end end - if APT.origin or APT.redir then APT.results["IPs"] = gatherIPs(pu.host) end - if not APT.logOpen(pu.host, arg[2], arg[3]) then return end I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", ")) - if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end - if nil ~= arg[3] then I(" Using file " .. arg[3]); end + if APT.origin or APT.redir then APT.results["IPs"] = gatherIPs(pu.host) end + if nil ~= arg[2] then I("   Using IP " .. arg[2]); ip = arg[2] end + if nil ~= arg[3] then I("   Using file " .. arg[3]); end APT.mirrors = loadfile("results/mirrors.lua")() APT.results = APT.padResults(APT.results) -- cgit v1.1

" .. lnk('FTP') .. "" .. lnk('HTTP') .. "" .. lnk('HTTPS') .. "" .. lnk('RSYNC') .. "" .. lnk('DNS round robin', 'DNS-RR') .. "" .. lnk('Protocol') .. "" .. lnk('URL sanity', 'URL-Sanity') .. "" .. lnk('Integrity') .. "" .. lnk('Updated') .. "" .. lnk('Speed range', 'Speed') .. "" .. lnk('Updated') .. "" .. lnk('Speed range', 'Speed') .. "" .. lnk('Weekly statistics', 'Weekly') .. "" .. lnk('Graphs') .. "
  ' .. percentUp .. '% up' .. percentUpdated .. '% updated ' .. percentUp .. '% up ' .. percentUpdated .. '% updated" .. ftp .. " " .. http .. " " .. https .. " " .. rsync .. " " .. dns .. - " " .. protocol .. " " .. sanity .. " " .. integrity .. " " .. updated .. - " " .. spd .. " " .. week .." " .. graph .. "
" .. protocol .. " " .. sanity .. " " .. integrity .. " ' .. rate .. + '' .. updated .. '" .. graph .. "
" .. active .. "