1 files changed, 51 insertions, 63 deletions

diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index b140831..889058b 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -163,7 +163,7 @@ local ip = ""
 local cor = nil
 
 local downloadLock = "flock -n results/curl-"
-
+local arw = ' &nbsp; <font color="magenta"><b>-></b></font> &nbsp; '
 
 local repoExists = function (r)
     r = r:match("([%a-]*)")
@@ -215,14 +215,15 @@ end
 
 local timeouts = 0;
 local totalTimeouts = 0
+local spcd = ' &nbsp; '
 checkHEAD = function (host, URL, r, retry, sanity)
     if nil == r then r = 0 end
     if nil == retry then retry = 0 end
-    if nil == sanity then sanity = false end
-    local check = "Checking file"
+    if true == sanity then sanity = 'URLSanity' else sanity = '' end
+    local check = "HEAD testing file"
     local PU = url.parse(URL, defaultURL)
     local pu = url.parse(PU.scheme .. "://" .. host, defaultURL)
-    if not APT.testing(PU.scheme, host) and APT.redir then I("Not supported, not tested " .. PU.scheme .. " " .. host .. " -> " .. URL, PU.scheme, "", host); return end
+    if not APT.testing(PU.scheme, host) and APT.redir then I(spcd .. string.upper(PU.scheme) .. " not supported, not tested. &nbsp; " .. URL, host); return end
     if 0 < r then
         check = "Redirecting to"
     end
@@ -231,23 +232,27 @@ checkHEAD = function (host, URL, r, retry, sanity)
         check = "Retry " .. retry .. " " .. check
     end
     if 2 <= timeouts then
-        E("too many timeouts!  " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
+        E(spcd .. spcd .. "too many timeouts!  " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host)
         return
     end
     if APT.options.timeouts.value <= (totalTimeouts) then
-        E("Way too many timeouts!", PU.scheme, "", host)
+        E(spcd .. spcd .. "Way too many timeouts!", PU.scheme, "", host)
         return
     end
     if 20 <= r then
-        E("too many redirects!  " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
+        E(spcd .. spcd .. "too many redirects!  " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host)
         return
     end
     if APT.options.retries.value <= retry then
-        E("too many retries!  " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
+        E(spcd .. spcd .. "too many retries!  " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host)
+        return
+    end
+    if "https" == PU.scheme and APT.options.roundRobin.value == host then
+        I(spcd .. "Not testing " .. APT.lnk(URL) .. " mirrors wont have the correct HTTPS certificate for the round robin.", host)
         return
+    else
+        I(spcd .. check .. " " .. APT.lnk(URL), host)
     end
-    D(PU.scheme .. " ://  " .. check .. " " .. host .. " -> " .. URL)
-    if "https" == PU.scheme and APT.options.roundRobin.value == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert.");  return end
 
     --[[ Using curl command line -
         -I - HEAD
@@ -282,20 +287,13 @@ checkHEAD = function (host, URL, r, retry, sanity)
         local msg = curlStatus[status]
         if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end
         if (28 == status) or (7 == status) then
-            if sanity then
-                T("  TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "URLSanity", host)
-            else
-                T("  TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "", host)
-            end
+            T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host)
             timeouts = timeouts + 1
         else
-            if sanity then
-                E("  The curl command return an error code of " .. status .. " - " .. msg, PU.scheme, "URLSanity", host)
-            else
-                E("  The curl command return an error code of " .. status .. " - " .. msg, PU.scheme, "", host)
-            end
+            E(spcd .. spcd .. "The curl command return an error code of " .. status .. " - " .. msg .. ' for '.. APT.lnk(URL), PU.scheme, sanity, host)
         end
-        checkHEAD(host, URL, r, retry + 1, sanity)
+        if 60 == status then return end -- Certificate is invalid, don't bother retrying.
+        checkHEAD(host, URL, r, retry + 1, '' ~= sanity)
         return
     end
     local code = "???"
@@ -303,17 +301,13 @@ checkHEAD = function (host, URL, r, retry, sanity)
     local location = nil
     local tmot = 1
     while not APT.checkFile('results/STATUS_' .. fname) do
-        I('Waiting for results/STATUS_' .. fname .. ' file.')
+        D(spcd .. spcd .. 'Waiting for results/STATUS_' .. fname .. ' file.')
         os.execute('sleep ' .. tmot)
         tmot = tmot * 2
         if 8 < tmot then
-            if sanity then
-                T("  TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "URLSanity", host)
-            else
-                T("  TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "", host)
-            end
+            T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host)
             timeouts = timeouts + 1
-            checkHEAD(host, URL, r, retry + 1, sanity)
+            checkHEAD(host, URL, r, retry + 1, '' ~= sanity)
             return
         end
     end
@@ -323,10 +317,10 @@ checkHEAD = function (host, URL, r, retry, sanity)
             if "#" == line:sub(1, 1) then
                 code = line:sub(2, 4)
                 if ("https" == PU.scheme) and ("0" ~= line:sub(6, 6)) then
-                    if sanity then
-                        E("  The certificate is invalid.", PU.scheme, "URLSanity", host)
+                    if '' ~= sanity then
+                        E(spcd .. spcd .. "The certificate is invalid.", PU.scheme, sanity, host)
                     else
-                        E("  The certificate is invalid.", PU.scheme, "https", host)
+                        E(spcd .. spcd .. "The certificate is invalid.", PU.scheme, "https", host)
                     end
                 end
             elseif "http" == line:sub(1, 4):lower() then
@@ -339,39 +333,35 @@ checkHEAD = function (host, URL, r, retry, sanity)
         os.execute('rm -f results/STATUS_' .. fname .. ' 2>/dev/null')
     end
     if ("4" == tostring(code):sub(1, 1)) or ("5" == tostring(code):sub(1, 1)) then
-        if sanity then
-            E("  " .. code .. " " .. cstr .. ".   " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "URLSanity", host)
-        else
-            E("  " .. code .. " " .. cstr .. ".   " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
-        end
+        E(spcd .. spcd .. code .. " " .. cstr .. ".   " .. check .. " " .. APT.lnk(URL), PU.scheme, sanity, host)
     else
         if not APT.testing(PU.scheme, host) then
-            W("Not supported, but works " .. PU.scheme .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
+            W(spcd .. spcd .. "Not supported, but works " .. PU.scheme .. " " .. APT.lnk(URL), PU.scheme, "", host)
         end
-        I("  " .. code .. " " .. cstr .. ".   " .. check .. " " .. host .. " -> " .. URL)
+        I(spcd .. spcd .. code .. " " .. cstr .. ".   " .. check .. " " .. APT.lnk(URL), host)
 --      timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through.
         if (1 <= APT.options.bandwidth.value) and (nil ~= location) then
             pu = url.parse(location, defaultURL)
             if ('http' == location:sub(1, 4)) and (pu.scheme ~= PU.scheme) then -- Sometimes a location sans scheme is returned, this is not a protocol change.
                 if APT.testing("Protocol") then
                     if APT.options.roundRobin.value == host then  -- Coz HTTPS shouldn't happen via the round robin.
-                        E("    protocol changed during redirect!  " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host)
+                        E(spcd .. spcd .. "Protocol changed during redirect!  " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "Protocol", host)
                     end
-                        W("    protocol changed during redirect!  " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host)
+                        W(spcd .. spcd .. "Protocol changed during redirect!  " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "Protocol", host)
                     else
                 end
             end
 
             if location == URL then
-                E("    redirect loop!  " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location, PU.scheme, "", host)
+                E(spcd .. spcd .. "Redirect loop!  " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "", host)
             elseif nil == pu.host then
-                I("    relative redirect.  " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location)
-                checkHEAD(host, PU.scheme .. "://" .. PU.host .. location, r + 1, retry, sanity)
+                I(spcd .. spcd .. "Relative redirect.  " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
+                checkHEAD(host, PU.scheme .. "://" .. PU.host .. location, r + 1, retry, '' ~= sanity)
             elseif (PU.host == pu.host) or (host == pu.host) then
-                I("    redirect to same host.  " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location)
-                checkHEAD(pu.host, location, r + 1, retry, sanity)
+                I(spcd .. spcd .. "Redirect to same host.  " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
+                checkHEAD(pu.host, location, r + 1, retry, '' ~= sanity)
             else
-                I("    redirect to different host.  " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location)
+                I(spcd .. spcd .. "Redirect to different host.  " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
                 --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location.
                     Not good for deb.debian.org, which we should only be throwing .debs at.
                     What we do is loop through the DNS entries, and only test the specific protocol & file being tested here.
@@ -384,10 +374,10 @@ checkHEAD = function (host, URL, r, retry, sanity)
                 if 'https' == pu.scheme then extraArgs = extraArgs .. ' --tests=-http' end
                 if 'http'  == pu.scheme then extraArgs = extraArgs .. ' --tests=-https' end
                 local pth = path:match('^(.*/pool/).*$')
-                if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E('Odd redirect path ' .. path) end
-                I("  Now checking redirected host " .. u)
+                if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E(spcd .. spcd .. 'Odd redirect path ' .. path) end
+                I(spcd .. spcd .. "Now checking redirected host " .. u .. ' &nbsp; for &nbsp; ' .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
                 APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork()
-                D('logging to ' .. APT.logName(pu.host, nil, file)[2])
+                D(spcd .. 'logging to ' .. APT.logName(pu.host, nil, file)[2])
             end
         end
     end
@@ -418,11 +408,10 @@ local checkFiles = function (host, ip, path, file)
     if nil == path then path = "" end
     if nil ~= file then
         if "redir" == ip then ip = host end
-        I("  Checking IP for file " .. host .. " -> " .. ip .. " " .. path .. " " .. file)
         if checkTimeouts(host, "http",  ip .. path .. "/" .. file) then return end
         if checkTimeouts(host, "https", ip .. path .. "/" .. file) then return end
     else
-        I("  Checking IP " .. host .. " -> " .. ip .. " " .. path)
+        I("  HEAD testing files for " .. host .. arw .. ip .. " " .. path, host)
         if 1 <= APT.options.bandwidth.value then
             -- Do these first, coz they are likely to fork off a different server.
             for i, s in pairs(referenceDebs) do
@@ -467,10 +456,10 @@ checkHost = function (orig, host, path, ip, file)
         end
     else
         if orig == host then
-            D("checkHost " .. orig .. "" .. file)
+            I("Testing mirror " .. orig .. "" .. file)
             APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork()
             D('logging to ' .. APT.logName(ph.host, nil, file)[2])
-        else D("checkHost " .. orig .. " -> " .. host) end
+        else D("checkHost " .. orig .. arw .. host) end
     end
 end
 
@@ -534,7 +523,7 @@ local downloads = function(host, URL, meta, release, list)
                     addDownload(host, URL, f, release, "/" .. l)
                 end
             else
-                D('Downloading http://' .. host .. URL .. '/merged/' .. list)
+                I('Downloading ' .. APT.lnk('http://' .. host .. URL .. '/merged/' .. list))
                 f:write('url "' .. 'http://' .. host .. URL .. '/merged/' .. list .. '"\n')
                 f:write('output "results/' .. host .. '/merged/' .. list .. '"\n')
             end
@@ -582,8 +571,8 @@ local getMirrors = function ()
     local host = ""
     local m = {}
     local active = true
-    local URL = "http://" .. APT.options.referenceSite.value .. "/mirror_list.txt"
-    I("getting mirrors.")
+    local URL = 'http://' .. APT.options.referenceSite.value .. '/mirror_list.txt'
+    I('Downloadin gand parsing http://' .. APT.options.referenceSite.value .. '/mirror_list.txt')
     local p, c, h = http.request(URL)
     if nil == p then E(c .. " fetching " .. URL) else
 
@@ -683,12 +672,12 @@ local parseDebs = function(host)
                         end
                         if APT.testing("Updated") then
                             if sz ~= fsz:sub(2, -2) then
-                                E('Package size mismatch - results/' .. host .. "/merged/" .. p, 'http', 'Updated', host)
+                                E('Package size mismatch for ' .. host .. "/merged/" .. p, 'http', 'Updated', host)
                             end
                         end
                         os.execute('rm -f results/' .. host .. "/merged/" .. p)
                     else
-                        E('Failed to download - results/' .. host .. "/merged/" .. p, 'http', 'Updated', host)
+                        E('Failed to download ' .. host .. "/merged/" .. p, 'http', 'Updated', host)
                     end
                 end
             end
@@ -803,7 +792,7 @@ local parseRelease = function(host)
                     if APT.testing("Integrity") then
                         local status = APT.exe( "gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. 
                                                 " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status
-                        if 0 ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end
+                        if 0 ~= status then E("GPG check failed for " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end
 -- TODO - should check the PGP sig of InRelease as well.
                     end
                     os.execute('rm results/' .. host .. '/merged/dists/' .. n .. '/' .. o)
@@ -992,12 +981,11 @@ if 0 < #arg then
         end
     end
 
-    if APT.origin or APT.redir then APT.results["IPs"] = gatherIPs(pu.host) end
-
     if not APT.logOpen(pu.host, arg[2], arg[3]) then return end
     I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", "))
-    if nil ~= arg[2] then I("  Using IP " .. arg[2]); ip = arg[2] end
-    if nil ~= arg[3] then I("  Using file " .. arg[3]); end
+    if APT.origin or APT.redir then APT.results["IPs"] = gatherIPs(pu.host) end
+    if nil ~= arg[2] then I(" &nbsp; Using IP " .. arg[2]); ip = arg[2] end
+    if nil ~= arg[3] then I(" &nbsp; Using file " .. arg[3]); end
     APT.mirrors = loadfile("results/mirrors.lua")()
     APT.results = APT.padResults(APT.results)