Add more report stuff.

TIMEOUT, report inactive mirrors and  why it's not active, add a * for
things tested but not supported.

3 files changed, 117 insertions, 82 deletions

diff --git a/apt-panopticon-report-email.lua b/apt-panopticon-report-email.lua
index c102b81..53db16f 100755
--- a/apt-panopticon-report-email.lua
+++ b/apt-panopticon-report-email.lua
@@ -234,8 +234,10 @@ if nil == file then C("opening mirrors file - " .. e) else
                 results = collate(k, i, results)
             end
         end
+        local ftp       = "[skip]"
         local http      = status(k, results, "http")
         local https     = status(k, results, "https")
+        local rsync     = "[skip]"
         local dns       = ""
         local protocol  = status(k, results, "Protocol")
         local sanity    = "[skip]"
@@ -275,8 +277,10 @@ if nil == file then C("opening mirrors file - " .. e) else
             dns = " DNS-RR: " .. dns
         end
 
-        file:write(     "      http: " .. http .. " https: " .. https .. dns .. "\n" ..
-                        "      Protocol: " .. protocol .. " URL-sanity: " .. sanity .. " Integrity: " .. integrity .. " Updated: " .. updated .. "\n")
+        file:write(     "      ftp: " .. ftp .. " http: " .. http .. " https: " .. https .." rsync: " .. rsync .. "\n" ..
+                        "     " .. dns .. "\n" ..
+                        "      Protocol: " .. protocol .. " URL-sanity: " .. sanity .. " Integrity: " .. integrity .. "\n" ..
+                        "      Updated: " .. updated .. "\n")
     end
     file:write( "\n==== faulty mirrors: ====\n" .. faulty)
     file:write( "\n---- END MIRROR-STATUS ----\n\n" ..
diff --git a/apt-panopticon-report-web.lua b/apt-panopticon-report-web.lua
index 61102c8..fb41341 100755
--- a/apt-panopticon-report-web.lua
+++ b/apt-panopticon-report-web.lua
@@ -155,24 +155,27 @@ local status = function(host, results, typ)
     local e = 0
     local w = 0
     local s = nil ~= mirrors[host].Protocols[typ]
+    local to = false
     if ('http' ~= typ) and ('https' ~= typ) and ('ftp' ~= typ) and ('rsync' ~= typ) then s = true end
     if nil ~= results[typ] then
         e = results[typ].errors
         w = results[typ].warnings
+--[[
         for k, v in pairs(results[typ]) do
             if "table" == type(v) then
-                e = e + v.errors
-                w = w + v.warnings
+                if 0 <= v.errors   then e = e + v.errors   else to = true end
+                if 0 <= v.warnings then w = w + v.warnings else to = true end
             end
         end
+]]
     else
         for k, v in pairs(results) do
             if "table" == type(v) then
                 for i, u in pairs(v) do
                     if "table" == type(u) then
                         if typ == i then
-                            e = e + u.errors
-                            w = w + u.warnings
+                            if 0 <= u.errors   then e = e + u.errors   else to = true end
+                            if 0 <= u.warnings then w = w + u.warnings else to = true end
                         end
                     end
                 end
@@ -180,18 +183,24 @@ local status = function(host, results, typ)
         end
     end
 
-    if 0 < e then
+    if to then
+        if s then
+            result = "[<font color='blue'><b>TIMEOUT</b></font>]"
+        else
+            result = "[<font color='darkblue'><b>TIMEOUT *</b></font>]"
+        end
+    elseif 0 < e then
         if s then
             result = "[<font color='red'><b>FAILED</b></font>]"
         else
-            result = "[<font color='darkred'><b>FAILED</b></font>]"
+            result = "[<font color='darkred'><b>FAILED *</b></font>]"
         end
         faulty = faulty .. host .. " (" .. typ .. ")<br>\n"
     else
         if s then
             result = "[<font color='lime'><b>OK</b></font>]"
         else
-            result = "[<font color='darkgreen'><b>OK</b></font>]"
+            result = "[<font color='darkgreen'><b>OK *</b></font>]"
         end
     end
     return result .. plurals(e, w)
@@ -258,7 +267,7 @@ end
 local file, e = io.open("results/Report-web.html", "w+")
 if nil == file then C("opening mirrors file - " .. e) else
     file:write( "<html><head><title>apt-panopticon results</title>\n" ..
-                '</head><body bgcolor="black" text="white" alink="red" link="blue" vlink="purple">' ..
+                '</head><body bgcolor="black" text="white">' ..
                 "<h1>Welcome to the apt-panopticon results page.</h1>\n" ..
                 "<p>This is the status of the mirror servers in the Devuan package mirror network.</p>\n" ..
                 "<p>The full list of Devuan package mirrors is available at the URL: " ..
@@ -269,21 +278,25 @@ if nil == file then C("opening mirrors file - " .. e) else
 
                 "<p>[<font color='red'><b>FAILED</b></font>] or [<font color='lime'><b>OK</b></font>]" ..
                 " means the tested thing is supported for that mirror.</p>\n" ..
-                "<p>[<font color='darkred'><b>FAILED</b></font>] or [<font color='darkgreen'><b>OK</b></font>]" ..
+                "<p>[<font color='darkred'><b>FAILED *</b></font>] or [<font color='darkgreen'><b>OK *</b></font>]" ..
                 " means the tested thing is unsupported for that mirror, but might have been tested anyway.</p>\n" ..
+                "<p>[<font color='blue'><b>TIMEOUT</b></font>] or [<font color='darkblue'><b>TIMEOUT</b></font>]" ..
+                " means the server had too many timeouts, and tests where aborted, so there is no result for this test.</p>" ..
                 "<p>The DNS round robin (DNS-RR) column shows the IPs for that mirror, or [<font color='grey'><b>no</b></font>] if it isn't part of the DNS-RR. &nbsp; " ..
                 "The IPs link to the testing log for that IP accessed via the DNS-RR. &nbsp; " ..
                 "deb.devuan.org is the DNS-RR itself, so it doesn't get tested directly.</p>\n" ..
                 "<p>The time in the Updated column is how often the mirror updates itself.</p>" ..
-                "<p>Mirrors with a <font style='background-color:dimgrey'>grey background</font> are not active.</p>\n" ..
+                "<p>Mirrors with a <font style='background-color:dimgrey'>grey background</font> are not active (though may be usable as part of the DNS-RR).</p>\n" ..
                 "<p>[<font color='grey'><b>skip</b></font>] means that the test hasn't been written yet.</p>\n" ..
                 "<tr><th></th><th>FTP</th><th>HTTP</th><th>HTTPS</th><th>RSYNC</th><th>DNS round robin</th><th>Protocol</th><th>URL sanity</th><th>Integrity</th><th>Updated</th></tr>\n"
                 )
     for k, v in orderedPairs(mirrors) do
         local results = loadfile("results/" .. k .. ".lua")()
+        local active = ""
         if "yes" == v.Active then
             file:write("     <tr><th>" .. k .. "</th> ")
         else
+            if nil == v.Active then active = 'nil' else active = v.Active end
             file:write("     <tr style='background-color:dimgrey'><th>" .. k .. "</th> ")
         end
         local IPs = v.IPs
@@ -343,6 +356,9 @@ if nil == file then C("opening mirrors file - " .. e) else
         file:write("<td>" .. ftp .. "&nbsp;</td><td>" .. http .. "&nbsp;</td><td>" .. https .. "&nbsp;</td><td>" .. rsync .. "&nbsp;</td><td>" .. dns ..
                     "&nbsp;</td><td>" .. protocol .. "&nbsp;</td><td>" .. sanity ..
                     "&nbsp;</td><td>" .. integrity .. "&nbsp;</td><td>" .. updated .. "</td></tr>\n")
+        if "" ~= v.Active then
+            file:write("<tr><td style='background-color:dimgrey'>" .. active .. "</td></tr>\n")
+        end
     end
     file:write( "</table>\n<br>\n<h2>==== faulty mirrors: ====</h2>\n" .. faulty)
     file:write( "<br>\n<br>\n<h2>==== DNS and logs: ====</h2>\n")
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index d66e32d..5d9ed82 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -756,7 +756,7 @@ local getMirrors = function ()
     local host = ""
     local m = {}
     local active = true
-    local URL = "https://" .. options.referenceSite.value .. "/mirror_list.txt"
+    local URL = "http://" .. options.referenceSite.value .. "/mirror_list.txt"
     I("getting mirrors.")
     local p, c, h = http.request(URL)
     if nil == p then E(c .. " fetching " .. URL) else
@@ -766,7 +766,8 @@ local getMirrors = function ()
             d = string.lower(d)
             if "FQDN" == t then
                 if "" ~= host then
-                    if active then mirrors[host] = m end
+--                  if active then mirrors[host] = m end
+                    mirrors[host] = m
                     m = {}
                     active = true
                 end
@@ -780,15 +781,16 @@ local getMirrors = function ()
                     prot[w] = true;
                 end
                 m[t] = prot
-            elseif "Active" == t and nil == d:find("yes", 1, true) then
+            elseif "Active" == t and nil == d:sub(1, 3):find("yes", 1, true) then
                 W("Mirror " .. host .. " is not active - " .. d, "", "", host)
                 active = false
+                m[t] = d
 -- TODO - Should do some input validation on BaseURL, and everything else.
             else
                 m[t] = d
             end
         end
-        if "" ~= host and active then
+        if "" ~= host --[[and active]] then
             mirrors[host] = m
         end
     end
@@ -933,88 +935,90 @@ if 0 < #arg then
     end
 
     if testing("Integrity") or testing("Updated") then
-        if origin and (options.roundRobin.value ~= pu.host) then
-            while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
-            os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock")
-            for i, n in pairs(releases) do
-                for l, o in pairs(releaseFiles) do
-                    if repoExists(i .. o) then
-                        postDownload(pu.host, n, o)
+        if 4 > (totalTimeouts) then
+            if origin and (options.roundRobin.value ~= pu.host) then
+                while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
+                os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock")
+                for i, n in pairs(releases) do
+                    for l, o in pairs(releaseFiles) do
+                        if repoExists(i .. o) then
+                            postDownload(pu.host, n, o)
+                        end
                     end
-                end
 
-                os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED')
-                if options.referenceSite.value == pu.host then
-                    execute('diff -U 0   results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
-                                        'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
-                                        '| grep -v "@@" | grep "^+" | grep "Packages.xz$" |  cut -c 77- >results/NEW_Release_' .. n .. '.txt')
-                else
+                    os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED')
+                    if options.referenceSite.value == pu.host then
+                        execute('diff -U 0   results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
+                                            'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
+                                            '| grep -v "@@" | grep "^+" | grep "Packages.xz$" |  cut -c 77- >results/NEW_Release_' .. n .. '.txt')
+                    else
 -- TODO - compare to the pkgmaster copy.
-                end
+                    end
 
-                local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r")
-                if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else
-                    local diff = dfile:read("*a")
-                    if "" ~= diff then
-                        downloads(pu.host, pu.path, n, diff)
+                    local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r")
+                    if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else
+                        local diff = dfile:read("*a")
+                        if "" ~= diff then
+                        downloads(pu.host, pu.path, n, diff)
+                        end
                     end
-                end
 
-            end
+                end
 
-            downloads(pu.host, pu.path, "", "")
-            while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
-            os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock")
+                downloads(pu.host, pu.path, "", "")
+                while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
+                os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock")
 
-            for i, n in pairs(releases) do
-                local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r")
-                if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else
-                    local diff = dfile:read("*a")
-                    for l in diff:gmatch("\n*([^\n]+)\n*") do
-                        postDownload(pu.host, n, "/" .. l)
+                for i, n in pairs(releases) do
+                    local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r")
+                    if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else
+                        local diff = dfile:read("*a")
+                        for l in diff:gmatch("\n*([^\n]+)\n*") do
+                            postDownload(pu.host, n, "/" .. l)
+                        end
+                    end
+                    if options.referenceSite.value == pu.host then
+                        -- In case it wasn't dealt with already.
+                        os.execute('touch results/NEW_Packages_' .. n .. '.test.txt')
                     end
                 end
-                if options.referenceSite.value == pu.host then
-                    -- In case it wasn't dealt with already.
-                    os.execute('touch results/NEW_Packages_' .. n .. '.test.txt')
-                end
-            end
 
 
-            for i, n in pairs(releases) do
-                local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
-                if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
-                    for l in nfile:lines() do
-                        local p = l:match('(pool/.*%.deb)')
-                        if nil ~= p then
-                            downloads(pu.host, pu.path, nil, p)
+                for i, n in pairs(releases) do
+                    local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
+                    if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
+                        for l in nfile:lines() do
+                            local p = l:match('(pool/.*%.deb)')
+                            if nil ~= p then
+                                downloads(pu.host, pu.path, nil, p)
+                            end
                         end
                     end
                 end
-            end
-            downloads(pu.host, pu.path, nil, "")
-            while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
-            for i, n in pairs(releases) do
-                local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
-                if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
-                    for l in nfile:lines() do
-                        local v, p, sz, m, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) | (%x.+) |')
-                        if nil ~= p then
-                            local status, fsz = execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5')
-                            if testing("Integrity") then
-                                if sz ~= fsz:sub(2, -2) then    -- The sub bit is to slice off the EOLs at each end.
-                                    E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host)
-                                    print('|' .. sz .. '~=' .. fsz:sub(2, -2) .. '|')
-                                else
-                                    local status, fm = execute('md5sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1')
-                                    if m ~= fm:sub(2, -2) then E('Package MD5 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end
-                                    local status, fsha = execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1')
-                                    if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end
+                downloads(pu.host, pu.path, nil, "")
+                while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
+                for i, n in pairs(releases) do
+                    local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
+                    if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
+                        for l in nfile:lines() do
+                            local v, p, sz, m, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) | (%x.+) |')
+                            if nil ~= p then
+                                local status, fsz = execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5')
+                                if testing("Integrity") then
+                                    if sz ~= fsz:sub(2, -2) then        -- The sub bit is to slice off the EOLs at each end.
+                                        E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host)
+                                        print('|' .. sz .. '~=' .. fsz:sub(2, -2) .. '|')
+                                    else
+                                        local status, fm = execute('md5sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1')
+                                        if m ~= fm:sub(2, -2) then E('Package MD5 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end
+                                        local status, fsha = execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1')
+                                        if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end
+                                    end
                                 end
-                            end
-                            if testing("Updated") then
-                                if sz ~= fsz:sub(2, -2) then
-                                    E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host)
+                                if testing("Updated") then
+                                    if sz ~= fsz:sub(2, -2) then
+                                        E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host)
+                                    end
                                 end
                             end
                         end
@@ -1022,6 +1026,17 @@ if 0 < #arg then
                 end
             end
 
+        else
+            for k, v in pairs{"ftp", "http", "https", "rsync"} do
+                if testing(v) then
+                    local tests = results[v]
+                    if testing("Integrity") then tests.Integrity = {errors = -1; warnings = -1} end
+                    if testing("Protocol")  then tests.Protocol  = {errors = -1; warnings = -1} end
+                    if testing("Updated")   then tests.Updated   = {errors = -1; warnings = -1} end
+                    if testing("URLSanity") then tests.URLSanity = {errors = -1; warnings = -1} end
+                    results[v] = tests
+                end
+            end
         end
     end