Track, store, and report info from the hosts that get redirected to.

1 files changed, 165 insertions, 36 deletions

diff --git a/apt-panopticon-report-email-web.lua b/apt-panopticon-report-email-web.lua
index 9faf476..23608d3 100755
--- a/apt-panopticon-report-email-web.lua
+++ b/apt-panopticon-report-email-web.lua
@@ -12,11 +12,12 @@ local arg, sendArgs = APT.parseArgs({...})
 
 local results = {}
 APT.mirrors = loadfile("results/mirrors.lua")()
+APT.debians = loadfile("results/debians.lua")()
 
-local revDNS = function(dom, IP)
+local revDNS = function(hosts, dom, IP)
     if "deb.devuan.org" ~= dom then
-        if nil ~= APT.mirrors["deb.devuan.org"] then
-            if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][IP] then
+        if nil ~= hosts["deb.devuan.org"] then
+            if nil ~= hosts["deb.devuan.org"].IPs["deb.roundr.devuan.org"][IP] then
                 if APT.html then
                     return "<font color='purple'><b>DNS-RR</b></font>"
                 else
@@ -25,7 +26,7 @@ local revDNS = function(dom, IP)
             end
         end
     else
-        for k, v in pairs(APT.mirrors) do
+        for k, v in pairs(hosts) do
             if "deb.devuan.org" ~= k then
                 local IPs = v.IPs
                 for i, u in pairs(IPs) do
@@ -44,12 +45,12 @@ local revDNS = function(dom, IP)
 end
 
 local faulty = ""
-local status = function(host, results, typ)
+local status = function(hosts, host, results, typ)
     local result = ""
     local e = 0
     local w = 0
     local t = 0
-    local s = nil ~= APT.mirrors[host].Protocols[typ]
+    local s = nil ~= hosts[host].Protocols[typ]
     local to = results.timeout
     if not APT.search(APT.protocols, typ) then s = true end
     if nil ~= results[typ] then
@@ -57,7 +58,7 @@ local status = function(host, results, typ)
         w = results[typ].warnings
         t = results[typ].timeouts
         for k, v in pairs(results[typ]) do
-            if "table" == type(v) then
+            if ("table" == type(v)) and ('redirects' ~= k) then
                 if 0 <= v.errors   then e = e + v.errors   else to = true end
                 if 0 <= v.warnings then w = w + v.warnings else to = true end
                 if 0 <= v.timeouts then t = t + v.timeouts else to = true end
@@ -139,31 +140,49 @@ local logCount = function(domain, ip)
     local nm = "LOG_" .. domain
     local log = ""
     local extra = ""
+    local errors = 0
+    local warnings = 0
+    local timeouts = 0
     if nil ~= ip then nm = nm .. "_" .. ip end
     nm = nm .. ".html"
     local rfile, e = io.open("results/" .. nm, "r")
     if nil ~= rfile then
-        local errors = 0
-        local warnings = 0
-        local timeouts = 0
         for l in rfile:lines() do
             if nil ~= l:match("><b>ERROR ") then errors = errors + 1 end
             if nil ~= l:match("><b>WARNING ") then warnings = warnings + 1 end
             if nil ~= l:match("><b>TIMEOUT ") then timeouts = timeouts + 1 end
         end
         rfile:close()
-        if APT.html then
-            if nil == ip then
-                log = "<a href='" .. nm .. "'>" .. domain .. "</a>"
-            else
-                log = "<a href='" .. nm .. "'>" .. ip .. "</a>"
-            end
+    end
+    if APT.html then
+        if nil == ip then
+            log = "<a href='" .. nm .. "'>" .. domain .. "</a>"
+        else
+            log = "<a href='" .. nm .. "'>" .. ip .. "</a>"
         end
-        log = log .. APT.plurals(errors, warnings, timeouts)
     end
+    log = log .. APT.plurals(errors, warnings, timeouts)
     return log
 end
 
+local redirs = function(hosts, host)
+    local results = APT.collateAll(hosts, 'results', host)
+    local rdr = {}
+    local redirs = ''
+    for p, pt in pairs(APT.protocols) do
+        if 0 ~= #(results[pt].redirects) then
+            table.sort(results[pt].redirects)
+            for r, rd in pairs(results[pt].redirects) do
+                rdr[rd] = rd
+            end
+        end
+    end
+    for r, rd in pairs(rdr) do
+        redirs = redirs .. ', &nbsp ' .. rd
+    end
+    if '' ~= redirs then redirs = '<br>\n &nbsp; &nbsp; (Redirects some packages to - ' .. redirs:sub(3) .. ')' end
+    return redirs
+end
 
 APT.html = false
 local email, e = io.open("results/Report-email.txt", "w+")
@@ -183,16 +202,16 @@ if nil == email then C("opening mirrors file - " .. e) else
                 "[skip] means that the test hasn't been written yet.\n\n")
     for k, v in APT.orderedPairs(APT.mirrors) do
         email:write(k .. "....\n")
-        local results = APT.collateAll('results', k)
+        local results = APT.collateAll(APT.mirrors, 'results', k)
         local ftp       = "[skip]"
-        local http      = status(k, results, "http")
-        local https     = status(k, results, "https")
+        local http      = status(APT.mirrors, k, results, "http")
+        local https     = status(APT.mirrors, k, results, "https")
         local rsync     = "[skip]"
         local dns       = ""
-        local protocol  = status(k, results, "Protocol")
-        local sanity    = status(k, results, "URLSanity")
-        local integrity = status(k, results, "Integrity")
-        local updated   = status(k, results, "Updated")
+        local protocol  = status(APT.mirrors, k, results, "Protocol")
+        local sanity    = status(APT.mirrors, k, results, "URLSanity")
+        local integrity = status(APT.mirrors, k, results, "Integrity")
+        local updated   = status(APT.mirrors, k, results, "Updated")
 
         -- DNS-RR test.
         if ("deb.devuan.org" ~= k) and (nil ~= APT.mirrors["deb.devuan.org"]) then
@@ -347,7 +366,7 @@ if nil == web then C("opening mirrors file - " .. e) else
                 "<th>Protocol</th><th>URL sanity</th><th>Integrity</th><th>Updated</th><th colspan='2'>Speed range</th></tr>\n"
                 )
     for k, v in APT.orderedPairs(APT.mirrors) do
-        local results = APT.collateAll('results', k)
+        local results = APT.collateAll(APT.mirrors, 'results', k)
         local active = ""
         if "yes" == v.Active then
             web:write("     <tr><th>" .. k .. "</th> ")
@@ -356,14 +375,14 @@ if nil == web then C("opening mirrors file - " .. e) else
             web:write("     <tr style='background-color:dimgrey'><th>" .. k .. "</th> ")
         end
         local ftp       = "[<font color='grey'><b>skip</b></font>]"
-        local http      = status(k, results, "http")
-        local https     = status(k, results, "https")
+        local http      = status(APT.mirrors, k, results, "http")
+        local https     = status(APT.mirrors, k, results, "https")
         local rsync     = "[<font color='grey'><b>skip</b></font>]"
         local dns       = ""
-        local protocol  = status(k, results, "Protocol")
-        local sanity    = status(k, results, "URLSanity")
-        local integrity = status(k, results, "Integrity")
-        local updated   = status(k, results, "Updated")
+        local protocol  = status(APT.mirrors, k, results, "Protocol")
+        local sanity    = status(APT.mirrors, k, results, "URLSanity")
+        local integrity = status(APT.mirrors, k, results, "Integrity")
+        local updated   = status(APT.mirrors, k, results, "Updated")
         local rate = v.Rate
         if nil ~= rate then updated = updated .. ' ' .. rate end
         local min       = tonumber(results.speed.min)
@@ -434,14 +453,14 @@ if nil == web then C("opening mirrors file - " .. e) else
                 n[l] = {}
                 for i, u in pairs(w) do
                     local log = logCount(k, i)
-                    if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(k, i)] = u end
+                    if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(APT.mirrors, k, i)] = u end
                 end
             else
                 local log = logCount(k, l)
-                if "" == log then n[l] = w else n[log .. " " .. revDNS(k, l)] = w end
+                if "" == log then n[l] = w else n[log .. " " .. revDNS(APT.mirrors, k, l)] = w end
             end
         end
-        m[log .. " DNS entries -"] = n
+        m[log .. " DNS entries -" .. redirs(APT.mirrors, k)] = n
     end
     web:write(  "<p>This lists each mirror, and the DNS entries for that mirror. &nbsp; " ..
                 "The links point to the testing log files for " .. logCount("apt-panopticon") .. " for each domain name / IP combination that was tested. &nbsp; " ..
@@ -453,8 +472,118 @@ if nil == web then C("opening mirrors file - " .. e) else
                 )
     web:write(APT.dumpTableHTML(m, "", ""))
     web:write( "\n<br>\n<br>\n<h2>==== graphs: ====</h2>\n" ..
-                "<img src='speed.png'>\n<br>\n<p><a href='../apt-panopticon_cgp/'>More graphs.</a> with greater detail.</p><hr>\n\n" ..
-                "<p>The <a href='Report-email.txt'>email report</a>. &nbsp; " ..
+                "<img src='speed.png'>\n<br>\n<p><a href='../apt-panopticon_cgp/'>More graphs.</a> with greater detail.</p><hr>\n\n")
+
+    results = {}
+    m = {}
+    faulty = ""
+    web:write(  "<hr>\n<h2>==== Debian mirror status ====</h2>\n" ..
+                "<p>NOTE - This is not fully probing the Debian mirrors, we just collect some data from any redirects to other servers. &nbsp; " ..
+                "So this isn't a full set of tests.</p>\n" ..
+                "<p><font style='background-color:red; color:black'>EXPERIMENTAL CODE - this is even more experimental than the rest.</font></p>\n" ..
+                "<table>\n<tr><th></th><th>FTP</th><th>HTTP</th><th>HTTPS</th><th>RSYNC</th><th>DNS round robin</th>" ..
+                "<th>Protocol</th><th>URL sanity</th><th>Integrity</th><th>Updated</th><th colspan='2'>Speed range</th></tr>\n")
+    for k, v in APT.orderedPairs(APT.debians) do
+        local results = APT.collateAll(APT.debians, 'results', k)
+        local active = ""
+        if "yes" == v.Active then
+            web:write("     <tr><th>" .. k .. "</th> ")
+        else
+            if nil == v.Active then active = 'nil' else active = v.Active end
+            web:write("     <tr style='background-color:dimgrey'><th>" .. k .. "</th> ")
+        end
+        local ftp       = "[<font color='grey'><b>skip</b></font>]"
+        local http      = status(APT.debians, k, results, "http")
+        local https     = status(APT.debians, k, results, "https")
+        local rsync     = "[<font color='grey'><b>skip</b></font>]"
+        local dns       = ""
+        local protocol  = status(APT.debians, k, results, "Protocol")
+        local sanity    = status(APT.debians, k, results, "URLSanity")
+        local integrity = status(APT.debians, k, results, "Integrity")
+        local updated   = status(APT.debians, k, results, "Updated")
+        local rate = v.Rate
+        if nil ~= rate then updated = updated .. ' ' .. rate end
+        local min       = tonumber(results.speed.min)
+        local max       = tonumber(results.speed.max)
+        local spd       = ''
+
+        -- DNS-RR test.
+        if ("deb.devuan.org" ~= k) and (nil ~= APT.debians["deb.devuan.org"]) then
+            for l, w in pairs(APT.debians[k].IPs) do
+                if type(w) == "table" then
+                    for i, u in pairs(w) do
+                        if nil ~= APT.debians["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then
+                            local log = logCount("deb.devuan.org", i)
+                            if "" ~= log then
+                                if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
+                                dns = dns .. logCount("deb.devuan.org", i)
+                            else
+                                if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
+                                dns = dns .. "<font color='maroon'><b>" .. i .. "</b></font>"
+                            end
+                        end
+                    end
+                else
+                    if nil ~= APT.debians["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then
+                        local log = logCount("deb.devuan.org", l)
+                        if "" ~= log then
+                            if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
+                            dns = dns .. log
+                        else
+                            if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
+                            dns = dns .. "<font color='maroon'><b>" .. l .. "</b></font>"
+                        end
+                    end
+                end
+            end
+            if "" == dns then dns = "[<font color='grey'><b>no</b></font>]" end
+
+            if 0 == max then
+                spd = '<td></td><td></td>'
+            else
+                spd = string.format('<td align="right">%d -></td><td align="right">%d</td>', min, max)
+            end
+        end
+
+        web:write("<td>" .. ftp .. "&nbsp;</td><td>" .. http .. "&nbsp;</td><td>" .. https .. "&nbsp;</td><td>" .. rsync .. "&nbsp;</td><td>" .. dns ..
+                    "&nbsp;</td><td>" .. protocol .. "&nbsp;</td><td>" .. sanity ..
+                    "&nbsp;</td><td>" .. integrity .. "&nbsp;</td><td>" .. updated .. "&nbsp;</td>" .. spd .. "</tr>\n")
+        if "" ~= active then
+            web:write("<tr><td style='background-color:dimgrey'>" .. active .. "</td></tr>\n")
+        end
+    end
+    web:write( "</table>\n<br>\n\n")
+
+    web:write( "<br>\n<br>\n<h2>==== Debian DNS and logs: ====</h2>\n")
+
+    for k, v in pairs(APT.debians) do
+        local log = k
+        local n = {}
+        log = logCount(k)
+        APT.debians[k].Protocols = nil
+        APT.debians[k].FQDN = nil
+        APT.debians[k].Active = nil
+        APT.debians[k].Rate = nil
+        APT.debians[k].BaseURL = nil
+        APT.debians[k].Country = nil
+        APT.debians[k].Bandwidth = nil
+        for l, w in pairs(APT.debians[k].IPs) do
+            if type(w) == "table" then
+                n[l] = {}
+                for i, u in pairs(w) do
+                    local log = logCount(k, i)
+                    if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(APT.debians, k, i)] = u end
+                end
+            else
+                local log = logCount(k, l)
+                if "" == log then n[l] = w else n[log .. " " .. revDNS(APT.debians, k, l)] = w end
+            end
+        end
+        m[log .. " DNS entries -" .. redirs(APT.mirrors, k)] = n
+    end
+    web:write(APT.dumpTableHTML(m, "", ""))
+
+    web:write( "<hr>\n<hr>\n<p>The <a href='Report-email.txt'>email report</a>. &nbsp; " ..
                 "All <a href='../results'>the logs and other output</a>. &nbsp; " ..
                 "You can get the <a href='https://sledjhamr.org/cgit/apt-panopticon/about/'>source code here</a>.</p>\n")
     local status, whn = APT.execute('TZ="GMT" ls -l1 --time-style="+%s" results/stamp | cut -d " " -f 6-6')