From 0f43ca83d630264e42f7875bc6af1d77125b5ebe Mon Sep 17 00:00:00 2001
From: onefang
Date: Wed, 25 Dec 2019 18:47:55 +1000
Subject: Track, store, and report info from the hosts that get redirected to.

---
 apt-panopticon-report-email-web.lua | 201 +++++++++++++++++++++++++++++-------
 1 file changed, 165 insertions(+), 36 deletions(-)

(limited to 'apt-panopticon-report-email-web.lua')

diff --git a/apt-panopticon-report-email-web.lua b/apt-panopticon-report-email-web.lua
index 9faf476..23608d3 100755
--- a/apt-panopticon-report-email-web.lua
+++ b/apt-panopticon-report-email-web.lua
@@ -12,11 +12,12 @@ local arg, sendArgs = APT.parseArgs({...})
 
 local results = {}
 APT.mirrors = loadfile("results/mirrors.lua")()
+APT.debians = loadfile("results/debians.lua")()
 
-local revDNS = function(dom, IP)
+local revDNS = function(hosts, dom, IP)
     if "deb.devuan.org" ~= dom then
-	if nil ~= APT.mirrors["deb.devuan.org"] then
-	    if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][IP] then
+	if nil ~= hosts["deb.devuan.org"] then
+	    if nil ~= hosts["deb.devuan.org"].IPs["deb.roundr.devuan.org"][IP] then
 		if APT.html then
 		    return "<font color='purple'><b>DNS-RR</b></font>"
 		else
@@ -25,7 +26,7 @@ local revDNS = function(dom, IP)
 	    end
 	end
     else
-	for k, v in pairs(APT.mirrors) do
+	for k, v in pairs(hosts) do
 	    if "deb.devuan.org" ~= k then
 		local IPs = v.IPs
 		for i, u in pairs(IPs) do
@@ -44,12 +45,12 @@ local revDNS = function(dom, IP)
 end
 
 local faulty = ""
-local status = function(host, results, typ)
+local status = function(hosts, host, results, typ)
     local result = ""
     local e = 0
     local w = 0
     local t = 0
-    local s = nil ~= APT.mirrors[host].Protocols[typ]
+    local s = nil ~= hosts[host].Protocols[typ]
     local to = results.timeout
     if not APT.search(APT.protocols, typ) then s = true end
     if nil ~= results[typ] then
@@ -57,7 +58,7 @@ local status = function(host, results, typ)
 	w = results[typ].warnings
 	t = results[typ].timeouts
 	for k, v in pairs(results[typ]) do
-	    if "table" == type(v) then
+	    if ("table" == type(v)) and ('redirects' ~= k) then
 		if 0 <= v.errors   then e = e + v.errors   else to = true end
 		if 0 <= v.warnings then w = w + v.warnings else to = true end
 		if 0 <= v.timeouts then t = t + v.timeouts else to = true end
@@ -139,31 +140,49 @@ local logCount = function(domain, ip)
     local nm = "LOG_" .. domain
     local log = ""
     local extra = ""
+    local errors = 0
+    local warnings = 0
+    local timeouts = 0
     if nil ~= ip then nm = nm .. "_" .. ip end
     nm = nm .. ".html"
     local rfile, e = io.open("results/" .. nm, "r")
     if nil ~= rfile then
-	local errors = 0
-	local warnings = 0
-	local timeouts = 0
 	for l in rfile:lines() do
 	    if nil ~= l:match("><b>ERROR ") then errors = errors + 1 end
 	    if nil ~= l:match("><b>WARNING ") then warnings = warnings + 1 end
 	    if nil ~= l:match("><b>TIMEOUT ") then timeouts = timeouts + 1 end
 	end
 	rfile:close()
-	if APT.html then
-	    if nil == ip then
-		log = "<a href='" .. nm .. "'>" .. domain .. "</a>"
-	    else
-		log = "<a href='" .. nm .. "'>" .. ip .. "</a>"
-	    end
+    end
+    if APT.html then
+	if nil == ip then
+	    log = "<a href='" .. nm .. "'>" .. domain .. "</a>"
+	else
+	    log = "<a href='" .. nm .. "'>" .. ip .. "</a>"
 	end
-	log = log .. APT.plurals(errors, warnings, timeouts)
     end
+    log = log .. APT.plurals(errors, warnings, timeouts)
     return log
 end
 
+local redirs = function(hosts, host)
+    local results = APT.collateAll(hosts, 'results', host)
+    local rdr = {}
+    local redirs = ''
+    for p, pt in pairs(APT.protocols) do
+	if 0 ~= #(results[pt].redirects) then
+	    table.sort(results[pt].redirects)
+	    for r, rd in pairs(results[pt].redirects) do
+		rdr[rd] = rd
+	    end
+	end
+    end
+    for r, rd in pairs(rdr) do
+	redirs = redirs .. ', &nbsp ' .. rd
+    end
+    if '' ~= redirs then redirs = '<br>\n &nbsp; &nbsp; (Redirects some packages to - ' .. redirs:sub(3) .. ')' end
+    return redirs
+end
 
 APT.html = false
 local email, e = io.open("results/Report-email.txt", "w+")
@@ -183,16 +202,16 @@ if nil == email then C("opening mirrors file - " .. e) else
 		"[skip] means that the test hasn't been written yet.\n\n")
     for k, v in APT.orderedPairs(APT.mirrors) do
 	email:write(k .. "....\n")
-	local results = APT.collateAll('results', k)
+	local results = APT.collateAll(APT.mirrors, 'results', k)
 	local ftp	= "[skip]"
-	local http	= status(k, results, "http")
-	local https	= status(k, results, "https")
+	local http	= status(APT.mirrors, k, results, "http")
+	local https	= status(APT.mirrors, k, results, "https")
 	local rsync	= "[skip]"
 	local dns	= ""
-	local protocol	= status(k, results, "Protocol")
-	local sanity	= status(k, results, "URLSanity")
-	local integrity = status(k, results, "Integrity")
-	local updated	= status(k, results, "Updated")
+	local protocol	= status(APT.mirrors, k, results, "Protocol")
+	local sanity	= status(APT.mirrors, k, results, "URLSanity")
+	local integrity = status(APT.mirrors, k, results, "Integrity")
+	local updated	= status(APT.mirrors, k, results, "Updated")
 
 	-- DNS-RR test.
 	if ("deb.devuan.org" ~= k) and (nil ~= APT.mirrors["deb.devuan.org"]) then
@@ -347,7 +366,7 @@ if nil == web then C("opening mirrors file - " .. e) else
 		"<th>Protocol</th><th>URL sanity</th><th>Integrity</th><th>Updated</th><th colspan='2'>Speed range</th></tr>\n"
 		)
     for k, v in APT.orderedPairs(APT.mirrors) do
-	local results = APT.collateAll('results', k)
+	local results = APT.collateAll(APT.mirrors, 'results', k)
 	local active = ""
 	if "yes" == v.Active then
 	    web:write("     <tr><th>" .. k .. "</th> ")
@@ -356,14 +375,14 @@ if nil == web then C("opening mirrors file - " .. e) else
 	    web:write("     <tr style='background-color:dimgrey'><th>" .. k .. "</th> ")
 	end
 	local ftp	= "[<font color='grey'><b>skip</b></font>]"
-	local http	= status(k, results, "http")
-	local https	= status(k, results, "https")
+	local http	= status(APT.mirrors, k, results, "http")
+	local https	= status(APT.mirrors, k, results, "https")
 	local rsync	= "[<font color='grey'><b>skip</b></font>]"
 	local dns	= ""
-	local protocol	= status(k, results, "Protocol")
-	local sanity	= status(k, results, "URLSanity")
-	local integrity = status(k, results, "Integrity")
-	local updated	= status(k, results, "Updated")
+	local protocol	= status(APT.mirrors, k, results, "Protocol")
+	local sanity	= status(APT.mirrors, k, results, "URLSanity")
+	local integrity = status(APT.mirrors, k, results, "Integrity")
+	local updated	= status(APT.mirrors, k, results, "Updated")
 	local rate = v.Rate
 	if nil ~= rate then updated = updated .. ' ' .. rate end
 	local min	= tonumber(results.speed.min)
@@ -434,14 +453,14 @@ if nil == web then C("opening mirrors file - " .. e) else
 		n[l] = {}
 		for i, u in pairs(w) do
 		    local log = logCount(k, i)
-		    if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(k, i)] = u end
+		    if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(APT.mirrors, k, i)] = u end
 		end
 	    else
 		local log = logCount(k, l)
-		if "" == log then n[l] = w else n[log .. " " .. revDNS(k, l)] = w end
+		if "" == log then n[l] = w else n[log .. " " .. revDNS(APT.mirrors, k, l)] = w end
 	    end
 	end
-	m[log .. " DNS entries -"] = n
+	m[log .. " DNS entries -" .. redirs(APT.mirrors, k)] = n
     end
     web:write(	"<p>This lists each mirror, and the DNS entries for that mirror. &nbsp; " ..
 		"The links point to the testing log files for " .. logCount("apt-panopticon") .. " for each domain name / IP combination that was tested. &nbsp; " ..
@@ -453,8 +472,118 @@ if nil == web then C("opening mirrors file - " .. e) else
 		)
     web:write(APT.dumpTableHTML(m, "", ""))
     web:write( "\n<br>\n<br>\n<h2>==== graphs: ====</h2>\n" ..
-		"<img src='speed.png'>\n<br>\n<p><a href='../apt-panopticon_cgp/'>More graphs.</a> with greater detail.</p><hr>\n\n" ..
-		"<p>The <a href='Report-email.txt'>email report</a>. &nbsp; " ..
+		"<img src='speed.png'>\n<br>\n<p><a href='../apt-panopticon_cgp/'>More graphs.</a> with greater detail.</p><hr>\n\n")
+
+    results = {}
+    m = {}
+    faulty = ""
+    web:write(  "<hr>\n<h2>==== Debian mirror status ====</h2>\n" ..
+		"<p>NOTE - This is not fully probing the Debian mirrors, we just collect some data from any redirects to other servers. &nbsp; " ..
+		"So this isn't a full set of tests.</p>\n" ..
+		"<p><font style='background-color:red; color:black'>EXPERIMENTAL CODE - this is even more experimental than the rest.</font></p>\n" ..
+		"<table>\n<tr><th></th><th>FTP</th><th>HTTP</th><th>HTTPS</th><th>RSYNC</th><th>DNS round robin</th>" ..
+		"<th>Protocol</th><th>URL sanity</th><th>Integrity</th><th>Updated</th><th colspan='2'>Speed range</th></tr>\n")
+    for k, v in APT.orderedPairs(APT.debians) do
+	local results = APT.collateAll(APT.debians, 'results', k)
+	local active = ""
+	if "yes" == v.Active then
+	    web:write("     <tr><th>" .. k .. "</th> ")
+	else
+	    if nil == v.Active then active = 'nil' else active = v.Active end
+	    web:write("     <tr style='background-color:dimgrey'><th>" .. k .. "</th> ")
+	end
+	local ftp	= "[<font color='grey'><b>skip</b></font>]"
+	local http	= status(APT.debians, k, results, "http")
+	local https	= status(APT.debians, k, results, "https")
+	local rsync	= "[<font color='grey'><b>skip</b></font>]"
+	local dns	= ""
+	local protocol	= status(APT.debians, k, results, "Protocol")
+	local sanity	= status(APT.debians, k, results, "URLSanity")
+	local integrity = status(APT.debians, k, results, "Integrity")
+	local updated	= status(APT.debians, k, results, "Updated")
+	local rate = v.Rate
+	if nil ~= rate then updated = updated .. ' ' .. rate end
+	local min	= tonumber(results.speed.min)
+	local max	= tonumber(results.speed.max)
+	local spd	= ''
+
+	-- DNS-RR test.
+	if ("deb.devuan.org" ~= k) and (nil ~= APT.debians["deb.devuan.org"]) then
+	    for l, w in pairs(APT.debians[k].IPs) do
+		if type(w) == "table" then
+		    for i, u in pairs(w) do
+			if nil ~= APT.debians["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then
+			    local log = logCount("deb.devuan.org", i)
+			    if "" ~= log then
+				if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
+				dns = dns .. logCount("deb.devuan.org", i)
+			    else
+				if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
+				dns = dns .. "<font color='maroon'><b>" .. i .. "</b></font>"
+			    end
+			end
+		    end
+		else
+		    if nil ~= APT.debians["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then
+			local log = logCount("deb.devuan.org", l)
+			if "" ~= log then
+			    if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
+			    dns = dns .. log
+			else
+			    if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
+			    dns = dns .. "<font color='maroon'><b>" .. l .. "</b></font>"
+			end
+		    end
+		end
+	    end
+	    if "" == dns then dns = "[<font color='grey'><b>no</b></font>]" end
+
+	    if 0 == max then
+		spd = '<td></td><td></td>'
+	    else
+		spd = string.format('<td align="right">%d -></td><td align="right">%d</td>', min, max)
+	    end
+	end
+
+	web:write("<td>" .. ftp .. "&nbsp;</td><td>" .. http .. "&nbsp;</td><td>" .. https .. "&nbsp;</td><td>" .. rsync .. "&nbsp;</td><td>" .. dns ..
+		    "&nbsp;</td><td>" .. protocol .. "&nbsp;</td><td>" .. sanity ..
+		    "&nbsp;</td><td>" .. integrity .. "&nbsp;</td><td>" .. updated .. "&nbsp;</td>" .. spd .. "</tr>\n")
+	if "" ~= active then
+	    web:write("<tr><td style='background-color:dimgrey'>" .. active .. "</td></tr>\n")
+	end
+    end
+    web:write( "</table>\n<br>\n\n")
+
+    web:write( "<br>\n<br>\n<h2>==== Debian DNS and logs: ====</h2>\n")
+
+    for k, v in pairs(APT.debians) do
+	local log = k
+	local n = {}
+	log = logCount(k)
+	APT.debians[k].Protocols = nil
+	APT.debians[k].FQDN = nil
+	APT.debians[k].Active = nil
+	APT.debians[k].Rate = nil
+	APT.debians[k].BaseURL = nil
+	APT.debians[k].Country = nil
+	APT.debians[k].Bandwidth = nil
+	for l, w in pairs(APT.debians[k].IPs) do
+	    if type(w) == "table" then
+		n[l] = {}
+		for i, u in pairs(w) do
+		    local log = logCount(k, i)
+		    if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(APT.debians, k, i)] = u end
+		end
+	    else
+		local log = logCount(k, l)
+		if "" == log then n[l] = w else n[log .. " " .. revDNS(APT.debians, k, l)] = w end
+	    end
+	end
+	m[log .. " DNS entries -" .. redirs(APT.mirrors, k)] = n
+    end
+    web:write(APT.dumpTableHTML(m, "", ""))
+
+    web:write( "<hr>\n<hr>\n<p>The <a href='Report-email.txt'>email report</a>. &nbsp; " ..
 		"All <a href='../results'>the logs and other output</a>. &nbsp; " ..
 		"You can get the <a href='https://sledjhamr.org/cgit/apt-panopticon/about/'>source code here</a>.</p>\n")
     local status, whn = APT.execute('TZ="GMT" ls -l1 --time-style="+%s" results/stamp | cut -d " " -f 6-6')
-- 
cgit v1.1