From 12f7868ef4a315893e65123d0fc3f60aaf0720ab Mon Sep 17 00:00:00 2001 From: onefang Date: Sun, 29 Dec 2019 17:30:51 +1000 Subject: Big refactor of the email / web report. --- apt-panopticon-report-email-web.lua | 440 +++++++++++++++--------------------- 1 file changed, 181 insertions(+), 259 deletions(-) diff --git a/apt-panopticon-report-email-web.lua b/apt-panopticon-report-email-web.lua index a08b901..97f7cef 100755 --- a/apt-panopticon-report-email-web.lua +++ b/apt-panopticon-report-email-web.lua @@ -14,10 +14,16 @@ local results = {} APT.mirrors = loadfile("results/mirrors.lua")() APT.debians = loadfile("results/debians.lua")() + +local lnk = function(name, link) + if nil == link then link = name end + return name .. " *" +end + local revDNS = function(hosts, dom, IP) - if "deb.devuan.org" ~= dom then - if nil ~= hosts["deb.devuan.org"] then - if nil ~= hosts["deb.devuan.org"].IPs["deb.roundr.devuan.org"][IP] then + if APT.options.roundRobin.value ~= dom then + if nil ~= hosts[APT.options.roundRobin.value] then + if nil ~= hosts[APT.options.roundRobin.value].IPs["deb.roundr.devuan.org"][IP] then if APT.html then return "DNS-RR" else @@ -27,7 +33,7 @@ local revDNS = function(hosts, dom, IP) end else for k, v in pairs(hosts) do - if "deb.devuan.org" ~= k then + if APT.options.roundRobin.value ~= k then local IPs = v.IPs for i, u in pairs(IPs) do if "table" == type(u) then @@ -174,6 +180,162 @@ local redirs = function(hosts, host) return redirs end + +local DNSrrTest = function(hosts, k) + local dns = '' + local space = ' ' + local no = 'no' + if APT.html then + space = '   ' + no = "no" + end + if (APT.options.roundRobin.value ~= k) and (nil ~= hosts[APT.options.roundRobin.value]) then + APT.allpairs(hosts[k].IPs, + function(i, w, k, v) + if nil ~= hosts[APT.options.roundRobin.value].IPs["deb.roundr.devuan.org"][i] then + local log = logCount(APT.options.roundRobin.value, i) + if "" ~= log then + if "" == dns then dns = " " else dns = dns .. space end + dns = dns .. logCount(APT.options.roundRobin.value, i) + else + if "" == dns then dns = " " else dns = dns .. space end + if APT.html then i = "" .. i .. "" end + dns = dns .. i + end + end + end + ) + if "" == dns then dns = no end + end + return dns +end + +local makeTable = function(web, hosts) + web:write("\n" .. + "" .. + "" .. + "\n") + local bg = '' + for k, v in APT.orderedPairs(hosts) do + if '' == bg then bg = " style='background-color:#111111'" else bg = '' end + local results = APT.collateAll(hosts, 'results', k) + local active = "" + if "yes" == v.Active then + web:write(" ") + else + if nil == v.Active then active = 'nil' else active = v.Active end + web:write(" ") + end + local ftp = "skip" + local http = status(hosts, k, results, "http") + local https = status(hosts, k, results, "https") + local rsync = "skip" + local dns = DNSrrTest(hosts, k) + local protocol = status(hosts, k, results, "Protocol") + local sanity = status(hosts, k, results, "URLSanity") + local integrity = status(hosts, k, results, "Integrity") + local updated = status(hosts, k, results, "Updated") + local rate = v.Rate + if nil ~= rate then updated = updated .. ' ' .. rate end + local min = tonumber(results.speed.min) + local max = tonumber(results.speed.max) + local spd = '' + local week = '' + local graph = 'graphs' + + -- DNS-RR test. + if (APT.options.roundRobin.value ~= k) and (nil ~= hosts[APT.options.roundRobin.value]) then + if 0 == max then + spd = '' + else + spd = string.format('', min, max) + end + end + + if (APT.options.roundRobin.value ~= k) then + local percentUp = '??' + local percentUpdated = '??' + if APT.checkFile('rrd/' .. k .. '/Speed/Speed.rrd') then + local start, step, names, data = APT.rrd.fetch('rrd/' .. k .. '/Speed/Speed.rrd', 'LAST', '-a', '-r', '10m', '-s', '-1w') + local count, up, down, unknown = 0, 0, 0, 0 + for i, dp in ipairs(data) do + for j,v in ipairs(dp) do + if 'max' == names[j] then + if 'nan' == tostring(v) then + unknown = unknown + 1 + else + count = count + 1 + if 0 == v then down = down + 1 else up = up + 1 end + end + end + end + end + percentUp = string.format('%.2f', up / count * 100) + end + if APT.checkFile('rrd/' .. k .. '/HTTP/Tests.rrd') then + local start, step, names, data = APT.rrd.fetch('rrd/' .. k .. '/HTTP/Tests.rrd', 'LAST', '-a', '-r', '10m', '-s', '-1w') + local count, up, down, unknown = 0, 0, 0, 0 + for i,dp in ipairs(data) do + for j,v in ipairs(dp) do + if 'UpdatedErrors' == names[j] then + if 'nan' == tostring(v) then + unknown = unknown + 1 + else + count = count + 1 + if 0 == v then down = down + 1 else up = up + 1 end + end + end + end + end + percentUpdated = string.format('%.2f', (down / count * 100)) + if '0.00' == percentUp then percentUpdated = '??' end -- We are counting errors, and you can't get an error if you can't check anything. + -- TODO - try to account for this better, this is just a quick hack. + end + week = '' + end + + web:write("\n") + if "" ~= active then + web:write("\n") + end + end + web:write( "
" .. lnk('FTP') .. "" .. lnk('HTTP') .. "" .. lnk('HTTPS') .. "" .. lnk('RSYNC') .. "" .. lnk('DNS round robin', 'DNS-RR') .. "" .. lnk('Protocol') .. "" .. lnk('URL sanity', 'URL-Sanity') .. "" .. lnk('Integrity') .. "" .. lnk('Updated') .. "" .. lnk('Speed range', 'Speed') .. "" .. lnk('Weekly statistics', 'Weekly') .. "" .. lnk('Graphs') .. "
" .. k .. "
" .. k .. "%d -%d' .. percentUp .. '% up' .. percentUpdated .. '% updated" .. ftp .. " " .. http .. " " .. https .. " " .. rsync .. " " .. dns .. + " " .. protocol .. " " .. sanity .. " " .. integrity .. " " .. updated .. + " " .. spd .. " " .. week .." " .. graph .. "
" .. active .. "
\n
\n") +end + +local makeIPlist = function(hosts) + local m = {} + for k, v in pairs(hosts) do + local log = k + local n = {} + log = logCount(k) + hosts[k].Protocols = nil + hosts[k].FQDN = nil + hosts[k].Active = nil + hosts[k].Rate = nil + hosts[k].BaseURL = nil + hosts[k].Country = nil + hosts[k].Bandwidth = nil + + for l, w in pairs(hosts[k].IPs) do + if type(w) == "table" then + n[l] = {} + for i, u in pairs(w) do + local log = logCount(k, i) + if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(hosts, k, i)] = u end + end + else + local log = logCount(k, l) + if "" == log then n[l] = w else n[log .. " " .. revDNS(hosts, k, l)] = w end + end + end + m[log .. " DNS entries -" .. redirs(hosts, k)] = n + end + return m +end + + APT.html = false local email, e = io.open("results/Report-email.txt", "w+") if nil == email then C("opening mirrors file - " .. e) else @@ -197,29 +359,14 @@ if nil == email then C("opening mirrors file - " .. e) else local http = status(APT.mirrors, k, results, "http") local https = status(APT.mirrors, k, results, "https") local rsync = "[skip]" - local dns = "" + local dns = DNSrrTest(APT.mirrors, k) local protocol = status(APT.mirrors, k, results, "Protocol") local sanity = status(APT.mirrors, k, results, "URLSanity") local integrity = status(APT.mirrors, k, results, "Integrity") local updated = status(APT.mirrors, k, results, "Updated") -- DNS-RR test. - if ("deb.devuan.org" ~= k) and (nil ~= APT.mirrors["deb.devuan.org"]) then - APT.allpairs(APT.mirrors[k].IPs, - function(i, w, k, v) - if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then - local log = logCount("deb.devuan.org", i) - if "" ~= log then - if "" == dns then dns = " " else dns = dns .. " " end - dns = dns .. logCount("deb.devuan.org", i) - else - if "" == dns then dns = " " else dns = dns .. " " end - dns = dns .. i - end - end - end - ) - if "" == dns then dns = "[no]" end + if (APT.options.roundRobin.value ~= k) and (nil ~= APT.mirrors[APT.options.roundRobin.value]) then dns = " DNS-RR: " .. dns end @@ -241,11 +388,6 @@ if nil == email then C("opening mirrors file - " .. e) else email:close() end -local lnk = function(name, link) - if nil == link then link = name end - return name .. " *" -end - local colours = { @@ -275,17 +417,17 @@ local colours = local g = {} local count = 0 for k, v in APT.orderedPairs(mirrors) do - if 'pkgmaster.devuan.org' ~= k then count = count + 1 end + if APT.options.referenceSite.value ~= k then count = count + 1 end end for i = 1, count do end count = 1 for k, v in APT.orderedPairs(mirrors) do - if 'deb.devuan.org' ~= k then + if APT.options.roundRobin.value ~= k then local c = colours[count] local name = string.format('%32s', k) - if 'pkgmaster.devuan.org' == k then c = 'ffffff' end + if APT.options.referenceSite.value == k then c = 'ffffff' end table.insert(g, 'DEF:speedn' .. count .. '=rrd/' .. k .. '/Speed/Speed.rrd:max:MIN') table.insert(g, 'DEF:speedx' .. count .. '=rrd/' .. k .. '/Speed/Speed.rrd:max:MAX') table.insert(g, 'DEF:speeda' .. count .. '=rrd/' .. k .. '/Speed/Speed.rrd:max:AVERAGE') @@ -343,153 +485,23 @@ if nil == web then C("opening mirrors file - " .. e) else "

NOTE: timeouts may be due to a problem on the testing computer.

" .. "

The DNS round robin (DNS-RR) column shows the IPs for that mirror, or no if it isn't part of the DNS-RR.   " .. "The IPs link to the testing log for that IP accessed via the DNS-RR.   " .. - "deb.devuan.org is the DNS-RR itself, so it doesn't get tested directly.

\n" .. + APT.options.roundRobin.value .. " is the DNS-RR itself, so it doesn't get tested directly.

\n" .. "

The time in the Updated column is how often the mirror updates itself.

" .. "

Mirrors with a grey background are not active (though may be usable as part of the DNS-RR).

\n" .. - "

skip means that the test hasn't been written yet.

\n" .. - "\n" .. - "" .. - "" .. - "\n" + "

skip means that the test hasn't been written yet.

\n" ) - local bg = '' - for k, v in APT.orderedPairs(APT.mirrors) do - local results = APT.collateAll(APT.mirrors, 'results', k) - if '' == bg then bg = " style='background-color:#111111'" else bg = '' end - local active = "" - if "yes" == v.Active then - web:write(" ") - else - if nil == v.Active then active = 'nil' else active = v.Active end - web:write(" ") - end - local ftp = "skip" - local http = status(APT.mirrors, k, results, "http") - local https = status(APT.mirrors, k, results, "https") - local rsync = "skip" - local dns = "" - local protocol = status(APT.mirrors, k, results, "Protocol") - local sanity = status(APT.mirrors, k, results, "URLSanity") - local integrity = status(APT.mirrors, k, results, "Integrity") - local updated = status(APT.mirrors, k, results, "Updated") - local rate = v.Rate - if nil ~= rate then updated = updated .. ' ' .. rate end - local min = tonumber(results.speed.min) - local max = tonumber(results.speed.max) - local spd = '' - local week = '' - local graph = 'graphs' - - -- DNS-RR test. - if ("deb.devuan.org" ~= k) and (nil ~= APT.mirrors["deb.devuan.org"]) then - APT.allpairs(APT.mirrors[k].IPs, - function(i, w, k, v) - if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then - local log = logCount("deb.devuan.org", i) - if "" ~= log then - if "" == dns then dns = " " else dns = dns .. "   " end - dns = dns .. logCount("deb.devuan.org", i) - else - if "" == dns then dns = " " else dns = dns .. "   " end - dns = dns .. "" .. i .. "" - end - end - end - ) - if "" == dns then dns = "no" end - - if 0 == max then - spd = '' - else - spd = string.format('', min, max) - end - end - - if ("deb.devuan.org" ~= k) then - local percentUp = '???' - local percentUpdated = '???' - if APT.checkFile('rrd/' .. k .. '/Speed/Speed.rrd') then - local start, step, names, data = APT.rrd.fetch('rrd/' .. k .. '/Speed/Speed.rrd', 'LAST', '-a', '-r', '10m', '-s', '-1w') - local count, up, down, unknown = 0, 0, 0, 0 - for i, dp in ipairs(data) do - for j,v in ipairs(dp) do - if 'max' == names[j] then - if 'nan' == tostring(v) then - unknown = unknown + 1 - else - count = count + 1 - if 0 == v then down = down + 1 else up = up + 1 end - end - end - end - end - percentUp = string.format('%.2f', up / count * 100) - end - if APT.checkFile('rrd/' .. k .. '/HTTP/Tests.rrd') then - local start, step, names, data = APT.rrd.fetch('rrd/' .. k .. '/HTTP/Tests.rrd', 'LAST', '-a', '-r', '10m', '-s', '-1w') - local count, up, down, unknown = 0, 0, 0, 0 - for i,dp in ipairs(data) do - for j,v in ipairs(dp) do - if 'UpdatedErrors' == names[j] then - if 'nan' == tostring(v) then - unknown = unknown + 1 - else - count = count + 1 - if 0 == v then down = down + 1 else up = up + 1 end - end - end - end - end - percentUpdated = string.format('%.2f', (down / count * 100)) - if '0.00' == percentUp then percentUpdated = '??' end -- We are counting errors, and you can't get an error if you can't check anything. - -- TODO - try to account for this better, this is just a quick hack. - end - week = '' - end - - web:write("\n") - if "" ~= active then - web:write("\n") - end - end - web:write( "
" .. lnk('FTP') .. "" .. lnk('HTTP') .. "" .. lnk('HTTPS') .. "" .. lnk('RSYNC') .. "" .. lnk('DNS round robin', 'DNS-RR') .. "" .. lnk('Protocol') .. "" .. lnk('URL sanity', 'URL-Sanity') .. "" .. lnk('Integrity') .. "" .. lnk('Updated') .. "" .. lnk('Speed range', 'Speed') .. "" .. lnk('Weekly statistics', 'Weekly') .. "" .. lnk('Graphs') .. "
" .. k .. "
" .. k .. "%d -%d' .. percentUp .. '% up' .. percentUpdated .. '% updated" .. ftp .. " " .. http .. " " .. https .. " " .. rsync .. " " .. dns .. - " " .. protocol .. " " .. sanity .. " " .. integrity .. " " .. updated .. - " " .. spd .. " " .. week .." " .. graph .. "
" .. active .. "
\n
\n

==== faulty mirrors: ====

\n" .. faulty) + makeTable(web, APT.mirrors) + web:write( "

==== faulty mirrors: ====

\n" .. faulty) web:write( "
\n
\n

==== DNS and logs: ====

\n") - for k, v in pairs(APT.mirrors) do - local log = k - local n = {} - log = logCount(k) - APT.mirrors[k].Protocols = nil - APT.mirrors[k].FQDN = nil - APT.mirrors[k].Active = nil - APT.mirrors[k].Rate = nil - APT.mirrors[k].BaseURL = nil - APT.mirrors[k].Country = nil - APT.mirrors[k].Bandwidth = nil - - for l, w in pairs(APT.mirrors[k].IPs) do - if type(w) == "table" then - n[l] = {} - for i, u in pairs(w) do - local log = logCount(k, i) - if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(APT.mirrors, k, i)] = u end - end - else - local log = logCount(k, l) - if "" == log then n[l] = w else n[log .. " " .. revDNS(APT.mirrors, k, l)] = w end - end - end - m[log .. " DNS entries -" .. redirs(APT.mirrors, k)] = n - end + m = makeIPlist(APT.mirrors) web:write( "

This lists each mirror, and the DNS entries for that mirror.   " .. "The links point to the testing log files for " .. logCount("apt-panopticon") .. " for each domain name / IP combination that was tested.   " .. "If a mirror has a CNAME, that CNAME is listed along with that CNAMEs DNS entries.   " .. - "deb.devuan.org is the DNS round robin, which points to the mirrors that are part of the DNS-RR.   " .. + APT.options.roundRobin.value .. " is the DNS round robin, which points to the mirrors that are part of the DNS-RR.   " .. "If an IP is part of the DNS-RR, it is marked with 'DNS-RR'   " .. - "pkgmaster.devuan.org is the master mirror, all the others sync to it.   " .. + APT.options.referenceSite.value .. " is the master mirror, all the others sync to it.   " .. "

\n" ) web:write(APT.dumpTableHTML(m, "")) @@ -499,106 +511,16 @@ if nil == web then C("opening mirrors file - " .. e) else results = {} m = {} faulty = "" - bg = '' web:write( "
\n

==== Debian mirror status ====

\n" .. "

NOTE - This is not fully probing the Debian mirrors, we just collect some data from any redirects to other servers.   " .. "So this isn't a full set of tests.   Basically we don't know the shape of the Debian mirror infrastructure.

\n" .. - "

EXPERIMENTAL CODE - this is even more experimental than the rest.

\n" .. - - "\n" .. - "" .. - "" .. - "\n" + "

EXPERIMENTAL CODE - this is even more experimental than the rest.

\n" ) - for k, v in APT.orderedPairs(APT.debians) do - if '' == bg then bg = " style='background-color:#111111'" else bg = '' end - local results = APT.collateAll(APT.debians, 'results', k) - local active = "" - if "yes" == v.Active then - web:write(" ") - else - if nil == v.Active then active = 'nil' else active = v.Active end - web:write(" ") - end - local ftp = "skip" - local http = status(APT.debians, k, results, "http") - local https = status(APT.debians, k, results, "https") - local rsync = "skip" - local dns = "" - local protocol = status(APT.debians, k, results, "Protocol") - local sanity = status(APT.debians, k, results, "URLSanity") - local integrity = status(APT.debians, k, results, "Integrity") - local updated = status(APT.debians, k, results, "Updated") - local rate = v.Rate - if nil ~= rate then updated = updated .. ' ' .. rate end - local min = tonumber(results.speed.min) - local max = tonumber(results.speed.max) - local spd = '' - local week = '' - local graph = 'graphs' - - -- DNS-RR test. - if ("deb.devuan.org" ~= k) and (nil ~= APT.debians["deb.devuan.org"]) then - APT.allpairs(APT.debians[k].IPs, - function(i, w, k, v) - if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then - local log = logCount("deb.devuan.org", i) - if "" ~= log then - if "" == dns then dns = " " else dns = dns .. "   " end - dns = dns .. logCount("deb.devuan.org", i) - else - if "" == dns then dns = " " else dns = dns .. "   " end - dns = dns .. "" .. i .. "" - end - end - end - ) - if "" == dns then dns = "no" end - - if 0 == max then - spd = '' - else - spd = string.format('', min, max) - end - end - - web:write("" .. spd .. " " .. week .." \n") - if "" ~= active then - web:write("\n") - end - end - web:write( "
" .. lnk('FTP') .. "" .. lnk('HTTP') .. "" .. lnk('HTTPS') .. "" .. lnk('RSYNC') .. "" .. lnk('DNS round robin', 'DNS-RR') .. "" .. lnk('Protocol') .. "" .. lnk('URL sanity', 'URL-Sanity') .. "" .. lnk('Integrity') .. "" .. lnk('Updated') .. "" .. lnk('Speed range', 'Speed') .. "" .. lnk('Weekly statistics', 'Weekly') .. "" .. lnk('Graphs') .. "
" .. k .. "
" .. k .. "??% up??% updated%d -%d" .. ftp .. " " .. http .. " " .. https .. " " .. rsync .. " " .. dns .. - " " .. protocol .. " " .. sanity .. " " .. integrity .. " " .. updated .. - " " .. graph .. "
" .. active .. "
\n
\n\n") + makeTable(web, APT.debians) web:write( "
\n
\n

==== Debian DNS and logs: ====

\n") - for k, v in pairs(APT.debians) do - local log = k - local n = {} - log = logCount(k) - APT.debians[k].Protocols = nil - APT.debians[k].FQDN = nil - APT.debians[k].Active = nil - APT.debians[k].Rate = nil - APT.debians[k].BaseURL = nil - APT.debians[k].Country = nil - APT.debians[k].Bandwidth = nil - for l, w in pairs(APT.debians[k].IPs) do - if type(w) == "table" then - n[l] = {} - for i, u in pairs(w) do - local log = logCount(k, i) - if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(APT.debians, k, i)] = u end - end - else - local log = logCount(k, l) - if "" == log then n[l] = w else n[log .. " " .. revDNS(APT.debians, k, l)] = w end - end - end - - m[log .. " DNS entries -" .. redirs(APT.mirrors, k)] = n - end + m = makeIPlist(APT.debians) web:write(APT.dumpTableHTML(m, "")) web:write( "
\n
\n

The email report.   " .. -- cgit v1.1