#!/usr/bin/env luajit local APT = require 'apt-panopticommon' local D = APT.D local I = APT.I local T = APT.T local W = APT.W local E = APT.E local C = APT.C local arg, sendArgs = APT.parseArgs({...}) local results = {} APT.mirrors = loadfile("results/mirrors.lua")() APT.debians = loadfile("results/debians.lua")() local revDNS = function(hosts, dom, IP) if "deb.devuan.org" ~= dom then if nil ~= hosts["deb.devuan.org"] then if nil ~= hosts["deb.devuan.org"].IPs["deb.roundr.devuan.org"][IP] then if APT.html then return "DNS-RR" else return "DNS-RR" end end end else for k, v in pairs(hosts) do if "deb.devuan.org" ~= k then local IPs = v.IPs for i, u in pairs(IPs) do if "table" == type(u) then for h, t in pairs(u) do if IP == h then return k end end else if IP == i then return k end end end end end end return "" end local faulty = "" local status = function(hosts, host, results, typ) local result = "" local e = 0 local w = 0 local t = 0 local s = nil ~= hosts[host].Protocols[typ] local to = results.timeout if not APT.search(APT.protocols, typ) then s = true end if nil ~= results[typ] then e = results[typ].errors w = results[typ].warnings t = results[typ].timeouts for k, v in pairs(results[typ]) do if ("table" == type(v)) and ('redirects' ~= k) then if 0 <= v.errors then e = e + v.errors else to = true end if 0 <= v.warnings then w = w + v.warnings else to = true end if 0 <= v.timeouts then t = t + v.timeouts else to = true end end end else for k, v in pairs(results) do if "table" == type(v) then for i, u in pairs(v) do if "table" == type(u) then if typ == i then if 0 <= u.errors then e = e + u.errors end if 0 <= u.warnings then w = w + u.warnings end if 0 <= u.timeouts then t = t + u.timeouts end end end end end end end if to then result = "[TIMEOUT" if not s then result = result .. "*" end if APT.html then if s then result = "[TIMEOUT" else result = "[TIMEOUT*" end end if APT.html then faulty = faulty .. host .. " (" .. typ .. ")
\n" else faulty = faulty .. host .. " (" .. typ .. ")\n" end elseif 0 < e then result = "[FAILED" if not s then result = result .. "*" end if APT.html then if s then result = "[FAILED" else result = "[FAILED*" end end if APT.html then faulty = faulty .. host .. " (" .. typ .. ")
\n" else faulty = faulty .. host .. " (" .. typ .. ")\n" end else result = "[OK" if not s then result = result .. "*" end if APT.html then if s then result = "[OK" else result = "[OK*" end end end return result .. APT.plurals(e, w, t) .. "]" end local m = {} local logCount = function(domain, ip) local nm = "LOG_" .. domain local log = "" local extra = "" local errors = 0 local warnings = 0 local timeouts = 0 if nil ~= ip then nm = nm .. "_" .. ip end nm = nm .. ".html" local rfile, e = io.open("results/" .. nm, "r") if nil ~= rfile then for l in rfile:lines() do if nil ~= l:match(">ERROR ") then errors = errors + 1 end if nil ~= l:match(">WARNING ") then warnings = warnings + 1 end if nil ~= l:match(">TIMEOUT ") then timeouts = timeouts + 1 end end rfile:close() end if APT.html then if nil == ip then log = "" .. domain .. "" else log = "" .. ip .. "" end end log = log .. APT.plurals(errors, warnings, timeouts) return log end local redirs = function(hosts, host) local results = APT.collateAll(hosts, 'results', host) local rdr = {} local redirs = '' for p, pt in pairs(APT.protocols) do if 0 ~= #(results[pt].redirects) then table.sort(results[pt].redirects) for r, rd in pairs(results[pt].redirects) do rdr[rd] = rd end end end for r, rd in pairs(rdr) do redirs = redirs .. ',   ' .. rd end if '' ~= redirs then redirs = '
\n     (Redirects some packages to - ' .. redirs:sub(3) .. ')' end return redirs end APT.html = false local email, e = io.open("results/Report-email.txt", "w+") if nil == email then C("opening mirrors file - " .. e) else email:write( "Dear Mirror Admins,\n\n" .. "This is the status of the mirror servers in the Devuan package mirror network.\n\n" .. "EXPERIMENTAL CODE - double check all results you see here, and read the logs if it's important." .. "The full list of Devuan package mirrors is available at the URL:\n\n" .. " https://pkgmaster.devuan.org/mirror_list.txt\n\n" .. 'Please contact "mirrors@devuan.org" if any of the information \nin the file above needs to be amended. \n\n' .. "The full results of the mirror checking is available at the URL:\n\n" .. " https://sledjhamr.org/apt-panopticon/results/Report-web.html\n\n" .. "Due to the nature of the tests, some errors or warnings will be \ncounted several times. " .. "Refer to the logs on the web page for details.\n\n" .. "Please see below the current status of the Devuan Package Mirror \nnetwork:\n\n" .. "==== package mirror status " .. os.date("!%F %H:%M") .. " GMT ====\n" .. "[skip] means that the test hasn't been written yet.\n\n") for k, v in APT.orderedPairs(APT.mirrors) do email:write(k .. "....\n") local results = APT.collateAll(APT.mirrors, 'results', k) local ftp = "[skip]" local http = status(APT.mirrors, k, results, "http") local https = status(APT.mirrors, k, results, "https") local rsync = "[skip]" local dns = "" local protocol = status(APT.mirrors, k, results, "Protocol") local sanity = status(APT.mirrors, k, results, "URLSanity") local integrity = status(APT.mirrors, k, results, "Integrity") local updated = status(APT.mirrors, k, results, "Updated") -- DNS-RR test. if ("deb.devuan.org" ~= k) and (nil ~= APT.mirrors["deb.devuan.org"]) then for l, w in pairs(APT.mirrors[k].IPs) do if type(w) == "table" then for i, u in pairs(w) do if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then local log = logCount("deb.devuan.org", i) if "" ~= log then if "" == dns then dns = " " else dns = dns .. " " end dns = dns .. logCount("deb.devuan.org", i) else if "" == dns then dns = " " else dns = dns .. " " end dns = dns .. i end end end else if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then local log = logCount("deb.devuan.org", l) if "" ~= log then if "" == dns then dns = " " else dns = dns .. " " end dns = dns .. log else if "" == dns then dns = " " else dns = dns .. " " end dns = dns .. l end end end end if "" == dns then dns = "[no]" end dns = " DNS-RR: " .. dns end email:write( " ftp: " .. ftp .. " http: " .. http .. " https: " .. https .." rsync: " .. rsync .. "\n" .. " " .. dns .. "\n" .. " Protocol: " .. protocol .. " URL-sanity: " .. sanity .. " Integrity: " .. integrity .. "\n" .. " Updated: " .. updated .. "\n") end email:write( "\n==== faulty mirrors: ====\n" .. faulty) email:write( "\n-------------------------\n\n" .. "* This means that this protocol isn't actually supported, but the test was run ayway.\n\n" .. "Thanks for your precious help in ensuring that Devuan GNU+Linux \nremains a universal, stable, dependable, free operating system.\n\n" .. "You can get the source code from https://sledjhamr.org/cgit/apt-panopticon/about/ (main repo)\n" .. "and from https://git.devuan.org/onefang/apt-panopticon' (Devuan repo).\n" .. "You can get the cgp graphing source code from https://sledjhamr.org/cgit/apt-panopticon_cgp/about/ (main repo)\n" .. "and https://git.devuan.org/onefang/apt-panopticon_cgp (Devuan repo)\n" .. "Love\n\n" .. "The Dev1Devs\n\n") email:close() end local lnk = function(name, link) if nil == link then link = name end return name .. " *" end local colours = { 'f0000080', '0f000080', '00f00080', '000f0080', '0000f080', '00000f80', '80000080', '08000080', '00800080', '00080080', '00008080', '00000880', 'ff000080', '0ff00080', '00ff0080', '000ff080', '0000ff80', '88000080', '08800080', '00880080', '00088080', '00008880', } local g = {} local count = 0 for k, v in APT.orderedPairs(mirrors) do if 'pkgmaster.devuan.org' ~= k then count = count + 1 end end for i = 1, count do end count = 1 for k, v in APT.orderedPairs(mirrors) do if 'deb.devuan.org' ~= k then local c = colours[count] local name = string.format('%32s', k) if 'pkgmaster.devuan.org' == k then c = 'ffffff' end table.insert(g, 'DEF:speedn' .. count .. '=rrd/' .. k .. '/Speed/Speed.rrd:max:MIN') table.insert(g, 'DEF:speedx' .. count .. '=rrd/' .. k .. '/Speed/Speed.rrd:max:MAX') table.insert(g, 'DEF:speeda' .. count .. '=rrd/' .. k .. '/Speed/Speed.rrd:max:AVERAGE') table.insert(g, 'DEF:speedl' .. count .. '=rrd/' .. k .. '/Speed/Speed.rrd:max:LAST') table.insert(g, 'VDEF:vspeedn' .. count .. '=speedn' .. count .. ',AVERAGE') table.insert(g, 'VDEF:vspeedx' .. count .. '=speedx' .. count .. ',AVERAGE') table.insert(g, 'VDEF:vspeeda' .. count .. '=speeda' .. count .. ',AVERAGE') table.insert(g, 'VDEF:vspeedl' .. count .. '=speedl' .. count .. ',AVERAGE') table.insert(g, 'LINE2:speedx' .. count .. '#' .. c .. ':' .. name .. ' ') table.insert(g, 'GPRINT:vspeedn' .. count .. ':Min %5.1lf%s,') table.insert(g, 'GPRINT:vspeeda' .. count .. ':Avg %5.1lf%s,') table.insert(g, 'GPRINT:vspeedx' .. count .. ':Max %5.1lf%s,') table.insert(g, 'GPRINT:vspeedl' .. count .. ':Last %5.1lf%s\\l') count = count + 1 end end APT.rrd.graph('results/speed.png', '--start', 'now-2w', '--end', 'now', '-t', 'Speed, rough maximum guess.', '-v', 'bytes per second', '-w', '900', '-h', '400', '-Z', '-c', 'BACK#000000', '-c', 'CANVAS#000000', '-c', 'FONT#FFFFFF', '-c', 'AXIS#FFFFFF', '-c', 'FRAME#FFFFFF', '-c', 'ARROW#FFFFFF', unpack(g)) results = {} m = {} faulty = "" APT.html = true local web, e = io.open("results/Report-web.html", "w+") if nil == web then C("opening mirrors file - " .. e) else web:write( "apt-panopticon results\n") if 0 < tonumber(APT.options.refresh.value) then web:write('\n') end web:write( '' .. "

Welcome to the apt-panopticon results page.

\n" .. "

Here the apt-panopticon system probes into every nook and cranny of the Devuan apt package mirror system, trying to live up to it's name.

" .. "

This is the status of the mirror servers in the Devuan package mirror network.

\n") if 0 < tonumber(APT.options.refresh.value) then web:write( '

This page will refresh every ' .. (APT.options.refresh.value / 60) .. ' minutes.

') end web:write( "

EXPERIMENTAL CODE - double check all results you see here, and read the logs if it's important.

" .. "

The full list of Devuan package mirrors is available at the URL: " .. "https://pkgmaster.devuan.org/mirror_list.txt

\n" .. "

Due to the nature of the tests, some errors or warnings will be counted several times.   " .. "The links in the table and DNS list go to the detailed testing logs.

\n\n" .. "
\n

==== package mirror status " .. os.date("!%F %H:%M") .. " GMT ====

\n" .. "

[FAILED] or [OK]" .. " means the tested thing is supported for that mirror.

\n" .. "

[FAILED*] or [OK*]" .. " means the tested thing is unsupported for that mirror, but might have been tested anyway.

\n" .. "

[timeout] or [timeout]" .. " means the mirror had some timeouts, and tests where not yet aborted. The darker colour means unsupported by the mirror, but tested anyway.

" .. "

[TIMEOUT] or [TIMEOUT]" .. " means the mirror had too many timeouts, and tests where aborted, so there is no result for this test.

" .. "

NOTE: timeouts may be due to a problem on the testing computer.

" .. "

The DNS round robin (DNS-RR) column shows the IPs for that mirror, or [no] if it isn't part of the DNS-RR.   " .. "The IPs link to the testing log for that IP accessed via the DNS-RR.   " .. "deb.devuan.org is the DNS-RR itself, so it doesn't get tested directly.

\n" .. "

The time in the Updated column is how often the mirror updates itself.

" .. "

Mirrors with a grey background are not active (though may be usable as part of the DNS-RR).

\n" .. "

[skip] means that the test hasn't been written yet.

\n" .. "\n" .. "" .. "\n" ) for k, v in APT.orderedPairs(APT.mirrors) do local results = APT.collateAll(APT.mirrors, 'results', k) local active = "" if "yes" == v.Active then web:write(" ") else if nil == v.Active then active = 'nil' else active = v.Active end web:write(" ") end local ftp = "[skip]" local http = status(APT.mirrors, k, results, "http") local https = status(APT.mirrors, k, results, "https") local rsync = "[skip]" local dns = "" local protocol = status(APT.mirrors, k, results, "Protocol") local sanity = status(APT.mirrors, k, results, "URLSanity") local integrity = status(APT.mirrors, k, results, "Integrity") local updated = status(APT.mirrors, k, results, "Updated") local rate = v.Rate if nil ~= rate then updated = updated .. ' ' .. rate end local min = tonumber(results.speed.min) local max = tonumber(results.speed.max) local spd = '' -- DNS-RR test. if ("deb.devuan.org" ~= k) and (nil ~= APT.mirrors["deb.devuan.org"]) then for l, w in pairs(APT.mirrors[k].IPs) do if type(w) == "table" then for i, u in pairs(w) do if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then local log = logCount("deb.devuan.org", i) if "" ~= log then if "" == dns then dns = " " else dns = dns .. "   " end dns = dns .. logCount("deb.devuan.org", i) else if "" == dns then dns = " " else dns = dns .. "   " end dns = dns .. "" .. i .. "" end end end else if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then local log = logCount("deb.devuan.org", l) if "" ~= log then if "" == dns then dns = " " else dns = dns .. "   " end dns = dns .. log else if "" == dns then dns = " " else dns = dns .. "   " end dns = dns .. "" .. l .. "" end end end end if "" == dns then dns = "[no]" end if 0 == max then spd = '' else spd = string.format('', min, max) end end web:write("" .. spd .. "\n") if "" ~= active then web:write("\n") end end web:write( "
" .. lnk('FTP') .. "" .. lnk('HTTP') .. "" .. lnk('HTTPS') .. "" .. lnk('RSYNC') .. "" .. lnk('DNS round robin', 'DNS-RR') .. "" .. lnk('Protocol') .. "" .. lnk('URL sanity', 'URL-Sanity') .. "" .. lnk('Integrity') .. "" .. lnk('Updated') .. "" .. lnk('Speed range', 'Speed') .. "
" .. k .. "
" .. k .. "%d ->%d" .. ftp .. " " .. http .. " " .. https .. " " .. rsync .. " " .. dns .. " " .. protocol .. " " .. sanity .. " " .. integrity .. " " .. updated .. " 
" .. active .. "
\n
\n

==== faulty mirrors: ====

\n" .. faulty) web:write( "
\n
\n

==== DNS and logs: ====

\n") for k, v in pairs(APT.mirrors) do local log = k local n = {} log = logCount(k) APT.mirrors[k].Protocols = nil APT.mirrors[k].FQDN = nil APT.mirrors[k].Active = nil APT.mirrors[k].Rate = nil APT.mirrors[k].BaseURL = nil APT.mirrors[k].Country = nil APT.mirrors[k].Bandwidth = nil for l, w in pairs(APT.mirrors[k].IPs) do if type(w) == "table" then n[l] = {} for i, u in pairs(w) do local log = logCount(k, i) if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(APT.mirrors, k, i)] = u end end else local log = logCount(k, l) if "" == log then n[l] = w else n[log .. " " .. revDNS(APT.mirrors, k, l)] = w end end end m[log .. " DNS entries -" .. redirs(APT.mirrors, k)] = n end web:write( "

This lists each mirror, and the DNS entries for that mirror.   " .. "The links point to the testing log files for " .. logCount("apt-panopticon") .. " for each domain name / IP combination that was tested.   " .. "If a mirror has a CNAME, that CNAME is listed along with that CNAMEs DNS entries.   " .. "deb.devuan.org is the DNS round robin, which points to the mirrors that are part of the DNS-RR.   " .. "If an IP is part of the DNS-RR, it is marked with 'DNS-RR'   " .. "pkgmaster.devuan.org is the master mirror, all the others sync to it.   " .. "

\n" ) web:write(APT.dumpTableHTML(m, "", "")) web:write( "\n
\n
\n

==== graphs: ====

\n" .. "\n
\n

More graphs. with greater detail.


\n\n") results = {} m = {} faulty = "" web:write( "
\n

==== Debian mirror status ====

\n" .. "

NOTE - This is not fully probing the Debian mirrors, we just collect some data from any redirects to other servers.   " .. "So this isn't a full set of tests.   Basically we don't know the shape of the Debian mirror infrastructure.

\n" .. "

EXPERIMENTAL CODE - this is even more experimental than the rest.

\n" .. "\n" .. "" .. "\n") for k, v in APT.orderedPairs(APT.debians) do local results = APT.collateAll(APT.debians, 'results', k) local active = "" if "yes" == v.Active then web:write(" ") else if nil == v.Active then active = 'nil' else active = v.Active end web:write(" ") end local ftp = "[skip]" local http = status(APT.debians, k, results, "http") local https = status(APT.debians, k, results, "https") local rsync = "[skip]" local dns = "" local protocol = status(APT.debians, k, results, "Protocol") local sanity = status(APT.debians, k, results, "URLSanity") local integrity = status(APT.debians, k, results, "Integrity") local updated = status(APT.debians, k, results, "Updated") local rate = v.Rate if nil ~= rate then updated = updated .. ' ' .. rate end local min = tonumber(results.speed.min) local max = tonumber(results.speed.max) local spd = '' -- DNS-RR test. if ("deb.devuan.org" ~= k) and (nil ~= APT.debians["deb.devuan.org"]) then for l, w in pairs(APT.debians[k].IPs) do if type(w) == "table" then for i, u in pairs(w) do if nil ~= APT.debians["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then local log = logCount("deb.devuan.org", i) if "" ~= log then if "" == dns then dns = " " else dns = dns .. "   " end dns = dns .. logCount("deb.devuan.org", i) else if "" == dns then dns = " " else dns = dns .. "   " end dns = dns .. "" .. i .. "" end end end else if nil ~= APT.debians["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then local log = logCount("deb.devuan.org", l) if "" ~= log then if "" == dns then dns = " " else dns = dns .. "   " end dns = dns .. log else if "" == dns then dns = " " else dns = dns .. "   " end dns = dns .. "" .. l .. "" end end end end if "" == dns then dns = "[no]" end if 0 == max then spd = '' else spd = string.format('', min, max) end end web:write("" .. spd .. "\n") if "" ~= active then web:write("\n") end end web:write( "
" .. lnk('FTP') .. "" .. lnk('HTTP') .. "" .. lnk('HTTPS') .. "" .. lnk('RSYNC') .. "" .. lnk('DNS round robin', 'DNS-RR') .. "" .. lnk('Protocol') .. "" .. lnk('URL sanity', 'URL-Sanity') .. "" .. lnk('Integrity') .. "" .. lnk('Updated') .. "" .. lnk('Speed range', 'Speed') .. "
" .. k .. "
" .. k .. "%d ->%d" .. ftp .. " " .. http .. " " .. https .. " " .. rsync .. " " .. dns .. " " .. protocol .. " " .. sanity .. " " .. integrity .. " " .. updated .. " 
" .. active .. "
\n
\n\n") web:write( "
\n
\n

==== Debian DNS and logs: ====

\n") for k, v in pairs(APT.debians) do local log = k local n = {} log = logCount(k) APT.debians[k].Protocols = nil APT.debians[k].FQDN = nil APT.debians[k].Active = nil APT.debians[k].Rate = nil APT.debians[k].BaseURL = nil APT.debians[k].Country = nil APT.debians[k].Bandwidth = nil for l, w in pairs(APT.debians[k].IPs) do if type(w) == "table" then n[l] = {} for i, u in pairs(w) do local log = logCount(k, i) if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(APT.debians, k, i)] = u end end else local log = logCount(k, l) if "" == log then n[l] = w else n[log .. " " .. revDNS(APT.debians, k, l)] = w end end end m[log .. " DNS entries -" .. redirs(APT.mirrors, k)] = n end web:write(APT.dumpTableHTML(m, "", "")) web:write( "
\n
\n

The email report.   " .. "All the logs and other output.   " .. "You can get the source code here (main repo)" .. "and here (Devuan repo).  " .. "You can get the cgp graphing source code here (main repo)" .. "and here (Devuan repo).

\n" ) local status, whn = APT.execute('TZ="GMT" ls -l1 --time-style="+%s" results/stamp | cut -d " " -f 6-6') web:write( "

This run took " .. (os.time() - tonumber("0" .. whn:sub(2, -2))) .. " seconds.

" .. "\n\n") web:close() end