#!/usr/bin/env luajit local APT = require 'apt-panopticommon' local D = APT.D local I = APT.I local T = APT.T local W = APT.W local E = APT.E local C = APT.C local arg, sendArgs = APT.parseArgs({...}) local results = {} APT.mirrors = loadfile("results/mirrors.lua")() APT.debians = loadfile("results/debians.lua")() local lnk = function(name, link) if nil == link then link = name end return name .. " *" end local revDNS = function(hosts, dom, IP) if APT.options.roundRobin.value ~= dom then if nil ~= hosts[APT.options.roundRobin.value] then if nil ~= hosts[APT.options.roundRobin.value].IPs["deb.roundr.devuan.org"][IP] then if APT.html then return "DNS-RR" else return "DNS-RR" end end end else for k, v in pairs(hosts) do if APT.options.roundRobin.value ~= k then local IPs = v.IPs for i, u in pairs(IPs) do if "table" == type(u) then for h, t in pairs(u) do if IP == h then return k end end else if IP == i then return k end end end end end end return "" end local faulty = "" local status = function(hosts, host, results, typ) local result = "" local e = 0 local w = 0 local t = 0 local s = nil ~= hosts[host].Protocols[typ] local to = results.timeout if not APT.search(APT.protocols, typ) then s = true end if nil ~= results[typ] then e = results[typ].errors w = results[typ].warnings t = results[typ].timeouts for k, v in pairs(results[typ]) do if ("table" == type(v)) and ('redirects' ~= k) then if 0 <= v.errors then e = e + v.errors else to = true end if 0 <= v.warnings then w = w + v.warnings else to = true end if 0 <= v.timeouts then t = t + v.timeouts else to = true end end end else for k, v in pairs(results) do if "table" == type(v) then for i, u in pairs(v) do if "table" == type(u) then if typ == i then if 0 <= u.errors then e = e + u.errors end if 0 <= u.warnings then w = w + u.warnings end if 0 <= u.timeouts then t = t + u.timeouts end end end end end end end if to then result = "TIMEOUT" if not s then result = result .. "*" end if APT.html then if s then result = "TIMEOUT" else result = "TIMEOUT*" end end if APT.html then faulty = faulty .. host .. " (" .. typ .. ")
\n" else faulty = faulty .. host .. " (" .. typ .. ")\n" end elseif 0 < e then result = "FAILED" if not s then result = result .. "*" end if APT.html then if s then result = "FAILED" else result = "FAILED*" end end if APT.html then faulty = faulty .. host .. " (" .. typ .. ")
\n" else faulty = faulty .. host .. " (" .. typ .. ")\n" end else result = "OK" if not s then result = result .. "*" end if APT.html then if s then result = "OK" else result = "OK*" end end end return result .. APT.plurals(e, w, t) end local m = {} local logCount = function(domain, ip) local nm = "LOG_" .. domain local log = "" local extra = "" local errors = 0 local warnings = 0 local timeouts = 0 if nil ~= ip then nm = nm .. "_" .. ip end nm = nm .. ".html" local rfile, e = io.open("results/" .. nm, "r") if nil ~= rfile then for l in rfile:lines() do if nil ~= l:match(">ERROR ") then errors = errors + 1 end if nil ~= l:match(">WARNING ") then warnings = warnings + 1 end if nil ~= l:match(">TIMEOUT ") then timeouts = timeouts + 1 end end rfile:close() end if APT.html then if nil == ip then log = "" .. domain .. "" else log = "" .. ip .. "" end end log = log .. APT.plurals(errors, warnings, timeouts) return log end local redirs = function(hosts, host) local results = APT.collateAll(hosts, 'results', host) local rdr = {} local redirs = '' for p, pt in pairs(APT.protocols) do if 0 ~= #(results[pt].redirects) then table.sort(results[pt].redirects) for r, rd in pairs(results[pt].redirects) do rdr[rd] = rd end end end for r, rd in pairs(rdr) do redirs = redirs .. ',   ' .. rd end if '' ~= redirs then redirs = '
\n     (Redirects some packages to - ' .. redirs:sub(3) .. ')' end return redirs end local DNSrrTest = function(hosts, k) local dns = '' local space = ' ' local no = 'no' if APT.html then space = '   ' no = "no" end if (APT.options.roundRobin.value ~= k) and (nil ~= hosts[APT.options.roundRobin.value]) then APT.allpairs(hosts[k].IPs, function(i, w, k, v) if nil ~= hosts[APT.options.roundRobin.value].IPs["deb.roundr.devuan.org"][i] then local log = logCount(APT.options.roundRobin.value, i) if "" ~= log then if "" == dns then dns = " " else dns = dns .. space end dns = dns .. logCount(APT.options.roundRobin.value, i) else if "" == dns then dns = " " else dns = dns .. space end if APT.html then i = "" .. i .. "" end dns = dns .. i end end end ) if "" == dns then dns = no end end return dns end local makeTable = function(web, hosts) web:write("\n" .. "" .. "" .. "\n") local bg = '' for k, v in APT.orderedPairs(hosts) do if '' == bg then bg = " style='background-color:#111111'" else bg = '' end local results = APT.collateAll(hosts, 'results', k) local active = "" if "yes" == v.Active then web:write(" ") else if nil == v.Active then active = 'nil' else active = v.Active end web:write(" ") end local ftp = "skip" local http = status(hosts, k, results, "http") local https = status(hosts, k, results, "https") local rsync = "skip" local dns = DNSrrTest(hosts, k) local protocol = status(hosts, k, results, "Protocol") local sanity = status(hosts, k, results, "URLSanity") local integrity = status(hosts, k, results, "Integrity") local redirects = status(hosts, k, results, "Redirects") local updated = status(hosts, k, results, "Updated") local rate = v.Rate local min = tonumber(results.speed.min) local max = tonumber(results.speed.max) local spd = '' local week = '' if nil == rate then rate = '' end -- DNS-RR test. if (APT.options.roundRobin.value ~= k) and (nil ~= hosts[APT.options.roundRobin.value]) then if 0 == max then spd = '' else spd = string.format('', min, max) end end if (APT.options.roundRobin.value ~= k) then local percentUp = '??' local percentUpdated = '??' if APT.checkFile('rrd/' .. k .. '/Speed/Speed.rrd') then local start, step, names, data = APT.rrd.fetch('rrd/' .. k .. '/Speed/Speed.rrd', 'LAST', '-a', '-r', '10m', '-s', '-1w') local count, up, down, unknown = 0, 0, 0, 0 for i, dp in ipairs(data) do for j,v in ipairs(dp) do if 'max' == names[j] then if 'nan' == tostring(v) then unknown = unknown + 1 else count = count + 1 if 0 == v then down = down + 1 else up = up + 1 end end end end end percentUp = string.format('%.2f', up / count * 100) end if APT.checkFile('rrd/' .. k .. '/HTTP/Tests.rrd') then local start, step, names, data = APT.rrd.fetch('rrd/' .. k .. '/HTTP/Tests.rrd', 'LAST', '-a', '-r', '10m', '-s', '-1w') local count, up, down, unknown = 0, 0, 0, 0 for i,dp in ipairs(data) do for j,v in ipairs(dp) do if 'UpdatedErrors' == names[j] then if 'nan' == tostring(v) then unknown = unknown + 1 else count = count + 1 if 0 == v then down = down + 1 else up = up + 1 end end end end end percentUpdated = string.format('%.2f', (down / count * 100)) if '0.00' == percentUp then percentUpdated = '??' end -- We are counting errors, and you can't get an error if you can't check anything. -- TODO - try to account for this better, this is just a quick hack. end week = '' end web:write("" .. ' ' .. spd .. " " .. week .." \n") if "" ~= active then web:write("\n") end end web:write( "
" .. lnk('FTP') .. "" .. lnk('HTTP') .. "" .. lnk('HTTPS') .. "" .. lnk('RSYNC') .. "" .. lnk('DNS round robin', 'DNS-RR') .. "" .. lnk('Protocol') .. "" .. lnk('Redirects') .. "" .. lnk('URL sanity', 'URL-Sanity') .. "" .. lnk('Integrity') .. "" .. lnk('Updated') .. "" .. lnk('Speed range', 'Speed') .. "" .. lnk('Weekly statistics', 'Weekly') .. "
" .. k .. "
" .. k .. "  %d -%d ' .. percentUp .. '% up ' .. percentUpdated .. '% updated" .. ftp .. " " .. http .. " " .. https .. " " .. rsync .. " " .. dns .. " " .. protocol .. " " .. redirects .. " " .. sanity .. " " .. integrity .. " ' .. rate .. '' .. updated .. '
" .. active .. "
\n
\n") end local makeIPlist = function(hosts) local m = {} for k, v in pairs(hosts) do local log = k local n = {} log = logCount(k) hosts[k].Protocols = nil hosts[k].FQDN = nil hosts[k].Active = nil hosts[k].Rate = nil hosts[k].BaseURL = nil hosts[k].Country = nil hosts[k].Bandwidth = nil for l, w in pairs(hosts[k].IPs) do if type(w) == "table" then n[l] = {} for i, u in pairs(w) do local log = '[graphs]   ' .. logCount(k, i) if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(hosts, k, i)] = u end end else local log = '[graphs]   ' .. logCount(k, l) if "" == log then n[l] = w else n[log .. " " .. revDNS(hosts, k, l)] = w end end end m['[graphs]   ' .. log .. " DNS entries -" .. redirs(hosts, k)] = n end return m end APT.html = false local email, e = io.open("results/Report-email.txt", "w+") if nil == email then C("opening mirrors file - " .. e) else email:write( "Dear Mirror Admins,\n\n" .. "This is the status of the mirror servers in the Devuan package mirror network.\n\n" .. "EXPERIMENTAL CODE - double check all results you see here, and read the logs if it's important." .. "The full list of Devuan package mirrors is available at the URL:\n\n" .. " https://pkgmaster.devuan.org/mirror_list.txt\n\n" .. 'Please contact "mirrors@devuan.org" if any of the information \nin the file above needs to be amended. \n\n' .. "The full results of the mirror checking is available at the URL:\n\n" .. " https://sledjhamr.org/apt-panopticon/results/Report-web.html\n\n" .. "Due to the nature of the tests, some errors or warnings will be \ncounted several times. " .. "Refer to the logs on the web page for details.\n\n" .. "Please see below the current status of the Devuan Package Mirror \nnetwork:\n\n" .. "==== package mirror status " .. os.date("!%F %H:%M") .. " GMT ====\n" .. "[skip] means that the test hasn't been written yet.\n\n") for k, v in APT.orderedPairs(APT.mirrors) do email:write(k .. "....\n") local results = APT.collateAll(APT.mirrors, 'results', k) local ftp = "[skip]" local http = status(APT.mirrors, k, results, "http") local https = status(APT.mirrors, k, results, "https") local rsync = "[skip]" local dns = DNSrrTest(APT.mirrors, k) local protocol = status(APT.mirrors, k, results, "Protocol") local redirects = status(APT.mirrors, k, results, "Redirects") local sanity = status(APT.mirrors, k, results, "URLSanity") local integrity = status(APT.mirrors, k, results, "Integrity") local updated = status(APT.mirrors, k, results, "Updated") -- DNS-RR test. if (APT.options.roundRobin.value ~= k) and (nil ~= APT.mirrors[APT.options.roundRobin.value]) then dns = " DNS-RR: " .. dns end email:write( " ftp: " .. ftp .. " http: " .. http .. " https: " .. https .." rsync: " .. rsync .. "\n" .. " " .. dns .. "\n" .. " Protocol: " .. protocol .. " URL-sanity: " .. sanity .. " Integrity: " .. integrity .. "\n" .. " Updated: " .. updated .. "\n") end email:write( "\n==== faulty mirrors: ====\n" .. faulty) email:write( "\n-------------------------\n\n" .. "* This means that this protocol isn't actually supported, but the test was run ayway.\n\n" .. "Thanks for your precious help in ensuring that Devuan GNU+Linux \nremains a universal, stable, dependable, free operating system.\n\n" .. "You can get the source code from https://sledjhamr.org/cgit/apt-panopticon/about/ (main repo)\n" .. "and from https://git.devuan.org/onefang/apt-panopticon' (Devuan repo).\n" .. "You can get the cgp graphing source code from https://sledjhamr.org/cgit/apt-panopticon_cgp/about/ (main repo)\n" .. "and https://git.devuan.org/onefang/apt-panopticon_cgp (Devuan repo)\n" .. "Love\n\n" .. "The Dev1Devs\n\n") email:close() end local colours = { 'f0000080', '0f000080', '00f00080', '000f0080', '0000f080', '00000f80', '80000080', '08000080', '00800080', '00080080', '00008080', '00000880', 'ff000080', '0ff00080', '00ff0080', '000ff080', '0000ff80', '88000080', '08800080', '00880080', '00088080', '00008880', } local g = {} local count = 0 for k, v in APT.orderedPairs(mirrors) do if APT.options.referenceSite.value ~= k then count = count + 1 end end for i = 1, count do end count = 1 for k, v in APT.orderedPairs(mirrors) do if APT.options.roundRobin.value ~= k then local c = colours[count] local name = string.format('%32s', k) if APT.options.referenceSite.value == k then c = 'ffffff' end table.insert(g, 'DEF:speedn' .. count .. '=rrd/' .. k .. '/Speed/Speed.rrd:max:MIN') table.insert(g, 'DEF:speedx' .. count .. '=rrd/' .. k .. '/Speed/Speed.rrd:max:MAX') table.insert(g, 'DEF:speeda' .. count .. '=rrd/' .. k .. '/Speed/Speed.rrd:max:AVERAGE') table.insert(g, 'DEF:speedl' .. count .. '=rrd/' .. k .. '/Speed/Speed.rrd:max:LAST') table.insert(g, 'VDEF:vspeedn' .. count .. '=speedn' .. count .. ',AVERAGE') table.insert(g, 'VDEF:vspeedx' .. count .. '=speedx' .. count .. ',AVERAGE') table.insert(g, 'VDEF:vspeeda' .. count .. '=speeda' .. count .. ',AVERAGE') table.insert(g, 'VDEF:vspeedl' .. count .. '=speedl' .. count .. ',AVERAGE') table.insert(g, 'LINE2:speedx' .. count .. '#' .. c .. ':' .. name .. ' ') table.insert(g, 'GPRINT:vspeedn' .. count .. ':Min %5.1lf%s,') table.insert(g, 'GPRINT:vspeeda' .. count .. ':Avg %5.1lf%s,') table.insert(g, 'GPRINT:vspeedx' .. count .. ':Max %5.1lf%s,') table.insert(g, 'GPRINT:vspeedl' .. count .. ':Last %5.1lf%s\\l') count = count + 1 end end APT.rrd.graph('results/speed.png', '--start', 'now-2w', '--end', 'now', '-t', 'Speed, rough maximum guess.', '-v', 'bytes per second', '-w', '900', '-h', '400', '-Z', '-c', 'BACK#000000', '-c', 'CANVAS#000000', '-c', 'FONT#FFFFFF', '-c', 'AXIS#FFFFFF', '-c', 'FRAME#FFFFFF', '-c', 'ARROW#FFFFFF', unpack(g)) results = {} m = {} faulty = "" APT.html = true local web, e = io.open("results/Report-web.html", "w+") if nil == web then C("opening mirrors file - " .. e) else web:write( "apt-panopticon results\n") if 0 < tonumber(APT.options.refresh.value) then web:write('\n') end web:write( '' .. "

Welcome to the apt-panopticon results page.

\n" .. "

Here the apt-panopticon system probes into every nook and cranny of the Devuan apt package mirror system, trying to live up to it's name.

" .. "

This is the status of the mirror servers in the Devuan package mirror network.

\n") if 0 < tonumber(APT.options.refresh.value) then web:write( '

This page will refresh every ' .. (APT.options.refresh.value / 60) .. ' minutes.

') end web:write( "

EXPERIMENTAL CODE - double check all results you see here, and read the logs if it's important.

" .. "

The full list of Devuan package mirrors is available at the URL: " .. "https://pkgmaster.devuan.org/mirror_list.txt

\n" .. "

Due to the nature of the tests, some errors or warnings will be counted several times.   " .. "The links in the table and DNS list go to the detailed testing logs.

\n\n" .. "
\n

==== package mirror status " .. os.date("!%F %H:%M") .. " GMT ====

\n" .. "

FAILED or OK" .. " means the tested thing is supported for that mirror.

\n" .. "

FAILED* or OK*" .. " means the tested thing is unsupported for that mirror, but might have been tested anyway.

\n" .. "

timeout or timeout" .. " means the mirror had some timeouts, and tests where not yet aborted. The darker colour means unsupported by the mirror, but tested anyway.

" .. "

TIMEOUT or TIMEOUT" .. " means the mirror had too many timeouts, and tests where aborted, so there is no result for this test.

" .. "

NOTE: timeouts may be due to a problem on the testing computer, it might be busy with other things, or be having it's own network problems..

" .. "

The DNS round robin (DNS-RR) column shows the IPs for that mirror, or no if it isn't part of the DNS-RR.   " .. "The IPs link to the testing log for that IP accessed via the DNS-RR.   " .. APT.options.roundRobin.value .. " is the DNS-RR itself, so it doesn't get tested directly.

\n" .. "

The time in the Updated column is how often the mirror updates itself.

" .. "

Mirrors with a grey background are not active (though may be usable as part of the DNS-RR).

\n" .. "

skip means that the test hasn't been written yet.

\n" ) makeTable(web, APT.mirrors) web:write( "

==== faulty mirrors: ====

\n" .. faulty) web:write( "
\n
\n

==== DNS, graphs *, and logs: ====

\n") m = makeIPlist(APT.mirrors) web:write( "

This lists each mirror, and the DNS entries for that mirror.   " .. "The links point to the testing log files (" .. logCount("apt-panopticon") .. ") for each domain name / IP combination that was tested.   " .. "If a mirror has a CNAME, that CNAME is listed along with that CNAMEs DNS entries.   " .. APT.options.roundRobin.value .. " is the DNS round robin, which points to the mirrors that are part of the DNS-RR.   " .. "If an IP is part of the DNS-RR, it is marked with 'DNS-RR'   " .. APT.options.referenceSite.value .. " is the master mirror, all the others sync to it.   " .. "

\n" ) web:write(APT.dumpTableHTML(m, "")) web:write( "\n
\n
\n

==== graphs: ====

\n" .. "\n
\n

More graphs. with greater detail.


\n\n") results = {} m = {} faulty = "" web:write( "
\n

==== Debian mirror status ====

\n" .. "

NOTE - This is not fully probing the Debian mirrors, we just collect some data from any redirects to other servers.   " .. "So this isn't a full set of tests.   Basically we don't know the shape of the Debian mirror infrastructure.

\n" .. "

EXPERIMENTAL CODE - this is even more experimental than the rest.

\n" ) makeTable(web, APT.debians) web:write( "
\n
\n

==== Debian DNS, graphs *, and logs: ====

\n") m = makeIPlist(APT.debians) web:write(APT.dumpTableHTML(m, "")) web:write( "
\n
\n

The email report.   " .. "All the logs and other output.   " .. "You can get the source code here (main repo)" .. "and here (Devuan repo).  " .. "You can get the cgp graphing source code here (main repo)" .. "and here (Devuan repo).

\n" ) local whn = APT.exe('TZ="GMT" ls -dl1 --time-style="+%s" results/stamp | cut -d " " -f 6-6'):Do().result:sub(2, -2) web:write( "

This run took " .. (os.time() - tonumber("0" .. whn)) .. " seconds.

" .. "\n\n") web:close() end