#!/usr/bin/env luajit
local APT = require 'apt-panopticommon'
local D = APT.D
local I = APT.I
local T = APT.T
local W = APT.W
local E = APT.E
local C = APT.C
local arg, sendArgs = APT.parseArgs({...})
local results = {}
APT.mirrors = loadfile("results/mirrors.lua")()
APT.debians = loadfile("results/debians.lua")()
local lnk = function(name, link)
if nil == link then link = name end
return name .. " *"
end
local revDNS = function(hosts, dom, IP)
if APT.options.roundRobin.value ~= dom then
if nil ~= hosts[APT.options.roundRobin.value] then
if nil ~= hosts[APT.options.roundRobin.value].IPs["deb.roundr.devuan.org"][IP] then
if APT.html then
return "DNS-RR"
else
return "DNS-RR"
end
end
end
else
for k, v in pairs(hosts) do
if APT.options.roundRobin.value ~= k then
local IPs = v.IPs
for i, u in pairs(IPs) do
if "table" == type(u) then
for h, t in pairs(u) do
if IP == h then return k end
end
else
if IP == i then return k end
end
end
end
end
end
return ""
end
local faulty = ""
local status = function(hosts, host, results, typ)
local result = ""
local e = 0
local w = 0
local t = 0
local s = nil ~= hosts[host].Protocols[typ]
local to = results.timeout
if not APT.search(APT.protocols, typ) then s = true end
if nil ~= results[typ] then
e = results[typ].errors
w = results[typ].warnings
t = results[typ].timeouts
for k, v in pairs(results[typ]) do
if ("table" == type(v)) and ('redirects' ~= k) then
if 0 <= v.errors then e = e + v.errors else to = true end
if 0 <= v.warnings then w = w + v.warnings else to = true end
if 0 <= v.timeouts then t = t + v.timeouts else to = true end
end
end
else
for k, v in pairs(results) do
if "table" == type(v) then
for i, u in pairs(v) do
if "table" == type(u) then
if typ == i then
if 0 <= u.errors then e = e + u.errors end
if 0 <= u.warnings then w = w + u.warnings end
if 0 <= u.timeouts then t = t + u.timeouts end
end
end
end
end
end
end
if to then
result = "TIMEOUT"
if not s then result = result .. "*" end
if APT.html then
if s then
result = "TIMEOUT"
else
result = "TIMEOUT*"
end
end
if APT.html then
faulty = faulty .. host .. " (" .. typ .. ")
\n"
else
faulty = faulty .. host .. " (" .. typ .. ")\n"
end
elseif 0 < e then
result = "FAILED"
if not s then result = result .. "*" end
if APT.html then
if s then
result = "FAILED"
else
result = "FAILED*"
end
end
if APT.html then
faulty = faulty .. host .. " (" .. typ .. ")
\n"
else
faulty = faulty .. host .. " (" .. typ .. ")\n"
end
else
result = "OK"
if not s then result = result .. "*" end
if APT.html then
if s then
result = "OK"
else
result = "OK*"
end
end
end
return result .. APT.plurals(e, w, t)
end
local m = {}
local logCount = function(domain, ip)
local nm = "LOG_" .. domain
local log = ""
local extra = ""
local errors = 0
local warnings = 0
local timeouts = 0
if nil ~= ip then nm = nm .. "_" .. ip end
nm = nm .. ".html"
local rfile, e = io.open("results/" .. nm, "r")
if nil ~= rfile then
for l in rfile:lines() do
if nil ~= l:match(">ERROR ") then errors = errors + 1 end
if nil ~= l:match(">WARNING ") then warnings = warnings + 1 end
if nil ~= l:match(">TIMEOUT ") then timeouts = timeouts + 1 end
end
rfile:close()
end
if APT.html then
if nil == ip then
log = "" .. domain .. ""
else
log = "" .. ip .. ""
end
end
log = log .. APT.plurals(errors, warnings, timeouts)
return log
end
local redirs = function(hosts, host)
local results = APT.collateAll(hosts, 'results', host)
local rdr = {}
local redirs = ''
for p, pt in pairs(APT.protocols) do
if 0 ~= #(results[pt].redirects) then
table.sort(results[pt].redirects)
for r, rd in pairs(results[pt].redirects) do
rdr[rd] = rd
end
end
end
for r, rd in pairs(rdr) do
redirs = redirs .. ',   ' .. rd
end
if '' ~= redirs then redirs = '
\n (Redirects some packages to - ' .. redirs:sub(3) .. ')' end
return redirs
end
local DNSrrTest = function(hosts, k)
local dns = ''
local space = ' '
local no = 'no'
if APT.html then
space = ' '
no = "no"
end
if (APT.options.roundRobin.value ~= k) and (nil ~= hosts[APT.options.roundRobin.value]) then
APT.allpairs(hosts[k].IPs,
function(i, w, k, v)
if nil ~= hosts[APT.options.roundRobin.value].IPs["deb.roundr.devuan.org"][i] then
local log = logCount(APT.options.roundRobin.value, i)
if "" ~= log then
if "" == dns then dns = " " else dns = dns .. space end
dns = dns .. logCount(APT.options.roundRobin.value, i)
else
if "" == dns then dns = " " else dns = dns .. space end
if APT.html then i = "" .. i .. "" end
dns = dns .. i
end
end
end
)
if "" == dns then dns = no end
end
return dns
end
local makeTable = function(web, hosts)
web:write("\n
\n \n")
local bg = ''
for k, v in APT.orderedPairs(hosts) do
if '' == bg then bg = " style='background-color:#111111'" else bg = '' end
local results = APT.collateAll(hosts, 'results', k)
local active = ""
if "yes" == v.Active then
web:write(" " .. lnk('FTP') .. " " .. lnk('HTTP') .. " " .. lnk('HTTPS') .. " " .. lnk('RSYNC') .. " " ..
"" .. lnk('DNS round robin', 'DNS-RR') .. " " .. lnk('Protocol') .. " " .. lnk('Redirects') ..
" " .. lnk('URL sanity', 'URL-Sanity') .. " " .. lnk('Integrity') .. " " ..
"" .. lnk('Updated') .. " " .. lnk('Speed range', 'Speed') .. " " ..
"" .. lnk('Weekly statistics', 'Weekly') .. " " .. k .. " ")
else
if nil == v.Active then active = 'nil' else active = v.Active end
web:write(" \n")
if "" ~= active then
web:write("" .. k .. " ")
end
local ftp = "skip"
local http = status(hosts, k, results, "http")
local https = status(hosts, k, results, "https")
local rsync = "skip"
local dns = DNSrrTest(hosts, k)
local protocol = status(hosts, k, results, "Protocol")
local sanity = status(hosts, k, results, "URLSanity")
local integrity = status(hosts, k, results, "Integrity")
local redirects = status(hosts, k, results, "Redirects")
local updated = status(hosts, k, results, "Updated")
local rate = v.Rate
local min = tonumber(results.speed.min)
local max = tonumber(results.speed.max)
local spd = ''
local week = ' '
if nil == rate then rate = '' end
-- DNS-RR test.
if (APT.options.roundRobin.value ~= k) and (nil ~= hosts[APT.options.roundRobin.value]) then
if 0 == max then
spd = ''
else
spd = string.format(' %d - %d ', min, max)
end
end
if (APT.options.roundRobin.value ~= k) then
local percentUp = '??'
local percentUpdated = '??'
if APT.checkFile('rrd/' .. k .. '/Speed/Speed.rrd') then
local start, step, names, data = APT.rrd.fetch('rrd/' .. k .. '/Speed/Speed.rrd', 'LAST', '-a', '-r', '10m', '-s', '-1w')
local count, up, down, unknown = 0, 0, 0, 0
for i, dp in ipairs(data) do
for j,v in ipairs(dp) do
if 'max' == names[j] then
if 'nan' == tostring(v) then
unknown = unknown + 1
else
count = count + 1
if 0 == v then down = down + 1 else up = up + 1 end
end
end
end
end
percentUp = string.format('%.2f', up / count * 100)
end
if APT.checkFile('rrd/' .. k .. '/HTTP/Tests.rrd') then
local start, step, names, data = APT.rrd.fetch('rrd/' .. k .. '/HTTP/Tests.rrd', 'LAST', '-a', '-r', '10m', '-s', '-1w')
local count, up, down, unknown = 0, 0, 0, 0
for i,dp in ipairs(data) do
for j,v in ipairs(dp) do
if 'UpdatedErrors' == names[j] then
if 'nan' == tostring(v) then
unknown = unknown + 1
else
count = count + 1
if 0 == v then down = down + 1 else up = up + 1 end
end
end
end
end
percentUpdated = string.format('%.2f', (down / count * 100))
if '0.00' == percentUp then percentUpdated = '??' end -- We are counting errors, and you can't get an error if you can't check anything.
-- TODO - try to account for this better, this is just a quick hack.
end
week = ' ' .. percentUp .. '% up ' .. percentUpdated .. '% updated '
end
web:write("" .. ftp .. " " .. http .. " " .. https .. " " .. rsync .. " " .. dns ..
" " .. protocol .. " " .. redirects .. " " .. sanity .. " " .. integrity .. " " .. '' .. rate ..
' ' .. updated .. ' ' .. spd .. " " .. week .." \n")
end
end
web:write( "" .. active .. "
\n")
end
local makeIPlist = function(hosts)
local m = {}
for k, v in pairs(hosts) do
local log = k
local n = {}
log = logCount(k)
hosts[k].Protocols = nil
hosts[k].FQDN = nil
hosts[k].Active = nil
hosts[k].Rate = nil
hosts[k].BaseURL = nil
hosts[k].Country = nil
hosts[k].Bandwidth = nil
for l, w in pairs(hosts[k].IPs) do
if type(w) == "table" then
n[l] = {}
for i, u in pairs(w) do
local log = '[graphs] ' .. logCount(k, i)
if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(hosts, k, i)] = u end
end
else
local log = '[graphs] ' .. logCount(k, l)
if "" == log then n[l] = w else n[log .. " " .. revDNS(hosts, k, l)] = w end
end
end
m['[graphs] ' .. log .. " DNS entries -" .. redirs(hosts, k)] = n
end
return m
end
APT.html = false
local email, e = io.open("results/Report-email.txt", "w+")
if nil == email then C("opening mirrors file - " .. e) else
email:write( "Dear Mirror Admins,\n\n" ..
"This is the status of the mirror servers in the Devuan package mirror network.\n\n" ..
"EXPERIMENTAL CODE - double check all results you see here, and read the logs if it's important." ..
"The full list of Devuan package mirrors is available at the URL:\n\n" ..
" https://pkgmaster.devuan.org/mirror_list.txt\n\n" ..
'Please contact "mirrors@devuan.org" if any of the information \nin the file above needs to be amended. \n\n' ..
"The full results of the mirror checking is available at the URL:\n\n" ..
" https://sledjhamr.org/apt-panopticon/results/Report-web.html\n\n" ..
"Due to the nature of the tests, some errors or warnings will be \ncounted several times. " ..
"Refer to the logs on the web page for details.\n\n" ..
"Please see below the current status of the Devuan Package Mirror \nnetwork:\n\n" ..
"==== package mirror status " .. os.date("!%F %H:%M") .. " GMT ====\n" ..
"[skip] means that the test hasn't been written yet.\n\n")
for k, v in APT.orderedPairs(APT.mirrors) do
email:write(k .. "....\n")
local results = APT.collateAll(APT.mirrors, 'results', k)
local ftp = "[skip]"
local http = status(APT.mirrors, k, results, "http")
local https = status(APT.mirrors, k, results, "https")
local rsync = "[skip]"
local dns = DNSrrTest(APT.mirrors, k)
local protocol = status(APT.mirrors, k, results, "Protocol")
local redirects = status(APT.mirrors, k, results, "Redirects")
local sanity = status(APT.mirrors, k, results, "URLSanity")
local integrity = status(APT.mirrors, k, results, "Integrity")
local updated = status(APT.mirrors, k, results, "Updated")
-- DNS-RR test.
if (APT.options.roundRobin.value ~= k) and (nil ~= APT.mirrors[APT.options.roundRobin.value]) then
dns = " DNS-RR: " .. dns
end
email:write( " ftp: " .. ftp .. " http: " .. http .. " https: " .. https .." rsync: " .. rsync .. "\n" ..
" " .. dns .. "\n" ..
" Protocol: " .. protocol .. " URL-sanity: " .. sanity .. " Integrity: " .. integrity .. "\n" ..
" Updated: " .. updated .. "\n")
end
email:write( "\n==== faulty mirrors: ====\n" .. faulty)
email:write( "\n-------------------------\n\n" ..
"* This means that this protocol isn't actually supported, but the test was run ayway.\n\n" ..
"Thanks for your precious help in ensuring that Devuan GNU+Linux \nremains a universal, stable, dependable, free operating system.\n\n" ..
"You can get the source code from https://sledjhamr.org/cgit/apt-panopticon/about/ (main repo)\n" ..
"and from https://git.devuan.org/onefang/apt-panopticon' (Devuan repo).\n" ..
"You can get the cgp graphing source code from https://sledjhamr.org/cgit/apt-panopticon_cgp/about/ (main repo)\n" ..
"and https://git.devuan.org/onefang/apt-panopticon_cgp (Devuan repo)\n" ..
"Love\n\n" ..
"The Dev1Devs\n\n")
email:close()
end
local colours =
{
'f0000080',
'0f000080',
'00f00080',
'000f0080',
'0000f080',
'00000f80',
'80000080',
'08000080',
'00800080',
'00080080',
'00008080',
'00000880',
'ff000080',
'0ff00080',
'00ff0080',
'000ff080',
'0000ff80',
'88000080',
'08800080',
'00880080',
'00088080',
'00008880',
}
local g = {}
local count = 0
for k, v in APT.orderedPairs(mirrors) do
if APT.options.referenceSite.value ~= k then count = count + 1 end
end
for i = 1, count do
end
count = 1
for k, v in APT.orderedPairs(mirrors) do
if APT.options.roundRobin.value ~= k then
local c = colours[count]
local name = string.format('%32s', k)
if APT.options.referenceSite.value == k then c = 'ffffff' end
table.insert(g, 'DEF:speedn' .. count .. '=rrd/' .. k .. '/Speed/Speed.rrd:max:MIN')
table.insert(g, 'DEF:speedx' .. count .. '=rrd/' .. k .. '/Speed/Speed.rrd:max:MAX')
table.insert(g, 'DEF:speeda' .. count .. '=rrd/' .. k .. '/Speed/Speed.rrd:max:AVERAGE')
table.insert(g, 'DEF:speedl' .. count .. '=rrd/' .. k .. '/Speed/Speed.rrd:max:LAST')
table.insert(g, 'VDEF:vspeedn' .. count .. '=speedn' .. count .. ',AVERAGE')
table.insert(g, 'VDEF:vspeedx' .. count .. '=speedx' .. count .. ',AVERAGE')
table.insert(g, 'VDEF:vspeeda' .. count .. '=speeda' .. count .. ',AVERAGE')
table.insert(g, 'VDEF:vspeedl' .. count .. '=speedl' .. count .. ',AVERAGE')
table.insert(g, 'LINE2:speedx' .. count .. '#' .. c .. ':' .. name .. ' ')
table.insert(g, 'GPRINT:vspeedn' .. count .. ':Min %5.1lf%s,')
table.insert(g, 'GPRINT:vspeeda' .. count .. ':Avg %5.1lf%s,')
table.insert(g, 'GPRINT:vspeedx' .. count .. ':Max %5.1lf%s,')
table.insert(g, 'GPRINT:vspeedl' .. count .. ':Last %5.1lf%s\\l')
count = count + 1
end
end
APT.rrd.graph('results/speed.png', '--start', 'now-2w', '--end', 'now', '-t', 'Speed, rough maximum guess.', '-v', 'bytes per second', '-w', '900', '-h', '400', '-Z',
'-c', 'BACK#000000', '-c', 'CANVAS#000000', '-c', 'FONT#FFFFFF', '-c', 'AXIS#FFFFFF', '-c', 'FRAME#FFFFFF', '-c', 'ARROW#FFFFFF',
unpack(g))
results = {}
m = {}
faulty = ""
APT.html = true
local web, e = io.open("results/Report-web.html", "w+")
if nil == web then C("opening mirrors file - " .. e) else
web:write( "Welcome to the apt-panopticon results page.
\n" ..
"
Here the apt-panopticon system probes into every nook and cranny of the Devuan apt package mirror system, trying to live up to it's name.
" .. "This is the status of the mirror servers in the Devuan package mirror network.
\n") if 0 < tonumber(APT.options.refresh.value) then web:write( 'This page will refresh every ' .. (APT.options.refresh.value / 60) .. ' minutes.
') end web:write( "EXPERIMENTAL CODE - double check all results you see here, and read the logs if it's important.
" .. "The full list of Devuan package mirrors is available at the URL: " .. "https://pkgmaster.devuan.org/mirror_list.txt
\n" .. "Due to the nature of the tests, some errors or warnings will be counted several times. " .. "The links in the table and DNS list go to the detailed testing logs.
\n\n" .. "FAILED or OK" .. " means the tested thing is supported for that mirror.
\n" .. "FAILED* or OK*" .. " means the tested thing is unsupported for that mirror, but might have been tested anyway.
\n" .. "timeout or timeout" .. " means the mirror had some timeouts, and tests where not yet aborted. The darker colour means unsupported by the mirror, but tested anyway.
" .. "TIMEOUT or TIMEOUT" .. " means the mirror had too many timeouts, and tests where aborted, so there is no result for this test.
" .. "NOTE: timeouts may be due to a problem on the testing computer, it might be busy with other things, or be having it's own network problems..
" .. "NOTE: the speed of the server this checking script is running on might be wildly exaggerated, it doesn't have to go through the Internet to download from itself.
" "The DNS round robin (DNS-RR) column shows the IPs for that mirror, or no if it isn't part of the DNS-RR. " .. "The IPs link to the testing log for that IP accessed via the DNS-RR. " .. APT.options.roundRobin.value .. " is the DNS-RR itself, so it doesn't get tested directly.
\n" .. "The time in the Updated column is how often the mirror updates itself.
" .. "Mirrors with a grey background are not active (though may be usable as part of the DNS-RR).
\n" .. "skip means that the test hasn't been written yet.
\n" ) makeTable(web, APT.mirrors) web:write( "This lists each mirror, and the DNS entries for that mirror. " .. "The links point to the testing log files (" .. logCount("apt-panopticon") .. ") for each domain name / IP combination that was tested. " .. "If a mirror has a CNAME, that CNAME is listed along with that CNAMEs DNS entries. " .. APT.options.roundRobin.value .. " is the DNS round robin, which points to the mirrors that are part of the DNS-RR. " .. "If an IP is part of the DNS-RR, it is marked with 'DNS-RR' " .. APT.options.referenceSite.value .. " is the master mirror, all the others sync to it. " .. "
\n" ) web:write(APT.dumpTableHTML(m, "")) web:write( "\nMore graphs. with greater detail.
NOTE - This is not fully probing the Debian mirrors, we just collect some data from any redirects to other servers. " .. "So this isn't a full set of tests.   Basically we don't know the shape of the Debian mirror infrastructure.
\n" .. "EXPERIMENTAL CODE - this is even more experimental than the rest.
\n" ) makeTable(web, APT.debians) web:write( "The email report. " .. "All the logs and other output. " .. "You can get the source code here (main repo)" .. "and here (Devuan repo). " .. "You can get the cgp graphing source code here (main repo)" .. "and here (Devuan repo).
\n" ) local whn = APT.exe('TZ="GMT" ls -dl1 --time-style="+%s" results/stamp | cut -d " " -f 6-6'):Do().result:sub(2, -2) web:write( "This run took " .. (os.time() - tonumber("0" .. whn)) .. " seconds.
" .. "\n\n") web:close() end