#!/usr/bin/env luajit local args = {...} verbosity = -1 local logFile local html = false --[[ Ordered table iterator, allow to iterate on the natural order of the keys of a table. From http://lua-users.org/wiki/SortedIteration ]] function __genOrderedIndex( t ) local orderedIndex = {} for key in pairs(t) do table.insert( orderedIndex, key ) end table.sort( orderedIndex ) return orderedIndex end function orderedNext(t, state) -- Equivalent of the next function, but returns the keys in the alphabetic -- order. We use a temporary ordered key table that is stored in the -- table being iterated. local key = nil --print("orderedNext: state = "..tostring(state) ) if state == nil then -- the first time, generate the index t.__orderedIndex = __genOrderedIndex( t ) key = t.__orderedIndex[1] else -- fetch the next value for i = 1,table.getn(t.__orderedIndex) do if t.__orderedIndex[i] == state then key = t.__orderedIndex[i+1] end end end if key then return key, t[key] end -- no more value to return, cleanup t.__orderedIndex = nil return end function orderedPairs(t) -- Equivalent of the pairs() function on tables. Allows to iterate -- in order return orderedNext, t, nil end -- Use this to dump a table to a string, with HTML. dumpTableHTML = function (table, space, name) local r = name .. "\n" r = r .. dumpTableHTMLSub(table, space .. " ") r = r .. space .. "" return r end dumpTableHTMLSub = function (table, space) local r = "" for k, v in orderedPairs(table) do if type(v) == "table" then if " " == space then r = r .. space .. dumpTableHTML(v, space, k .. "\n" else r = r .. "
  • " .. space .. dumpTableHTML(v, space, k .. "
  • \n" end else r = r .. space .. "
  • " .. k .. "
  • \n" end end return r end local checkFile = function(f) local h, e = io.open(f, "r") if nil == h then return false else h:close(); return true end end local plurals = function(e, w) local result = "" if 1 == e then result = e .. " error" elseif e ~= 0 then result = e .. " errors" end if ("" ~= result) and html then result = "" .. result .. "" end -- result = " " .. result if 0 < w then if 0 < e then result = result .. ", " end if 1 == w then result = result .. w .. " warning" else result = result .. w .. " warnings" end if ("" ~= result) and html then result = "" .. result .. "" end -- result = " " .. result end if "" ~= result then result = " (" .. result .. ")" end return result end local results = {} local log = function(v, t, s, prot, test, host) local x = "" if nil == prot then prot = "" end if nil ~= test then x = x .. test else test = "" end if nil ~= host then if #x > 0 then x = x .. " " end x = x .. host end if #x > 0 then t = t .. "(" .. x .. ")" end if v <= verbosity then if 3 <= verbosity then t = os.date() .. " " .. t end print(t .. ": " .. s) end if nil ~= logFile then logFile:write(os.date() .. " " .. t .. ": " .. s .. "\n") logFile:flush() end end local D = function(s) log(3, "DEBUG ", s) end local I = function(s) log(2, "INFO ", s) end local W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end local E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end local C = function(s) log(-1, "CRITICAL", s) end local mirrors = loadfile("results/mirrors.lua")() local revDNS = function(dom, IP) if "deb.devuan.org" ~= dom then if nil ~= mirrors["deb.devuan.org"] then if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][IP] then if html then return "DNS-RR" else return "DNS-RR" end end end else for k, v in pairs(mirrors) do if "deb.devuan.org" ~= k then local IPs = v.IPs for i, u in pairs(IPs) do if "table" == type(u) then for h, t in pairs(u) do if IP == h then return k end end else if IP == i then return k end end end end end end return "" end local faulty = "" local status = function(host, results, typ) local result = "" local e = 0 local w = 0 local s = nil ~= mirrors[host].Protocols[typ] local to = results.timeout if ('http' ~= typ) and ('https' ~= typ) and ('ftp' ~= typ) and ('rsync' ~= typ) then s = true end if nil ~= results[typ] then e = results[typ].errors w = results[typ].warnings for k, v in pairs(results[typ]) do if "table" == type(v) then if 0 <= v.errors then e = e + v.errors else to = true end if 0 <= v.warnings then w = w + v.warnings else to = true end end end else for k, v in pairs(results) do if "table" == type(v) then for i, u in pairs(v) do if "table" == type(u) then if typ == i then if 0 <= u.errors then e = e + u.errors end if 0 <= u.warnings then w = w + u.warnings end end end end end end end if to then result = "[TIMEOUT" if not s then result = result .. "*" end if html then if s then result = "[TIMEOUT" else result = "[TIMEOUT*" end end elseif 0 < e then result = "[FAILED" if not s then result = result .. "*" end if html then if s then result = "[FAILED" else result = "[FAILED*" end end if html then faulty = faulty .. host .. " (" .. typ .. ")
    \n" else faulty = faulty .. host .. " (" .. typ .. ")\n" end else result = "[OK" if not s then result = result .. "*" end if html then if s then result = "[OK" else result = "[OK*" end end end return result .. plurals(e, w) .. "]" end local collate = function(host, ip, results) local f = "results/" .. host .. "_" .. ip .. ".lua" local rfile, e = io.open(f, "r") if nil == rfile then I("opening " .. f .. " file - " .. e) else rfile:close() local rs = loadfile(f)() for k, v in pairs(rs) do if "table" == type(v) then if "speed" == k then if v.min < results.speed.min then results.speed.min = v.min end if v.max > results.speed.max then results.speed.max = v.max end else for i, u in pairs(v) do if "table" == type(u) then for h, t in pairs(u) do local a = results[k][i][h] if nil == a then a = 0 end results[k][i][h] = a + t end else local a = results[k][i] if nil == a then a = 0 end results[k][i] = a + u end end end elseif "timeout" ~= k then local a = results[k] if nil == a then a = 0 end results[k] = a + v end end end return results end local m = {} local logCount = function(domain, ip) local nm = "LOG_" .. domain local log = "" local extra = "" if nil ~= ip then nm = nm .. "_" .. ip end nm = nm .. ".html" local rfile, e = io.open("results/" .. nm, "r") if nil ~= rfile then local errors = 0 local warnings = 0 for l in rfile:lines() do if nil ~= l:match(">ERROR ") then errors = errors + 1 end if nil ~= l:match(">WARNING ") then warnings = warnings + 1 end end rfile:close() if html then if nil == ip then log = "" .. domain .. "" else log = "" .. ip .. "" end end log = log .. plurals(errors, warnings) end return log end html = false local email, e = io.open("results/Report-email.txt", "w+") if nil == email then C("opening mirrors file - " .. e) else email:write( "Dear Mirror Admins,\n\n" .. "This is the status of the mirror servers in the Devuan package mirror network.\n\n" .. "EXPERIMENTAL CODE - double check all results you see here, and read the logs if it's important." .. "The full list of Devuan package mirrors is available at the URL:\n\n" .. " https://pkgmaster.devuan.org/mirror_list.txt\n\n" .. 'Please contact "mirrors@devuan.org" if any of the information \nin the file above needs to be amended. \n\n' .. "The full results of the mirror checking is available at the URL:\n\n" .. " https://sledjhamr.org/apt-panopticon/results/Report-web.html\n\n" .. "Due to the nature of the tests, some errors or warnings will be \ncounted several times. " .. "Refer to the logs on the web page for details.\n\n" .. "Please see below the current status of the Devuan Package Mirror \nnetwork:\n\n" .. "==== package mirror status " .. os.date("!%Y-%m-%d %H:%M") .. " GMT ====\n" .. "[skip] means that the test hasn't been written yet.\n\n") for k, v in orderedPairs(mirrors) do local results = loadfile("results/" .. k .. ".lua")() email:write(k .. "....\n") local IPs = v.IPs for i, u in pairs(IPs) do if "table" == type(u) then for h, t in pairs(u) do results = collate(k, h, results) end else results = collate(k, i, results) end end local ftp = "[skip]" local http = status(k, results, "http") local https = status(k, results, "https") local rsync = "[skip]" local dns = "" local protocol = status(k, results, "Protocol") local sanity = status(k, results, "URLSanity") local integrity = status(k, results, "Integrity") local updated = status(k, results, "Updated") -- DNS-RR test. if ("deb.devuan.org" ~= k) and (nil ~= mirrors["deb.devuan.org"]) then for l, w in pairs(mirrors[k].IPs) do if type(w) == "table" then for i, u in pairs(w) do if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then local log = logCount("deb.devuan.org", i) if "" ~= log then if "" == dns then dns = " " else dns = dns .. " " end dns = dns .. logCount("deb.devuan.org", i) else if "" == dns then dns = " " else dns = dns .. " " end dns = dns .. i end end end else if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then local log = logCount("deb.devuan.org", l) if "" ~= log then if "" == dns then dns = " " else dns = dns .. " " end dns = dns .. log else if "" == dns then dns = " " else dns = dns .. " " end dns = dns .. l end end end end if "" == dns then dns = "[no]" end dns = " DNS-RR: " .. dns end email:write( " ftp: " .. ftp .. " http: " .. http .. " https: " .. https .." rsync: " .. rsync .. "\n" .. " " .. dns .. "\n" .. " Protocol: " .. protocol .. " URL-sanity: " .. sanity .. " Integrity: " .. integrity .. "\n" .. " Updated: " .. updated .. "\n") end email:write( "\n==== faulty mirrors: ====\n" .. faulty) email:write( "\n-------------------------\n\n" .. "* This means that this protocol isn't actually supported, but the test was run ayway.\n\n" .. "Thanks for your precious help in ensuring that Devuan GNU+Linux \nremains a universal, stable, dependable, free operating system.\n\n" .. "You can get the source code from https://sledjhamr.org/cgit/apt-panopticon/about/ .\n\n" .. "Love\n\n" .. "The Dev1Devs\n\n") email:close() end results = {} m = {} faulty = "" html = true local web, e = io.open("results/Report-web.html", "w+") if nil == web then C("opening mirrors file - " .. e) else web:write( "apt-panopticon results\n" .. '' .. "

    Welcome to the apt-panopticon results page.

    \n" .. "

    This is the status of the mirror servers in the Devuan package mirror network.

    \n" .. "

    EXPERIMENTAL CODE - double check all results you see here, and read the logs if it's important.

    " .. "

    The full list of Devuan package mirrors is available at the URL: " .. "https://pkgmaster.devuan.org/mirror_list.txt

    \n" .. "

    Due to the nature of the tests, some errors or warnings will be counted several times.   " .. "The links in the table and DNS list go to the detailed testing logs.

    \n\n" .. "
    \n

    ==== package mirror status " .. os.date("!%Y-%m-%d %H:%M") .. " GMT ====

    \n" .. "

    [FAILED] or [OK]" .. " means the tested thing is supported for that mirror.

    \n" .. "

    [FAILED*] or [OK*]" .. " means the tested thing is unsupported for that mirror, but might have been tested anyway.

    \n" .. "

    [TIMEOUT] or [TIMEOUT]" .. " means the server had too many timeouts, and tests where aborted, so there is no result for this test.

    " .. "

    The DNS round robin (DNS-RR) column shows the IPs for that mirror, or [no] if it isn't part of the DNS-RR.   " .. "The IPs link to the testing log for that IP accessed via the DNS-RR.   " .. "deb.devuan.org is the DNS-RR itself, so it doesn't get tested directly.

    \n" .. "

    The time in the Updated column is how often the mirror updates itself.

    " .. "

    Mirrors with a grey background are not active (though may be usable as part of the DNS-RR).

    \n" .. "

    [skip] means that the test hasn't been written yet.

    \n" .. "\n" .. "\n" ) for k, v in orderedPairs(mirrors) do local results = loadfile("results/" .. k .. ".lua")() local active = "" if "yes" == v.Active then web:write(" ") else if nil == v.Active then active = 'nil' else active = v.Active end web:write(" ") end local IPs = v.IPs for i, u in pairs(IPs) do if "table" == type(u) then for h, t in pairs(u) do results = collate(k, h, results) end else results = collate(k, i, results) end end local ftp = "[skip]" local http = status(k, results, "http") local https = status(k, results, "https") local rsync = "[skip]" local dns = "" local protocol = status(k, results, "Protocol") local sanity = status(k, results, "URLSanity") local integrity = status(k, results, "Integrity") local updated = status(k, results, "Updated") local rate = v.Rate if nil ~= rate then updated = updated .. ' ' .. rate end local min = tonumber(results.speed.min) local max = tonumber(results.speed.max) local spd = '' -- DNS-RR test. if ("deb.devuan.org" ~= k) and (nil ~= mirrors["deb.devuan.org"]) then for l, w in pairs(mirrors[k].IPs) do if type(w) == "table" then for i, u in pairs(w) do if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then local log = logCount("deb.devuan.org", i) if "" ~= log then if "" == dns then dns = " " else dns = dns .. "   " end dns = dns .. logCount("deb.devuan.org", i) else if "" == dns then dns = " " else dns = dns .. "   " end dns = dns .. "" .. i .. "" end end end else if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then local log = logCount("deb.devuan.org", l) if "" ~= log then if "" == dns then dns = " " else dns = dns .. "   " end dns = dns .. log else if "" == dns then dns = " " else dns = dns .. "   " end dns = dns .. "" .. l .. "" end end end end if "" == dns then dns = "[no]" end if 0 == max then spd = '' else spd = string.format('', min, max) end end web:write("" .. spd .. "\n") if "" ~= active then web:write("\n") end end web:write( "
    FTPHTTPHTTPSRSYNCDNS round robinProtocolURL sanityIntegrityUpdatedSpeed range
    " .. k .. "
    " .. k .. "%d ->%d" .. ftp .. " " .. http .. " " .. https .. " " .. rsync .. " " .. dns .. " " .. protocol .. " " .. sanity .. " " .. integrity .. " " .. updated .. " 
    " .. active .. "
    \n
    \n

    ==== faulty mirrors: ====

    \n" .. faulty) web:write( "
    \n
    \n

    ==== DNS and logs: ====

    \n") for k, v in pairs(mirrors) do local log = k local n = {} log = logCount(k) mirrors[k].Protocols = nil mirrors[k].FQDN = nil mirrors[k].Active = nil mirrors[k].Rate = nil mirrors[k].BaseURL = nil mirrors[k].Country = nil mirrors[k].Bandwidth = nil for l, w in pairs(mirrors[k].IPs) do if type(w) == "table" then n[l] = {} for i, u in pairs(w) do local log = logCount(k, i) if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(k, i)] = u end end else local log = logCount(k, l) if "" == log then n[l] = w else n[log .. " " .. revDNS(k, l)] = w end end end m[log .. " DNS entries -"] = n end web:write( "

    This lists each mirror, and the DNS entries for that mirror.   " .. "The links point to the testing log files for " .. logCount("apt-panopticon") .. " for each domain name / IP combination that was tested.   " .. "If a mirror has a CNAME, that CNAME is listed along with that CNAMEs DNS entries.   " .. "deb.devuan.org is the DNS round robin, which points to the mirrors that are part of the DNS-RR.   " .. "If an IP is part of the DNS-RR, it is marked with 'DNS-RR'   " .. "pkgmaster.devuan.org is the master mirror, all the others sync to it.   " .. "

    \n" ) web:write(dumpTableHTML(m, "", "")) web:write( "\n
    \n
    \n\n" .. "

    The email report.   " .. "All the logs and other output.   " .. "You can get the source code here.

    " .. "\n") web:close() end