#!/usr/bin/env luajit local args = {...} verbosity = -1 local logFile local html = false --[[ Ordered table iterator, allow to iterate on the natural order of the keys of a table. From http://lua-users.org/wiki/SortedIteration ]] function __genOrderedIndex( t ) local orderedIndex = {} for key in pairs(t) do table.insert( orderedIndex, key ) end table.sort( orderedIndex ) return orderedIndex end function orderedNext(t, state) -- Equivalent of the next function, but returns the keys in the alphabetic -- order. We use a temporary ordered key table that is stored in the -- table being iterated. local key = nil --print("orderedNext: state = "..tostring(state) ) if state == nil then -- the first time, generate the index t.__orderedIndex = __genOrderedIndex( t ) key = t.__orderedIndex[1] else -- fetch the next value for i = 1,table.getn(t.__orderedIndex) do if t.__orderedIndex[i] == state then key = t.__orderedIndex[i+1] end end end if key then return key, t[key] end -- no more value to return, cleanup t.__orderedIndex = nil return end function orderedPairs(t) -- Equivalent of the pairs() function on tables. Allows to iterate -- in order return orderedNext, t, nil end -- Use this to dump a table to a string, with HTML. dumpTableHTML = function (table, space, name) local r = name .. "\n" r = r .. dumpTableHTMLSub(table, space .. " ") r = r .. space .. "" return r end dumpTableHTMLSub = function (table, space) local r = "" for k, v in orderedPairs(table) do if type(v) == "table" then if " " == space then r = r .. space .. dumpTableHTML(v, space, k .. "\n" else r = r .. "
  • " .. space .. dumpTableHTML(v, space, k .. "
  • \n" end else r = r .. space .. "
  • " .. k .. "
  • \n" end end return r end local plurals = function(e, w) local result = "" if 1 == e then result = e .. " error" elseif e ~= 0 then result = e .. " errors" end if ("" ~= result) and html then result = "" .. result .. "" end -- result = " " .. result if 0 < w then if 0 < e then result = result .. ", " end if 1 == w then result = result .. w .. " warning" else result = result .. w .. " warnings" end if ("" ~= result) and html then result = "" .. result .. "" end -- result = " " .. result end if "" ~= result then result = " (" .. result .. ")" end return result end local results = {} local log = function(v, t, s, prot, test, host) local x = "" if nil == prot then prot = "" end if nil ~= test then x = x .. test else test = "" end if nil ~= host then if #x > 0 then x = x .. " " end x = x .. host end if #x > 0 then t = t .. "(" .. x .. ")" end if v <= verbosity then if 3 <= verbosity then t = os.date() .. " " .. t end print(t .. ": " .. s) end if nil ~= logFile then logFile:write(os.date() .. " " .. t .. ": " .. s .. "\n") logFile:flush() end end local D = function(s) log(3, "DEBUG ", s) end local I = function(s) log(2, "INFO ", s) end local W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end local E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end local C = function(s) log(-1, "CRITICAL", s) end local mirrors = loadfile("results/mirrors.lua")() local revDNS = function(dom, IP) if "deb.devuan.org" ~= dom then if nil ~= mirrors["deb.devuan.org"] then if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][IP] then if html then return "DNS-RR" else return "DNS-RR" end end end else for k, v in pairs(mirrors) do if "deb.devuan.org" ~= k then local IPs = v.IPs for i, u in pairs(IPs) do if "table" == type(u) then for h, t in pairs(u) do if IP == h then return k end end else if IP == i then return k end end end end end end return "" end local faulty = "" local status = function(host, results, typ) local result = "" local e = 0 local w = 0 local s = nil ~= mirrors[host].Protocols[typ] local to = false if ('http' ~= typ) and ('https' ~= typ) and ('ftp' ~= typ) and ('rsync' ~= typ) then s = true end if nil ~= results[typ] then e = results[typ].errors w = results[typ].warnings --[[ for k, v in pairs(results[typ]) do if "table" == type(v) then if 0 <= v.errors then e = e + v.errors else to = true end if 0 <= v.warnings then w = w + v.warnings else to = true end end end ]] else for k, v in pairs(results) do if "table" == type(v) then for i, u in pairs(v) do if "table" == type(u) then if typ == i then if 0 <= u.errors then e = e + u.errors else to = true end if 0 <= u.warnings then w = w + u.warnings else to = true end end end end end end end if to then result = "[TIMEOUT" if not s then result = result .. "*" end if html then if s then result = "[TIMEOUT" else result = "[TIMEOUT*" end end elseif 0 < e then result = "[FAILED" if not s then result = result .. "*" end if html then if s then result = "[FAILED" else result = "[FAILED*" end end if html then faulty = faulty .. host .. " (" .. typ .. ")
    \n" else faulty = faulty .. host .. " (" .. typ .. ")\n" end else result = "[OK" if not s then result = result .. "*" end if html then if s then result = "[OK" else result = "[OK*" end end end return result .. plurals(e, w) .. "]" end local collate = function(host, ip, results) local f = "results/" .. host .. "_" .. ip .. ".lua" local rfile, e = io.open(f, "r") if nil == rfile then I("opening " .. f .. " file - " .. e) else rfile:close() local rs = loadfile(f)() for k, v in pairs(rs) do if "table" == type(v) then for i, u in pairs(v) do if "table" == type(u) then for h, t in pairs(u) do local a = results[k][h] if nil == a then a = 0 end results[k][h] = a + t end else local a = results[k][i] if nil == a then a = 0 end results[k][i] = a + u end end else local a = results[k] if nil == a then a = 0 end results[k] = a + v end end end return results end local m = {} local logCount = function(domain, ip) local nm = "LOG_" .. domain local log = "" local extra = "" if nil ~= ip then nm = nm .. "_" .. ip end nm = nm .. ".html" local rfile, e = io.open("results/" .. nm, "r") if nil ~= rfile then local errors = 0 local warnings = 0 for l in rfile:lines() do if nil ~= l:match(">ERROR ") then errors = errors + 1 end if nil ~= l:match(">WARNING ") then warnings = warnings + 1 end end rfile:close() if html then if nil == ip then log = "" .. domain .. "" else log = "" .. ip .. "" end end log = log .. plurals(errors, warnings) end return log end html = false local email, e = io.open("results/Report-email.txt", "w+") if nil == email then C("opening mirrors file - " .. e) else email:write( "Dear Mirror Admins,\n\n" .. "This is the status of the mirror servers in the Devuan package mirror network.\n\n" .. "The full list of Devuan package mirrors is available at the URL:\n\n" .. " https://pkgmaster.devuan.org/mirror_list.txt\n\n" .. 'Please contact "mirrors@devuan.org" if any of the information \nin the file above needs to be amended. \n\n' .. "The full results of the mirror checking is available at the URL:\n\n" .. " https://sledjhamr.org/apt-panopticon/results/Report-web.html\n\n" .. "Due to the nature of the tests, some errors or warnings will be \ncounted several times. " .. "Refer to the logs on the web page for details.\n\n" .. "Please see below the current status of the Devuan Package Mirror \nnetwork:\n\n" .. "==== package mirror status " .. os.date("!%Y-%m-%d %H:%M") .. " GMT ====\n" .. "[skip] means that the test hasn't been written yet.\n\n") for k, v in orderedPairs(mirrors) do local results = loadfile("results/" .. k .. ".lua")() email:write(k .. "....\n") local IPs = v.IPs for i, u in pairs(IPs) do if "table" == type(u) then for h, t in pairs(u) do results = collate(k, h, results) end else results = collate(k, i, results) end end local ftp = "[skip]" local http = status(k, results, "http") local https = status(k, results, "https") local rsync = "[skip]" local dns = "" local protocol = status(k, results, "Protocol") local sanity = "[skip]" local integrity = status(k, results, "Integrity") local updated = status(k, results, "Updated") -- DNS-RR test. if ("deb.devuan.org" ~= k) and (nil ~= mirrors["deb.devuan.org"]) then for l, w in pairs(mirrors[k].IPs) do if type(w) == "table" then for i, u in pairs(w) do if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then local log = logCount("deb.devuan.org", i) if "" ~= log then if "" == dns then dns = " " else dns = dns .. " " end dns = dns .. logCount("deb.devuan.org", i) else if "" == dns then dns = " " else dns = dns .. " " end dns = dns .. i end end end else if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then local log = logCount("deb.devuan.org", l) if "" ~= log then if "" == dns then dns = " " else dns = dns .. " " end dns = dns .. log else if "" == dns then dns = " " else dns = dns .. " " end dns = dns .. l end end end end if "" == dns then dns = "[no]" end dns = " DNS-RR: " .. dns end email:write( " ftp: " .. ftp .. " http: " .. http .. " https: " .. https .." rsync: " .. rsync .. "\n" .. " " .. dns .. "\n" .. " Protocol: " .. protocol .. " URL-sanity: " .. sanity .. " Integrity: " .. integrity .. "\n" .. " Updated: " .. updated .. "\n") end email:write( "\n==== faulty mirrors: ====\n" .. faulty) email:write( "\n-------------------------\n\n" .. "* This means that this protocol isn't actually supported, but the test was run ayway.\n\n" .. "Thanks for your precious help in ensuring that Devuan GNU+Linux \nremains a universal, stable, dependable, free operating system.\n\n" .. "You can get the source code from https://sledjhamr.org/cgit/apt-panopticon/about/ .\n\n" .. "Love\n\n" .. "The Dev1Devs\n\n") email:close() end results = {} m = {} faulty = "" html = true local web, e = io.open("results/Report-web.html", "w+") if nil == web then C("opening mirrors file - " .. e) else web:write( "apt-panopticon results\n" .. '' .. "

    Welcome to the apt-panopticon results page.

    \n" .. "

    This is the status of the mirror servers in the Devuan package mirror network.

    \n" .. "

    The full list of Devuan package mirrors is available at the URL: " .. "https://pkgmaster.devuan.org/mirror_list.txt

    \n" .. "

    Due to the nature of the tests, some errors or warnings will be counted several times.   " .. "The links in the table and DNS list go to the detailed testing logs.

    \n\n" .. "
    \n

    ==== package mirror status " .. os.date("!%Y-%m-%d %H:%M") .. " GMT ====

    \n\n" .. "

    [FAILED] or [OK]" .. " means the tested thing is supported for that mirror.

    \n" .. "

    [FAILED*] or [OK*]" .. " means the tested thing is unsupported for that mirror, but might have been tested anyway.

    \n" .. "

    [TIMEOUT] or [TIMEOUT]" .. " means the server had too many timeouts, and tests where aborted, so there is no result for this test.

    " .. "

    The DNS round robin (DNS-RR) column shows the IPs for that mirror, or [no] if it isn't part of the DNS-RR.   " .. "The IPs link to the testing log for that IP accessed via the DNS-RR.   " .. "deb.devuan.org is the DNS-RR itself, so it doesn't get tested directly.

    \n" .. "

    The time in the Updated column is how often the mirror updates itself.

    " .. "

    Mirrors with a grey background are not active (though may be usable as part of the DNS-RR).

    \n" .. "

    [skip] means that the test hasn't been written yet.

    \n" .. "\n" ) for k, v in orderedPairs(mirrors) do local results = loadfile("results/" .. k .. ".lua")() local active = "" if "yes" == v.Active then web:write(" ") else if nil == v.Active then active = 'nil' else active = v.Active end web:write(" ") end local IPs = v.IPs for i, u in pairs(IPs) do if "table" == type(u) then for h, t in pairs(u) do results = collate(k, h, results) end else results = collate(k, i, results) end end local ftp = "[skip]" local http = status(k, results, "http") local https = status(k, results, "https") local rsync = "[skip]" local dns = "" local protocol = status(k, results, "Protocol") local sanity = "[skip]" local integrity = status(k, results, "Integrity") local updated = status(k, results, "Updated") local rate = v.Rate if nil ~= rate then updated = updated .. ' ' .. rate end -- DNS-RR test. if ("deb.devuan.org" ~= k) and (nil ~= mirrors["deb.devuan.org"]) then for l, w in pairs(mirrors[k].IPs) do if type(w) == "table" then for i, u in pairs(w) do if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then local log = logCount("deb.devuan.org", i) if "" ~= log then if "" == dns then dns = " " else dns = dns .. "   " end dns = dns .. logCount("deb.devuan.org", i) else if "" == dns then dns = " " else dns = dns .. "   " end dns = dns .. "" .. i .. "" end end end else if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then local log = logCount("deb.devuan.org", l) if "" ~= log then if "" == dns then dns = " " else dns = dns .. "   " end dns = dns .. log else if "" == dns then dns = " " else dns = dns .. "   " end dns = dns .. "" .. l .. "" end end end end if "" == dns then dns = "[no]" end end web:write("\n") if "" ~= v.Active then web:write("\n") end end web:write( "
    FTPHTTPHTTPSRSYNCDNS round robinProtocolURL sanityIntegrityUpdated
    " .. k .. "
    " .. k .. "" .. ftp .. " " .. http .. " " .. https .. " " .. rsync .. " " .. dns .. " " .. protocol .. " " .. sanity .. " " .. integrity .. " " .. updated .. "
    " .. active .. "
    \n
    \n

    ==== faulty mirrors: ====

    \n" .. faulty) web:write( "
    \n
    \n

    ==== DNS and logs: ====

    \n") for k, v in pairs(mirrors) do local log = k local n = {} log = logCount(k) mirrors[k].Protocols = nil mirrors[k].FQDN = nil mirrors[k].Active = nil mirrors[k].Rate = nil mirrors[k].BaseURL = nil mirrors[k].Country = nil mirrors[k].Bandwidth = nil for l, w in pairs(mirrors[k].IPs) do if type(w) == "table" then n[l] = {} for i, u in pairs(w) do local log = logCount(k, i) if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(k, i)] = u end end else local log = logCount(k, l) if "" == log then n[l] = w else n[log .. " " .. revDNS(k, l)] = w end end end m[log .. " DNS entries -"] = n end web:write( "

    This lists each mirror, and the DNS entries for that mirror.   " .. "The links point to the testing log files for " .. logCount("apt-panopticon") .. " for each domain name / IP combination that was tested.   " .. "If a mirror has a CNAME, that CNAME is listed along with that CNAMEs DNS entries.   " .. "deb.devuan.org is the DNS round robin, which points to the mirrors that are part of the DNS-RR.   " .. "If an IP is part of the DNS-RR, it is marked with 'DNS-RR'   " .. "pkgmaster.devuan.org is the master mirror, all the others sync to it.   " .. "

    \n" ) web:write(dumpTableHTML(m, "", "")) web:write( "\n
    \n
    \n\n" .. "

    The email report.   " .. "All the logs and other output.   " .. "You can get the source code here.

    " .. "\n") web:close() end