From 3829c9b7962f11e760361deb764fc96af1345c62 Mon Sep 17 00:00:00 2001 From: onefang Date: Mon, 2 Dec 2019 06:28:30 +1000 Subject: Merge email and web report scripts. --- apt-panopticon-report-email-web.lua | 525 ++++++++++++++++++++++++++++++++++++ apt-panopticon-report-email.lua | 292 -------------------- apt-panopticon-report-web.lua | 406 ---------------------------- apt-panopticon.lua | 3 +- 4 files changed, 526 insertions(+), 700 deletions(-) create mode 100755 apt-panopticon-report-email-web.lua delete mode 100755 apt-panopticon-report-email.lua delete mode 100755 apt-panopticon-report-web.lua diff --git a/apt-panopticon-report-email-web.lua b/apt-panopticon-report-email-web.lua new file mode 100755 index 0000000..60027d0 --- /dev/null +++ b/apt-panopticon-report-email-web.lua @@ -0,0 +1,525 @@ +#!/usr/bin/env luajit + +local args = {...} + +verbosity = -1 +local logFile +local html = false + + +--[[ Ordered table iterator, allow to iterate on the natural order of the keys of a table. + From http://lua-users.org/wiki/SortedIteration + ]] +function __genOrderedIndex( t ) + local orderedIndex = {} + for key in pairs(t) do + table.insert( orderedIndex, key ) + end + table.sort( orderedIndex ) + return orderedIndex +end +function orderedNext(t, state) + -- Equivalent of the next function, but returns the keys in the alphabetic + -- order. We use a temporary ordered key table that is stored in the + -- table being iterated. + + local key = nil + --print("orderedNext: state = "..tostring(state) ) + if state == nil then + -- the first time, generate the index + t.__orderedIndex = __genOrderedIndex( t ) + key = t.__orderedIndex[1] + else + -- fetch the next value + for i = 1,table.getn(t.__orderedIndex) do + if t.__orderedIndex[i] == state then + key = t.__orderedIndex[i+1] + end + end + end + + if key then + return key, t[key] + end + + -- no more value to return, cleanup + t.__orderedIndex = nil + return +end +function orderedPairs(t) + -- Equivalent of the pairs() function on tables. Allows to iterate + -- in order + return orderedNext, t, nil +end + +-- Use this to dump a table to a string, with HTML. +dumpTableHTML = function (table, space, name) + local r = name .. "\n" + r = r .. dumpTableHTMLSub(table, space .. " ") + r = r .. space .. "" + return r +end +dumpTableHTMLSub = function (table, space) + local r = "" + for k, v in orderedPairs(table) do + if type(v) == "table" then + if " " == space then + r = r .. space .. dumpTableHTML(v, space, k .. "\n" + else + r = r .. "
  • " .. space .. dumpTableHTML(v, space, k .. "
  • \n" + end + else + r = r .. space .. "
  • " .. k .. "
  • \n" + end + end + return r +end + +local plurals = function(e, w) + local result = "" + if 1 == e then + result = e .. " error" + elseif e ~= 0 then + result = e .. " errors" + end + if ("" ~= result) and html then result = "" .. result .. "" end +-- result = " " .. result + if 0 < w then + if 0 < e then result = result .. ", " end + if 1 == w then + result = result .. w .. " warning" + else + result = result .. w .. " warnings" + end + if ("" ~= result) and html then result = "" .. result .. "" end +-- result = " " .. result + end + if "" ~= result then result = " (" .. result .. ")" end + return result +end + +local results = {} + +local log = function(v, t, s, prot, test, host) + local x = "" + if nil == prot then prot = "" end + if nil ~= test then x = x .. test else test = "" end + if nil ~= host then + if #x > 0 then x = x .. " " end + x = x .. host + end + if #x > 0 then + t = t .. "(" .. x .. ")" + end + if v <= verbosity then + if 3 <= verbosity then t = os.date() .. " " .. t end + print(t .. ": " .. s) + end + if nil ~= logFile then + logFile:write(os.date() .. " " .. t .. ": " .. s .. "\n") + logFile:flush() + end +end +local D = function(s) log(3, "DEBUG ", s) end +local I = function(s) log(2, "INFO ", s) end +local W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end +local E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end +local C = function(s) log(-1, "CRITICAL", s) end + +local mirrors = loadfile("results/mirrors.lua")() + +local revDNS = function(dom, IP) + if "deb.devuan.org" ~= dom then + if nil ~= mirrors["deb.devuan.org"] then + if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][IP] then + if html then + return "DNS-RR" + else + return "DNS-RR" + end + end + end + else + for k, v in pairs(mirrors) do + if "deb.devuan.org" ~= k then + local IPs = v.IPs + for i, u in pairs(IPs) do + if "table" == type(u) then + for h, t in pairs(u) do + if IP == h then return k end + end + else + if IP == i then return k end + end + end + end + end + end + return "" +end + +local faulty = "" +local status = function(host, results, typ) + local result = "" + local e = 0 + local w = 0 + local s = nil ~= mirrors[host].Protocols[typ] + local to = false + if ('http' ~= typ) and ('https' ~= typ) and ('ftp' ~= typ) and ('rsync' ~= typ) then s = true end + if nil ~= results[typ] then + e = results[typ].errors + w = results[typ].warnings +--[[ + for k, v in pairs(results[typ]) do + if "table" == type(v) then + if 0 <= v.errors then e = e + v.errors else to = true end + if 0 <= v.warnings then w = w + v.warnings else to = true end + end + end +]] + else + for k, v in pairs(results) do + if "table" == type(v) then + for i, u in pairs(v) do + if "table" == type(u) then + if typ == i then + if 0 <= u.errors then e = e + u.errors else to = true end + if 0 <= u.warnings then w = w + u.warnings else to = true end + end + end + end + end + end + end + + if to then + result = "[TIMEOUT" + if not s then result = result .. "*" end + if html then + if s then + result = "[TIMEOUT" + else + result = "[TIMEOUT*" + end + end + elseif 0 < e then + result = "[FAILED" + if not s then result = result .. "*" end + if html then + if s then + result = "[FAILED" + else + result = "[FAILED*" + end + end + if html then + faulty = faulty .. host .. " (" .. typ .. ")
    \n" + else + faulty = faulty .. host .. " (" .. typ .. ")\n" + end + else + result = "[OK" + if not s then result = result .. "*" end + if html then + if s then + result = "[OK" + else + result = "[OK*" + end + end + end + return result .. plurals(e, w) .. "]" +end + +local collate = function(host, ip, results) + local f = "results/" .. host .. "_" .. ip .. ".lua" + local rfile, e = io.open(f, "r") + if nil == rfile then I("opening " .. f .. " file - " .. e) else + rfile:close() + local rs = loadfile(f)() + for k, v in pairs(rs) do + if "table" == type(v) then + for i, u in pairs(v) do + if "table" == type(u) then + for h, t in pairs(u) do + local a = results[k][h] + if nil == a then a = 0 end + results[k][h] = a + t + end + else + local a = results[k][i] + if nil == a then a = 0 end + results[k][i] = a + u + end + end + else + local a = results[k] + if nil == a then a = 0 end + results[k] = a + v + end + end + end + return results +end + +local m = {} + +local logCount = function(domain, ip) + local nm = "LOG_" .. domain + local log = "" + local extra = "" + if nil ~= ip then nm = nm .. "_" .. ip end + nm = nm .. ".html" + local rfile, e = io.open("results/" .. nm, "r") + if nil ~= rfile then + local errors = 0 + local warnings = 0 + for l in rfile:lines() do + if nil ~= l:match(">ERROR ") then errors = errors + 1 end + if nil ~= l:match(">WARNING ") then warnings = warnings + 1 end + end + rfile:close() + if html then + if nil == ip then + log = "" .. domain .. "" + else + log = "" .. ip .. "" + end + end + log = log .. plurals(errors, warnings) + end + return log +end + + +html = false +local email, e = io.open("results/Report-email.txt", "w+") +if nil == email then C("opening mirrors file - " .. e) else + email:write( "Dear Mirror Admins,\n\n" .. + "This is the status of the mirror servers in the Devuan package mirror network.\n\n" .. + "The full list of Devuan package mirrors is available at the URL:\n\n" .. + " https://pkgmaster.devuan.org/mirror_list.txt\n\n" .. + 'Please contact "mirrors@devuan.org" if any of the information \nin the file above needs to be amended. \n\n' .. + "The full results of the mirror checking is available at the URL:\n\n" .. + " https://sledjhamr.org/apt-panopticon/results/Report-web.html\n\n" .. + "Due to the nature of the tests, some errors or warnings will be \ncounted several times. " .. + "Refer to the logs on the web page for details.\n\n" .. + "Please see below the current status of the Devuan Package Mirror \nnetwork:\n\n" .. + "==== package mirror status " .. os.date("!%Y-%m-%d %H:%M") .. " GMT ====\n" .. + "[skip] means that the test hasn't been written yet.\n\n") + for k, v in orderedPairs(mirrors) do + local results = loadfile("results/" .. k .. ".lua")() + email:write(k .. "....\n") + local IPs = v.IPs + for i, u in pairs(IPs) do + if "table" == type(u) then + for h, t in pairs(u) do + results = collate(k, h, results) + end + else + results = collate(k, i, results) + end + end + local ftp = "[skip]" + local http = status(k, results, "http") + local https = status(k, results, "https") + local rsync = "[skip]" + local dns = "" + local protocol = status(k, results, "Protocol") + local sanity = "[skip]" + local integrity = status(k, results, "Integrity") + local updated = status(k, results, "Updated") + + -- DNS-RR test. + if ("deb.devuan.org" ~= k) and (nil ~= mirrors["deb.devuan.org"]) then + for l, w in pairs(mirrors[k].IPs) do + if type(w) == "table" then + for i, u in pairs(w) do + if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then + local log = logCount("deb.devuan.org", i) + if "" ~= log then + if "" == dns then dns = " " else dns = dns .. " " end + dns = dns .. logCount("deb.devuan.org", i) + else + if "" == dns then dns = " " else dns = dns .. " " end + dns = dns .. i + end + end + end + else + if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then + local log = logCount("deb.devuan.org", l) + if "" ~= log then + if "" == dns then dns = " " else dns = dns .. " " end + dns = dns .. log + else + if "" == dns then dns = " " else dns = dns .. " " end + dns = dns .. l + end + end + end + end + if "" == dns then dns = "[no]" end + dns = " DNS-RR: " .. dns + end + + email:write( " ftp: " .. ftp .. " http: " .. http .. " https: " .. https .." rsync: " .. rsync .. "\n" .. + " " .. dns .. "\n" .. + " Protocol: " .. protocol .. " URL-sanity: " .. sanity .. " Integrity: " .. integrity .. "\n" .. + " Updated: " .. updated .. "\n") + end + email:write( "\n==== faulty mirrors: ====\n" .. faulty) + email:write( "\n-------------------------\n\n" .. + "* This means that this protocol isn't actually supported, but the test was run ayway.\n\n" .. + "Thanks for your precious help in ensuring that Devuan GNU+Linux \nremains a universal, stable, dependable, free operating system.\n\n" .. + "You can get the source code from https://sledjhamr.org/cgit/apt-panopticon/about/ .\n\n" .. + "Love\n\n" .. + "The Dev1Devs\n\n") + email:close() +end + + +results = {} +m = {} +faulty = "" +html = true +local web, e = io.open("results/Report-web.html", "w+") +if nil == web then C("opening mirrors file - " .. e) else + web:write( "apt-panopticon results\n" .. + '' .. + "

    Welcome to the apt-panopticon results page.

    \n" .. + "

    This is the status of the mirror servers in the Devuan package mirror network.

    \n" .. + "

    The full list of Devuan package mirrors is available at the URL: " .. + "https://pkgmaster.devuan.org/mirror_list.txt

    \n" .. + "

    Due to the nature of the tests, some errors or warnings will be counted several times.   " .. + "The links in the table and DNS list go to the detailed testing logs.

    \n\n" .. + "
    \n

    ==== package mirror status " .. os.date("!%Y-%m-%d %H:%M") .. " GMT ====

    \n\n" .. + + "

    [FAILED] or [OK]" .. + " means the tested thing is supported for that mirror.

    \n" .. + "

    [FAILED*] or [OK*]" .. + " means the tested thing is unsupported for that mirror, but might have been tested anyway.

    \n" .. + "

    [TIMEOUT] or [TIMEOUT]" .. + " means the server had too many timeouts, and tests where aborted, so there is no result for this test.

    " .. + "

    The DNS round robin (DNS-RR) column shows the IPs for that mirror, or [no] if it isn't part of the DNS-RR.   " .. + "The IPs link to the testing log for that IP accessed via the DNS-RR.   " .. + "deb.devuan.org is the DNS-RR itself, so it doesn't get tested directly.

    \n" .. + "

    The time in the Updated column is how often the mirror updates itself.

    " .. + "

    Mirrors with a grey background are not active (though may be usable as part of the DNS-RR).

    \n" .. + "

    [skip] means that the test hasn't been written yet.

    \n" .. + "\n" + ) + for k, v in orderedPairs(mirrors) do + local results = loadfile("results/" .. k .. ".lua")() + local active = "" + if "yes" == v.Active then + web:write(" ") + else + if nil == v.Active then active = 'nil' else active = v.Active end + web:write(" ") + end + local IPs = v.IPs + for i, u in pairs(IPs) do + if "table" == type(u) then + for h, t in pairs(u) do + results = collate(k, h, results) + end + else + results = collate(k, i, results) + end + end + local ftp = "[skip]" + local http = status(k, results, "http") + local https = status(k, results, "https") + local rsync = "[skip]" + local dns = "" + local protocol = status(k, results, "Protocol") + local sanity = "[skip]" + local integrity = status(k, results, "Integrity") + local updated = status(k, results, "Updated") + local rate = v.Rate + if nil ~= rate then updated = updated .. ' ' .. rate end + + -- DNS-RR test. + if ("deb.devuan.org" ~= k) and (nil ~= mirrors["deb.devuan.org"]) then + for l, w in pairs(mirrors[k].IPs) do + if type(w) == "table" then + for i, u in pairs(w) do + if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then + local log = logCount("deb.devuan.org", i) + if "" ~= log then + if "" == dns then dns = " " else dns = dns .. "   " end + dns = dns .. logCount("deb.devuan.org", i) + else + if "" == dns then dns = " " else dns = dns .. "   " end + dns = dns .. "" .. i .. "" + end + end + end + else + if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then + local log = logCount("deb.devuan.org", l) + if "" ~= log then + if "" == dns then dns = " " else dns = dns .. "   " end + dns = dns .. log + else + if "" == dns then dns = " " else dns = dns .. "   " end + dns = dns .. "" .. l .. "" + end + end + end + end + if "" == dns then dns = "[no]" end + end + + web:write("\n") + if "" ~= v.Active then + web:write("\n") + end + end + web:write( "
    FTPHTTPHTTPSRSYNCDNS round robinProtocolURL sanityIntegrityUpdated
    " .. k .. "
    " .. k .. "" .. ftp .. " " .. http .. " " .. https .. " " .. rsync .. " " .. dns .. + " " .. protocol .. " " .. sanity .. + " " .. integrity .. " " .. updated .. "
    " .. active .. "
    \n
    \n

    ==== faulty mirrors: ====

    \n" .. faulty) + web:write( "
    \n
    \n

    ==== DNS and logs: ====

    \n") + + for k, v in pairs(mirrors) do + local log = k + local n = {} + log = logCount(k) + mirrors[k].Protocols = nil + mirrors[k].FQDN = nil + mirrors[k].Active = nil + mirrors[k].Rate = nil + mirrors[k].BaseURL = nil + mirrors[k].Country = nil + mirrors[k].Bandwidth = nil + for l, w in pairs(mirrors[k].IPs) do + if type(w) == "table" then + n[l] = {} + for i, u in pairs(w) do + local log = logCount(k, i) + if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(k, i)] = u end + end + else + local log = logCount(k, l) + if "" == log then n[l] = w else n[log .. " " .. revDNS(k, l)] = w end + end + end + m[log .. " DNS entries -"] = n + end + web:write( "

    This lists each mirror, and the DNS entries for that mirror.   " .. + "The links point to the testing log files for " .. logCount("apt-panopticon") .. " for each domain name / IP combination that was tested.   " .. + "If a mirror has a CNAME, that CNAME is listed along with that CNAMEs DNS entries.   " .. + "deb.devuan.org is the DNS round robin, which points to the mirrors that are part of the DNS-RR.   " .. + "If an IP is part of the DNS-RR, it is marked with 'DNS-RR'   " .. + "pkgmaster.devuan.org is the master mirror, all the others sync to it.   " .. + "

    \n" + ) + web:write(dumpTableHTML(m, "", "")) + web:write( "\n
    \n
    \n\n" .. + "

    The email report.   " .. + "All the logs and other output.   " .. + "You can get the source code here.

    " .. + "\n") + web:close() +end diff --git a/apt-panopticon-report-email.lua b/apt-panopticon-report-email.lua deleted file mode 100755 index 53db16f..0000000 --- a/apt-panopticon-report-email.lua +++ /dev/null @@ -1,292 +0,0 @@ -#!/usr/bin/env luajit - -local args = {...} - -verbosity = -1 -local logFile - - ---[[ Ordered table iterator, allow to iterate on the natural order of the keys of a table. - From http://lua-users.org/wiki/SortedIteration - ]] -function __genOrderedIndex( t ) - local orderedIndex = {} - for key in pairs(t) do - table.insert( orderedIndex, key ) - end - table.sort( orderedIndex ) - return orderedIndex -end -function orderedNext(t, state) - -- Equivalent of the next function, but returns the keys in the alphabetic - -- order. We use a temporary ordered key table that is stored in the - -- table being iterated. - - local key = nil - --print("orderedNext: state = "..tostring(state) ) - if state == nil then - -- the first time, generate the index - t.__orderedIndex = __genOrderedIndex( t ) - key = t.__orderedIndex[1] - else - -- fetch the next value - for i = 1,table.getn(t.__orderedIndex) do - if t.__orderedIndex[i] == state then - key = t.__orderedIndex[i+1] - end - end - end - - if key then - return key, t[key] - end - - -- no more value to return, cleanup - t.__orderedIndex = nil - return -end -function orderedPairs(t) - -- Equivalent of the pairs() function on tables. Allows to iterate - -- in order - return orderedNext, t, nil -end - - -local plurals = function(e, w) - local result = "" - if 1 == e then - result = e .. " error" - elseif e ~= 0 then - result = e .. " errors" - end - if 0 < w then - if 0 < e then result = result .. ", " end - if 1 == w then - result = result .. w .. " warning" - else - result = result .. w .. " warnings" - end - end - if "" ~= result then result = " (" .. result .. ")" end - return result -end - -local results = {} - -local log = function(v, t, s, prot, test, host) - local x = "" - if nil == prot then prot = "" end - if nil ~= test then x = x .. test else test = "" end - if nil ~= host then - if #x > 0 then x = x .. " " end - x = x .. host - end - if #x > 0 then - t = t .. "(" .. x .. ")" - end - if v <= verbosity then - if 3 <= verbosity then t = os.date() .. " " .. t end - print(t .. ": " .. s) - end - if nil ~= logFile then - logFile:write(os.date() .. " " .. t .. ": " .. s .. "\n") - logFile:flush() - end -end -local D = function(s) log(3, "DEBUG ", s) end -local I = function(s) log(2, "INFO ", s) end -local W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end -local E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end -local C = function(s) log(-1, "CRITICAL", s) end - -local faulty = "" -local status = function(host, results, typ) - local result = "" - local e = 0 - local w = 0 - if nil ~= results[typ] then - e = results[typ].errors - w = results[typ].warnings - for k, v in pairs(results[typ]) do - if "table" == type(v) then - e = e + v.errors - w = w + v.warnings - end - end - else - for k, v in pairs(results) do - if "table" == type(v) then - for i, u in pairs(v) do - if "table" == type(u) then - if typ == i then - e = e + u.errors - w = w + u.warnings - end - end - end - end - end - end - - if 0 < e then result = e .. " errors" end - if 1 == e then result = e .. " error" end - if 0 < w then - if 0 < e then result = result .. ", " end - if 1 == w then - result = result .. w .. " warning" - else - result = result .. w .. " warnings" - end - end - if "[OK]" ~= result then - if 0 < e then - result = "[FAILED] (" .. result .. ")" - faulty = faulty .. host .. " (" .. typ .. ")\n" - elseif 0 < w then - result = "[OK] (" .. result .. ")" - else - result = "[OK]" - end - end - return result -end - -local collate = function(host, ip, results) - local f = "results/" .. host .. "_" .. ip .. ".lua" - local rfile, e = io.open(f, "r") - if nil == rfile then I("opening " .. f .. " file - " .. e) else - rfile:close() - local rs = loadfile(f)() - for k, v in pairs(rs) do - if "table" == type(v) then - for i, u in pairs(v) do - if "table" == type(u) then - for h, t in pairs(u) do - local a = results[k][h] - if nil == a then a = 0 end - results[k][h] = a + t - end - else - local a = results[k][i] - if nil == a then a = 0 end - results[k][i] = a + u - end - end - else - local a = results[k] - if nil == a then a = 0 end - results[k] = a + v - end - end - end - return results -end - -local mirrors = loadfile("results/mirrors.lua")() - -local logCount = function(domain, ip) - local nm = "LOG_" .. domain - local log = "" - local extra = "" - if nil ~= ip then nm = nm .. "_" .. ip end - nm = nm .. ".html" - local rfile, e = io.open("results/" .. nm, "r") - if nil ~= rfile then - local errors = 0 - local warnings = 0 - for l in rfile:lines() do - if nil ~= l:match(">ERROR ") then errors = errors + 1 end - if nil ~= l:match(">WARNING ") then warnings = warnings + 1 end - end - rfile:close() --- if nil == ip then --- log = "" .. domain .. "" --- else --- log = "" .. ip .. "" --- end - log = log .. plurals(errors, warnings) - end - return log -end - -local file, e = io.open("results/Report-email.txt", "w+") -if nil == file then C("opening mirrors file - " .. e) else - file:write( "Dear Mirror Admins,\n\n" .. - "The full list of Devuan package mirrors is available at the URL:\n\n" .. - " https://pkgmaster.devuan.org/mirror_list.txt\n\n" .. - 'Please contact "mirrors@devuan.org" if any of the information \nin the file above needs to be amended. \n\n' .. - "The full results of the mirror checking is available at the URL:\n\n" .. - " https://sledjhamr.org/apt-panopticon/results/Report-web.html\n\n" .. - "Due to the nature of the tests, some errors or warnings will be \ncounted several times. " .. - "Refer to the logs on the web page for details.\n\n" .. - "Please see below the current status of the Devuan Package Mirror \nnetwork:\n\n" .. - "---- BEGIN MIRROR-STATUS " .. os.date("!%Y-%m-%d %H:%M") .. " GMT ----\n") - for k, v in orderedPairs(mirrors) do - local results = loadfile("results/" .. k .. ".lua")() - file:write(k .. "....\n") - local IPs = v.IPs - for i, u in pairs(IPs) do - if "table" == type(u) then - for h, t in pairs(u) do - results = collate(k, h, results) - end - else - results = collate(k, i, results) - end - end - local ftp = "[skip]" - local http = status(k, results, "http") - local https = status(k, results, "https") - local rsync = "[skip]" - local dns = "" - local protocol = status(k, results, "Protocol") - local sanity = "[skip]" - local integrity = status(k, results, "Integrity") - local updated = status(k, results, "Updated") - - -- DNS-RR test. - if ("deb.devuan.org" ~= k) and (nil ~= mirrors["deb.devuan.org"]) then - for l, w in pairs(mirrors[k].IPs) do - if type(w) == "table" then - for i, u in pairs(w) do - if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then - local log = logCount("deb.devuan.org", i) - if "" ~= log then - if "" == dns then dns = " " else dns = dns .. " " end - dns = dns .. logCount("deb.devuan.org", i) - else - if "" == dns then dns = " " else dns = dns .. " " end - dns = dns .. i - end - end - end - else - if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then - local log = logCount("deb.devuan.org", l) - if "" ~= log then - if "" == dns then dns = " " else dns = dns .. " " end - dns = dns .. log - else - if "" == dns then dns = " " else dns = dns .. " " end - dns = dns .. l - end - end - end - end - if "" == dns then dns = "[no]" end - dns = " DNS-RR: " .. dns - end - - file:write( " ftp: " .. ftp .. " http: " .. http .. " https: " .. https .." rsync: " .. rsync .. "\n" .. - " " .. dns .. "\n" .. - " Protocol: " .. protocol .. " URL-sanity: " .. sanity .. " Integrity: " .. integrity .. "\n" .. - " Updated: " .. updated .. "\n") - end - file:write( "\n==== faulty mirrors: ====\n" .. faulty) - file:write( "\n---- END MIRROR-STATUS ----\n\n" .. - "Thanks for your precious help in ensuring that Devuan GNU+Linux \nremains a universal, stable, dependable, free operating system.\n\n" .. - "You can get the source code from https://sledjhamr.org/cgit/apt-panopticon/about/ ." .. - "Love\n\n" .. - "The Dev1Devs\n\n") - file:close() -end diff --git a/apt-panopticon-report-web.lua b/apt-panopticon-report-web.lua deleted file mode 100755 index fb41341..0000000 --- a/apt-panopticon-report-web.lua +++ /dev/null @@ -1,406 +0,0 @@ -#!/usr/bin/env luajit - -local args = {...} - -verbosity = -1 -local logFile - - ---[[ Ordered table iterator, allow to iterate on the natural order of the keys of a table. - From http://lua-users.org/wiki/SortedIteration - ]] -function __genOrderedIndex( t ) - local orderedIndex = {} - for key in pairs(t) do - table.insert( orderedIndex, key ) - end - table.sort( orderedIndex ) - return orderedIndex -end -function orderedNext(t, state) - -- Equivalent of the next function, but returns the keys in the alphabetic - -- order. We use a temporary ordered key table that is stored in the - -- table being iterated. - - local key = nil - --print("orderedNext: state = "..tostring(state) ) - if state == nil then - -- the first time, generate the index - t.__orderedIndex = __genOrderedIndex( t ) - key = t.__orderedIndex[1] - else - -- fetch the next value - for i = 1,table.getn(t.__orderedIndex) do - if t.__orderedIndex[i] == state then - key = t.__orderedIndex[i+1] - end - end - end - - if key then - return key, t[key] - end - - -- no more value to return, cleanup - t.__orderedIndex = nil - return -end -function orderedPairs(t) - -- Equivalent of the pairs() function on tables. Allows to iterate - -- in order - return orderedNext, t, nil -end - --- Use this to dump a table to a string, with HTML. -dumpTableHTML = function (table, space, name) - local r = name .. "\n" - r = r .. dumpTableHTMLSub(table, space .. " ") - r = r .. space .. "" - return r -end -dumpTableHTMLSub = function (table, space) - local r = "" - for k, v in orderedPairs(table) do - if type(v) == "table" then - if " " == space then - r = r .. space .. dumpTableHTML(v, space, k .. "
      ") .. "
    \n" - else - r = r .. "
  • " .. space .. dumpTableHTML(v, space, k .. "
      ") .. "
  • \n" - end - else - r = r .. space .. "
  • " .. k .. "
  • \n" - end - end - return r -end - -local plurals = function(e, w) - local result = "" - if 1 == e then - result = e .. " error" - elseif e ~= 0 then - result = e .. " errors" - end - if 0 < w then - if 0 < e then result = result .. ", " end - if 1 == w then - result = result .. w .. " warning" - else - result = result .. w .. " warnings" - end - end - if "" ~= result then result = " (" .. result .. ")" end - return result -end - -local results = {} - -local log = function(v, t, s, prot, test, host) - local x = "" - if nil == prot then prot = "" end - if nil ~= test then x = x .. test else test = "" end - if nil ~= host then - if #x > 0 then x = x .. " " end - x = x .. host - end - if #x > 0 then - t = t .. "(" .. x .. ")" - end - if v <= verbosity then - if 3 <= verbosity then t = os.date() .. " " .. t end - print(t .. ": " .. s) - end - if nil ~= logFile then - logFile:write(os.date() .. " " .. t .. ": " .. s .. "\n") - logFile:flush() - end -end -local D = function(s) log(3, "DEBUG ", s) end -local I = function(s) log(2, "INFO ", s) end -local W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end -local E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end -local C = function(s) log(-1, "CRITICAL", s) end - -local mirrors = loadfile("results/mirrors.lua")() - -local revDNS = function(dom, IP) - if "deb.devuan.org" ~= dom then - if nil ~= mirrors["deb.devuan.org"] then - if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][IP] then - return "DNS-RR" - end - end - else - for k, v in pairs(mirrors) do - if "deb.devuan.org" ~= k then - local IPs = v.IPs - for i, u in pairs(IPs) do - if "table" == type(u) then - for h, t in pairs(u) do - if IP == h then return k end - end - else - if IP == i then return k end - end - end - end - end - end - return "" -end - -local faulty = "" -local status = function(host, results, typ) - local result = "" - local e = 0 - local w = 0 - local s = nil ~= mirrors[host].Protocols[typ] - local to = false - if ('http' ~= typ) and ('https' ~= typ) and ('ftp' ~= typ) and ('rsync' ~= typ) then s = true end - if nil ~= results[typ] then - e = results[typ].errors - w = results[typ].warnings ---[[ - for k, v in pairs(results[typ]) do - if "table" == type(v) then - if 0 <= v.errors then e = e + v.errors else to = true end - if 0 <= v.warnings then w = w + v.warnings else to = true end - end - end -]] - else - for k, v in pairs(results) do - if "table" == type(v) then - for i, u in pairs(v) do - if "table" == type(u) then - if typ == i then - if 0 <= u.errors then e = e + u.errors else to = true end - if 0 <= u.warnings then w = w + u.warnings else to = true end - end - end - end - end - end - end - - if to then - if s then - result = "[TIMEOUT]" - else - result = "[TIMEOUT *]" - end - elseif 0 < e then - if s then - result = "[FAILED]" - else - result = "[FAILED *]" - end - faulty = faulty .. host .. " (" .. typ .. ")
    \n" - else - if s then - result = "[OK]" - else - result = "[OK *]" - end - end - return result .. plurals(e, w) -end - -local collate = function(host, ip, results) - local f = "results/" .. host .. "_" .. ip .. ".lua" - local rfile, e = io.open(f, "r") - if nil == rfile then I("opening " .. f .. " file - " .. e) else - rfile:close() - local rs = loadfile(f)() - for k, v in pairs(rs) do - if "table" == type(v) then - for i, u in pairs(v) do - if "table" == type(u) then - for h, t in pairs(u) do - local a = results[k][h] - if nil == a then a = 0 end - results[k][h] = a + t - end - else - local a = results[k][i] - if nil == a then a = 0 end - results[k][i] = a + u - end - end - else - local a = results[k] - if nil == a then a = 0 end - results[k] = a + v - end - end - end - return results -end - -local m = {} - -local logCount = function(domain, ip) - local nm = "LOG_" .. domain - local log = "" - local extra = "" - if nil ~= ip then nm = nm .. "_" .. ip end - nm = nm .. ".html" - local rfile, e = io.open("results/" .. nm, "r") - if nil ~= rfile then - local errors = 0 - local warnings = 0 - for l in rfile:lines() do - if nil ~= l:match(">ERROR ") then errors = errors + 1 end - if nil ~= l:match(">WARNING ") then warnings = warnings + 1 end - end - rfile:close() - if nil == ip then - log = "" .. domain .. "" - else - log = "" .. ip .. "" - end - log = log .. plurals(errors, warnings) - end - return log -end - -local file, e = io.open("results/Report-web.html", "w+") -if nil == file then C("opening mirrors file - " .. e) else - file:write( "apt-panopticon results\n" .. - '' .. - "

    Welcome to the apt-panopticon results page.

    \n" .. - "

    This is the status of the mirror servers in the Devuan package mirror network.

    \n" .. - "

    The full list of Devuan package mirrors is available at the URL: " .. - "https://pkgmaster.devuan.org/mirror_list.txt

    \n" .. - "

    Due to the nature of the tests, some errors or warnings will be counted several times.   " .. - "The links in the table and DNS list go to the detailed testing logs.

    \n\n" .. - "
    \n

    ==== package mirror status " .. os.date("!%Y-%m-%d %H:%M") .. " GMT: ====

    \n\n" .. - - "

    [FAILED] or [OK]" .. - " means the tested thing is supported for that mirror.

    \n" .. - "

    [FAILED *] or [OK *]" .. - " means the tested thing is unsupported for that mirror, but might have been tested anyway.

    \n" .. - "

    [TIMEOUT] or [TIMEOUT]" .. - " means the server had too many timeouts, and tests where aborted, so there is no result for this test.

    " .. - "

    The DNS round robin (DNS-RR) column shows the IPs for that mirror, or [no] if it isn't part of the DNS-RR.   " .. - "The IPs link to the testing log for that IP accessed via the DNS-RR.   " .. - "deb.devuan.org is the DNS-RR itself, so it doesn't get tested directly.

    \n" .. - "

    The time in the Updated column is how often the mirror updates itself.

    " .. - "

    Mirrors with a grey background are not active (though may be usable as part of the DNS-RR).

    \n" .. - "

    [skip] means that the test hasn't been written yet.

    \n" .. - "\n" - ) - for k, v in orderedPairs(mirrors) do - local results = loadfile("results/" .. k .. ".lua")() - local active = "" - if "yes" == v.Active then - file:write(" ") - else - if nil == v.Active then active = 'nil' else active = v.Active end - file:write(" ") - end - local IPs = v.IPs - for i, u in pairs(IPs) do - if "table" == type(u) then - for h, t in pairs(u) do - results = collate(k, h, results) - end - else - results = collate(k, i, results) - end - end - local ftp = "[skip]" - local http = status(k, results, "http") - local https = status(k, results, "https") - local rsync = "[skip]" - local dns = "" - local protocol = status(k, results, "Protocol") - local sanity = "[skip]" - local integrity = status(k, results, "Integrity") - local updated = status(k, results, "Updated") - local rate = v.Rate - if nil ~= rate then updated = updated .. ' ' .. rate end - - -- DNS-RR test. - if ("deb.devuan.org" ~= k) and (nil ~= mirrors["deb.devuan.org"]) then - for l, w in pairs(mirrors[k].IPs) do - if type(w) == "table" then - for i, u in pairs(w) do - if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then - local log = logCount("deb.devuan.org", i) - if "" ~= log then - if "" == dns then dns = " " else dns = dns .. "   " end - dns = dns .. logCount("deb.devuan.org", i) - else - if "" == dns then dns = " " else dns = dns .. "   " end - dns = dns .. "" .. i .. "" - end - end - end - else - if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then - local log = logCount("deb.devuan.org", l) - if "" ~= log then - if "" == dns then dns = " " else dns = dns .. "   " end - dns = dns .. log - else - if "" == dns then dns = " " else dns = dns .. "   " end - dns = dns .. "" .. l .. "" - end - end - end - end - if "" == dns then dns = "[no]" end - end - - file:write("\n") - if "" ~= v.Active then - file:write("\n") - end - end - file:write( "
    FTPHTTPHTTPSRSYNCDNS round robinProtocolURL sanityIntegrityUpdated
    " .. k .. "
    " .. k .. "" .. ftp .. " " .. http .. " " .. https .. " " .. rsync .. " " .. dns .. - " " .. protocol .. " " .. sanity .. - " " .. integrity .. " " .. updated .. "
    " .. active .. "
    \n
    \n

    ==== faulty mirrors: ====

    \n" .. faulty) - file:write( "
    \n
    \n

    ==== DNS and logs: ====

    \n") - - for k, v in pairs(mirrors) do - local log = k - local n = {} - log = logCount(k) - mirrors[k].Protocols = nil - mirrors[k].FQDN = nil - mirrors[k].Active = nil - mirrors[k].Rate = nil - mirrors[k].BaseURL = nil - mirrors[k].Country = nil - mirrors[k].Bandwidth = nil - for l, w in pairs(mirrors[k].IPs) do - if type(w) == "table" then - n[l] = {} - for i, u in pairs(w) do - local log = logCount(k, i) - if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(k, i)] = u end - end - else - local log = logCount(k, l) - if "" == log then n[l] = w else n[log .. " " .. revDNS(k, l)] = w end - end - end - m[log .. " DNS entries -"] = n - end - file:write( "

    This lists each mirror, and the DNS entries for that mirror.   " .. - "The links point to the testing log files for " .. logCount("apt-panopticon") .. " for each domain name / IP combination that was tested.   " .. - "If a mirror has a CNAME, that CNAME is listed along with that CNAMEs DNS entries.   " .. - "deb.devuan.org is the DNS round robin, which points to the mirrors that are part of the DNS-RR.   " .. - "If an IP is part of the DNS-RR, it is marked with 'DNS-RR'   " .. - "pkgmaster.devuan.org is the master mirror, all the others sync to it.   " .. - "

    \n" - ) - file:write(dumpTableHTML(m, "", "")) - file:write( "\n
    \n
    \n\n" .. - "

    The email report.   " .. - "All the logs and other output.   " .. - "You can get the source code here.

    " .. - "\n") - file:close() -end diff --git a/apt-panopticon.lua b/apt-panopticon.lua index 5d9ed82..96f6d62 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua @@ -60,11 +60,10 @@ options = help = "", value = { - "email", + "email-web", -- "Nagios", -- "Prometheus", -- "RRD", - "web", }, }, } -- cgit v1.1