From 4bb51520521d9ab612d50dd2bcf904d70cf15735 Mon Sep 17 00:00:00 2001 From: onefang Date: Tue, 10 Dec 2019 15:45:11 +1000 Subject: Move common code to it's own module. Some minor clean ups and test tweaks likely came for the ride. --- apt-panopticommon.lua | 563 ++++++++++++++++++++++++++++++++++++ apt-panopticon-report-email-web.lua | 250 +++------------- apt-panopticon.lua | 507 +++++++------------------------- 3 files changed, 722 insertions(+), 598 deletions(-) create mode 100644 apt-panopticommon.lua diff --git a/apt-panopticommon.lua b/apt-panopticommon.lua new file mode 100644 index 0000000..b38595c --- /dev/null +++ b/apt-panopticommon.lua @@ -0,0 +1,563 @@ +local APT = {} + +-- https://oss.oetiker.ch/rrdtool/prog/rrdlua.en.html +APT.rrd = require 'rrd' + +verbosity = -1 +APT.origin = false +APT.keep = false +-- TODO - Should actually implement this. +APT.fork = true + +APT.options = +{ + referenceSite = + { + typ = "string", + help = "", + value = "pkgmaster.devuan.org", + }, + roundRobin = + { + typ = "string", + help = "", + value = "deb.devuan.org", + }, + tests = + { + typ = "table", + help = "", + value = + { + "IPv4", + "IPv6", +-- "ftp", + "http", + "https", +-- "rsync", + "DNSRR", + "Protocol", + "URLSanity", + "Integrity", + "Updated", + }, + }, + maxtime = + { + typ = "number", + help = "", + value = 300, + }, + timeout = + { + typ = "number", + help = "", + value = 15, + }, + reports = + { + typ = "table", + help = "", + value = + { + "RRD", -- RRD has to be before web, coz web creates a graph from the RRD data. + "email-web", +-- "Nagios", +-- "Prometheus", + }, + }, +} + +APT.parseArgs = function(args) + local arg = {} + local sendArgs = "" + if 0 ~= #(args) then + local option = "" + for i, a in pairs(args) do + if ("--help" == a) or ("-h" == a) then + print("I should write some docs, huh? Read README.md for instructions.") + elseif "--version" == a then + print("apt-panopticon version 0.1 WIP development version") + elseif "-v" == a then + verbosity = verbosity + 1 + sendArgs = sendArgs .. a .. " " + elseif "-q" == a then + verbosity = -1 + sendArgs = sendArgs .. a .. " " + elseif "-k" == a then + APT.keep = true + elseif "-n" == a then + APT.fork = false + elseif "-o" == a then + APT.origin = true + elseif "--" == a:sub(1, 2) then + local s, e = a:find("=") + if nil == s then e = -1 end + option = a:sub(3, e - 1) + local o = APT.options[option] + if nil == o then + print("Unknown option --" .. option) + option = "" + else + option = a + sendArgs = sendArgs .. a .. " " + local s, e = a:find("=") + if nil == s then e = 0 end + option = a:sub(3, e - 1) + if "table" == APT.options[option].typ then + local result = {} + for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do + local f = t:sub(1, 1) + local n = t:sub(2, -1) + if ("+" ~= f) and ("-" ~= f) then + table.insert(result, t) + end + end + if 0 ~= #result then + APT.options[option].value = result + else + for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do + local f = t:sub(1, 1) + local n = t:sub(2, -1) + if "+" == f then + table.insert(APT.options[option].value, n) + elseif "-" == f then + local r = {} + for i, k in pairs(APT.options[option].value) do + if k ~= n then table.insert(r, k) end + end + APT.options[option].value = r + end + end + end + else + APT.options[option].value = a:sub(e + 1, -1) + end + option = "" + end + elseif "-" == a:sub(1, 1) then + print("Unknown option " .. a) + else + table.insert(arg, a) + end + end + end + return arg, sendArgs +end + +--print(APT.dumpTable(APT.options, "", "options")) + + + +--[[ Ordered table iterator, allow to iterate on the natural order of the keys of a table. + From http://lua-users.org/wiki/SortedIteration + ]] +function __genOrderedIndex( t ) + local orderedIndex = {} + for key in pairs(t) do + table.insert( orderedIndex, key ) + end + table.sort( orderedIndex ) + return orderedIndex +end +function orderedNext(t, state) + -- Equivalent of the next function, but returns the keys in the alphabetic + -- order. We use a temporary ordered key table that is stored in the + -- table being iterated. + + local key = nil + --print("orderedNext: state = "..tostring(state) ) + if state == nil then + -- the first time, generate the index + t.__orderedIndex = __genOrderedIndex( t ) + key = t.__orderedIndex[1] + else + -- fetch the next value + for i = 1,table.getn(t.__orderedIndex) do + if t.__orderedIndex[i] == state then + key = t.__orderedIndex[i+1] + end + end + end + + if key then + return key, t[key] + end + + -- no more value to return, cleanup + t.__orderedIndex = nil + return +end +function APT.orderedPairs(t) + -- Equivalent of the pairs() function on tables. Allows to iterate + -- in order + return orderedNext, t, nil +end + +-- Use this to dump a table to a string, with HTML. +APT.dumpTableHTML = function (table, space, name) + local r = name .. "\n" + r = r .. dumpTableHTMLSub(table, space .. " ") + r = r .. space .. "" + return r +end +dumpTableHTMLSub = function (table, space) + local r = "" + for k, v in APT.orderedPairs(table) do + if type(v) == "table" then + if " " == space then + r = r .. space .. APT.dumpTableHTML(v, space, k .. "\n" + else + r = r .. "
  • " .. space .. APT.dumpTableHTML(v, space, k .. "
    • \n" + end + else + r = r .. space .. "
  • " .. k .. "
    • \n" + end + end + return r +end + +-- Use this to dump a table to a string. +APT.dumpTable = function (table, space, name) + local r = "" + if "" == space then r = r .. space .. name .. " =\n" else r = r .. space .. "[" .. name .. "] =\n" end + r = r .. space .. "{\n" + r = r .. dumpTableSub(table, space .. " ") + if "" == space then r = r .. space .. "}\n" else r = r .. space .. "},\n" end + return r +end +dumpTableSub = function (table, space) + local r = "" + for k, v in pairs(table) do + if type(k) == "string" then k = '"' .. k .. '"' end + if type(v) == "table" then + r = r .. APT.dumpTable(v, space, k) + elseif type(v) == "string" then + r = r .. space .. "[" .. k .. "] = '" .. v .. "';\n" + elseif type(v) == "function" then + r = r .. space .. "[" .. k .. "] = function ();\n" + elseif type(v) == "userdata" then + r = r .. space .. "userdata " .. "[" .. k .. "];\n" + elseif type(v) == "boolean" then + if (v) then + r = r .. space .. "[" .. k .. "] = true;\n" + else + r = r .. space .. "[" .. k .. "] = false;\n" + end + else + r = r .. space .. "[" .. k .. "] = " .. v .. ";\n" + end + end + return r +end + +APT.results = {} +APT.logFile = nil +APT.html = false + +APT.logPre = function() + if nil ~= APT.logFile then + APT.logFile:write("\n") + APT.logFile:write("\n") + end +end +APT.logPost = function() + if nil ~= APT.logFile then + APT.logFile:write(" \n") + end +end + +local log = function(v, t, s, prot, test, host) + local x = "" + if nil == prot then prot = "" end + if nil == test then test = "" end + x = x .. prot + if "" ~= test then + if #x > 0 then x = x .. " " end + x = x .. test + end + if nil ~= host then + if #x > 0 then x = x .. " " end + x = x .. host + end + if #x > 0 then + t = t .. "(" .. x .. ")" + if "" ~= prot then + if "" == test then + if nil == APT.results[prot] then APT.results[prot] = {errors = 0; warnings = 0} end + if v == 0 then APT.results[prot].errors = APT.results[prot].errors + 1 end + if v == 1 then APT.results[prot].warnings = APT.results[prot].warnings + 1 end + else + if nil == APT.results[prot] then APT.results[prot] = {errors = 0; warnings = 0} end + if nil == APT.results[prot][test] then APT.results[prot][test] = {errors = 0; warnings = 0} end + if v == 0 then APT.results[prot][test].errors = APT.results[prot][test].errors + 1 end + if v == 1 then APT.results[prot][test].warnings = APT.results[prot][test].warnings + 1 end + end + end + end + if v <= verbosity then + if 3 <= verbosity then t = os.date() .. " " .. t end + print(t .. ": " .. s) + end + if nil ~= APT.logFile then + if APT.html then + local colour = "white" + if -1 == v then colour = "fuchsia" end -- CRITICAL + if 0 == v then colour = "red " end -- ERROR + if 1 == v then colour = "yellow " end -- WARNING + if 2 == v then colour = "white " end -- INFO + if 3 == v then colour = "gray " end -- DEBUG + APT.logFile:write(os.date() .. " " .. t .. ": " .. s .. "
    \n") + else + APT.logFile:write(os.date() .. " " .. t .. ": " .. s .. "\n") + end + APT.logFile:flush() + end +end +APT.D = function(s) log(3, "DEBUG ", s) end +APT.I = function(s) log(2, "INFO ", s) end +APT.W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end +APT.E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end +APT.C = function(s) log(-1, "CRITICAL", s) end +local D = APT.D +local I = APT.I +local W = APT.W +local E = APT.E +local C = APT.C + + +APT.mirrors = {} + +APT.testing = function(t, host) + for i, v in pairs(APT.options.tests.value) do + if t == v then + local h = APT.mirrors[host] + if nil == h then return true end + if true == h["Protocols"][t] then return true else D("Skipping " .. t .. " checks for " .. host) end + end + end + return false +end + +APT.execute = function (s) + D(" executing 
    " .. s .. "
    ") + --[[ Damn os.execute() + Lua 5.1 says it returns "a status code, which is system-dependent" + Lua 5.2 says it returns true/nil, "exit"/"signal", the status code. + I'm getting 7168 or 0. No idea what the fuck that is. + local ok, rslt, status = os.execute(s) + ]] + local f = io.popen(s .. ' ; echo "$?"', 'r') + local status = "" + local result = "" + -- The last line will be the command's returned status, collect everything else in result. + for l in f:lines() do + result = result .. status .. "\n" + status = l + end + return status, result +end + +APT.fork = function(s) + D(" forking 
    " .. s .. "
    ") + os.execute(s .. " &") +end + +APT.checkExes = function (exe) + local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | grep -v "flock -n apt-panopticon.lock " | wc -l'):read("*l") + D(count .. " " .. exe .. " commands still running.") + return tonumber(count) +end + +APT.checkFile = function(f) + local h, e = io.open(f, "r") + if nil == h then return false else h:close(); return true end +end + +APT.plurals = function(e, w) + local result = "" + if 1 == e then + result = e .. " error" + elseif e ~= 0 then + result = e .. " errors" + end + if ("" ~= result) and APT.html then result = "" .. result .. "" end +-- result = " " .. result + if 0 < w then + if 0 < e then result = result .. ", " end + if 1 == w then + result = result .. w .. " warning" + else + result = result .. w .. " warnings" + end + if ("" ~= result) and APT.html then result = "" .. result .. "" end +-- result = " " .. result + end + if "" ~= result then result = " (" .. result .. ")" end + return result +end + +APT.collate = function(host, ip, results) + local f = "results/" .. host .. "_" .. ip .. ".lua" + local rfile, e = io.open(f, "r") + if nil == rfile then I("opening " .. f .. " file - " .. e) else + rfile:close() + local rs = loadfile(f)() + for k, v in pairs(rs) do + if "table" == type(v) then + if ("speed" == k) and (nil ~= results.speed) then + if v.min < results.speed.min then results.speed.min = v.min end + if v.max > results.speed.max then results.speed.max = v.max end + else + for i, u in pairs(v) do + if "table" == type(u) then + for h, t in pairs(u) do + local a = results[k][i][h] + if nil == a then a = 0 end + results[k][i][h] = a + t + end + else + local a = results[k] + if nil == a then a = 0; results[k] = {} else a = a[i] end + if nil == a then a = 0 end + results[k][i] = a + u + end + end + end + elseif "timeout" ~= k then + local a = results[k] + if nil == a then a = 0 end + results[k] = a + v + end + end + end + return results +end + + + + +APT.now = 0 +local status +status, APT.now = APT.execute('TZ="GMT" ls -l --time-style="+%s" results/stamp | cut -d " " -f 6-6') +APT.now = tonumber(APT.now) +APT.protocols = {'ftp', 'http', 'https', 'rsync'} +APT.tests = {'raw', 'Integrity', 'Protocol', 'Updated', 'URLSanity', 'Speed'} +local start = 'now-2week' +local step = '10min' +local hb = '150min' +local DSIe = 'DS:IntegrityErrors:GAUGE:' .. hb .. ':0:U' +local DSIw = 'DS:IntegrityWarnings:GAUGE:' .. hb .. ':0:U' +local DSPe = 'DS:ProtocolErrors:GAUGE:' .. hb .. ':0:U' +local DSPw = 'DS:ProtocolWarnings:GAUGE:' .. hb .. ':0:U' +local DSUe = 'DS:UpdatedErrors:GAUGE:' .. hb .. ':0:U' +local DSUw = 'DS:UpdatedWarnings:GAUGE:' .. hb .. ':0:U' +local DSSe = 'DS:URLSanityErrors:GAUGE:' .. hb .. ':0:U' +local DSSw = 'DS:URLSanityWarnings:GAUGE:' .. hb .. ':0:U' +local DSx = 'DS:max:GAUGE:' .. hb .. ':0:U' +local DSn = 'DS:min:GAUGE:' .. hb .. ':0:U' + +-- What Collectd uses. +local RRAc0 = 'RRA:AVERAGE:0.9:1:1200' +local RRAc1 = 'RRA:MIN:0.9:1:1200' +local RRAc2 = 'RRA:MAX:0.9:1:1200' +local RRAc3 = 'RRA:AVERAGE:0.9:7:1235' +local RRAc4 = 'RRA:MIN:0.9:7:1235' +local RRAc5 = 'RRA:MAX:0.9:7:1235' +local RRAc6 = 'RRA:AVERAGE:0.9:50:1210' +local RRAc7 = 'RRA:MIN:0.9:50:1210' +local RRAc8 = 'RRA:MAX:0.9:50:1210' +local RRAc9 = 'RRA:AVERAGE:0.9:223:1202' +local RRAc10 = 'RRA:MIN:0.9:223:1202' +local RRAc11 = 'RRA:MAX:0.9:223:1202' +local RRAc12 = 'RRA:AVERAGE:0.9:2635:1201' +local RRAc13 = 'RRA:MIN:0.9:2635:1201' +local RRAc14 = 'RRA:MAX:0.9:2635:1201' + +-- Try LAST. +local RRAl0 = 'RRA:LAST:0.9:1:1200' +local RRAl1 = 'RRA:LAST:0.9:7:1235' +local RRAl2 = 'RRA:LAST:0.9:50:1210' +local RRAl3 = 'RRA:LAST:0.9:223:1202' +local RRAl4 = 'RRA:LAST:0.9:2635:1201' + +--[[ +/var/lib/collectd/rrd/onefang/interface-eth0/if_packets.rrd + + rrd/host.IP/protocol.rrd test-error count + rrd/host.IP/protocol.rrd test-warning count + + rrh/host.IP/protocol-speed.rrd min + rrh/host.IP/protocol-speed.rrd max +]] + +APT.createRRD = function(host, ip) + if nil ~= ip then host = host .. '_' .. ip end + for i, p in pairs(APT.protocols) do +-- for j, t in pairs(tests) do + os.execute( 'mkdir -p rrd/' .. host .. '/' .. p:upper()) + if not APT.checkFile('rrd/' .. host .. '/' .. p:upper() .. '/Speed.rrd') then + D('Creating ' .. 'rrd/' .. host .. '/' .. p:upper() .. '/Speed.rrd') +-- if 'Speed' == t then + APT.rrd.create( 'rrd/' .. host .. '/' .. p:upper() .. '/Speed.rrd', '--start', start, '--step', step, DSx, DSn, + RRAc0, RRAc1, RRAc2, RRAl0, RRAc3, RRAc4, RRAc5, RRAl1, RRAc6, RRAc7, RRAc8, RRAl2, RRAc9, RRAc10, RRAc11, RRAl3, RRAc12, RRAc13, RRAc14, RRAl4) +-- else + end + if not APT.checkFile('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd') then + D('Creating ' .. 'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd') + APT.rrd.create( 'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', '--start', start, '--step', step, DSIe, DSIw, DSPe, DSPw, DSUe, DSUw, DSSe, DSSw, + RRAc0, RRAc1, RRAc2, RRAl0, RRAc3, RRAc4, RRAc5, RRAl1, RRAc6, RRAc7, RRAc8, RRAl2, RRAc9, RRAc10, RRAc11, RRAl3, RRAc12, RRAc13, RRAc14, RRAl4) +-- end + -- Start them at 0 so the average has something to work on. + APT.rrd.update( 'rrd/' .. host .. '/' .. p:upper() .. '/Speed.rrd', (APT.now - 600) .. ':0:0') + APT.rrd.update( 'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', (APT.now - 600) .. ':0:0:0:0:0:0:0:0') + end +-- end + end +end + +APT.updateRRD = function(results, host, ip) + if nil ~= ip then host = host .. '_' .. ip end + for i, p in pairs(APT.protocols) do + if nil ~= results.speed then + APT.rrd.update( 'rrd/' .. host .. '/' .. p:upper() .. '/Speed.rrd', APT.now .. ':' .. results.speed.max .. ':' .. results.speed.min) + end + if nil ~= results[p] then + APT.rrd.update('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', APT.now .. ':' .. + results[p]['Integrity'].errors .. ':' .. results[p]['Integrity'].warnings .. ':' .. + results[p]['Protocol'].errors .. ':' .. results[p]['Protocol'].warnings .. ':' .. + results[p]['Updated'].errors .. ':' .. results[p]['Updated'].warnings .. ':' .. + results[p]['URLSanity'].errors .. ':' .. results[p]['URLSanity'].warnings) + else + APT.rrd.update('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', APT.now .. ':U:U:U:U:U:U:U:U') + end + end +end + + +APT.doRRD = function(l, k, v) + if APT.checkFile(l .. "/" .. k .. ".lua") then + local results = loadfile(l .. "/" .. k .. ".lua")() + APT.createRRD(k) + APT.updateRRD(results, k) + if til ~= v then + local IPs = v.IPs + for i, u in pairs(IPs) do + if "table" == type(u) then + for h, t in pairs(u) do + APT.createRRD(k, h) + results = APT.collate(k, h, results) + APT.updateRRD(results, k, h) + end + else + APT.createRRD(k, i) + results = APT.collate(k, i, results) + APT.updateRRD(results, k, i) + end + end + end + end +end + + +return APT diff --git a/apt-panopticon-report-email-web.lua b/apt-panopticon-report-email-web.lua index 1144550..ef61183 100755 --- a/apt-panopticon-report-email-web.lua +++ b/apt-panopticon-report-email-web.lua @@ -1,143 +1,22 @@ #!/usr/bin/env luajit -local args = {...} +local APT = require 'apt-panopticommon' +local D = APT.D +local I = APT.I +local W = APT.W +local E = APT.E +local C = APT.C +local arg, sendArgs = APT.parseArgs({...}) -verbosity = -1 -local logFile -local html = false - - ---[[ Ordered table iterator, allow to iterate on the natural order of the keys of a table. - From http://lua-users.org/wiki/SortedIteration - ]] -function __genOrderedIndex( t ) - local orderedIndex = {} - for key in pairs(t) do - table.insert( orderedIndex, key ) - end - table.sort( orderedIndex ) - return orderedIndex -end -function orderedNext(t, state) - -- Equivalent of the next function, but returns the keys in the alphabetic - -- order. We use a temporary ordered key table that is stored in the - -- table being iterated. - - local key = nil - --print("orderedNext: state = "..tostring(state) ) - if state == nil then - -- the first time, generate the index - t.__orderedIndex = __genOrderedIndex( t ) - key = t.__orderedIndex[1] - else - -- fetch the next value - for i = 1,table.getn(t.__orderedIndex) do - if t.__orderedIndex[i] == state then - key = t.__orderedIndex[i+1] - end - end - end - - if key then - return key, t[key] - end - - -- no more value to return, cleanup - t.__orderedIndex = nil - return -end -function orderedPairs(t) - -- Equivalent of the pairs() function on tables. Allows to iterate - -- in order - return orderedNext, t, nil -end - --- Use this to dump a table to a string, with HTML. -dumpTableHTML = function (table, space, name) - local r = name .. "\n" - r = r .. dumpTableHTMLSub(table, space .. " ") - r = r .. space .. "" - return r -end -dumpTableHTMLSub = function (table, space) - local r = "" - for k, v in orderedPairs(table) do - if type(v) == "table" then - if " " == space then - r = r .. space .. dumpTableHTML(v, space, k .. "\n" - else - r = r .. "
  • " .. space .. dumpTableHTML(v, space, k .. "
    • \n" - end - else - r = r .. space .. "
  • " .. k .. "
    • \n" - end - end - return r -end - -local checkFile = function(f) - local h, e = io.open(f, "r") - if nil == h then return false else h:close(); return true end -end - -local plurals = function(e, w) - local result = "" - if 1 == e then - result = e .. " error" - elseif e ~= 0 then - result = e .. " errors" - end - if ("" ~= result) and html then result = "" .. result .. "" end --- result = " " .. result - if 0 < w then - if 0 < e then result = result .. ", " end - if 1 == w then - result = result .. w .. " warning" - else - result = result .. w .. " warnings" - end - if ("" ~= result) and html then result = "" .. result .. "" end --- result = " " .. result - end - if "" ~= result then result = " (" .. result .. ")" end - return result -end local results = {} - -local log = function(v, t, s, prot, test, host) - local x = "" - if nil == prot then prot = "" end - if nil ~= test then x = x .. test else test = "" end - if nil ~= host then - if #x > 0 then x = x .. " " end - x = x .. host - end - if #x > 0 then - t = t .. "(" .. x .. ")" - end - if v <= verbosity then - if 3 <= verbosity then t = os.date() .. " " .. t end - print(t .. ": " .. s) - end - if nil ~= logFile then - logFile:write(os.date() .. " " .. t .. ": " .. s .. "\n") - logFile:flush() - end -end -local D = function(s) log(3, "DEBUG ", s) end -local I = function(s) log(2, "INFO ", s) end -local W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end -local E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end -local C = function(s) log(-1, "CRITICAL", s) end - -local mirrors = loadfile("results/mirrors.lua")() +APT.mirrors = loadfile("results/mirrors.lua")() local revDNS = function(dom, IP) if "deb.devuan.org" ~= dom then - if nil ~= mirrors["deb.devuan.org"] then - if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][IP] then - if html then + if nil ~= APT.mirrors["deb.devuan.org"] then + if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][IP] then + if APT.html then return "DNS-RR" else return "DNS-RR" @@ -145,7 +24,7 @@ local revDNS = function(dom, IP) end end else - for k, v in pairs(mirrors) do + for k, v in pairs(APT.mirrors) do if "deb.devuan.org" ~= k then local IPs = v.IPs for i, u in pairs(IPs) do @@ -168,7 +47,7 @@ local status = function(host, results, typ) local result = "" local e = 0 local w = 0 - local s = nil ~= mirrors[host].Protocols[typ] + local s = nil ~= APT.mirrors[host].Protocols[typ] local to = results.timeout if ('http' ~= typ) and ('https' ~= typ) and ('ftp' ~= typ) and ('rsync' ~= typ) then s = true end if nil ~= results[typ] then @@ -198,7 +77,7 @@ local status = function(host, results, typ) if to then result = "[TIMEOUT" if not s then result = result .. "*" end - if html then + if APT.html then if s then result = "[TIMEOUT" else @@ -208,14 +87,14 @@ local status = function(host, results, typ) elseif 0 < e then result = "[FAILED" if not s then result = result .. "*" end - if html then + if APT.html then if s then result = "[FAILED" else result = "[FAILED*" end end - if html then + if APT.html then faulty = faulty .. host .. " (" .. typ .. ")
    \n" else faulty = faulty .. host .. " (" .. typ .. ")\n" @@ -223,7 +102,7 @@ local status = function(host, results, typ) else result = "[OK" if not s then result = result .. "*" end - if html then + if APT.html then if s then result = "[OK" else @@ -231,43 +110,7 @@ local status = function(host, results, typ) end end end - return result .. plurals(e, w) .. "]" -end - -local collate = function(host, ip, results) - local f = "results/" .. host .. "_" .. ip .. ".lua" - local rfile, e = io.open(f, "r") - if nil == rfile then I("opening " .. f .. " file - " .. e) else - rfile:close() - local rs = loadfile(f)() - for k, v in pairs(rs) do - if "table" == type(v) then - if "speed" == k then - if v.min < results.speed.min then results.speed.min = v.min end - if v.max > results.speed.max then results.speed.max = v.max end - else - for i, u in pairs(v) do - if "table" == type(u) then - for h, t in pairs(u) do - local a = results[k][i][h] - if nil == a then a = 0 end - results[k][i][h] = a + t - end - else - local a = results[k][i] - if nil == a then a = 0 end - results[k][i] = a + u - end - end - end - elseif "timeout" ~= k then - local a = results[k] - if nil == a then a = 0 end - results[k] = a + v - end - end - end - return results + return result .. APT.plurals(e, w) .. "]" end local m = {} @@ -287,20 +130,20 @@ local logCount = function(domain, ip) if nil ~= l:match(">WARNING ") then warnings = warnings + 1 end end rfile:close() - if html then + if APT.html then if nil == ip then log = "" .. domain .. "" else log = "" .. ip .. "" end end - log = log .. plurals(errors, warnings) + log = log .. APT.plurals(errors, warnings) end return log end -html = false +APT.html = false local email, e = io.open("results/Report-email.txt", "w+") if nil == email then C("opening mirrors file - " .. e) else email:write( "Dear Mirror Admins,\n\n" .. @@ -316,17 +159,17 @@ if nil == email then C("opening mirrors file - " .. e) else "Please see below the current status of the Devuan Package Mirror \nnetwork:\n\n" .. "==== package mirror status " .. os.date("!%Y-%m-%d %H:%M") .. " GMT ====\n" .. "[skip] means that the test hasn't been written yet.\n\n") - for k, v in orderedPairs(mirrors) do + for k, v in APT.orderedPairs(APT.mirrors) do local results = loadfile("results/" .. k .. ".lua")() email:write(k .. "....\n") local IPs = v.IPs for i, u in pairs(IPs) do if "table" == type(u) then for h, t in pairs(u) do - results = collate(k, h, results) + results = APT.collate(k, h, results) end else - results = collate(k, i, results) + results = APT.collate(k, i, results) end end local ftp = "[skip]" @@ -340,11 +183,11 @@ if nil == email then C("opening mirrors file - " .. e) else local updated = status(k, results, "Updated") -- DNS-RR test. - if ("deb.devuan.org" ~= k) and (nil ~= mirrors["deb.devuan.org"]) then - for l, w in pairs(mirrors[k].IPs) do + if ("deb.devuan.org" ~= k) and (nil ~= APT.mirrors["deb.devuan.org"]) then + for l, w in pairs(APT.mirrors[k].IPs) do if type(w) == "table" then for i, u in pairs(w) do - if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then + if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then local log = logCount("deb.devuan.org", i) if "" ~= log then if "" == dns then dns = " " else dns = dns .. " " end @@ -356,7 +199,7 @@ if nil == email then C("opening mirrors file - " .. e) else end end else - if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then + if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then local log = logCount("deb.devuan.org", l) if "" ~= log then if "" == dns then dns = " " else dns = dns .. " " end @@ -391,7 +234,7 @@ end results = {} m = {} faulty = "" -html = true +APT.html = true local web, e = io.open("results/Report-web.html", "w+") if nil == web then C("opening mirrors file - " .. e) else web:write( "apt-panopticon results\n" .. @@ -420,7 +263,7 @@ if nil == web then C("opening mirrors file - " .. e) else "\n" .. "\n" ) - for k, v in orderedPairs(mirrors) do + for k, v in APT.orderedPairs(APT.mirrors) do local results = loadfile("results/" .. k .. ".lua")() local active = "" if "yes" == v.Active then @@ -433,10 +276,10 @@ if nil == web then C("opening mirrors file - " .. e) else for i, u in pairs(IPs) do if "table" == type(u) then for h, t in pairs(u) do - results = collate(k, h, results) + results = APT.collate(k, h, results) end else - results = collate(k, i, results) + results = APT.collate(k, i, results) end end local ftp = "[skip]" @@ -455,11 +298,11 @@ if nil == web then C("opening mirrors file - " .. e) else local spd = '' -- DNS-RR test. - if ("deb.devuan.org" ~= k) and (nil ~= mirrors["deb.devuan.org"]) then - for l, w in pairs(mirrors[k].IPs) do + if ("deb.devuan.org" ~= k) and (nil ~= APT.mirrors["deb.devuan.org"]) then + for l, w in pairs(APT.mirrors[k].IPs) do if type(w) == "table" then for i, u in pairs(w) do - if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then + if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then local log = logCount("deb.devuan.org", i) if "" ~= log then if "" == dns then dns = " " else dns = dns .. "   " end @@ -471,7 +314,7 @@ if nil == web then C("opening mirrors file - " .. e) else end end else - if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then + if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then local log = logCount("deb.devuan.org", l) if "" ~= log then if "" == dns then dns = " " else dns = dns .. "   " end @@ -502,18 +345,18 @@ if nil == web then C("opening mirrors file - " .. e) else web:write( "
    FTPHTTPHTTPSRSYNCDNS round robinProtocolURL sanityIntegrityUpdatedSpeed range
    \n
    \n

    ==== faulty mirrors: ====

    \n" .. faulty) web:write( "
    \n
    \n

    ==== DNS and logs: ====

    \n") - for k, v in pairs(mirrors) do + for k, v in pairs(APT.mirrors) do local log = k local n = {} log = logCount(k) - mirrors[k].Protocols = nil - mirrors[k].FQDN = nil - mirrors[k].Active = nil - mirrors[k].Rate = nil - mirrors[k].BaseURL = nil - mirrors[k].Country = nil - mirrors[k].Bandwidth = nil - for l, w in pairs(mirrors[k].IPs) do + APT.mirrors[k].Protocols = nil + APT.mirrors[k].FQDN = nil + APT.mirrors[k].Active = nil + APT.mirrors[k].Rate = nil + APT.mirrors[k].BaseURL = nil + APT.mirrors[k].Country = nil + APT.mirrors[k].Bandwidth = nil + for l, w in pairs(APT.mirrors[k].IPs) do if type(w) == "table" then n[l] = {} for i, u in pairs(w) do @@ -535,11 +378,12 @@ if nil == web then C("opening mirrors file - " .. e) else "pkgmaster.devuan.org is the master mirror, all the others sync to it.   " .. "

    \n" ) - web:write(dumpTableHTML(m, "", "")) web:write( "\n
    \n
    \n\n" .. + web:write(APT.dumpTableHTML(m, "", "")) "

    The email report.   " .. "All the logs and other output.   " .. "You can get the source code here.

    " .. "\n") web:close() end + diff --git a/apt-panopticon.lua b/apt-panopticon.lua index 37778ac..e454740 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua @@ -1,7 +1,13 @@ #!/usr/bin/env luajit - -local args = {...} +local APT = require 'apt-panopticommon' +local D = APT.D +local I = APT.I +local W = APT.W +local E = APT.E +local C = APT.C +local arg, sendArgs = APT.parseArgs({...}) +APT.html = true --[[ TODO - What to do about HTTPS://deb.devuan.org/ redirects. Some mirrors give a 404. @@ -10,70 +16,6 @@ local args = {...} They shouldn't have the proper certificate, but are giving a result anyway. ]] -origin = false -verbosity = -1 -keep = false --- TODO - Should actually implement this. -fork = true -options = -{ - referenceSite = - { - typ = "string", - help = "", - value = "pkgmaster.devuan.org", - }, - roundRobin = - { - typ = "string", - help = "", - value = "deb.devuan.org", - }, - tests = - { - typ = "table", - help = "", - value = - { - "IPv4", - "IPv6", --- "ftp", - "http", - "https", --- "rsync", - "DNSRR", - "Protocol", - "URLSanity", - "Integrity", - "Updated", - }, - }, - maxtime = - { - typ = "number", - help = "", - value = 300, - }, - timeout = - { - typ = "number", - help = "", - value = 15, - }, - reports = - { - typ = "table", - help = "", - value = - { - "email-web", --- "Nagios", --- "Prometheus", --- "RRD", - }, - }, -} - local defaultURL = {scheme = "http"} local releases = {"jessie", "ascii", "beowulf", "ceres"} local releaseFiles = @@ -108,9 +50,6 @@ local referenceDevs = "merged/pool/DEVUAN/main/d/desktop-base/desktop-base_3.0_all.deb", "merged/pool/DEVUAN/main/u/util-linux/util-linux_2.32.1-0.1+devuan2.1_amd64.deb", } -local arg = {} -local sendArgs = "" -local logFile local curlStatus = { @@ -218,151 +157,8 @@ local http = require 'socket.http' local url = require 'socket.url' --- Use this to dump a table to a string. -dumpTable = function (table, space, name) - local r = "" - if "" == space then r = r .. space .. name .. " =\n" else r = r .. space .. "[" .. name .. "] =\n" end - r = r .. space .. "{\n" - r = r .. dumpTableSub(table, space .. " ") - if "" == space then r = r .. space .. "}\n" else r = r .. space .. "},\n" end - return r -end -dumpTableSub = function (table, space) - local r = "" - for k, v in pairs(table) do - if type(k) == "string" then k = '"' .. k .. '"' end - if type(v) == "table" then - r = r .. dumpTable(v, space, k) - elseif type(v) == "string" then - r = r .. space .. "[" .. k .. "] = '" .. v .. "';\n" - elseif type(v) == "function" then - r = r .. space .. "[" .. k .. "] = function ();\n" - elseif type(v) == "userdata" then - r = r .. space .. "userdata " .. "[" .. k .. "];\n" - elseif type(v) == "boolean" then - if (v) then - r = r .. space .. "[" .. k .. "] = true;\n" - else - r = r .. space .. "[" .. k .. "] = false;\n" - end - else - r = r .. space .. "[" .. k .. "] = " .. v .. ";\n" - end - end - return r -end - local ip = "" -local results = {} - -local logPre = function() - if nil ~= logFile then - logFile:write("\n") - logFile:write("\n") - end -end -local logPost = function() - if nil ~= logFile then - logFile:write(" \n") - end -end - -local log = function(v, t, s, prot, test, host) - local x = "" - if nil == prot then prot = "" end - if nil == test then test = "" end - x = x .. prot - if "" ~= test then - if #x > 0 then x = x .. " " end - x = x .. test - end - if nil ~= host then - if #x > 0 then x = x .. " " end - x = x .. host - end - if #x > 0 then - t = t .. "(" .. x .. ")" - if "" ~= prot then - if "" == test then - if nil == results[prot] then results[prot] = {errors = 0; warnings = 0} end - if v == 0 then results[prot].errors = results[prot].errors + 1 end - if v == 1 then results[prot].warnings = results[prot].warnings + 1 end - else - if nil == results[prot] then results[prot] = {errors = 0; warnings = 0} end - if nil == results[prot][test] then results[prot][test] = {errors = 0; warnings = 0} end - if v == 0 then results[prot][test].errors = results[prot][test].errors + 1 end - if v == 1 then results[prot][test].warnings = results[prot][test].warnings + 1 end - end - end - end - if v <= verbosity then - if 3 <= verbosity then t = os.date() .. " " .. t end - print(t .. ": " .. s) - end - if nil ~= logFile then - local colour = "white" - if -1 == v then colour = "fuchsia" end -- CRITICAL - if 0 == v then colour = "red " end -- ERROR - if 1 == v then colour = "yellow " end -- WARNING - if 2 == v then colour = "white " end -- INFO - if 3 == v then colour = "gray " end -- DEBUG - logFile:write(os.date() .. " " .. t .. ": " .. s .. "
    \n") - logFile:flush() - end -end -local D = function(s) log(3, "DEBUG ", s) end -local I = function(s) log(2, "INFO ", s) end -local W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end -local E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end -local C = function(s) log(-1, "CRITICAL", s) end - -local mirrors = {} - -local testing = function(t, host) - for i, v in pairs(options.tests.value) do - if t == v then - local h = mirrors[host] - if nil == h then return true end - if true == h["Protocols"][t] then return true else D("Skipping " .. t .. " checks for " .. host) end - end - end - return false -end - -local execute = function (s) - D(" executing 
    " .. s .. "
    ") - --[[ Damn os.execute() - Lua 5.1 says it returns "a status code, which is system-dependent" - Lua 5.2 says it returns true/nil, "exit"/"signal", the status code. - I'm getting 7168 or 0. No idea what the fuck that is. - local ok, rslt, status = os.execute(s) - ]] - local f = io.popen(s .. ' ; echo "$?"', 'r') - local status = "" - local result = "" - -- The last line will be the command's returned status, collect everything else in result. - for l in f:lines() do - result = result .. status .. "\n" - status = l - end - return status, result -end - -local fork = function(s) - D(" forking 
    " .. s .. "
    ") - os.execute(s .. " &") -end -local checkExes = function (exe) - local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | grep -v "flock -n apt-panopticon.lock " | wc -l'):read("*l") - D(count .. " " .. exe .. " commands still running.") - return tonumber(count) -end - -local checkFile = function(f) - local h, e = io.open(f, "r") - if nil == h then return false else h:close(); return true end -end local repoExists = function (r) r = r:match("([%a-]*)") @@ -441,9 +237,9 @@ checkHEAD = function (host, URL, r, retry, sanity) return end D(PU.scheme .. " :// " .. check .. " " .. host .. " -> " .. URL) - if not testing(PU.scheme, host) then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL); return end + if not APT.testing(PU.scheme, host) then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL); return end -- TODO - Perhaps we should try it anyway, and mark it as a warning if it DOES work? - if "https" == PU.scheme and options.roundRobin.value == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert."); return end + if "https" == PU.scheme and APT.options.roundRobin.value == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert."); return end --[[ Using curl command line - -I - HEAD @@ -468,10 +264,10 @@ checkHEAD = function (host, URL, r, retry, sanity) end IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"' end - local cmd = 'ionice -c3 nice -n 19 curl -I --retry 0 -s --path-as-is --connect-timeout ' .. options.timeout.value .. ' --max-redirs 0 ' .. + local cmd = 'ionice -c3 nice -n 19 curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' .. IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' .. hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"' - local status, result = execute(cmd) + local status, result = APT.execute(cmd) os.execute('cat results/"HEADERS_' .. fname .. '" >>results/"STATUS_' .. fname .. '" 2>/dev/null; rm -f results/"HEADERS_' .. fname .. '" 2>/dev/null') if "0" ~= status then local msg = curlStatus[0 + status] @@ -522,7 +318,7 @@ checkHEAD = function (host, URL, r, retry, sanity) if nil ~= location then pu = url.parse(location, defaultURL) if ('http' == location:sub(1, 4)) and (pu.scheme ~= PU.scheme) then -- Sometimes a location sans scheme is returned, this is not a protocol change. - if testing("Protocol") then W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host) end + if APT.testing("Protocol") then W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host) end if (pu.host == host) and pu.path == PU.path then D("Not testing protocol change " .. URL .. " -> " .. location); return end end @@ -555,7 +351,7 @@ checkHEAD = function (host, URL, r, retry, sanity) end local checkTimeouts = function(host, scheme, URL) - if testing(scheme) then + if APT.testing(scheme) then totalTimeouts = totalTimeouts + timeouts; timeouts = 0 checkHEAD(host, scheme .. "://" .. URL) if 4 <= (totalTimeouts) then @@ -563,7 +359,7 @@ local checkTimeouts = function(host, scheme, URL) return true end end - if testing("URLSanity") then + if APT.testing("URLSanity") then URL = URL:gsub("merged/", "merged///") totalTimeouts = totalTimeouts + timeouts; timeouts = 0 checkHEAD(host, scheme .. "://" .. URL, 0, 0, true) @@ -617,24 +413,24 @@ checkHost = function (orig, host, path, ip, file) else if orig == host then D("checkHost " .. orig .. "" .. file) - if testing("IPv4") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file) end + if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file) end else D("checkHost " .. orig .. " -> " .. host) end - local h = mirrors[ph.host] + local h = APT.mirrors[ph.host] if nil == h then return end for k, v in pairs(h.IPs) do if "table" == type(v) then for k1, v1 in pairs(v) do if v1 == "A" then - if testing("IPv4") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end + if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end elseif v1 == "AAAA" then - if testing("IPv6") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end + if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end end end else if v == "A" then - if testing("IPv4") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end + if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end elseif v == "AAAA" then - if testing("IPv6") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end + if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end end end end @@ -644,14 +440,12 @@ end local addDownload = function(host, URL, f, r, k) local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename. - local o, e = io.open("results/" .. host .. "/merged/dists/" .. r .. k, "r") - if nil ~= o then - o:close() + if APT.checkFile("results/" .. host .. "/merged/dists/" .. r .. k) then -- Curls "check timestamp and overwrite file" stuff sucks. -- -R means the destination file gets the timestamp of the remote file. -- Can only do ONE timestamp check per command. -- This doesn't work either. All downloads get all these headers. Pffft --- local status, ts = execute('TZ="GMT" ls -l --time-style="+%a, %d %b %Y %T %Z" results/' .. host .. "/merged/dists/" .. r .. k .. ' | cut -d " " -f 6-11') +-- local status, ts = APT.execute('TZ="GMT" ls -l --time-style="+%a, %d %b %Y %T %Z" results/' .. host .. "/merged/dists/" .. r .. k .. ' | cut -d " " -f 6-11') -- f:write('header "If-Modified-Since: ' .. ts:sub(2, -2) .. '"\n') -- Curl will DELETE the existing file if the timestamp fails to download a new one, unless we change directory first, -- which wont work with multiple files in multiple directories. WTF? @@ -671,21 +465,21 @@ local postDownload = function(host, r, k) " && [ ! -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then cp -a" .. " results/" .. host .. "/merged/dists/" .. r .. k .. ".old" .. " results/" .. host .. "/merged/dists/" .. r .. k .. "; fi") - if ".gz" == k:sub(-3, -1) then execute("ionice -c3 nice -n 19 gzip -dfk results/" .. host .. "/merged/dists/" .. r .. k) end - if ".xz" == k:sub(-3, -1) then execute("ionice -c3 nice -n 19 xz -dfk results/" .. host .. "/merged/dists/" .. r .. k .. " 2>/dev/null") end - if testing("Integrity") then + if ".gz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 gzip -dfk results/" .. host .. "/merged/dists/" .. r .. k) end + if ".xz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 xz -dfk results/" .. host .. "/merged/dists/" .. r .. k .. " 2>/dev/null") end + if APT.testing("Integrity") then if ".gpg" == k:sub(-4, -1) then - local status, out = execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. r .. k .. + local status, out = APT.execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. r .. k .. " results/" .. host .. "/merged/dists/" .. r .. k:sub(1, -5) .. " 2>/dev/null") if "0" ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. r .. k, "http", "Integrity", host) end end -- TODO - should check the PGP sig of InRelease as well. end - if testing("Integrity") or testing("Updated") then + if APT.testing("Integrity") or APT.testing("Updated") then if "Packages." == file:sub(1, 9) then -- TODO - compare the SHA256 sums in pkgmaster's Release for both the packed and unpacked versions. -- Also note that this might get only a partial download due to maxtime. - if options.referenceSite.value == host then + if APT.options.referenceSite.value == host then local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed', "w+") if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed' .. ' file - ' .. e) else local pp = {} @@ -711,7 +505,7 @@ local postDownload = function(host, r, k) end Pp:close() os.execute('sort results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed >results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages_parsed-sorted') - if checkFile('Packages/' .. r .. dir .. 'Packages_parsed-sorted') then + if APT.checkFile('Packages/' .. r .. dir .. 'Packages_parsed-sorted') then os.execute('diff -U 0 Packages/' .. r .. dir .. 'Packages_parsed-sorted ' .. 'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages_parsed-sorted ' .. ' | grep -E "^-" | grep -Ev "^\\+\\+\\+|^---" >>results/OLD_PACKAGES_' .. r .. '.txt') @@ -737,14 +531,14 @@ local postDownload = function(host, r, k) end local downloadLock = "flock -n results/curl-" -local download = "curl --connect-timeout " .. options.timeout.value .. " --create-dirs -f -L --max-time " .. options.maxtime.value .. " -z 'results/stamp.old' -v -R " +local download = "curl --connect-timeout " .. APT.options.timeout.value .. " --create-dirs -f -L --max-time " .. APT.options.maxtime.value .. " -z 'results/stamp.old' -v -R " local downloads = function(host, URL, release, list) if nil == URL then URL = "" end local lock = "META-" .. host .. ".lock" local log = " --stderr results/curl-META-" .. host .. ".log" local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock .. " " .. download .. log .. " -K results/" .. host .. ".curl" - if testing("IPv4") and (not testing("IPv6")) then cm = cm .. ' -4' end - if (not testing("IPv4")) and testing("IPv6") then cm = cm .. ' -6' end + if APT.testing("IPv4") and (not APT.testing("IPv6")) then cm = cm .. ' -4' end + if (not APT.testing("IPv4")) and APT.testing("IPv6") then cm = cm .. ' -6' end f, e = io.open("results/" .. host .. ".curl", "a+") if nil == f then C("opening curl file - " .. e); return end @@ -772,7 +566,7 @@ local downloads = function(host, URL, release, list) end end f:close() - fork(cm) + APT.fork(cm) end @@ -781,7 +575,7 @@ local getMirrors = function () local host = "" local m = {} local active = true - local URL = "http://" .. options.referenceSite.value .. "/mirror_list.txt" + local URL = "http://" .. APT.options.referenceSite.value .. "/mirror_list.txt" I("getting mirrors.") local p, c, h = http.request(URL) if nil == p then E(c .. " fetching " .. URL) else @@ -818,95 +612,20 @@ local getMirrors = function () mirrors[host] = m end end - if testing("DNSRR") then - mirrors[options.roundRobin.value] = { ["Protocols"] = { ["http"] = true; ["https"] = true; }; ["FQDN"] = 'deb.devuan.org'; ["Active"] = 'yes'; ["BaseURL"] = 'deb.devuan.org'; } - gatherIPs(options.roundRobin.value) - mirrors[options.roundRobin.value].IPs = IP[options.roundRobin.value] + if APT.testing("DNSRR") then + mirrors[APT.options.roundRobin.value] = { ["Protocols"] = { ["http"] = true; ["https"] = true; }; ["FQDN"] = 'deb.devuan.org'; ["Active"] = 'yes'; ["BaseURL"] = 'deb.devuan.org'; } + gatherIPs(APT.options.roundRobin.value) + mirrors[APT.options.roundRobin.value].IPs = IP[APT.options.roundRobin.value] end local file, e = io.open("results/mirrors.lua", "w+") if nil == file then C("opening mirrors file - " .. e) else - file:write(dumpTable(mirrors, "", "mirrors") .. "\nreturn mirrors\n") + file:write(APT.dumpTable(mirrors, "", "mirrors") .. "\nreturn mirrors\n") file:close() end return mirrors end -if 0 ~= #args then - local option = "" - for i, a in pairs(args) do - if ("--help" == a) or ("-h" == a) then - print("I should write some docs, huh? Read README.md for instructions.") - elseif "--version" == a then - print("apt-panopticon version 0.1 WIP development version") - elseif "-v" == a then - verbosity = verbosity + 1 - sendArgs = sendArgs .. a .. " " - elseif "-q" == a then - verbosity = -1 - sendArgs = sendArgs .. a .. " " - elseif "-k" == a then - keep = true - elseif "-n" == a then - fork = false - elseif "-o" == a then - origin = true - elseif "--" == a:sub(1, 2) then - local s, e = a:find("=") - if nil == s then e = -1 end - option = a:sub(3, e - 1) - local o = options[option] - if nil == o then - print("Unknown option --" .. option) - option = "" - else - option = a - sendArgs = sendArgs .. a .. " " - local s, e = a:find("=") - if nil == s then e = 0 end - option = a:sub(3, e - 1) - if "table" == options[option].typ then - local result = {} - for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do - local f = t:sub(1, 1) - local n = t:sub(2, -1) - if ("+" ~= f) and ("-" ~= f) then - table.insert(result, t) - end - end - if 0 ~= #result then - options[option].value = result - else - for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do - local f = t:sub(1, 1) - local n = t:sub(2, -1) - if "+" == f then - table.insert(options[option].value, n) - elseif "-" == f then - local r = {} - for i, k in pairs(options[option].value) do - if k ~= n then table.insert(r, k) end - end - options[option].value = r - end - end - end - else - options[option].value = a:sub(e + 1, -1) - end - option = "" - end - elseif "-" == a:sub(1, 1) then - print("Unknown option " .. a) - else - table.insert(arg, a) - end - end -end - ---print(dumpTable(options.tests.value, "", "tests")) - - if 0 < #arg then if "/" == arg[1]:sub(-1, -1) then W("slash at end of path! " .. arg[1]) @@ -918,37 +637,37 @@ if 0 < #arg then end local pu = url.parse("http://" .. arg[1]) - if testing("Integrity") or testing("Updated") then - if origin and options.referenceSite.value == pu.host then --- if not keep then execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end + if APT.testing("Integrity") or APT.testing("Updated") then + if APT.origin and APT.options.referenceSite.value == pu.host then +-- if not APT.keep then os.execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end end end if nil ~= arg[2] then - logFile, e = io.open("results/LOG_" .. pu.host .. "_" .. arg[2] .. ".html", "a+") + APT.logFile, e = io.open("results/LOG_" .. pu.host .. "_" .. arg[2] .. ".html", "a+") else - logFile, e = io.open("results/LOG_" .. pu.host .. ".html", "a+") + APT.logFile, e = io.open("results/LOG_" .. pu.host .. ".html", "a+") end - if nil == logFile then C("opening log file - " .. e); return end - logPre() - I("Starting tests for " ..arg[1] .. " with these tests - " .. table.concat(options.tests.value, ", ")) - mirrors = loadfile("results/mirrors.lua")() + if nil == APT.logFile then C("opening log file - " .. e); return end + APT.logPre() + I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", ")) + APT.mirrors = loadfile("results/mirrors.lua")() if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end if nil ~= arg[3] then I(" Using file " .. arg[3]); end for k, v in pairs{"ftp", "http", "https", "rsync"} do - if testing(v) then + if APT.testing(v) then local tests = {errors = 0; warnings = 0} - if testing("Integrity") then tests.Integrity = {errors = 0; warnings = 0} end - if testing("Protocol") then tests.Protocol = {errors = 0; warnings = 0} end - if testing("Updated") then tests.Updated = {errors = 0; warnings = 0} end - if testing("URLSanity") then tests.URLSanity = {errors = 0; warnings = 0} end - results[v] = tests + if APT.testing("Integrity") then tests.Integrity = {errors = 0; warnings = 0} end + if APT.testing("Protocol") then tests.Protocol = {errors = 0; warnings = 0} end + if APT.testing("Updated") then tests.Updated = {errors = 0; warnings = 0} end + if APT.testing("URLSanity") then tests.URLSanity = {errors = 0; warnings = 0} end + APT.results[v] = tests end end - if origin then - if testing("Integrity") or testing("Updated") then - if origin and (options.roundRobin.value ~= pu.host) then + if APT.origin then + if APT.testing("Integrity") or APT.testing("Updated") then + if APT.origin and (APT.options.roundRobin.value ~= pu.host) then I("Starting file downloads for " .. pu.host) downloads(pu.host, pu.path) end @@ -958,10 +677,10 @@ if 0 < #arg then checkHost(pu.host, pu.host, pu.path, arg[2], arg[3]) end - if testing("Integrity") or testing("Updated") then + if APT.testing("Integrity") or APT.testing("Updated") then if 4 > (totalTimeouts) then - if origin and (options.roundRobin.value ~= pu.host) then - while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end + if APT.origin and (APT.options.roundRobin.value ~= pu.host) then + while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " .. "mv results/curl-META-" .. pu.host .. ".log results/curl-Release-" .. pu.host .. ".log") for i, n in pairs(releases) do @@ -971,24 +690,25 @@ if 0 < #arg then end end - if checkFile('results/' .. pu.host .. '/merged/dists/' .. n .. '/Release') and - checkFile('results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED') then + if APT.checkFile('results/' .. pu.host .. '/merged/dists/' .. n .. '/Release') then os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED') - if options.referenceSite.value == pu.host then - os.execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. - 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. - '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt') - os.execute('rm -f results/' .. pu.host .. '/merged/dists/' .. n .. '/Release 2>/dev/null') + if APT.checkFile('results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED') then + if APT.options.referenceSite.value == pu.host then + os.execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. + 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. + '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt') -- TODO - Maybe check the date in Release, though since they are updated daily, is there any point? Perhaps it's for checking amprolla got run? - else + os.execute('rm -f results/' .. pu.host .. '/merged/dists/' .. n .. '/Release 2>/dev/null') + else -- TODO - compare to the pkgmaster copy. - end + end - local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") - if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else - local diff = dfile:read("*a") - if "" ~= diff then - downloads(pu.host, pu.path, n, diff) + local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") + if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else + local diff = dfile:read("*a") + if "" ~= diff then + downloads(pu.host, pu.path, n, diff) + end end end end @@ -996,7 +716,7 @@ if 0 < #arg then end downloads(pu.host, pu.path, "", "") - while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end + while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " .. "mv results/curl-META-" .. pu.host .. ".log results/curl-Packages-" .. pu.host .. ".log") @@ -1008,7 +728,7 @@ if 0 < #arg then postDownload(pu.host, n, "/" .. l) end end - if options.referenceSite.value == pu.host then + if APT.options.referenceSite.value == pu.host then -- In case it wasn't dealt with already. os.execute('touch results/NEW_Packages_' .. n .. '.test.txt') end @@ -1027,26 +747,26 @@ if 0 < #arg then end end downloads(pu.host, pu.path, nil, "") - while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end + while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end for i, n in pairs(releases) do local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else for l in nfile:lines() do local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |') if nil ~= p then - if checkFile('results/' .. pu.host .. "/merged/" .. p) then - local status, fsz = execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5') - if testing("Integrity") then + if APT.checkFile('results/' .. pu.host .. "/merged/" .. p) then + local status, fsz = APT.execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5') + if APT.testing("Integrity") then if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) print('|' .. sz .. '~=' .. fsz:sub(2, -2) .. '|') else - local status, fsha = execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') + local status, fsha = APT.execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end -- TODO - maybe check the PGP key, though packages are mostly not signed. end end - if testing("Updated") then + if APT.testing("Updated") then if sz ~= fsz:sub(2, -2) then E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) end @@ -1060,20 +780,20 @@ if 0 < #arg then end end - results["timeout"] = false + APT.results["timeout"] = false else - results["timeout"] = true + APT.results["timeout"] = true end end - if origin and options.referenceSite.value ~= pu.host then - if not keep then os.execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end + if APT.origin and APT.options.referenceSite.value ~= pu.host then + if not APT.keep then os.execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end os.execute('rm STATUS_' .. pu.host .. '* 2>/dev/null') end local min, max, spd = 999999999999, 0 for i, mt in pairs({'Release', 'Packages', 'META'}) do - if checkFile("results/curl-" .. mt .. "-" .. pu.host .. ".log") then + if APT.checkFile("results/curl-" .. mt .. "-" .. pu.host .. ".log") then for l in io.lines("results/curl-" .. mt .. "-" .. pu.host .. ".log") do local speed, crrnt = l:match('^%c *%d+ +%d+k? +%d+ +%d+k? +%d+ +%d+ +(%d+k?) +%d+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +(%d+k?)') if nil ~= speed then @@ -1089,17 +809,17 @@ if 0 < #arg then end end end - results["speed"] = {min = min, max = max} + APT.results["speed"] = {min = min, max = max} local f = pu.host if "" ~= ip then f = f .. "_" .. ip end local rfile, e = io.open("results/" .. f .. ".lua", "w+") if nil == rfile then C("opening results file - " .. e) else - rfile:write(dumpTable(results, "", "results") .. "\nreturn results\n") + rfile:write(APT.dumpTable(APT.results, "", "results") .. "\nreturn results\n") rfile:close() end - logPost() - logFile:close() + APT.logPost() + APT.logFile:close() else local fadt = io.popen("ls -dl results_old 2>/dev/null | cut -d '>' -f 2 | cut -d ' ' -f 2") local adt = fadt:read('*l') @@ -1113,25 +833,25 @@ else if nil ~= dt then os.execute('mkdir -p results_' .. dt .. '; rm -f results; ln -s results_' .. dt .. ' results 2>/dev/null') end os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp') os.execute("rm -f results/*.check 2>/dev/null") - if not keep then + if not APT.keep then os.execute("rm -f results/*.curl 2>/dev/null") os.execute("rm -f results/*.log 2>/dev/null") os.execute("rm -f results/*.html 2>/dev/null") os.execute("rm -f results/*.txt 2>/dev/null") end - logFile, e = io.open("results/LOG_apt-panopticon.html", "a+") - if nil == logFile then C("opening log file - " .. e); return end - logPre() - I("Starting tests " .. table.concat(options.tests.value, ", ")) + APT.logFile, e = io.open("results/LOG_apt-panopticon.html", "a+") + if nil == APT.logFile then C("opening log file - " .. e); return end + APT.logPre() + I("Starting tests " .. table.concat(APT.options.tests.value, ", ")) os.execute("mkdir -p results") - mirrors = getMirrors() - checkHost(options.referenceSite.value) + APT.mirrors = getMirrors() + checkHost(APT.options.referenceSite.value) for i, n in pairs(releases) do - while not checkFile('results/NEW_Packages_' .. n .. '.test.txt') do os.execute("sleep 10") end + while not APT.checkFile('results/NEW_Packages_' .. n .. '.test.txt') do os.execute("sleep 10") end end - for k, m in pairs(mirrors) do + for k, m in pairs(APT.mirrors) do if "/" == m.BaseURL:sub(-1, -1) then W("slash at end of BaseURL in mirror_list.txt! " .. m.BaseURL, "", "", m.FQDN) m.BaseURL = m.BaseURL:sub(1, -2) @@ -1141,30 +861,27 @@ else m.BaseURL = m.BaseURL:sub(1, -2) end local pu = url.parse("http://" .. m.BaseURL) - if options.referenceSite.value ~= pu.host then + if APT.options.referenceSite.value ~= pu.host then checkHost(m.BaseURL) - checkExes("apt-panopticon.lua " .. sendArgs) - if testing("Integrity") or testing("Updated") then checkExes(downloadLock) end + APT.checkExes("apt-panopticon.lua " .. sendArgs) + if APT.testing("Integrity") or APT.testing("Updated") then APT.checkExes(downloadLock) end end end - while 1 <= checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end + while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end os.execute("rm -f results/*.check; rm -f results/*.lock 2>/dev/null") -- Create the reports. - for n, r in pairs(options.reports.value) do - local report = "apt-panopticon-report-" .. r .. ".lua" - local rfile, e = io.open(report, "r") - if nil == rfile then C("opening " .. report .. " file - " .. e) else - rfile:close() - I("Creating " .. report .. " report.") - execute("./" .. report .. " ") + for n, r in pairs(APT.options.reports.value) do + if APT.checkFile("apt-panopticon-report-" .. r .. ".lua") then + I("Creating " .. r .. " report.") + APT.execute("./apt-panopticon-report-" .. r .. ".lua") end end if nil ~= adt then os.execute('rm -fr ' .. adt .. ' 2>/dev/null') end - logPost() - logFile:close() + APT.logPost() + APT.logFile:close() end -- cgit v1.1