local APT = {} -- https://oss.oetiker.ch/rrdtool/prog/rrdlua.en.html APT.rrd = require 'rrd' APT.protocols = {"ftp", "http", "https", "rsync"} APT.tests = {'raw', 'Integrity', 'Protocol', 'Updated', 'URLSanity', 'Speed'} APT.releases = {"jessie", "ascii", "beowulf", "ceres"} APT.subRels = {'backports', 'proposed-updates', 'security', 'updates'} APT.notExist = { 'jessie-backports', -- No longer existing as this is old stable. 'jessie-proposed-updates', 'jessie-updates', 'beowulf-backports', -- Wont exist until Beowulf goes live. 'ceres-backports', -- These will never exist, it's our code name for the testing suite. 'ceres-proposed-updates', 'ceres-updates', "ceres-security", } APT.verbosity = -1 APT.origin = false APT.redir = false APT.keep = false APT.options = { bandwidth = { typ = "number", help = ' 0 = low - HTTP tests for all IPs of all mirrors (HEAD tests), but not URL sanity tests.\n' .. ' 1 = medium - Also HTTP(S) redirect tests, other protocols, download Release files over HTTP, and check them.\n' .. ' 2 = high - Also download Packages.xz files that changed, unpack and check them.\n' .. ' Also download and check InRelease files\n.' .. ' Pick a few small packages, download them, check their SHA512.\n' .. ' 3 = more - Also pick more packages, some for each arch.\n' .. ' 4 = all - Do absolutely everything.\n' .. ' Actually download some Contents files, and some more Packages.xz, and package files.', value = 2, }, referenceSite = { typ = "string", help = "The mirror that is used as a reference, coz the oters sync to it.", value = "pkgmaster.devuan.org", }, roundRobin = { typ = "string", help = "The round robin DNS name.", value = "deb.devuan.org", }, tests = { typ = "table", help = "A list of the tests, see the documentation for details.", value = { "IPv4", "IPv6", -- "ftp", "http", "https", -- "rsync", "DNSRR", "Protocol", "URLSanity", "Integrity", "Updated", }, }, maxtime = { typ = "number", help = "", value = 240, }, timeout = { typ = "number", help = "", value = 5, }, timeouts = { typ = "number", help = "", value = 3, }, refresh = { typ = "number", help = "", value = 300, }, retries = { typ = "number", help = "", value = 2, }, reports = { typ = "table", help = "A list of the reports generated.", value = { "RRD", -- RRD has to be before web, coz web creates a graph from the RRD data. "email-web", -- "Prometheus", }, }, } APT.args = {} APT.parseArgs = function(args) APT.args = args local arg = {} local sendArgs = "" if 0 ~= #(args) then local option = "" for i, a in pairs(args) do if ("--help" == a) or ("-h" == a) then print("I should write some docs, huh? Read README.md for instructions.") elseif "--version" == a then print("apt-panopticon version 0.1 WIP development version") elseif "-v" == a then APT.verbosity = APT.verbosity + 1 sendArgs = sendArgs .. a .. " " elseif "-q" == a then APT.verbosity = -1 sendArgs = sendArgs .. a .. " " elseif "-k" == a then APT.keep = true sendArgs = sendArgs .. a .. " " elseif "-o" == a then APT.origin = true -- Not adding to sendArgs. elseif "-r" == a then APT.redir = true sendArgs = sendArgs .. a .. " " elseif "--low" == a then APT.options.bandwidth.value = 0 APT.options.timeout.value = 2 APT.options.timeouts.value = 1 APT.options.retries.value = 1 sendArgs = sendArgs .. a .. " " elseif "--medium" == a then APT.options.bandwidth.value = 1 sendArgs = sendArgs .. a .. " " elseif "--high" == a then APT.options.bandwidth.value = 2 sendArgs = sendArgs .. a .. " " elseif "--more" == a then APT.options.bandwidth.value = 3 sendArgs = sendArgs .. a .. " " elseif "--all" == a then APT.options.bandwidth.value = 4 sendArgs = sendArgs .. a .. " " elseif "--" == a:sub(1, 2) then local s, e = a:find("=") if nil == s then e = -1 end option = a:sub(3, e - 1) local o = APT.options[option] if nil == o then print("Unknown option --" .. option) option = "" else option = a sendArgs = sendArgs .. a .. " " local s, e = a:find("=") if nil == s then e = 0 end option = a:sub(3, e - 1) if "table" == APT.options[option].typ then local result = {} for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do local f = t:sub(1, 1) local n = t:sub(2, -1) if ("+" ~= f) and ("-" ~= f) then table.insert(result, t) end end if 0 ~= #result then APT.options[option].value = result else for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do local f = t:sub(1, 1) local n = t:sub(2, -1) if "+" == f then table.insert(APT.options[option].value, n) elseif "-" == f then local r = {} for i, k in pairs(APT.options[option].value) do if k ~= n then table.insert(r, k) end end APT.options[option].value = r end end end else APT.options[option].value = a:sub(e + 1, -1) end option = "" end elseif "-" == a:sub(1, 1) then print("Unknown option " .. a) else table.insert(arg, a) end end end return arg, sendArgs end --[[ Ordered table iterator, allow to iterate on the natural order of the keys of a table. From http://lua-users.org/wiki/SortedIteration ]] function __genOrderedIndex( t ) local orderedIndex = {} for key in pairs(t) do table.insert( orderedIndex, key ) end table.sort( orderedIndex ) return orderedIndex end function orderedNext(t, state) -- Equivalent of the next function, but returns the keys in the alphabetic -- order. We use a temporary ordered key table that is stored in the -- table being iterated. local key = nil --print("orderedNext: state = "..tostring(state) ) if state == nil then -- the first time, generate the index t.__orderedIndex = __genOrderedIndex( t ) key = t.__orderedIndex[1] else -- fetch the next value for i = 1,table.getn(t.__orderedIndex) do if t.__orderedIndex[i] == state then key = t.__orderedIndex[i+1] end end end if key then return key, t[key] end -- no more value to return, cleanup t.__orderedIndex = nil return end function APT.orderedPairs(t) -- Equivalent of the pairs() function on tables. Allows to iterate -- in order return orderedNext, t, nil end -- Use this to dump a table to a string, with HTML. APT.dumpTableHTML = function (table, name, space) if nil == space then space = '' end local r = name .. "\n" r = r .. dumpTableHTMLSub(table, space .. " ") r = r .. space .. "" return r end dumpTableHTMLSub = function (table, space) local r = "" for k, v in APT.orderedPairs(table) do if type(v) == "table" then if " " == space then r = r .. space .. APT.dumpTableHTML(v, k .. "
\n") APT.logFile:write(APT.dumpTable(APT.args, 'Arguments')) APT.logFile:write("\n") else return false end return true end APT.logPost = function() if nil ~= APT.logFile then APT.logFile:write(" \n") APT.logFile:close() end end local log = function(v, t, s, prot, test, host) local x = "" if nil == prot then prot = "" end if nil == test then test = "" end x = x .. prot if "" ~= test then if #x > 0 then x = x .. " " end x = x .. test end if nil ~= host then if #x > 0 then x = x .. " " end x = x .. host end if #x > 0 then t = t .. "(" .. x .. ")" if "" ~= prot then if "" == test then if nil == APT.results[prot] then APT.results[prot] = {errors = 0; warnings = 0; timeouts = 0} end if v == 0 then APT.results[prot].errors = APT.results[prot].errors + 1 end if v == 1 then APT.results[prot].warnings = APT.results[prot].warnings + 1 end if v == 2 then APT.results[prot].timeouts = APT.results[prot].timeouts + 1 end else if nil == APT.results[prot] then APT.results[prot] = {errors = 0; warnings = 0; timeouts = 0} end if nil == APT.results[prot][test] then APT.results[prot][test] = {errors = 0; warnings = 0; timeouts = 0} end if v == 0 then APT.results[prot][test].errors = APT.results[prot][test].errors + 1 end if v == 1 then APT.results[prot][test].warnings = APT.results[prot][test].warnings + 1 end if v == 2 then APT.results[prot][test].timeouts = APT.results[prot][test].timeouts + 1 end end end end if v <= APT.verbosity then if 3 <= APT.verbosity then t = os.date('!%F %T') .. " " .. t end print(t .. ": " .. s) end if nil ~= APT.logFile then if APT.html then local colour = "white" if -1 == v then colour = "fuchsia" end -- CRITICAL if 0 == v then colour = "red " end -- ERROR if 1 == v then colour = "yellow " end -- WARNING if 2 == v then colour = "blue " end -- TIMEOUT if 3 == v then colour = "white " end -- INFO if 4 == v then colour = "gray " end -- DEBUG APT.logFile:write(os.date('!%F %T') .. " " .. t .. ": " .. s .. "\n") else APT.logFile:write(os.date('!%F %T') .. " " .. t .. ": " .. s .. "\n") end APT.logFile:flush() end end APT.D = function(s) log(4, "DEBUG ", s) end APT.I = function(s) log(3, "INFO ", s) end APT.T = function(s, p, t, h) log(2, "TIMEOUT ", s, p, t, h) end APT.W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end APT.E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end APT.C = function(s) log(-1, "CRITICAL", s) end local D = APT.D local I = APT.I local T = APT.T local W = APT.W local E = APT.E local C = APT.C APT.debians = {} APT.mirrors = {} APT.testing = function(t, host) for i, v in pairs(APT.options.tests.value) do if t == v then local h = APT.mirrors[host] if nil == h then return true end if true == h["Protocols"][t] then return true else D("Skipping " .. t .. " checks for " .. host) end end end return false end APT.exe = function(c) local exe = {status = 0, result = '', log = true, cmd = c .. ' '} function exe:log() self.log = true return self end function exe:Nice(c) if nil == c then self.cmd = 'ionice -c3 nice -n 19 ' .. self.cmd else self.cmd = self.cmd .. 'ionice -c3 nice -n 19 ' .. c .. ' ' end return self end function exe:also(c) if nil == c then c = '' else c = ' ' .. c end self.cmd = self.cmd .. ';' .. c .. ' ' return self end function exe:And(c) if nil == c then c = '' else c = ' ' .. c end self.cmd = self.cmd .. '&&' .. c .. ' ' return self end function exe:Or(c) if nil == c then c = '' end self.cmd = self.cmd .. '|| ' .. c .. ' ' return self end function exe:noErr() self.cmd = self.cmd .. '2>/dev/null ' return self end function exe:wait(w) self.cmd = self.cmd .. '&& touch ' .. w .. ' ' return self end function exe:Do() --[[ "The condition expression of a control structure can return any value. Both false and nil are considered false. All values different from nil and false are considered true (in particular, the number 0 and the empty string are also true)." says the docs, I beg to differ.]] if true == self.log then D(" executing -
" .. self.cmd .. "
") end
--[[ Damn os.execute()
Lua 5.1 says it returns "a status code, which is system-dependent"
Lua 5.2 says it returns true/nil, "exit"/"signal", the status code.
I'm getting 7168 or 0. No idea what the fuck that is.
local ok, rslt, status = os.execute(s)
]]
local f = io.popen(self.cmd .. ' ; echo "$?"', 'r')
-- The last line will be the command's returned status, collect everything else in result.
self.status = '' -- Otherwise the result starts with 0.
for l in f:lines() do
self.result = self.result .. self.status .. "\n"
self.status = l
end
self.status = tonumber(self.status)
return self
end
function exe:fork()
self.cmd = '{ ' .. self.cmd .. '; } &'
if true == self.log then D(" forking - " .. self.cmd .. "
") end
os.execute(self.cmd)
return self
end
return exe
end
APT.checkExes = function (exe)
local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | grep -v "flock -n apt-panopticon.lock " | wc -l'):read("*l")
D(count .. " " .. exe .. " commands still running.")
return tonumber(count)
end
APT.checkFile = function(f)
local h, e = io.open(f, "r")
if nil == h then return false else h:close(); return true end
end
APT.plurals = function(e, w, t)
local result = ""
if 1 == e then
result = e .. " error"
elseif e ~= 0 then
result = e .. " errors"
end
if ("" ~= result) and APT.html then result = "" .. result .. "" end
if 0 < w then
if 0 < e then result = result .. ", " end
if 1 == w then
result = result .. w .. " warning"
else
result = result .. w .. " warnings"
end
if ("" ~= result) and APT.html then result = "" .. result .. "" end
end
if 0 < t then
if (0 < e) or (0 < w) then result = result .. ", " end
if 1 == t then
result = result .. t .. " timeout"
else
result = result .. t .. " timeouts"
end
if ("" ~= result) and APT.html then result = "" .. result .. "" end
end
if "" ~= result then result = " " .. result end
return result
end
local typs = {'errors', 'warnings', 'timeouts'}
APT.padResults = function(results)
local c = 0
if nil == results then results = {}; c = 999 end
for k, v in pairs(APT.protocols) do
tests = results[v]
if nil == tests then tests = {} end
for m, x in pairs(typs) do
if nil == tests[x] then tests[x] = c end
end
for l, w in pairs(APT.tests) do
if ('raw' ~= w) and ('Speed' ~= w) then
if nil == tests[w] then tests[w] = {} end
for m, x in pairs(typs) do
if nil == tests[w][x] then tests[w][x] = c end
end
end
end
if nil == tests.redirects then
tests.redirects = {}
end
results[v] = tests
end
if nil == results.timeout then results.timeout = false end
if nil == results.speed then results.speed = {min = 999999999999; max = 0} end
return results
end
APT.collate = function(l, host, ip, results)
results = APT.padResults(results)
local f = l .. "/" .. host .. "_" .. ip .. ".lua"
if APT.checkFile(f) then
local rs = loadfile(f)()
rs = APT.padResults(rs)
for k, v in pairs(rs) do
if "table" == type(v) then
if ("speed" == k) and (nil ~= results.speed) then
if v.min < results.speed.min then results.speed.min = v.min end
if v.max > results.speed.max then results.speed.max = v.max end
elseif 'IPs' ~= k then
for i, u in pairs(v) do
if 'redirects' ~= i then
if "table" == type(u) then
for h, t in pairs(u) do
local a = results[k]
if nil == a then results[k] = {i = {}}; a = results[k] end
a = a[i]
if nil == a then results[k][i] = {h = {}}; a = results[k][i] end
a = a[h]
if nil == a then a = 0 end
results[k][i][h] = a + t
end
else
local a = results[k]
if nil == a then a = 0; results[k] = {} else a = a[i] end
if nil == a then a = 0 end
results[k][i] = a + u
end
end
end
end
elseif "timeout" ~= k then
local a = results[k]
if nil == a then a = 0 end
results[k] = a + v
end
end
end
return results
end
APT.collateAll = function(hosts, l, host, func)
results = {}
local f = l .. "/" .. host .. ".lua"
if APT.checkFile(f) then
results = loadfile(f)()
results = APT.padResults(results)
if nil ~= func then func(results) end
local v = hosts[host]
if nil ~= v then
local IPs = results.IPs
if nil == IPs then W('No IPs for ' .. host .. ' in ' .. l) else
for i, u in pairs(IPs) do
if "table" == type(u) then
for h, t in pairs(u) do
results = APT.collate(l, host, h, results)
if nil ~= func then func(results, h) end
end
else
results = APT.collate(l, host, i, results)
if nil ~= func then func(results, i) end
end
end
end
end
end
return results
end
APT.now = 0
local status
APT.now = APT.exe('TZ="GMT" ls -l --time-style="+%s" results/stamp | cut -d " " -f 6-6'):Do().result
APT.now = tonumber(APT.now)
local start = 'now-1month'
local step = '10min'
local hb = '150min'
local DSIe = 'DS:IntegrityErrors:GAUGE:' .. hb .. ':0:U'
local DSIw = 'DS:IntegrityWarnings:GAUGE:' .. hb .. ':0:U'
local DSIt = 'DS:IntegrityTimeouts:GAUGE:' .. hb .. ':0:U'
local DSPe = 'DS:ProtocolErrors:GAUGE:' .. hb .. ':0:U'
local DSPw = 'DS:ProtocolWarnings:GAUGE:' .. hb .. ':0:U'
local DSPt = 'DS:ProtocolTimeouts:GAUGE:' .. hb .. ':0:U'
local DSUe = 'DS:UpdatedErrors:GAUGE:' .. hb .. ':0:U'
local DSUw = 'DS:UpdatedWarnings:GAUGE:' .. hb .. ':0:U'
local DSUt = 'DS:UpdatedTimeouts:GAUGE:' .. hb .. ':0:U'
local DSSe = 'DS:URLSanityErrors:GAUGE:' .. hb .. ':0:U'
local DSSw = 'DS:URLSanityWarnings:GAUGE:' .. hb .. ':0:U'
local DSSt = 'DS:URLSanityTimeouts:GAUGE:' .. hb .. ':0:U'
local DSx = 'DS:max:GAUGE:' .. hb .. ':0:U'
local DSn = 'DS:min:GAUGE:' .. hb .. ':0:U'
-- What Collectd uses.
local RRAc0 = 'RRA:AVERAGE:0.9:1:1200'
local RRAc1 = 'RRA:MIN:0.9:1:1200'
local RRAc2 = 'RRA:MAX:0.9:1:1200'
local RRAc3 = 'RRA:AVERAGE:0.9:7:1235'
local RRAc4 = 'RRA:MIN:0.9:7:1235'
local RRAc5 = 'RRA:MAX:0.9:7:1235'
local RRAc6 = 'RRA:AVERAGE:0.9:50:1210'
local RRAc7 = 'RRA:MIN:0.9:50:1210'
local RRAc8 = 'RRA:MAX:0.9:50:1210'
local RRAc9 = 'RRA:AVERAGE:0.9:223:1202'
local RRAc10 = 'RRA:MIN:0.9:223:1202'
local RRAc11 = 'RRA:MAX:0.9:223:1202'
local RRAc12 = 'RRA:AVERAGE:0.9:2635:1201'
local RRAc13 = 'RRA:MIN:0.9:2635:1201'
local RRAc14 = 'RRA:MAX:0.9:2635:1201'
-- Try LAST.
local RRAl0 = 'RRA:LAST:0.9:1:1200'
local RRAl1 = 'RRA:LAST:0.9:7:1235'
local RRAl2 = 'RRA:LAST:0.9:50:1210'
local RRAl3 = 'RRA:LAST:0.9:223:1202'
local RRAl4 = 'RRA:LAST:0.9:2635:1201'
APT.createRRD = function(host, ip, o)
if nil ~= o then start = o end
if nil ~= ip then host = host .. '_' .. ip end
for i, p in pairs(APT.protocols) do
os.execute( 'mkdir -p rrd/' .. host .. '/' .. p:upper())
if not APT.checkFile('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd') then
D('Creating ' .. 'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd')
APT.rrd.create( 'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', '--start', start, '--step', step, DSIe, DSIw, DSIt, DSPe, DSPw, DSPt, DSUe, DSUw, DSUt, DSSe, DSSw, DSSt,
RRAc0, RRAc1, RRAc2, RRAl0, RRAc3, RRAc4, RRAc5, RRAl1, RRAc6, RRAc7, RRAc8, RRAl2, RRAc9, RRAc10, RRAc11, RRAl3, RRAc12, RRAc13, RRAc14, RRAl4)
-- Start them at 0 so the average has something to work on.
APT.rrd.update( 'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', (APT.now - 600) .. ':0:0:0:0:0:0:0:0:0:0:0:0')
end
end
os.execute( 'mkdir -p rrd/' .. host .. '/Speed')
if not APT.checkFile('rrd/' .. host .. '/Speed/Speed.rrd') then
D('Creating ' .. 'rrd/' .. host .. '/Speed/Speed.rrd')
APT.rrd.create( 'rrd/' .. host .. '/Speed/Speed.rrd', '--start', start, '--step', step, DSx, DSn,
RRAc0, RRAc1, RRAc2, RRAl0, RRAc3, RRAc4, RRAc5, RRAl1, RRAc6, RRAc7, RRAc8, RRAl2, RRAc9, RRAc10, RRAc11, RRAl3, RRAc12, RRAc13, RRAc14, RRAl4)
APT.rrd.update( 'rrd/' .. host .. '/Speed/Speed.rrd', (APT.now - 600) .. ':0:0')
end
end
APT.updateRRD = function(results, host, ip)
if nil ~= ip then host = host .. '_' .. ip end
for i, p in pairs(APT.protocols) do
if nil ~= results[p] then
APT.rrd.update('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', APT.now .. ':' ..
results[p]['Integrity'].errors .. ':' .. results[p]['Integrity'].warnings .. ':' .. results[p]['Integrity'].timeouts .. ':' ..
results[p]['Protocol'].errors .. ':' .. results[p]['Protocol'].warnings .. ':' .. results[p]['Protocol'].timeouts .. ':' ..
results[p]['Updated'].errors .. ':' .. results[p]['Updated'].warnings .. ':' .. results[p]['Updated'].timeouts .. ':' ..
results[p]['URLSanity'].errors .. ':' .. results[p]['URLSanity'].warnings .. ':' .. results[p]['URLSanity'].timeouts)
else
APT.rrd.update('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', APT.now .. ':U:U:U:U:U:U:U:U:U:U:U:U')
end
end
if nil ~= results.speed then
if 0 ~= results.speed.max then
APT.rrd.update('rrd/' .. host .. '/Speed/Speed.rrd', APT.now .. ':' .. results.speed.max .. ':' .. results.speed.min)
else
APT.rrd.update('rrd/' .. host .. '/Speed/Speed.rrd', APT.now .. ':0:0')
end
else
APT.rrd.update( 'rrd/' .. host .. '/Speed/Speed.rrd', APT.now .. ':U:U')
end
end
APT.doRRD = function(l, k, v, o)
APT.collateAll(APT.mirrors, l, k,
function(results, ip)
APT.createRRD(k, ip, o)
APT.updateRRD(results, k, ip)
end)
end
return APT