path: root/apt-panopticommon.lua

local APT = {}

-- https://oss.oetiker.ch/rrdtool/prog/rrdlua.en.html
APT.rrd = require 'rrd'

APT.protocols = {"ftp", "http", "https", "rsync"}
APT.tests = {'raw', 'Integrity', 'Protocol', 'Updated', 'URLSanity', 'Speed'}
APT.releases = {"jessie", "ascii", "beowulf", "ceres"}
APT.subRels = {'backports', 'proposed-updates', 'security', 'updates'}
APT.notExist =
{
    'jessie-backports',		-- No longer existing as this is old stable.
    'jessie-proposed-updates',
    'jessie-updates',
    'beowulf-backports',	-- Wont exist until Beowulf goes live.
    'ceres-backports',		-- These will never exist, it's our code name for the testing suite.
    'ceres-proposed-updates',
    'ceres-updates',
    "ceres-security",
}

APT.verbosity = -1
APT.origin = false
APT.redir = false
APT.keep = false

APT.options =
{
    referenceSite =
	{
	    typ = "string",
	    help = "",
	    value = "pkgmaster.devuan.org",
	},
    roundRobin =
	{
	    typ = "string",
	    help = "",
	    value = "deb.devuan.org",
	},
    tests =
	{
	    typ = "table",
	    help = "",
	    value =
	    {
	        "IPv4",
	        "IPv6",
--	        "ftp",
	        "http",
	        "https",
--	        "rsync",
	        "DNSRR",
	        "Protocol",
	        "URLSanity",
--	        "Integrity",
	        "Updated",
	    },
	},
    maxtime =
	{
	    typ = "number",
	    help = "",
	    value = 300,
	},
    timeout =
	{
	    typ = "number",
	    help = "",
	    value = 5,
	},
    timeouts =
	{
	    typ = "number",
	    help = "",
	    value = 3,
	},
    refresh =
	{
	    typ = "number",
	    help = "",
	    value = 300,
	},
    retries =
	{
	    typ = "number",
	    help = "",
	    value = 3,
	},
    reports =
	{
	    typ = "table",
	    help = "",
	    value =
	    {
		"RRD",		-- RRD has to be before web, coz web creates a graph from the RRD data.
		"email-web",
--		"Nagios",
--		"Prometheus",
	    },
	},
}

APT.args = {}
APT.parseArgs = function(args)
    APT.args = args
    local arg = {}
    local sendArgs = ""
    if 0 ~= #(args) then
	local option = ""
	for i, a in pairs(args) do
	    if ("--help" == a) or ("-h" == a) then
		print("I should write some docs, huh?  Read README.md for instructions.")
	    elseif "--version" == a then
		print("apt-panopticon version 0.1 WIP development version")
	    elseif "-v" == a then
		APT.verbosity = APT.verbosity + 1
		sendArgs = sendArgs .. a .. " "
	    elseif "-q" == a then
		APT.verbosity = -1
		sendArgs = sendArgs .. a .. " "
	    elseif "-k" == a then
		APT.keep = true
		sendArgs = sendArgs .. a .. " "
	    elseif "-o" == a then
		APT.origin = true
		-- Not adding to sendArgs.
	    elseif "-r" == a then
		APT.redir = true
		sendArgs = sendArgs .. a .. " "
	    elseif "--" == a:sub(1, 2) then
		local s, e = a:find("=")
		if nil == s then e = -1 end
		option = a:sub(3, e - 1)
		local o = APT.options[option]
		if nil == o then
		    print("Unknown option --" .. option)
		    option = ""
		else
		    option = a
		    sendArgs = sendArgs .. a .. " "
		    local s, e = a:find("=")
		    if nil == s then e = 0 end
		    option = a:sub(3, e - 1)
		    if "table" == APT.options[option].typ then
			local result = {}
			for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do
			    local f = t:sub(1, 1)
			    local n = t:sub(2, -1)
			    if ("+" ~= f) and ("-" ~= f) then
				table.insert(result, t)
			    end
			end
			if 0 ~= #result then
			    APT.options[option].value = result
			else
			    for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do
				local f = t:sub(1, 1)
				local n = t:sub(2, -1)
				if "+" == f then
				    table.insert(APT.options[option].value, n)
				elseif "-" == f then
				    local r = {}
				    for i, k in pairs(APT.options[option].value) do
					if k ~= n then table.insert(r, k) end
				    end
				    APT.options[option].value = r
				end
			    end
			end
		    else
			APT.options[option].value = a:sub(e + 1, -1)
		    end
		    option = ""
		end
	    elseif "-" == a:sub(1, 1) then
		print("Unknown option " .. a)
	    else
		table.insert(arg, a)
	    end
	end
    end
    return arg, sendArgs
end

--print(APT.dumpTable(APT.options, "", "options"))



--[[ Ordered table iterator, allow to iterate on the natural order of the keys of a table.
    From http://lua-users.org/wiki/SortedIteration
  ]]
function __genOrderedIndex( t )
    local orderedIndex = {}
    for key in pairs(t) do
        table.insert( orderedIndex, key )
    end
    table.sort( orderedIndex )
    return orderedIndex
end
function orderedNext(t, state)
    -- Equivalent of the next function, but returns the keys in the alphabetic
    -- order. We use a temporary ordered key table that is stored in the
    -- table being iterated.

    local key = nil
    --print("orderedNext: state = "..tostring(state) )
    if state == nil then
        -- the first time, generate the index
        t.__orderedIndex = __genOrderedIndex( t )
        key = t.__orderedIndex[1]
    else
        -- fetch the next value
        for i = 1,table.getn(t.__orderedIndex) do
            if t.__orderedIndex[i] == state then
                key = t.__orderedIndex[i+1]
            end
        end
    end

    if key then
        return key, t[key]
    end

    -- no more value to return, cleanup
    t.__orderedIndex = nil
    return
end
function APT.orderedPairs(t)
    -- Equivalent of the pairs() function on tables. Allows to iterate
    -- in order
    return orderedNext, t, nil
end

-- Use this to dump a table to a string, with HTML.
APT.dumpTableHTML = function (table, space, name)
    local r = name .. "\n"
    r = r .. dumpTableHTMLSub(table, space .. " ")
    r = r .. space .. ""
    return r
end
dumpTableHTMLSub = function (table, space)
    local r = ""
    for k, v in APT.orderedPairs(table) do 
	if type(v) == "table" then
	    if " " == space then
		r = r ..           space .. APT.dumpTableHTML(v, space, k .. "<ul>") .. "</ul>\n"
	    else
		r = r .. "<li>" .. space .. APT.dumpTableHTML(v, space, k .. "<ul>") .. "</ul></li>\n"
	    end
	else
	    r = r .. space .. "<li>" .. k .. "</li>\n"
	end
    end
    return r
end

-- Use this to dump a table to a string.
APT.dumpTable = function (table, space, name)
    local r = ""
    if "" == space then r = r .. space .. name .. " =\n" else r = r .. space .. "[" .. name .. "] =\n" end
    r = r .. space .. "{\n"
    r = r .. dumpTableSub(table, space .. "  ")
    if "" == space then r = r .. space .. "}\n" else r = r .. space .. "},\n" end
    return r
end
dumpTableSub = function (table, space)
    local r = ""
    for k, v in pairs(table) do 
        if type(k) == "string" then k = '"' .. k .. '"' end
	if type(v) == "table" then
	    r = r .. APT.dumpTable(v, space, k)
	elseif type(v) == "string" then
	    local bq = '"'
	    local eq = '"'
	    if nil ~= v:match(bq) then
		bq, eq = '"', '"'
		if nil ~= v:match(bq) then
		    bq, eq = '[[', ']]'
		    mbq, meq = '%[%[', '%]%]'
		    while (nil ~= v:match(mbq)) or (nil ~= v:match(meq)) do
			bq = '[' .. '=' .. bq:sub(2, -1)
			eq = ']' .. '=' .. eq:sub(2, -1)
			mbq = '%[' .. '=' .. bq:sub(3, -1)
			meq = '%]' .. '=' .. eq:sub(3, -1)
		    end
		end
	    end
	    r = r .. space .. "[" .. k .. "] = " .. bq .. v .. eq .. ";\n"
	elseif type(v) == "function" then
	    r = r .. space .. "[" .. k .. "] = function ();\n"
	elseif type(v) == "userdata" then
	    r = r .. space .. "userdata " .. "[" .. k .. "];\n"
	elseif type(v) == "boolean" then
	    if (v) then
		r = r .. space .. "[" .. k .. "] = true;\n"
	    else
		r = r .. space .. "[" .. k .. "] = false;\n"
	    end
	else
	    r = r .. space .. "[" .. k .. "] = " .. v .. ";\n"
	end
    end
    return r
end

APT.search = function(t, s)
    for i, v in pairs(t) do
	if v == s then return true end
    end
    return false
end

APT.results = {}
APT.logFile = nil
APT.html = false

APT.logName = function(host, a2, a3)
    local name = host
    if (nil ~= a2) and ('' ~= a2) then name = name .. "_" .. a2 end
    if (nil ~= a3) and ('' ~= a3) then name = name .. "_" .. a3 end
    name = 'LOG_' .. name .. '.html'
    return {'results/' .. name, '<a href="' .. name:gsub("/", "_") .. '">' .. name:gsub("/", "_") .. '</a>'}
end

APT.logOpen = function(host, a2, a3)
    local name = APT.logName(host, a2, a3)[1]
    if APT.checkFile(name) then return false end
    APT.logFile, e = io.open(name, "a+")
    if nil == APT.logFile then C('opening log file (' .. name .. ') - ' .. e); return false end
    if nil ~= APT.logFile then
	APT.logFile:write("<html><head>\n")
	APT.logFile:write("</head><body bgcolor='black' text='white' alink='red' link='blue' vlink='purple'>\n")
	APT.logFile:write("<pre>\n")
	APT.logFile:write(APT.dumpTable(APT.args, '', 'Arguments'))
	APT.logFile:write("</pre>\n")
    else
	return false
    end
    return true
end

APT.logPost = function()
    if nil ~= APT.logFile then
	APT.logFile:write("</body></html> \n")
	APT.logFile:close()
    end
end

local log = function(v, t, s, prot, test, host)
    local x = ""
    if nil == prot then prot = "" end
    if nil == test then test = "" end
    x = x .. prot
    if "" ~= test then
	if #x > 0 then x = x .. " " end
	x = x .. test
    end
    if nil ~= host then
	if #x > 0 then x = x .. " " end
	x = x .. host
    end
    if #x > 0 then
	t = t .. "(" .. x ..  ")"
	if "" ~= prot then
	    if "" == test then
		if nil == APT.results[prot] then APT.results[prot] = {errors = 0; warnings = 0; timeouts = 0} end
		if v == 0 then APT.results[prot].errors = APT.results[prot].errors + 1 end
		if v == 1 then APT.results[prot].warnings = APT.results[prot].warnings + 1 end
		if v == 2 then APT.results[prot].timeouts = APT.results[prot].timeouts + 1 end
	    else
		if nil == APT.results[prot] then APT.results[prot] = {errors = 0; warnings = 0; timeouts = 0} end
		if nil == APT.results[prot][test] then APT.results[prot][test] = {errors = 0; warnings = 0; timeouts = 0} end
		if v == 0 then APT.results[prot][test].errors = APT.results[prot][test].errors + 1 end
		if v == 1 then APT.results[prot][test].warnings = APT.results[prot][test].warnings + 1 end
		if v == 2 then APT.results[prot][test].timeouts = APT.results[prot][test].timeouts + 1 end
	    end
	end
    end
    if v <= APT.verbosity then
	if 3 <= APT.verbosity then t = os.date('!%F %T') .. " " .. t end
	print(t .. ": " .. s)
    end
    if nil ~= APT.logFile then
	if APT.html then
	    local colour = "white"
	    if -1 == v then colour = "fuchsia" end	-- CRITICAL
	    if  0 == v then colour = "red    " end	-- ERROR
	    if  1 == v then colour = "yellow " end	-- WARNING
	    if  2 == v then colour = "blue   " end	-- TIMEOUT
	    if  3 == v then colour = "white  " end	-- INFO
	    if  4 == v then colour = "gray   " end	-- DEBUG
	    APT.logFile:write(os.date('!%F %T') .. " <font color='" .. colour .. "'><b>" .. t .. "</b></font>: " .. s .. "</br>\n")
	else
	    APT.logFile:write(os.date('!%F %T') .. " " .. t .. ": " .. s .. "\n")
	end
	APT.logFile:flush()
    end
end
APT.D = function(s) log(4,  "DEBUG   ", s) end
APT.I = function(s) log(3,  "INFO    ", s) end
APT.T = function(s, p, t, h) log(2,  "TIMEOUT ", s, p, t, h) end
APT.W = function(s, p, t, h) log(1,  "WARNING ", s, p, t, h) end
APT.E = function(s, p, t, h) log(0,  "ERROR   ", s, p, t, h) end
APT.C = function(s) log(-1, "CRITICAL", s) end
local D = APT.D
local I = APT.I
local T = APT.T
local W = APT.W
local E = APT.E
local C = APT.C


APT.debians = {}
APT.mirrors = {}

APT.testing = function(t, host)
    for i, v in pairs(APT.options.tests.value) do
	if t == v then
	    local h = APT.mirrors[host]
	    if nil == h then return true end
	    if true == h["Protocols"][t] then return true else D("Skipping " .. t .. " checks for " .. host) end
	end
    end
    return false
end

APT.execute = function (s)
    D("  executing - &nbsp; <code>" .. s .. "</code>")
    --[[ Damn os.execute()
	Lua 5.1 says it returns "a status code, which is system-dependent"
	Lua 5.2 says it returns true/nil, "exit"/"signal", the status code.
	I'm getting 7168 or 0.  No idea what the fuck that is.
    local ok, rslt, status = os.execute(s)
    ]]
    local f = io.popen(s .. ' ; echo "$?"', 'r')
    local status = ""
    local result = ""
    -- The last line will be the command's returned status, collect everything else in result.
    for l in f:lines() do
	result = result .. status .. "\n"
	status = l
    end
    return status, result
end

APT.fork = function(s)
    D("  forking - &nbsp; <code>" .. s .. "</code>")
    os.execute(s .. " &")
end

APT.checkExes = function (exe)
    local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | grep -v "flock -n apt-panopticon.lock " | wc -l'):read("*l")
    D(count .. " " .. exe .. " commands still running.")
    return tonumber(count)
end

APT.checkFile = function(f)
    local h, e = io.open(f, "r")
    if nil == h then return false else h:close();  return true end
end

APT.plurals = function(e, w, t)
    local result = ""
    if 1 == e then
	result = e .. " error"
    elseif e ~= 0 then
	result = e .. " errors"
    end
    if ("" ~= result) and APT.html then result = "<font color='red'><b>" .. result .. "</b></font>" end
    if 0 < w then
	if 0 < e then result = result .. ", " end
	if 1 == w then 
	    result = result .. w .. " warning"
	else
	    result = result .. w .. " warnings"
	end
	if ("" ~= result) and APT.html then result = "<font color='yellow'><b>" .. result .. "</b></font>" end
    end
    if 0 < t then
	if (0 < e) or (0 < w) then result = result .. ", " end
	if 1 == t then 
	    result = result .. t .. " timeout"
	else
	    result = result .. t .. " timeouts"
	end
	if ("" ~= result) and APT.html then result = "<font color='blue'><b>" .. result .. "</b></font>" end
    end
    if "" ~= result then result = " (" .. result .. ")" end
    return result
end

APT.padResults = function(results)
    local c = 0
    if nil == results then results = {}; c = 999 end
    for k, v in pairs(APT.protocols) do
	tests = results[v]
	if nil == tests then tests = {errors = c; warnings = c; timeouts = c} end
	if nil == tests.Integrity then tests.Integrity = {errors = c; warnings = c; timeouts = c} end
	if nil == tests.Protocol  then tests.Protocol  = {errors = c; warnings = c; timeouts = c} end
	if nil == tests.Updated   then tests.Updated   = {errors = c; warnings = c; timeouts = c} end
	if nil == tests.redirects then tests.redirects = {} end
	if nil == tests.URLSanity then tests.URLSanity = {errors = c; warnings = c; timeouts = c} end
	results[v] = tests
    end
    if nil == results.timeout then results.timeout = false end
    if nil == results.speed   then results.speed   = {min = 999999999999; max = 0} end
    return results
end

APT.collate = function(l, host, ip, results)
    results = APT.padResults(results)
    local f = l .. "/" .. host .. "_" .. ip .. ".lua"
    if APT.checkFile(f) then
	local rs = loadfile(f)()
	rs = APT.padResults(rs)
	for k, v in pairs(rs) do
	    if "table" == type(v) then
		if ("speed" == k) and (nil ~= results.speed) then
		    if v.min < results.speed.min then results.speed.min = v.min end
		    if v.max > results.speed.max then results.speed.max = v.max end
		elseif 'IPs' ~= k then
		    for i, u in pairs(v) do
			if 'redirects' ~= i then
			    if "table" == type(u) then
				for h, t in pairs(u) do
				        local a = results[k]
				    if nil == a then results[k]    = {i = {}}; a = results[k]    end
				    a = a[i]
				    if nil == a then results[k][i] = {h = {}}; a = results[k][i] end
				    a = a[h]
				    if nil == a then a = 0 end
				    results[k][i][h] = a + t
				end
			    else
				local a = results[k]
				if nil == a then a = 0; results[k] = {} else a = a[i] end
				if nil == a then a = 0 end
				results[k][i] = a + u
			    end
			end
		    end
		end
	    elseif "timeout" ~= k then
		local a = results[k]
		if nil == a then a = 0 end
		results[k] = a + v
	    end
	end
    end
    return results
end
APT.collateAll = function(hosts, l, host, func)
    results = {}
    local f = l .. "/" .. host .. ".lua"
    if APT.checkFile(f) then
	results = loadfile(f)()
	results = APT.padResults(results)
	if nil ~= func then func(results) end
	local v = hosts[host]
	if nil ~= v then
	    local IPs = results.IPs
	    if nil == IPs then W('No IPs for ' .. host .. ' in ' .. l) else
		for i, u in pairs(IPs) do
		    if "table" == type(u) then
			for h, t in pairs(u) do
			    results = APT.collate(l, host, h, results)
			    if nil ~= func then func(results, h) end
			end
		    else
			results = APT.collate(l, host, i, results)
			if nil ~= func then func(results, i) end
		    end
		end
	    end
	end
    end
    return results
end


APT.now = 0
local status
status, APT.now = APT.execute('TZ="GMT" ls -l --time-style="+%s" results/stamp | cut -d " " -f 6-6')
APT.now = tonumber(APT.now)
local start = 'now-1month'
local step = '10min'
local hb = '150min'
local DSIe = 'DS:IntegrityErrors:GAUGE:' .. hb .. ':0:U'
local DSIw = 'DS:IntegrityWarnings:GAUGE:' .. hb .. ':0:U'
local DSPe = 'DS:ProtocolErrors:GAUGE:' .. hb .. ':0:U'
local DSPw = 'DS:ProtocolWarnings:GAUGE:' .. hb .. ':0:U'
local DSUe = 'DS:UpdatedErrors:GAUGE:' .. hb .. ':0:U'
local DSUw = 'DS:UpdatedWarnings:GAUGE:' .. hb .. ':0:U'
local DSSe = 'DS:URLSanityErrors:GAUGE:' .. hb .. ':0:U'
local DSSw = 'DS:URLSanityWarnings:GAUGE:' .. hb .. ':0:U'
local DSx = 'DS:max:GAUGE:' .. hb .. ':0:U'
local DSn = 'DS:min:GAUGE:' .. hb .. ':0:U'
-- What Collectd uses.
local RRAc0  = 'RRA:AVERAGE:0.9:1:1200'
local RRAc1  = 'RRA:MIN:0.9:1:1200'
local RRAc2  = 'RRA:MAX:0.9:1:1200'
local RRAc3  = 'RRA:AVERAGE:0.9:7:1235'
local RRAc4  = 'RRA:MIN:0.9:7:1235'
local RRAc5  = 'RRA:MAX:0.9:7:1235'
local RRAc6  = 'RRA:AVERAGE:0.9:50:1210'
local RRAc7  = 'RRA:MIN:0.9:50:1210'
local RRAc8  = 'RRA:MAX:0.9:50:1210'
local RRAc9  = 'RRA:AVERAGE:0.9:223:1202'
local RRAc10 = 'RRA:MIN:0.9:223:1202'
local RRAc11 = 'RRA:MAX:0.9:223:1202'
local RRAc12 = 'RRA:AVERAGE:0.9:2635:1201'
local RRAc13 = 'RRA:MIN:0.9:2635:1201'
local RRAc14 = 'RRA:MAX:0.9:2635:1201'
-- Try LAST.
local RRAl0  = 'RRA:LAST:0.9:1:1200'
local RRAl1  = 'RRA:LAST:0.9:7:1235'
local RRAl2  = 'RRA:LAST:0.9:50:1210'
local RRAl3  = 'RRA:LAST:0.9:223:1202'
local RRAl4  = 'RRA:LAST:0.9:2635:1201'

APT.createRRD = function(host, ip, o)
    if nil ~= o then start = o end
    if nil ~= ip then host = host .. '_' .. ip end
    for i, p in pairs(APT.protocols) do
	os.execute( 'mkdir -p rrd/' .. host .. '/' .. p:upper())
	if not APT.checkFile('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd') then
	    D('Creating ' .. 'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd')
	    APT.rrd.create(  'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', '--start', start, '--step', step, DSIe, DSIw, DSPe, DSPw, DSUe, DSUw, DSSe, DSSw,
			    RRAc0, RRAc1, RRAc2, RRAl0, RRAc3, RRAc4, RRAc5, RRAl1, RRAc6, RRAc7, RRAc8, RRAl2, RRAc9, RRAc10, RRAc11, RRAl3, RRAc12, RRAc13, RRAc14, RRAl4)
	    -- Start them at 0 so the average has something to work on.
	    APT.rrd.update(      'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', (APT.now - 600) .. ':0:0:0:0:0:0:0:0')
	end
    end
    os.execute( 'mkdir -p rrd/' .. host .. '/Speed')
    if not APT.checkFile('rrd/' .. host .. '/Speed/Speed.rrd') then
	D('Creating ' .. 'rrd/' .. host .. '/Speed/Speed.rrd')
	APT.rrd.create(  'rrd/' .. host .. '/Speed/Speed.rrd', '--start', start, '--step', step, DSx, DSn,
			RRAc0, RRAc1, RRAc2, RRAl0, RRAc3, RRAc4, RRAc5, RRAl1, RRAc6, RRAc7, RRAc8, RRAl2, RRAc9, RRAc10, RRAc11, RRAl3, RRAc12, RRAc13, RRAc14, RRAl4)
	APT.rrd.update(      'rrd/' .. host .. '/Speed/Speed.rrd', (APT.now - 600) .. ':0:0')
    end
end

APT.updateRRD = function(results, host, ip)
    if nil ~= ip then host = host .. '_' .. ip end
    for i, p in pairs(APT.protocols) do
	if nil ~= results[p] then
	    APT.rrd.update('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', APT.now .. ':' ..
	    results[p]['Integrity'].errors .. ':'  .. results[p]['Integrity'].warnings .. ':' ..
	    results[p]['Protocol'].errors  .. ':'  .. results[p]['Protocol'].warnings  .. ':' ..
	    results[p]['Updated'].errors   .. ':'  .. results[p]['Updated'].warnings   .. ':' ..
	    results[p]['URLSanity'].errors .. ':'  .. results[p]['URLSanity'].warnings)
	else
	    APT.rrd.update('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', APT.now .. ':U:U:U:U:U:U:U:U')
	end
    end
    if nil ~= results.speed then
	if 0 ~= results.speed.max then
	    APT.rrd.update('rrd/' .. host .. '/Speed/Speed.rrd', APT.now .. ':' .. results.speed.max .. ':'  .. results.speed.min)
	else
	    APT.rrd.update('rrd/' .. host .. '/Speed/Speed.rrd', APT.now .. ':0:0')
	end
    else
	APT.rrd.update(    'rrd/' .. host .. '/Speed/Speed.rrd', APT.now .. ':U:U')
    end
end

APT.doRRD = function(l, k, v, o)
    APT.collateAll(APT.mirrors, l, k,
	function(results, ip)
	    APT.createRRD(k, ip, o)
	    APT.updateRRD(results, k, ip)
	end)
end


return APT