local APT = {} -- https://oss.oetiker.ch/rrdtool/prog/rrdlua.en.html APT.rrd = require 'rrd' APT.protocols = {"ftp", "http", "https", "rsync"} APT.tests = {'raw', 'Integrity', 'Protocol', 'Updated', 'URLSanity', 'Speed'} APT.releases = {"jessie", "ascii", "beowulf", "ceres"} APT.subRels = {'backports', 'proposed-updates', 'security', 'updates'} APT.notExist = { 'jessie-backports', -- No longer existing as this is old stable. 'jessie-proposed-updates', 'jessie-updates', 'beowulf-backports', -- Wont exist until Beowulf goes live. 'ceres-backports', -- These will never exist, it's our code name for the testing suite. 'ceres-proposed-updates', 'ceres-updates', "ceres-security", } APT.verbosity = -1 APT.origin = false APT.redir = false APT.keep = false APT.options = { bandwidth = { typ = "number", help = ' 0 = low - HTTP tests for all IPs of all mirrors (HEAD tests), but not URL sanity tests.\n' .. ' 1 = medium - Also HTTP(S) redirect tests, other protocols, download Release files over HTTP, and check them.\n' .. ' 2 = high - Also download Packages.xz files that changed, unpack and check them.\n' .. ' Also download and check InRelease files\n.' .. ' Pick a few small packages, download them, check their SHA512.\n' .. ' 3 = more - Also pick more packages, some for each arch.\n' .. ' 4 = all - Do absolutely everything.\n' .. ' Actually download some Contents files, and some more Packages.xz, and package files.', value = 2, }, referenceSite = { typ = "string", help = "The mirror that is used as a reference, coz the oters sync to it.", value = "pkgmaster.devuan.org", }, roundRobin = { typ = "string", help = "The round robin DNS name.", value = "deb.devuan.org", }, tests = { typ = "table", help = "A list of the tests, see the documentation for details.", value = { "IPv4", "IPv6", -- "ftp", "http", "https", -- "rsync", "DNSRR", "Protocol", "URLSanity", "Integrity", "Updated", }, }, maxtime = { typ = "number", help = "", value = 240, }, timeout = { typ = "number", help = "", value = 5, }, timeouts = { typ = "number", help = "", value = 3, }, refresh = { typ = "number", help = "", value = 300, }, retries = { typ = "number", help = "", value = 2, }, reports = { typ = "table", help = "A list of the reports generated.", value = { "RRD", -- RRD has to be before web, coz web creates a graph from the RRD data. "email-web", -- "Prometheus", }, }, } APT.args = {} APT.parseArgs = function(args) APT.args = args local arg = {} local sendArgs = "" if 0 ~= #(args) then local option = "" for i, a in pairs(args) do if ("--help" == a) or ("-h" == a) then print("I should write some docs, huh? Read README.md for instructions.") elseif "--version" == a then print("apt-panopticon version 0.1 WIP development version") elseif "-v" == a then APT.verbosity = APT.verbosity + 1 sendArgs = sendArgs .. a .. " " elseif "-q" == a then APT.verbosity = -1 sendArgs = sendArgs .. a .. " " elseif "-k" == a then APT.keep = true sendArgs = sendArgs .. a .. " " elseif "-o" == a then APT.origin = true -- Not adding to sendArgs. elseif "-r" == a then APT.redir = true sendArgs = sendArgs .. a .. " " elseif "--low" == a then APT.options.bandwidth.value = 0 APT.options.timeout.value = 2 APT.options.timeouts.value = 1 APT.options.retries.value = 1 sendArgs = sendArgs .. a .. " " elseif "--medium" == a then APT.options.bandwidth.value = 1 sendArgs = sendArgs .. a .. " " elseif "--high" == a then APT.options.bandwidth.value = 2 sendArgs = sendArgs .. a .. " " elseif "--more" == a then APT.options.bandwidth.value = 3 sendArgs = sendArgs .. a .. " " elseif "--all" == a then APT.options.bandwidth.value = 4 sendArgs = sendArgs .. a .. " " elseif "--" == a:sub(1, 2) then local s, e = a:find("=") if nil == s then e = -1 end option = a:sub(3, e - 1) local o = APT.options[option] if nil == o then print("Unknown option --" .. option) option = "" else option = a sendArgs = sendArgs .. a .. " " local s, e = a:find("=") if nil == s then e = 0 end option = a:sub(3, e - 1) if "table" == APT.options[option].typ then local result = {} for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do local f = t:sub(1, 1) local n = t:sub(2, -1) if ("+" ~= f) and ("-" ~= f) then table.insert(result, t) end end if 0 ~= #result then APT.options[option].value = result else for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do local f = t:sub(1, 1) local n = t:sub(2, -1) if "+" == f then table.insert(APT.options[option].value, n) elseif "-" == f then local r = {} for i, k in pairs(APT.options[option].value) do if k ~= n then table.insert(r, k) end end APT.options[option].value = r end end end else APT.options[option].value = a:sub(e + 1, -1) end option = "" end elseif "-" == a:sub(1, 1) then print("Unknown option " .. a) else table.insert(arg, a) end end end return arg, sendArgs end --[[ Ordered table iterator, allow to iterate on the natural order of the keys of a table. From http://lua-users.org/wiki/SortedIteration ]] function __genOrderedIndex( t ) local orderedIndex = {} for key in pairs(t) do table.insert( orderedIndex, key ) end table.sort( orderedIndex ) return orderedIndex end function orderedNext(t, state) -- Equivalent of the next function, but returns the keys in the alphabetic -- order. We use a temporary ordered key table that is stored in the -- table being iterated. local key = nil --print("orderedNext: state = "..tostring(state) ) if state == nil then -- the first time, generate the index t.__orderedIndex = __genOrderedIndex( t ) key = t.__orderedIndex[1] else -- fetch the next value for i = 1,table.getn(t.__orderedIndex) do if t.__orderedIndex[i] == state then key = t.__orderedIndex[i+1] end end end if key then return key, t[key] end -- no more value to return, cleanup t.__orderedIndex = nil return end function APT.orderedPairs(t) -- Equivalent of the pairs() function on tables. Allows to iterate -- in order return orderedNext, t, nil end -- Use this to dump a table to a string, with HTML. APT.dumpTableHTML = function (table, name, space) if nil == space then space = '' end local r = name .. "\n" r = r .. dumpTableHTMLSub(table, space .. " ") r = r .. space .. "" return r end dumpTableHTMLSub = function (table, space) local r = "" for k, v in APT.orderedPairs(table) do if type(v) == "table" then if " " == space then r = r .. space .. APT.dumpTableHTML(v, k .. "\n" else r = r .. "
  • " .. space .. APT.dumpTableHTML(v, k .. "
  • \n" end else r = r .. space .. "
  • " .. k .. "
  • \n" end end return r end -- Use this to dump a table to a string. APT.dumpTable = function (table, name, space) if nil == space then space = '' end local r = "" if "" == space then r = r .. space .. name .. " =\n" else r = r .. space .. "[" .. name .. "] =\n" end r = r .. space .. "{\n" r = r .. dumpTableSub(table, space .. " ") if "" == space then r = r .. space .. "}\n" else r = r .. space .. "},\n" end return r end dumpTableSub = function (table, space) local r = "" for k, v in pairs(table) do if type(k) == "string" then k = '"' .. k .. '"' end if type(v) == "table" then r = r .. APT.dumpTable(v, k, space) elseif type(v) == "string" then local bq = '"' local eq = '"' if nil ~= v:match(bq) then bq, eq = '"', '"' if nil ~= v:match(bq) then bq, eq = '[[', ']]' mbq, meq = '%[%[', '%]%]' while (nil ~= v:match(mbq)) or (nil ~= v:match(meq)) do bq = '[' .. '=' .. bq:sub(2, -1) eq = ']' .. '=' .. eq:sub(2, -1) mbq = '%[' .. '=' .. bq:sub(3, -1) meq = '%]' .. '=' .. eq:sub(3, -1) end end end r = r .. space .. "[" .. k .. "] = " .. bq .. v .. eq .. ";\n" elseif type(v) == "function" then r = r .. space .. "[" .. k .. "] = function ();\n" elseif type(v) == "userdata" then r = r .. space .. "userdata " .. "[" .. k .. "];\n" elseif type(v) == "boolean" then if (v) then r = r .. space .. "[" .. k .. "] = true;\n" else r = r .. space .. "[" .. k .. "] = false;\n" end else r = r .. space .. "[" .. k .. "] = " .. v .. ";\n" end end return r end APT.allpairs = function(tbl, func) for k, v in pairs(tbl) do if 'table' == type(v) then for i, w in pairs(v) do func(i, w, k, v) end else func(k, v) end end end APT.search = function(t, s) for i, v in pairs(t) do if v == s then return true end end return false end APT.results = {} APT.logFile = nil APT.html = false APT.logName = function(host, a2, a3) local name = host if (nil ~= a2) and ('' ~= a2) then name = name .. "_" .. a2 end if (nil ~= a3) and ('' ~= a3) then name = name .. "_" .. a3 end name = 'LOG_' .. name .. '.html' return {'results/' .. name, '' .. name:gsub("/", "_") .. ''} end APT.logOpen = function(host, a2, a3) local name = APT.logName(host, a2, a3)[1] if APT.checkFile(name) then return false end APT.logFile, e = io.open(name, "a+") if nil == APT.logFile then C('opening log file (' .. name .. ') - ' .. e); return false end if nil ~= APT.logFile then APT.logFile:write("\n") APT.logFile:write("\n") APT.logFile:write("
    \n")
    	APT.logFile:write(APT.dumpTable(APT.args, 'Arguments'))
    	APT.logFile:write("
    \n") else return false end return true end APT.logPost = function() if nil ~= APT.logFile then APT.logFile:write(" \n") APT.logFile:close() end end local log = function(v, t, s, prot, test, host) local x = "" if nil == prot then prot = "" end if nil == test then test = "" end x = x .. prot if "" ~= test then if #x > 0 then x = x .. " " end x = x .. test end if nil ~= host then if #x > 0 then x = x .. " " end x = x .. host end if #x > 0 then t = t .. "(" .. x .. ")" if "" ~= prot then if "" == test then if nil == APT.results[prot] then APT.results[prot] = {errors = 0; warnings = 0; timeouts = 0} end if v == 0 then APT.results[prot].errors = APT.results[prot].errors + 1 end if v == 1 then APT.results[prot].warnings = APT.results[prot].warnings + 1 end if v == 2 then APT.results[prot].timeouts = APT.results[prot].timeouts + 1 end else if nil == APT.results[prot] then APT.results[prot] = {errors = 0; warnings = 0; timeouts = 0} end if nil == APT.results[prot][test] then APT.results[prot][test] = {errors = 0; warnings = 0; timeouts = 0} end if v == 0 then APT.results[prot][test].errors = APT.results[prot][test].errors + 1 end if v == 1 then APT.results[prot][test].warnings = APT.results[prot][test].warnings + 1 end if v == 2 then APT.results[prot][test].timeouts = APT.results[prot][test].timeouts + 1 end end end end if v <= APT.verbosity then if 3 <= APT.verbosity then t = os.date('!%F %T') .. " " .. t end print(t .. ": " .. s) end if nil ~= APT.logFile then if APT.html then local colour = "white" if -1 == v then colour = "fuchsia" end -- CRITICAL if 0 == v then colour = "red " end -- ERROR if 1 == v then colour = "yellow " end -- WARNING if 2 == v then colour = "blue " end -- TIMEOUT if 3 == v then colour = "white " end -- INFO if 4 == v then colour = "gray " end -- DEBUG APT.logFile:write(os.date('!%F %T') .. " " .. t .. ": " .. s .. "
    \n") else APT.logFile:write(os.date('!%F %T') .. " " .. t .. ": " .. s .. "\n") end APT.logFile:flush() end end APT.D = function(s) log(4, "DEBUG ", s) end APT.I = function(s) log(3, "INFO ", s) end APT.T = function(s, p, t, h) log(2, "TIMEOUT ", s, p, t, h) end APT.W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end APT.E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end APT.C = function(s) log(-1, "CRITICAL", s) end local D = APT.D local I = APT.I local T = APT.T local W = APT.W local E = APT.E local C = APT.C APT.debians = {} APT.mirrors = {} APT.testing = function(t, host) for i, v in pairs(APT.options.tests.value) do if t == v then local h = APT.mirrors[host] if nil == h then return true end if true == h["Protocols"][t] then return true else D("Skipping " .. t .. " checks for " .. host) end end end return false end APT.exe = function(c) local exe = {status = 0, result = '', log = true, cmd = c .. ' '} function exe:log() self.log = true return self end function exe:Nice(c) if nil == c then self.cmd = 'ionice -c3 nice -n 19 ' .. self.cmd else self.cmd = self.cmd .. 'ionice -c3 nice -n 19 ' .. c .. ' ' end return self end function exe:also(c) if nil == c then c = '' else c = ' ' .. c end self.cmd = self.cmd .. ';' .. c .. ' ' return self end function exe:And(c) if nil == c then c = '' else c = ' ' .. c end self.cmd = self.cmd .. '&&' .. c .. ' ' return self end function exe:Or(c) if nil == c then c = '' end self.cmd = self.cmd .. '|| ' .. c .. ' ' return self end function exe:noErr() self.cmd = self.cmd .. '2>/dev/null ' return self end function exe:wait(w) self.cmd = self.cmd .. '&& touch ' .. w .. ' ' return self end function exe:Do() --[[ "The condition expression of a control structure can return any value. Both false and nil are considered false. All values different from nil and false are considered true (in particular, the number 0 and the empty string are also true)." says the docs, I beg to differ.]] if true == self.log then D(" executing -   " .. self.cmd .. "") end --[[ Damn os.execute() Lua 5.1 says it returns "a status code, which is system-dependent" Lua 5.2 says it returns true/nil, "exit"/"signal", the status code. I'm getting 7168 or 0. No idea what the fuck that is. local ok, rslt, status = os.execute(s) ]] local f = io.popen(self.cmd .. ' ; echo "$?"', 'r') -- The last line will be the command's returned status, collect everything else in result. self.status = '' -- Otherwise the result starts with 0. for l in f:lines() do self.result = self.result .. self.status .. "\n" self.status = l end self.status = tonumber(self.status) return self end function exe:fork() self.cmd = '{ ' .. self.cmd .. '; } &' if true == self.log then D(" forking -   " .. self.cmd .. "") end os.execute(self.cmd) return self end return exe end APT.checkExes = function (exe) local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | grep -v "flock -n apt-panopticon.lock " | wc -l'):read("*l") D(count .. " " .. exe .. " commands still running.") return tonumber(count) end APT.checkFile = function(f) local h, e = io.open(f, "r") if nil == h then return false else h:close(); return true end end APT.plurals = function(e, w, t) local result = "" if 1 == e then result = e .. " error" elseif e ~= 0 then result = e .. " errors" end if ("" ~= result) and APT.html then result = "" .. result .. "" end if 0 < w then if 0 < e then result = result .. ", " end if 1 == w then result = result .. w .. " warning" else result = result .. w .. " warnings" end if ("" ~= result) and APT.html then result = "" .. result .. "" end end if 0 < t then if (0 < e) or (0 < w) then result = result .. ", " end if 1 == t then result = result .. t .. " timeout" else result = result .. t .. " timeouts" end if ("" ~= result) and APT.html then result = "" .. result .. "" end end if "" ~= result then result = " " .. result end return result end local typs = {'errors', 'warnings', 'timeouts'} APT.padResults = function(results) local c = 0 if nil == results then results = {}; c = 999 end for k, v in pairs(APT.protocols) do tests = results[v] if nil == tests then tests = {} end for m, x in pairs(typs) do if nil == tests[x] then tests[x] = c end end for l, w in pairs(APT.tests) do if ('raw' ~= w) and ('Speed' ~= w) then if nil == tests[w] then tests[w] = {} end for m, x in pairs(typs) do if nil == tests[w][x] then tests[w][x] = c end end end end if nil == tests.redirects then tests.redirects = {} end results[v] = tests end if nil == results.timeout then results.timeout = false end if nil == results.speed then results.speed = {min = 999999999999; max = 0} end return results end APT.collate = function(l, host, ip, results) results = APT.padResults(results) local f = l .. "/" .. host .. "_" .. ip .. ".lua" if APT.checkFile(f) then local rs = loadfile(f)() rs = APT.padResults(rs) for k, v in pairs(rs) do if "table" == type(v) then if ("speed" == k) and (nil ~= results.speed) then if v.min < results.speed.min then results.speed.min = v.min end if v.max > results.speed.max then results.speed.max = v.max end elseif 'IPs' ~= k then for i, u in pairs(v) do if 'redirects' ~= i then if "table" == type(u) then for h, t in pairs(u) do local a = results[k] if nil == a then results[k] = {i = {}}; a = results[k] end a = a[i] if nil == a then results[k][i] = {h = {}}; a = results[k][i] end a = a[h] if nil == a then a = 0 end results[k][i][h] = a + t end else local a = results[k] if nil == a then a = 0; results[k] = {} else a = a[i] end if nil == a then a = 0 end results[k][i] = a + u end end end end elseif "timeout" ~= k then local a = results[k] if nil == a then a = 0 end results[k] = a + v end end end return results end APT.collateAll = function(hosts, l, host, func) results = {} local f = l .. "/" .. host .. ".lua" if APT.checkFile(f) then results = loadfile(f)() results = APT.padResults(results) if nil ~= func then func(results) end local v = hosts[host] if nil ~= v then local IPs = results.IPs if nil == IPs then W('No IPs for ' .. host .. ' in ' .. l) else for i, u in pairs(IPs) do if "table" == type(u) then for h, t in pairs(u) do results = APT.collate(l, host, h, results) if nil ~= func then func(results, h) end end else results = APT.collate(l, host, i, results) if nil ~= func then func(results, i) end end end end end end return results end APT.now = 0 local status APT.now = APT.exe('TZ="GMT" ls -l --time-style="+%s" results/stamp | cut -d " " -f 6-6'):Do().result APT.now = tonumber(APT.now) local start = 'now-1month' local step = '10min' local hb = '150min' local DSIe = 'DS:IntegrityErrors:GAUGE:' .. hb .. ':0:U' local DSIw = 'DS:IntegrityWarnings:GAUGE:' .. hb .. ':0:U' local DSIt = 'DS:IntegrityTimeouts:GAUGE:' .. hb .. ':0:U' local DSPe = 'DS:ProtocolErrors:GAUGE:' .. hb .. ':0:U' local DSPw = 'DS:ProtocolWarnings:GAUGE:' .. hb .. ':0:U' local DSPt = 'DS:ProtocolTimeouts:GAUGE:' .. hb .. ':0:U' local DSUe = 'DS:UpdatedErrors:GAUGE:' .. hb .. ':0:U' local DSUw = 'DS:UpdatedWarnings:GAUGE:' .. hb .. ':0:U' local DSUt = 'DS:UpdatedTimeouts:GAUGE:' .. hb .. ':0:U' local DSSe = 'DS:URLSanityErrors:GAUGE:' .. hb .. ':0:U' local DSSw = 'DS:URLSanityWarnings:GAUGE:' .. hb .. ':0:U' local DSSt = 'DS:URLSanityTimeouts:GAUGE:' .. hb .. ':0:U' local DSx = 'DS:max:GAUGE:' .. hb .. ':0:U' local DSn = 'DS:min:GAUGE:' .. hb .. ':0:U' -- What Collectd uses. local RRAc0 = 'RRA:AVERAGE:0.9:1:1200' local RRAc1 = 'RRA:MIN:0.9:1:1200' local RRAc2 = 'RRA:MAX:0.9:1:1200' local RRAc3 = 'RRA:AVERAGE:0.9:7:1235' local RRAc4 = 'RRA:MIN:0.9:7:1235' local RRAc5 = 'RRA:MAX:0.9:7:1235' local RRAc6 = 'RRA:AVERAGE:0.9:50:1210' local RRAc7 = 'RRA:MIN:0.9:50:1210' local RRAc8 = 'RRA:MAX:0.9:50:1210' local RRAc9 = 'RRA:AVERAGE:0.9:223:1202' local RRAc10 = 'RRA:MIN:0.9:223:1202' local RRAc11 = 'RRA:MAX:0.9:223:1202' local RRAc12 = 'RRA:AVERAGE:0.9:2635:1201' local RRAc13 = 'RRA:MIN:0.9:2635:1201' local RRAc14 = 'RRA:MAX:0.9:2635:1201' -- Try LAST. local RRAl0 = 'RRA:LAST:0.9:1:1200' local RRAl1 = 'RRA:LAST:0.9:7:1235' local RRAl2 = 'RRA:LAST:0.9:50:1210' local RRAl3 = 'RRA:LAST:0.9:223:1202' local RRAl4 = 'RRA:LAST:0.9:2635:1201' APT.createRRD = function(host, ip, o) if nil ~= o then start = o end if nil ~= ip then host = host .. '_' .. ip end for i, p in pairs(APT.protocols) do os.execute( 'mkdir -p rrd/' .. host .. '/' .. p:upper()) if not APT.checkFile('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd') then D('Creating ' .. 'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd') APT.rrd.create( 'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', '--start', start, '--step', step, DSIe, DSIw, DSIt, DSPe, DSPw, DSPt, DSUe, DSUw, DSUt, DSSe, DSSw, DSSt, RRAc0, RRAc1, RRAc2, RRAl0, RRAc3, RRAc4, RRAc5, RRAl1, RRAc6, RRAc7, RRAc8, RRAl2, RRAc9, RRAc10, RRAc11, RRAl3, RRAc12, RRAc13, RRAc14, RRAl4) -- Start them at 0 so the average has something to work on. APT.rrd.update( 'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', (APT.now - 600) .. ':0:0:0:0:0:0:0:0:0:0:0:0') end end os.execute( 'mkdir -p rrd/' .. host .. '/Speed') if not APT.checkFile('rrd/' .. host .. '/Speed/Speed.rrd') then D('Creating ' .. 'rrd/' .. host .. '/Speed/Speed.rrd') APT.rrd.create( 'rrd/' .. host .. '/Speed/Speed.rrd', '--start', start, '--step', step, DSx, DSn, RRAc0, RRAc1, RRAc2, RRAl0, RRAc3, RRAc4, RRAc5, RRAl1, RRAc6, RRAc7, RRAc8, RRAl2, RRAc9, RRAc10, RRAc11, RRAl3, RRAc12, RRAc13, RRAc14, RRAl4) APT.rrd.update( 'rrd/' .. host .. '/Speed/Speed.rrd', (APT.now - 600) .. ':0:0') end end APT.updateRRD = function(results, host, ip) if nil ~= ip then host = host .. '_' .. ip end for i, p in pairs(APT.protocols) do if nil ~= results[p] then APT.rrd.update('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', APT.now .. ':' .. results[p]['Integrity'].errors .. ':' .. results[p]['Integrity'].warnings .. ':' .. results[p]['Integrity'].timeouts .. ':' .. results[p]['Protocol'].errors .. ':' .. results[p]['Protocol'].warnings .. ':' .. results[p]['Protocol'].timeouts .. ':' .. results[p]['Updated'].errors .. ':' .. results[p]['Updated'].warnings .. ':' .. results[p]['Updated'].timeouts .. ':' .. results[p]['URLSanity'].errors .. ':' .. results[p]['URLSanity'].warnings .. ':' .. results[p]['URLSanity'].timeouts) else APT.rrd.update('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', APT.now .. ':U:U:U:U:U:U:U:U:U:U:U:U') end end if nil ~= results.speed then if 0 ~= results.speed.max then APT.rrd.update('rrd/' .. host .. '/Speed/Speed.rrd', APT.now .. ':' .. results.speed.max .. ':' .. results.speed.min) else APT.rrd.update('rrd/' .. host .. '/Speed/Speed.rrd', APT.now .. ':0:0') end else APT.rrd.update( 'rrd/' .. host .. '/Speed/Speed.rrd', APT.now .. ':U:U') end end APT.doRRD = function(l, k, v, o) APT.collateAll(APT.mirrors, l, k, function(results, ip) APT.createRRD(k, ip, o) APT.updateRRD(results, k, ip) end) end return APT