From 4bb51520521d9ab612d50dd2bcf904d70cf15735 Mon Sep 17 00:00:00 2001
From: onefang
Date: Tue, 10 Dec 2019 15:45:11 +1000
Subject: Move common code to it's own module.
Some minor clean ups and test tweaks likely came for the ride.
---
apt-panopticon.lua | 507 ++++++++++++-----------------------------------------
1 file changed, 112 insertions(+), 395 deletions(-)
(limited to 'apt-panopticon.lua')
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index 37778ac..e454740 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -1,7 +1,13 @@
#!/usr/bin/env luajit
-
-local args = {...}
+local APT = require 'apt-panopticommon'
+local D = APT.D
+local I = APT.I
+local W = APT.W
+local E = APT.E
+local C = APT.C
+local arg, sendArgs = APT.parseArgs({...})
+APT.html = true
--[[ TODO - What to do about HTTPS://deb.devuan.org/ redirects.
Some mirrors give a 404.
@@ -10,70 +16,6 @@ local args = {...}
They shouldn't have the proper certificate, but are giving a result anyway.
]]
-origin = false
-verbosity = -1
-keep = false
--- TODO - Should actually implement this.
-fork = true
-options =
-{
- referenceSite =
- {
- typ = "string",
- help = "",
- value = "pkgmaster.devuan.org",
- },
- roundRobin =
- {
- typ = "string",
- help = "",
- value = "deb.devuan.org",
- },
- tests =
- {
- typ = "table",
- help = "",
- value =
- {
- "IPv4",
- "IPv6",
--- "ftp",
- "http",
- "https",
--- "rsync",
- "DNSRR",
- "Protocol",
- "URLSanity",
- "Integrity",
- "Updated",
- },
- },
- maxtime =
- {
- typ = "number",
- help = "",
- value = 300,
- },
- timeout =
- {
- typ = "number",
- help = "",
- value = 15,
- },
- reports =
- {
- typ = "table",
- help = "",
- value =
- {
- "email-web",
--- "Nagios",
--- "Prometheus",
--- "RRD",
- },
- },
-}
-
local defaultURL = {scheme = "http"}
local releases = {"jessie", "ascii", "beowulf", "ceres"}
local releaseFiles =
@@ -108,9 +50,6 @@ local referenceDevs =
"merged/pool/DEVUAN/main/d/desktop-base/desktop-base_3.0_all.deb",
"merged/pool/DEVUAN/main/u/util-linux/util-linux_2.32.1-0.1+devuan2.1_amd64.deb",
}
-local arg = {}
-local sendArgs = ""
-local logFile
local curlStatus =
{
@@ -218,151 +157,8 @@ local http = require 'socket.http'
local url = require 'socket.url'
--- Use this to dump a table to a string.
-dumpTable = function (table, space, name)
- local r = ""
- if "" == space then r = r .. space .. name .. " =\n" else r = r .. space .. "[" .. name .. "] =\n" end
- r = r .. space .. "{\n"
- r = r .. dumpTableSub(table, space .. " ")
- if "" == space then r = r .. space .. "}\n" else r = r .. space .. "},\n" end
- return r
-end
-dumpTableSub = function (table, space)
- local r = ""
- for k, v in pairs(table) do
- if type(k) == "string" then k = '"' .. k .. '"' end
- if type(v) == "table" then
- r = r .. dumpTable(v, space, k)
- elseif type(v) == "string" then
- r = r .. space .. "[" .. k .. "] = '" .. v .. "';\n"
- elseif type(v) == "function" then
- r = r .. space .. "[" .. k .. "] = function ();\n"
- elseif type(v) == "userdata" then
- r = r .. space .. "userdata " .. "[" .. k .. "];\n"
- elseif type(v) == "boolean" then
- if (v) then
- r = r .. space .. "[" .. k .. "] = true;\n"
- else
- r = r .. space .. "[" .. k .. "] = false;\n"
- end
- else
- r = r .. space .. "[" .. k .. "] = " .. v .. ";\n"
- end
- end
- return r
-end
-
local ip = ""
-local results = {}
-
-local logPre = function()
- if nil ~= logFile then
- logFile:write("\n")
- logFile:write("\n")
- end
-end
-local logPost = function()
- if nil ~= logFile then
- logFile:write(" \n")
- end
-end
-
-local log = function(v, t, s, prot, test, host)
- local x = ""
- if nil == prot then prot = "" end
- if nil == test then test = "" end
- x = x .. prot
- if "" ~= test then
- if #x > 0 then x = x .. " " end
- x = x .. test
- end
- if nil ~= host then
- if #x > 0 then x = x .. " " end
- x = x .. host
- end
- if #x > 0 then
- t = t .. "(" .. x .. ")"
- if "" ~= prot then
- if "" == test then
- if nil == results[prot] then results[prot] = {errors = 0; warnings = 0} end
- if v == 0 then results[prot].errors = results[prot].errors + 1 end
- if v == 1 then results[prot].warnings = results[prot].warnings + 1 end
- else
- if nil == results[prot] then results[prot] = {errors = 0; warnings = 0} end
- if nil == results[prot][test] then results[prot][test] = {errors = 0; warnings = 0} end
- if v == 0 then results[prot][test].errors = results[prot][test].errors + 1 end
- if v == 1 then results[prot][test].warnings = results[prot][test].warnings + 1 end
- end
- end
- end
- if v <= verbosity then
- if 3 <= verbosity then t = os.date() .. " " .. t end
- print(t .. ": " .. s)
- end
- if nil ~= logFile then
- local colour = "white"
- if -1 == v then colour = "fuchsia" end -- CRITICAL
- if 0 == v then colour = "red " end -- ERROR
- if 1 == v then colour = "yellow " end -- WARNING
- if 2 == v then colour = "white " end -- INFO
- if 3 == v then colour = "gray " end -- DEBUG
- logFile:write(os.date() .. " " .. t .. ": " .. s .. "\n")
- logFile:flush()
- end
-end
-local D = function(s) log(3, "DEBUG ", s) end
-local I = function(s) log(2, "INFO ", s) end
-local W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end
-local E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end
-local C = function(s) log(-1, "CRITICAL", s) end
-
-local mirrors = {}
-
-local testing = function(t, host)
- for i, v in pairs(options.tests.value) do
- if t == v then
- local h = mirrors[host]
- if nil == h then return true end
- if true == h["Protocols"][t] then return true else D("Skipping " .. t .. " checks for " .. host) end
- end
- end
- return false
-end
-
-local execute = function (s)
- D(" executing " .. s .. "
")
- --[[ Damn os.execute()
- Lua 5.1 says it returns "a status code, which is system-dependent"
- Lua 5.2 says it returns true/nil, "exit"/"signal", the status code.
- I'm getting 7168 or 0. No idea what the fuck that is.
- local ok, rslt, status = os.execute(s)
- ]]
- local f = io.popen(s .. ' ; echo "$?"', 'r')
- local status = ""
- local result = ""
- -- The last line will be the command's returned status, collect everything else in result.
- for l in f:lines() do
- result = result .. status .. "\n"
- status = l
- end
- return status, result
-end
-
-local fork = function(s)
- D(" forking " .. s .. "
")
- os.execute(s .. " &")
-end
-local checkExes = function (exe)
- local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | grep -v "flock -n apt-panopticon.lock " | wc -l'):read("*l")
- D(count .. " " .. exe .. " commands still running.")
- return tonumber(count)
-end
-
-local checkFile = function(f)
- local h, e = io.open(f, "r")
- if nil == h then return false else h:close(); return true end
-end
local repoExists = function (r)
r = r:match("([%a-]*)")
@@ -441,9 +237,9 @@ checkHEAD = function (host, URL, r, retry, sanity)
return
end
D(PU.scheme .. " :// " .. check .. " " .. host .. " -> " .. URL)
- if not testing(PU.scheme, host) then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL); return end
+ if not APT.testing(PU.scheme, host) then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL); return end
-- TODO - Perhaps we should try it anyway, and mark it as a warning if it DOES work?
- if "https" == PU.scheme and options.roundRobin.value == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert."); return end
+ if "https" == PU.scheme and APT.options.roundRobin.value == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert."); return end
--[[ Using curl command line -
-I - HEAD
@@ -468,10 +264,10 @@ checkHEAD = function (host, URL, r, retry, sanity)
end
IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"'
end
- local cmd = 'ionice -c3 nice -n 19 curl -I --retry 0 -s --path-as-is --connect-timeout ' .. options.timeout.value .. ' --max-redirs 0 ' ..
+ local cmd = 'ionice -c3 nice -n 19 curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' ..
IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' ..
hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"'
- local status, result = execute(cmd)
+ local status, result = APT.execute(cmd)
os.execute('cat results/"HEADERS_' .. fname .. '" >>results/"STATUS_' .. fname .. '" 2>/dev/null; rm -f results/"HEADERS_' .. fname .. '" 2>/dev/null')
if "0" ~= status then
local msg = curlStatus[0 + status]
@@ -522,7 +318,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
if nil ~= location then
pu = url.parse(location, defaultURL)
if ('http' == location:sub(1, 4)) and (pu.scheme ~= PU.scheme) then -- Sometimes a location sans scheme is returned, this is not a protocol change.
- if testing("Protocol") then W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host) end
+ if APT.testing("Protocol") then W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host) end
if (pu.host == host) and pu.path == PU.path then D("Not testing protocol change " .. URL .. " -> " .. location); return end
end
@@ -555,7 +351,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
end
local checkTimeouts = function(host, scheme, URL)
- if testing(scheme) then
+ if APT.testing(scheme) then
totalTimeouts = totalTimeouts + timeouts; timeouts = 0
checkHEAD(host, scheme .. "://" .. URL)
if 4 <= (totalTimeouts) then
@@ -563,7 +359,7 @@ local checkTimeouts = function(host, scheme, URL)
return true
end
end
- if testing("URLSanity") then
+ if APT.testing("URLSanity") then
URL = URL:gsub("merged/", "merged///")
totalTimeouts = totalTimeouts + timeouts; timeouts = 0
checkHEAD(host, scheme .. "://" .. URL, 0, 0, true)
@@ -617,24 +413,24 @@ checkHost = function (orig, host, path, ip, file)
else
if orig == host then
D("checkHost " .. orig .. "" .. file)
- if testing("IPv4") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file) end
+ if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file) end
else D("checkHost " .. orig .. " -> " .. host) end
- local h = mirrors[ph.host]
+ local h = APT.mirrors[ph.host]
if nil == h then return end
for k, v in pairs(h.IPs) do
if "table" == type(v) then
for k1, v1 in pairs(v) do
if v1 == "A" then
- if testing("IPv4") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end
+ if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end
elseif v1 == "AAAA" then
- if testing("IPv6") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end
+ if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end
end
end
else
if v == "A" then
- if testing("IPv4") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end
+ if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end
elseif v == "AAAA" then
- if testing("IPv6") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end
+ if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end
end
end
end
@@ -644,14 +440,12 @@ end
local addDownload = function(host, URL, f, r, k)
local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename.
- local o, e = io.open("results/" .. host .. "/merged/dists/" .. r .. k, "r")
- if nil ~= o then
- o:close()
+ if APT.checkFile("results/" .. host .. "/merged/dists/" .. r .. k) then
-- Curls "check timestamp and overwrite file" stuff sucks.
-- -R means the destination file gets the timestamp of the remote file.
-- Can only do ONE timestamp check per command.
-- This doesn't work either. All downloads get all these headers. Pffft
--- local status, ts = execute('TZ="GMT" ls -l --time-style="+%a, %d %b %Y %T %Z" results/' .. host .. "/merged/dists/" .. r .. k .. ' | cut -d " " -f 6-11')
+-- local status, ts = APT.execute('TZ="GMT" ls -l --time-style="+%a, %d %b %Y %T %Z" results/' .. host .. "/merged/dists/" .. r .. k .. ' | cut -d " " -f 6-11')
-- f:write('header "If-Modified-Since: ' .. ts:sub(2, -2) .. '"\n')
-- Curl will DELETE the existing file if the timestamp fails to download a new one, unless we change directory first,
-- which wont work with multiple files in multiple directories. WTF?
@@ -671,21 +465,21 @@ local postDownload = function(host, r, k)
" && [ ! -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then cp -a" ..
" results/" .. host .. "/merged/dists/" .. r .. k .. ".old" ..
" results/" .. host .. "/merged/dists/" .. r .. k .. "; fi")
- if ".gz" == k:sub(-3, -1) then execute("ionice -c3 nice -n 19 gzip -dfk results/" .. host .. "/merged/dists/" .. r .. k) end
- if ".xz" == k:sub(-3, -1) then execute("ionice -c3 nice -n 19 xz -dfk results/" .. host .. "/merged/dists/" .. r .. k .. " 2>/dev/null") end
- if testing("Integrity") then
+ if ".gz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 gzip -dfk results/" .. host .. "/merged/dists/" .. r .. k) end
+ if ".xz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 xz -dfk results/" .. host .. "/merged/dists/" .. r .. k .. " 2>/dev/null") end
+ if APT.testing("Integrity") then
if ".gpg" == k:sub(-4, -1) then
- local status, out = execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. r .. k ..
+ local status, out = APT.execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. r .. k ..
" results/" .. host .. "/merged/dists/" .. r .. k:sub(1, -5) .. " 2>/dev/null")
if "0" ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. r .. k, "http", "Integrity", host) end
end
-- TODO - should check the PGP sig of InRelease as well.
end
- if testing("Integrity") or testing("Updated") then
+ if APT.testing("Integrity") or APT.testing("Updated") then
if "Packages." == file:sub(1, 9) then
-- TODO - compare the SHA256 sums in pkgmaster's Release for both the packed and unpacked versions.
-- Also note that this might get only a partial download due to maxtime.
- if options.referenceSite.value == host then
+ if APT.options.referenceSite.value == host then
local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed', "w+")
if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed' .. ' file - ' .. e) else
local pp = {}
@@ -711,7 +505,7 @@ local postDownload = function(host, r, k)
end
Pp:close()
os.execute('sort results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed >results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages_parsed-sorted')
- if checkFile('Packages/' .. r .. dir .. 'Packages_parsed-sorted') then
+ if APT.checkFile('Packages/' .. r .. dir .. 'Packages_parsed-sorted') then
os.execute('diff -U 0 Packages/' .. r .. dir .. 'Packages_parsed-sorted ' ..
'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages_parsed-sorted ' ..
' | grep -E "^-" | grep -Ev "^\\+\\+\\+|^---" >>results/OLD_PACKAGES_' .. r .. '.txt')
@@ -737,14 +531,14 @@ local postDownload = function(host, r, k)
end
local downloadLock = "flock -n results/curl-"
-local download = "curl --connect-timeout " .. options.timeout.value .. " --create-dirs -f -L --max-time " .. options.maxtime.value .. " -z 'results/stamp.old' -v -R "
+local download = "curl --connect-timeout " .. APT.options.timeout.value .. " --create-dirs -f -L --max-time " .. APT.options.maxtime.value .. " -z 'results/stamp.old' -v -R "
local downloads = function(host, URL, release, list)
if nil == URL then URL = "" end
local lock = "META-" .. host .. ".lock"
local log = " --stderr results/curl-META-" .. host .. ".log"
local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock .. " " .. download .. log .. " -K results/" .. host .. ".curl"
- if testing("IPv4") and (not testing("IPv6")) then cm = cm .. ' -4' end
- if (not testing("IPv4")) and testing("IPv6") then cm = cm .. ' -6' end
+ if APT.testing("IPv4") and (not APT.testing("IPv6")) then cm = cm .. ' -4' end
+ if (not APT.testing("IPv4")) and APT.testing("IPv6") then cm = cm .. ' -6' end
f, e = io.open("results/" .. host .. ".curl", "a+")
if nil == f then C("opening curl file - " .. e); return end
@@ -772,7 +566,7 @@ local downloads = function(host, URL, release, list)
end
end
f:close()
- fork(cm)
+ APT.fork(cm)
end
@@ -781,7 +575,7 @@ local getMirrors = function ()
local host = ""
local m = {}
local active = true
- local URL = "http://" .. options.referenceSite.value .. "/mirror_list.txt"
+ local URL = "http://" .. APT.options.referenceSite.value .. "/mirror_list.txt"
I("getting mirrors.")
local p, c, h = http.request(URL)
if nil == p then E(c .. " fetching " .. URL) else
@@ -818,95 +612,20 @@ local getMirrors = function ()
mirrors[host] = m
end
end
- if testing("DNSRR") then
- mirrors[options.roundRobin.value] = { ["Protocols"] = { ["http"] = true; ["https"] = true; }; ["FQDN"] = 'deb.devuan.org'; ["Active"] = 'yes'; ["BaseURL"] = 'deb.devuan.org'; }
- gatherIPs(options.roundRobin.value)
- mirrors[options.roundRobin.value].IPs = IP[options.roundRobin.value]
+ if APT.testing("DNSRR") then
+ mirrors[APT.options.roundRobin.value] = { ["Protocols"] = { ["http"] = true; ["https"] = true; }; ["FQDN"] = 'deb.devuan.org'; ["Active"] = 'yes'; ["BaseURL"] = 'deb.devuan.org'; }
+ gatherIPs(APT.options.roundRobin.value)
+ mirrors[APT.options.roundRobin.value].IPs = IP[APT.options.roundRobin.value]
end
local file, e = io.open("results/mirrors.lua", "w+")
if nil == file then C("opening mirrors file - " .. e) else
- file:write(dumpTable(mirrors, "", "mirrors") .. "\nreturn mirrors\n")
+ file:write(APT.dumpTable(mirrors, "", "mirrors") .. "\nreturn mirrors\n")
file:close()
end
return mirrors
end
-if 0 ~= #args then
- local option = ""
- for i, a in pairs(args) do
- if ("--help" == a) or ("-h" == a) then
- print("I should write some docs, huh? Read README.md for instructions.")
- elseif "--version" == a then
- print("apt-panopticon version 0.1 WIP development version")
- elseif "-v" == a then
- verbosity = verbosity + 1
- sendArgs = sendArgs .. a .. " "
- elseif "-q" == a then
- verbosity = -1
- sendArgs = sendArgs .. a .. " "
- elseif "-k" == a then
- keep = true
- elseif "-n" == a then
- fork = false
- elseif "-o" == a then
- origin = true
- elseif "--" == a:sub(1, 2) then
- local s, e = a:find("=")
- if nil == s then e = -1 end
- option = a:sub(3, e - 1)
- local o = options[option]
- if nil == o then
- print("Unknown option --" .. option)
- option = ""
- else
- option = a
- sendArgs = sendArgs .. a .. " "
- local s, e = a:find("=")
- if nil == s then e = 0 end
- option = a:sub(3, e - 1)
- if "table" == options[option].typ then
- local result = {}
- for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do
- local f = t:sub(1, 1)
- local n = t:sub(2, -1)
- if ("+" ~= f) and ("-" ~= f) then
- table.insert(result, t)
- end
- end
- if 0 ~= #result then
- options[option].value = result
- else
- for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do
- local f = t:sub(1, 1)
- local n = t:sub(2, -1)
- if "+" == f then
- table.insert(options[option].value, n)
- elseif "-" == f then
- local r = {}
- for i, k in pairs(options[option].value) do
- if k ~= n then table.insert(r, k) end
- end
- options[option].value = r
- end
- end
- end
- else
- options[option].value = a:sub(e + 1, -1)
- end
- option = ""
- end
- elseif "-" == a:sub(1, 1) then
- print("Unknown option " .. a)
- else
- table.insert(arg, a)
- end
- end
-end
-
---print(dumpTable(options.tests.value, "", "tests"))
-
-
if 0 < #arg then
if "/" == arg[1]:sub(-1, -1) then
W("slash at end of path! " .. arg[1])
@@ -918,37 +637,37 @@ if 0 < #arg then
end
local pu = url.parse("http://" .. arg[1])
- if testing("Integrity") or testing("Updated") then
- if origin and options.referenceSite.value == pu.host then
--- if not keep then execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end
+ if APT.testing("Integrity") or APT.testing("Updated") then
+ if APT.origin and APT.options.referenceSite.value == pu.host then
+-- if not APT.keep then os.execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end
end
end
if nil ~= arg[2] then
- logFile, e = io.open("results/LOG_" .. pu.host .. "_" .. arg[2] .. ".html", "a+")
+ APT.logFile, e = io.open("results/LOG_" .. pu.host .. "_" .. arg[2] .. ".html", "a+")
else
- logFile, e = io.open("results/LOG_" .. pu.host .. ".html", "a+")
+ APT.logFile, e = io.open("results/LOG_" .. pu.host .. ".html", "a+")
end
- if nil == logFile then C("opening log file - " .. e); return end
- logPre()
- I("Starting tests for " ..arg[1] .. " with these tests - " .. table.concat(options.tests.value, ", "))
- mirrors = loadfile("results/mirrors.lua")()
+ if nil == APT.logFile then C("opening log file - " .. e); return end
+ APT.logPre()
+ I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", "))
+ APT.mirrors = loadfile("results/mirrors.lua")()
if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end
if nil ~= arg[3] then I(" Using file " .. arg[3]); end
for k, v in pairs{"ftp", "http", "https", "rsync"} do
- if testing(v) then
+ if APT.testing(v) then
local tests = {errors = 0; warnings = 0}
- if testing("Integrity") then tests.Integrity = {errors = 0; warnings = 0} end
- if testing("Protocol") then tests.Protocol = {errors = 0; warnings = 0} end
- if testing("Updated") then tests.Updated = {errors = 0; warnings = 0} end
- if testing("URLSanity") then tests.URLSanity = {errors = 0; warnings = 0} end
- results[v] = tests
+ if APT.testing("Integrity") then tests.Integrity = {errors = 0; warnings = 0} end
+ if APT.testing("Protocol") then tests.Protocol = {errors = 0; warnings = 0} end
+ if APT.testing("Updated") then tests.Updated = {errors = 0; warnings = 0} end
+ if APT.testing("URLSanity") then tests.URLSanity = {errors = 0; warnings = 0} end
+ APT.results[v] = tests
end
end
- if origin then
- if testing("Integrity") or testing("Updated") then
- if origin and (options.roundRobin.value ~= pu.host) then
+ if APT.origin then
+ if APT.testing("Integrity") or APT.testing("Updated") then
+ if APT.origin and (APT.options.roundRobin.value ~= pu.host) then
I("Starting file downloads for " .. pu.host)
downloads(pu.host, pu.path)
end
@@ -958,10 +677,10 @@ if 0 < #arg then
checkHost(pu.host, pu.host, pu.path, arg[2], arg[3])
end
- if testing("Integrity") or testing("Updated") then
+ if APT.testing("Integrity") or APT.testing("Updated") then
if 4 > (totalTimeouts) then
- if origin and (options.roundRobin.value ~= pu.host) then
- while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
+ if APT.origin and (APT.options.roundRobin.value ~= pu.host) then
+ while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " ..
"mv results/curl-META-" .. pu.host .. ".log results/curl-Release-" .. pu.host .. ".log")
for i, n in pairs(releases) do
@@ -971,24 +690,25 @@ if 0 < #arg then
end
end
- if checkFile('results/' .. pu.host .. '/merged/dists/' .. n .. '/Release') and
- checkFile('results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED') then
+ if APT.checkFile('results/' .. pu.host .. '/merged/dists/' .. n .. '/Release') then
os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED')
- if options.referenceSite.value == pu.host then
- os.execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
- 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
- '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt')
- os.execute('rm -f results/' .. pu.host .. '/merged/dists/' .. n .. '/Release 2>/dev/null')
+ if APT.checkFile('results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED') then
+ if APT.options.referenceSite.value == pu.host then
+ os.execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
+ 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
+ '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt')
-- TODO - Maybe check the date in Release, though since they are updated daily, is there any point? Perhaps it's for checking amprolla got run?
- else
+ os.execute('rm -f results/' .. pu.host .. '/merged/dists/' .. n .. '/Release 2>/dev/null')
+ else
-- TODO - compare to the pkgmaster copy.
- end
+ end
- local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r")
- if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else
- local diff = dfile:read("*a")
- if "" ~= diff then
- downloads(pu.host, pu.path, n, diff)
+ local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r")
+ if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else
+ local diff = dfile:read("*a")
+ if "" ~= diff then
+ downloads(pu.host, pu.path, n, diff)
+ end
end
end
end
@@ -996,7 +716,7 @@ if 0 < #arg then
end
downloads(pu.host, pu.path, "", "")
- while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
+ while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " ..
"mv results/curl-META-" .. pu.host .. ".log results/curl-Packages-" .. pu.host .. ".log")
@@ -1008,7 +728,7 @@ if 0 < #arg then
postDownload(pu.host, n, "/" .. l)
end
end
- if options.referenceSite.value == pu.host then
+ if APT.options.referenceSite.value == pu.host then
-- In case it wasn't dealt with already.
os.execute('touch results/NEW_Packages_' .. n .. '.test.txt')
end
@@ -1027,26 +747,26 @@ if 0 < #arg then
end
end
downloads(pu.host, pu.path, nil, "")
- while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
+ while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
for i, n in pairs(releases) do
local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
for l in nfile:lines() do
local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |')
if nil ~= p then
- if checkFile('results/' .. pu.host .. "/merged/" .. p) then
- local status, fsz = execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5')
- if testing("Integrity") then
+ if APT.checkFile('results/' .. pu.host .. "/merged/" .. p) then
+ local status, fsz = APT.execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5')
+ if APT.testing("Integrity") then
if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end.
E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host)
print('|' .. sz .. '~=' .. fsz:sub(2, -2) .. '|')
else
- local status, fsha = execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1')
+ local status, fsha = APT.execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1')
if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end
-- TODO - maybe check the PGP key, though packages are mostly not signed.
end
end
- if testing("Updated") then
+ if APT.testing("Updated") then
if sz ~= fsz:sub(2, -2) then
E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host)
end
@@ -1060,20 +780,20 @@ if 0 < #arg then
end
end
- results["timeout"] = false
+ APT.results["timeout"] = false
else
- results["timeout"] = true
+ APT.results["timeout"] = true
end
end
- if origin and options.referenceSite.value ~= pu.host then
- if not keep then os.execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end
+ if APT.origin and APT.options.referenceSite.value ~= pu.host then
+ if not APT.keep then os.execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end
os.execute('rm STATUS_' .. pu.host .. '* 2>/dev/null')
end
local min, max, spd = 999999999999, 0
for i, mt in pairs({'Release', 'Packages', 'META'}) do
- if checkFile("results/curl-" .. mt .. "-" .. pu.host .. ".log") then
+ if APT.checkFile("results/curl-" .. mt .. "-" .. pu.host .. ".log") then
for l in io.lines("results/curl-" .. mt .. "-" .. pu.host .. ".log") do
local speed, crrnt = l:match('^%c *%d+ +%d+k? +%d+ +%d+k? +%d+ +%d+ +(%d+k?) +%d+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +(%d+k?)')
if nil ~= speed then
@@ -1089,17 +809,17 @@ if 0 < #arg then
end
end
end
- results["speed"] = {min = min, max = max}
+ APT.results["speed"] = {min = min, max = max}
local f = pu.host
if "" ~= ip then f = f .. "_" .. ip end
local rfile, e = io.open("results/" .. f .. ".lua", "w+")
if nil == rfile then C("opening results file - " .. e) else
- rfile:write(dumpTable(results, "", "results") .. "\nreturn results\n")
+ rfile:write(APT.dumpTable(APT.results, "", "results") .. "\nreturn results\n")
rfile:close()
end
- logPost()
- logFile:close()
+ APT.logPost()
+ APT.logFile:close()
else
local fadt = io.popen("ls -dl results_old 2>/dev/null | cut -d '>' -f 2 | cut -d ' ' -f 2")
local adt = fadt:read('*l')
@@ -1113,25 +833,25 @@ else
if nil ~= dt then os.execute('mkdir -p results_' .. dt .. '; rm -f results; ln -s results_' .. dt .. ' results 2>/dev/null') end
os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp')
os.execute("rm -f results/*.check 2>/dev/null")
- if not keep then
+ if not APT.keep then
os.execute("rm -f results/*.curl 2>/dev/null")
os.execute("rm -f results/*.log 2>/dev/null")
os.execute("rm -f results/*.html 2>/dev/null")
os.execute("rm -f results/*.txt 2>/dev/null")
end
- logFile, e = io.open("results/LOG_apt-panopticon.html", "a+")
- if nil == logFile then C("opening log file - " .. e); return end
- logPre()
- I("Starting tests " .. table.concat(options.tests.value, ", "))
+ APT.logFile, e = io.open("results/LOG_apt-panopticon.html", "a+")
+ if nil == APT.logFile then C("opening log file - " .. e); return end
+ APT.logPre()
+ I("Starting tests " .. table.concat(APT.options.tests.value, ", "))
os.execute("mkdir -p results")
- mirrors = getMirrors()
- checkHost(options.referenceSite.value)
+ APT.mirrors = getMirrors()
+ checkHost(APT.options.referenceSite.value)
for i, n in pairs(releases) do
- while not checkFile('results/NEW_Packages_' .. n .. '.test.txt') do os.execute("sleep 10") end
+ while not APT.checkFile('results/NEW_Packages_' .. n .. '.test.txt') do os.execute("sleep 10") end
end
- for k, m in pairs(mirrors) do
+ for k, m in pairs(APT.mirrors) do
if "/" == m.BaseURL:sub(-1, -1) then
W("slash at end of BaseURL in mirror_list.txt! " .. m.BaseURL, "", "", m.FQDN)
m.BaseURL = m.BaseURL:sub(1, -2)
@@ -1141,30 +861,27 @@ else
m.BaseURL = m.BaseURL:sub(1, -2)
end
local pu = url.parse("http://" .. m.BaseURL)
- if options.referenceSite.value ~= pu.host then
+ if APT.options.referenceSite.value ~= pu.host then
checkHost(m.BaseURL)
- checkExes("apt-panopticon.lua " .. sendArgs)
- if testing("Integrity") or testing("Updated") then checkExes(downloadLock) end
+ APT.checkExes("apt-panopticon.lua " .. sendArgs)
+ if APT.testing("Integrity") or APT.testing("Updated") then APT.checkExes(downloadLock) end
end
end
- while 1 <= checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end
+ while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end
os.execute("rm -f results/*.check; rm -f results/*.lock 2>/dev/null")
-- Create the reports.
- for n, r in pairs(options.reports.value) do
- local report = "apt-panopticon-report-" .. r .. ".lua"
- local rfile, e = io.open(report, "r")
- if nil == rfile then C("opening " .. report .. " file - " .. e) else
- rfile:close()
- I("Creating " .. report .. " report.")
- execute("./" .. report .. " ")
+ for n, r in pairs(APT.options.reports.value) do
+ if APT.checkFile("apt-panopticon-report-" .. r .. ".lua") then
+ I("Creating " .. r .. " report.")
+ APT.execute("./apt-panopticon-report-" .. r .. ".lua")
end
end
if nil ~= adt then os.execute('rm -fr ' .. adt .. ' 2>/dev/null') end
- logPost()
- logFile:close()
+ APT.logPost()
+ APT.logFile:close()
end
--
cgit v1.1