diff options
Diffstat (limited to 'apt-panopticon.lua')
-rwxr-xr-x | apt-panopticon.lua | 507 |
1 files changed, 112 insertions, 395 deletions
diff --git a/apt-panopticon.lua b/apt-panopticon.lua index 37778ac..e454740 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua | |||
@@ -1,7 +1,13 @@ | |||
1 | #!/usr/bin/env luajit | 1 | #!/usr/bin/env luajit |
2 | 2 | ||
3 | 3 | local APT = require 'apt-panopticommon' | |
4 | local args = {...} | 4 | local D = APT.D |
5 | local I = APT.I | ||
6 | local W = APT.W | ||
7 | local E = APT.E | ||
8 | local C = APT.C | ||
9 | local arg, sendArgs = APT.parseArgs({...}) | ||
10 | APT.html = true | ||
5 | 11 | ||
6 | --[[ TODO - What to do about HTTPS://deb.devuan.org/ redirects. | 12 | --[[ TODO - What to do about HTTPS://deb.devuan.org/ redirects. |
7 | Some mirrors give a 404. | 13 | Some mirrors give a 404. |
@@ -10,70 +16,6 @@ local args = {...} | |||
10 | They shouldn't have the proper certificate, but are giving a result anyway. | 16 | They shouldn't have the proper certificate, but are giving a result anyway. |
11 | ]] | 17 | ]] |
12 | 18 | ||
13 | origin = false | ||
14 | verbosity = -1 | ||
15 | keep = false | ||
16 | -- TODO - Should actually implement this. | ||
17 | fork = true | ||
18 | options = | ||
19 | { | ||
20 | referenceSite = | ||
21 | { | ||
22 | typ = "string", | ||
23 | help = "", | ||
24 | value = "pkgmaster.devuan.org", | ||
25 | }, | ||
26 | roundRobin = | ||
27 | { | ||
28 | typ = "string", | ||
29 | help = "", | ||
30 | value = "deb.devuan.org", | ||
31 | }, | ||
32 | tests = | ||
33 | { | ||
34 | typ = "table", | ||
35 | help = "", | ||
36 | value = | ||
37 | { | ||
38 | "IPv4", | ||
39 | "IPv6", | ||
40 | -- "ftp", | ||
41 | "http", | ||
42 | "https", | ||
43 | -- "rsync", | ||
44 | "DNSRR", | ||
45 | "Protocol", | ||
46 | "URLSanity", | ||
47 | "Integrity", | ||
48 | "Updated", | ||
49 | }, | ||
50 | }, | ||
51 | maxtime = | ||
52 | { | ||
53 | typ = "number", | ||
54 | help = "", | ||
55 | value = 300, | ||
56 | }, | ||
57 | timeout = | ||
58 | { | ||
59 | typ = "number", | ||
60 | help = "", | ||
61 | value = 15, | ||
62 | }, | ||
63 | reports = | ||
64 | { | ||
65 | typ = "table", | ||
66 | help = "", | ||
67 | value = | ||
68 | { | ||
69 | "email-web", | ||
70 | -- "Nagios", | ||
71 | -- "Prometheus", | ||
72 | -- "RRD", | ||
73 | }, | ||
74 | }, | ||
75 | } | ||
76 | |||
77 | local defaultURL = {scheme = "http"} | 19 | local defaultURL = {scheme = "http"} |
78 | local releases = {"jessie", "ascii", "beowulf", "ceres"} | 20 | local releases = {"jessie", "ascii", "beowulf", "ceres"} |
79 | local releaseFiles = | 21 | local releaseFiles = |
@@ -108,9 +50,6 @@ local referenceDevs = | |||
108 | "merged/pool/DEVUAN/main/d/desktop-base/desktop-base_3.0_all.deb", | 50 | "merged/pool/DEVUAN/main/d/desktop-base/desktop-base_3.0_all.deb", |
109 | "merged/pool/DEVUAN/main/u/util-linux/util-linux_2.32.1-0.1+devuan2.1_amd64.deb", | 51 | "merged/pool/DEVUAN/main/u/util-linux/util-linux_2.32.1-0.1+devuan2.1_amd64.deb", |
110 | } | 52 | } |
111 | local arg = {} | ||
112 | local sendArgs = "" | ||
113 | local logFile | ||
114 | 53 | ||
115 | local curlStatus = | 54 | local curlStatus = |
116 | { | 55 | { |
@@ -218,151 +157,8 @@ local http = require 'socket.http' | |||
218 | local url = require 'socket.url' | 157 | local url = require 'socket.url' |
219 | 158 | ||
220 | 159 | ||
221 | -- Use this to dump a table to a string. | ||
222 | dumpTable = function (table, space, name) | ||
223 | local r = "" | ||
224 | if "" == space then r = r .. space .. name .. " =\n" else r = r .. space .. "[" .. name .. "] =\n" end | ||
225 | r = r .. space .. "{\n" | ||
226 | r = r .. dumpTableSub(table, space .. " ") | ||
227 | if "" == space then r = r .. space .. "}\n" else r = r .. space .. "},\n" end | ||
228 | return r | ||
229 | end | ||
230 | dumpTableSub = function (table, space) | ||
231 | local r = "" | ||
232 | for k, v in pairs(table) do | ||
233 | if type(k) == "string" then k = '"' .. k .. '"' end | ||
234 | if type(v) == "table" then | ||
235 | r = r .. dumpTable(v, space, k) | ||
236 | elseif type(v) == "string" then | ||
237 | r = r .. space .. "[" .. k .. "] = '" .. v .. "';\n" | ||
238 | elseif type(v) == "function" then | ||
239 | r = r .. space .. "[" .. k .. "] = function ();\n" | ||
240 | elseif type(v) == "userdata" then | ||
241 | r = r .. space .. "userdata " .. "[" .. k .. "];\n" | ||
242 | elseif type(v) == "boolean" then | ||
243 | if (v) then | ||
244 | r = r .. space .. "[" .. k .. "] = true;\n" | ||
245 | else | ||
246 | r = r .. space .. "[" .. k .. "] = false;\n" | ||
247 | end | ||
248 | else | ||
249 | r = r .. space .. "[" .. k .. "] = " .. v .. ";\n" | ||
250 | end | ||
251 | end | ||
252 | return r | ||
253 | end | ||
254 | |||
255 | local ip = "" | 160 | local ip = "" |
256 | local results = {} | ||
257 | |||
258 | local logPre = function() | ||
259 | if nil ~= logFile then | ||
260 | logFile:write("<html><head>\n") | ||
261 | logFile:write("</head><body bgcolor='black' text='white' alink='red' link='blue' vlink='purple'>\n") | ||
262 | end | ||
263 | end | ||
264 | local logPost = function() | ||
265 | if nil ~= logFile then | ||
266 | logFile:write("</body></html> \n") | ||
267 | end | ||
268 | end | ||
269 | |||
270 | local log = function(v, t, s, prot, test, host) | ||
271 | local x = "" | ||
272 | if nil == prot then prot = "" end | ||
273 | if nil == test then test = "" end | ||
274 | x = x .. prot | ||
275 | if "" ~= test then | ||
276 | if #x > 0 then x = x .. " " end | ||
277 | x = x .. test | ||
278 | end | ||
279 | if nil ~= host then | ||
280 | if #x > 0 then x = x .. " " end | ||
281 | x = x .. host | ||
282 | end | ||
283 | if #x > 0 then | ||
284 | t = t .. "(" .. x .. ")" | ||
285 | if "" ~= prot then | ||
286 | if "" == test then | ||
287 | if nil == results[prot] then results[prot] = {errors = 0; warnings = 0} end | ||
288 | if v == 0 then results[prot].errors = results[prot].errors + 1 end | ||
289 | if v == 1 then results[prot].warnings = results[prot].warnings + 1 end | ||
290 | else | ||
291 | if nil == results[prot] then results[prot] = {errors = 0; warnings = 0} end | ||
292 | if nil == results[prot][test] then results[prot][test] = {errors = 0; warnings = 0} end | ||
293 | if v == 0 then results[prot][test].errors = results[prot][test].errors + 1 end | ||
294 | if v == 1 then results[prot][test].warnings = results[prot][test].warnings + 1 end | ||
295 | end | ||
296 | end | ||
297 | end | ||
298 | if v <= verbosity then | ||
299 | if 3 <= verbosity then t = os.date() .. " " .. t end | ||
300 | print(t .. ": " .. s) | ||
301 | end | ||
302 | if nil ~= logFile then | ||
303 | local colour = "white" | ||
304 | if -1 == v then colour = "fuchsia" end -- CRITICAL | ||
305 | if 0 == v then colour = "red " end -- ERROR | ||
306 | if 1 == v then colour = "yellow " end -- WARNING | ||
307 | if 2 == v then colour = "white " end -- INFO | ||
308 | if 3 == v then colour = "gray " end -- DEBUG | ||
309 | logFile:write(os.date() .. " <font color='" .. colour .. "'><b>" .. t .. "</b></font>: " .. s .. "</br>\n") | ||
310 | logFile:flush() | ||
311 | end | ||
312 | end | ||
313 | local D = function(s) log(3, "DEBUG ", s) end | ||
314 | local I = function(s) log(2, "INFO ", s) end | ||
315 | local W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end | ||
316 | local E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end | ||
317 | local C = function(s) log(-1, "CRITICAL", s) end | ||
318 | |||
319 | local mirrors = {} | ||
320 | |||
321 | local testing = function(t, host) | ||
322 | for i, v in pairs(options.tests.value) do | ||
323 | if t == v then | ||
324 | local h = mirrors[host] | ||
325 | if nil == h then return true end | ||
326 | if true == h["Protocols"][t] then return true else D("Skipping " .. t .. " checks for " .. host) end | ||
327 | end | ||
328 | end | ||
329 | return false | ||
330 | end | ||
331 | |||
332 | local execute = function (s) | ||
333 | D(" executing <pre><code>" .. s .. "</code></pre>") | ||
334 | --[[ Damn os.execute() | ||
335 | Lua 5.1 says it returns "a status code, which is system-dependent" | ||
336 | Lua 5.2 says it returns true/nil, "exit"/"signal", the status code. | ||
337 | I'm getting 7168 or 0. No idea what the fuck that is. | ||
338 | local ok, rslt, status = os.execute(s) | ||
339 | ]] | ||
340 | local f = io.popen(s .. ' ; echo "$?"', 'r') | ||
341 | local status = "" | ||
342 | local result = "" | ||
343 | -- The last line will be the command's returned status, collect everything else in result. | ||
344 | for l in f:lines() do | ||
345 | result = result .. status .. "\n" | ||
346 | status = l | ||
347 | end | ||
348 | return status, result | ||
349 | end | ||
350 | |||
351 | local fork = function(s) | ||
352 | D(" forking <pre><code>" .. s .. "</code></pre>") | ||
353 | os.execute(s .. " &") | ||
354 | end | ||
355 | 161 | ||
356 | local checkExes = function (exe) | ||
357 | local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | grep -v "flock -n apt-panopticon.lock " | wc -l'):read("*l") | ||
358 | D(count .. " " .. exe .. " commands still running.") | ||
359 | return tonumber(count) | ||
360 | end | ||
361 | |||
362 | local checkFile = function(f) | ||
363 | local h, e = io.open(f, "r") | ||
364 | if nil == h then return false else h:close(); return true end | ||
365 | end | ||
366 | 162 | ||
367 | local repoExists = function (r) | 163 | local repoExists = function (r) |
368 | r = r:match("([%a-]*)") | 164 | r = r:match("([%a-]*)") |
@@ -441,9 +237,9 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
441 | return | 237 | return |
442 | end | 238 | end |
443 | D(PU.scheme .. " :// " .. check .. " " .. host .. " -> " .. URL) | 239 | D(PU.scheme .. " :// " .. check .. " " .. host .. " -> " .. URL) |
444 | if not testing(PU.scheme, host) then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL); return end | 240 | if not APT.testing(PU.scheme, host) then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL); return end |
445 | -- TODO - Perhaps we should try it anyway, and mark it as a warning if it DOES work? | 241 | -- TODO - Perhaps we should try it anyway, and mark it as a warning if it DOES work? |
446 | if "https" == PU.scheme and options.roundRobin.value == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert."); return end | 242 | if "https" == PU.scheme and APT.options.roundRobin.value == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert."); return end |
447 | 243 | ||
448 | --[[ Using curl command line - | 244 | --[[ Using curl command line - |
449 | -I - HEAD | 245 | -I - HEAD |
@@ -468,10 +264,10 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
468 | end | 264 | end |
469 | IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"' | 265 | IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"' |
470 | end | 266 | end |
471 | local cmd = 'ionice -c3 nice -n 19 curl -I --retry 0 -s --path-as-is --connect-timeout ' .. options.timeout.value .. ' --max-redirs 0 ' .. | 267 | local cmd = 'ionice -c3 nice -n 19 curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' .. |
472 | IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' .. | 268 | IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' .. |
473 | hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"' | 269 | hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"' |
474 | local status, result = execute(cmd) | 270 | local status, result = APT.execute(cmd) |
475 | os.execute('cat results/"HEADERS_' .. fname .. '" >>results/"STATUS_' .. fname .. '" 2>/dev/null; rm -f results/"HEADERS_' .. fname .. '" 2>/dev/null') | 271 | os.execute('cat results/"HEADERS_' .. fname .. '" >>results/"STATUS_' .. fname .. '" 2>/dev/null; rm -f results/"HEADERS_' .. fname .. '" 2>/dev/null') |
476 | if "0" ~= status then | 272 | if "0" ~= status then |
477 | local msg = curlStatus[0 + status] | 273 | local msg = curlStatus[0 + status] |
@@ -522,7 +318,7 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
522 | if nil ~= location then | 318 | if nil ~= location then |
523 | pu = url.parse(location, defaultURL) | 319 | pu = url.parse(location, defaultURL) |
524 | if ('http' == location:sub(1, 4)) and (pu.scheme ~= PU.scheme) then -- Sometimes a location sans scheme is returned, this is not a protocol change. | 320 | if ('http' == location:sub(1, 4)) and (pu.scheme ~= PU.scheme) then -- Sometimes a location sans scheme is returned, this is not a protocol change. |
525 | if testing("Protocol") then W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host) end | 321 | if APT.testing("Protocol") then W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host) end |
526 | if (pu.host == host) and pu.path == PU.path then D("Not testing protocol change " .. URL .. " -> " .. location); return end | 322 | if (pu.host == host) and pu.path == PU.path then D("Not testing protocol change " .. URL .. " -> " .. location); return end |
527 | end | 323 | end |
528 | 324 | ||
@@ -555,7 +351,7 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
555 | end | 351 | end |
556 | 352 | ||
557 | local checkTimeouts = function(host, scheme, URL) | 353 | local checkTimeouts = function(host, scheme, URL) |
558 | if testing(scheme) then | 354 | if APT.testing(scheme) then |
559 | totalTimeouts = totalTimeouts + timeouts; timeouts = 0 | 355 | totalTimeouts = totalTimeouts + timeouts; timeouts = 0 |
560 | checkHEAD(host, scheme .. "://" .. URL) | 356 | checkHEAD(host, scheme .. "://" .. URL) |
561 | if 4 <= (totalTimeouts) then | 357 | if 4 <= (totalTimeouts) then |
@@ -563,7 +359,7 @@ local checkTimeouts = function(host, scheme, URL) | |||
563 | return true | 359 | return true |
564 | end | 360 | end |
565 | end | 361 | end |
566 | if testing("URLSanity") then | 362 | if APT.testing("URLSanity") then |
567 | URL = URL:gsub("merged/", "merged///") | 363 | URL = URL:gsub("merged/", "merged///") |
568 | totalTimeouts = totalTimeouts + timeouts; timeouts = 0 | 364 | totalTimeouts = totalTimeouts + timeouts; timeouts = 0 |
569 | checkHEAD(host, scheme .. "://" .. URL, 0, 0, true) | 365 | checkHEAD(host, scheme .. "://" .. URL, 0, 0, true) |
@@ -617,24 +413,24 @@ checkHost = function (orig, host, path, ip, file) | |||
617 | else | 413 | else |
618 | if orig == host then | 414 | if orig == host then |
619 | D("checkHost " .. orig .. "" .. file) | 415 | D("checkHost " .. orig .. "" .. file) |
620 | if testing("IPv4") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file) end | 416 | if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file) end |
621 | else D("checkHost " .. orig .. " -> " .. host) end | 417 | else D("checkHost " .. orig .. " -> " .. host) end |
622 | local h = mirrors[ph.host] | 418 | local h = APT.mirrors[ph.host] |
623 | if nil == h then return end | 419 | if nil == h then return end |
624 | for k, v in pairs(h.IPs) do | 420 | for k, v in pairs(h.IPs) do |
625 | if "table" == type(v) then | 421 | if "table" == type(v) then |
626 | for k1, v1 in pairs(v) do | 422 | for k1, v1 in pairs(v) do |
627 | if v1 == "A" then | 423 | if v1 == "A" then |
628 | if testing("IPv4") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end | 424 | if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end |
629 | elseif v1 == "AAAA" then | 425 | elseif v1 == "AAAA" then |
630 | if testing("IPv6") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end | 426 | if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end |
631 | end | 427 | end |
632 | end | 428 | end |
633 | else | 429 | else |
634 | if v == "A" then | 430 | if v == "A" then |
635 | if testing("IPv4") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end | 431 | if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end |
636 | elseif v == "AAAA" then | 432 | elseif v == "AAAA" then |
637 | if testing("IPv6") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end | 433 | if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end |
638 | end | 434 | end |
639 | end | 435 | end |
640 | end | 436 | end |
@@ -644,14 +440,12 @@ end | |||
644 | 440 | ||
645 | local addDownload = function(host, URL, f, r, k) | 441 | local addDownload = function(host, URL, f, r, k) |
646 | local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename. | 442 | local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename. |
647 | local o, e = io.open("results/" .. host .. "/merged/dists/" .. r .. k, "r") | 443 | if APT.checkFile("results/" .. host .. "/merged/dists/" .. r .. k) then |
648 | if nil ~= o then | ||
649 | o:close() | ||
650 | -- Curls "check timestamp and overwrite file" stuff sucks. | 444 | -- Curls "check timestamp and overwrite file" stuff sucks. |
651 | -- -R means the destination file gets the timestamp of the remote file. | 445 | -- -R means the destination file gets the timestamp of the remote file. |
652 | -- Can only do ONE timestamp check per command. | 446 | -- Can only do ONE timestamp check per command. |
653 | -- This doesn't work either. All downloads get all these headers. Pffft | 447 | -- This doesn't work either. All downloads get all these headers. Pffft |
654 | -- local status, ts = execute('TZ="GMT" ls -l --time-style="+%a, %d %b %Y %T %Z" results/' .. host .. "/merged/dists/" .. r .. k .. ' | cut -d " " -f 6-11') | 448 | -- local status, ts = APT.execute('TZ="GMT" ls -l --time-style="+%a, %d %b %Y %T %Z" results/' .. host .. "/merged/dists/" .. r .. k .. ' | cut -d " " -f 6-11') |
655 | -- f:write('header "If-Modified-Since: ' .. ts:sub(2, -2) .. '"\n') | 449 | -- f:write('header "If-Modified-Since: ' .. ts:sub(2, -2) .. '"\n') |
656 | -- Curl will DELETE the existing file if the timestamp fails to download a new one, unless we change directory first, | 450 | -- Curl will DELETE the existing file if the timestamp fails to download a new one, unless we change directory first, |
657 | -- which wont work with multiple files in multiple directories. WTF? | 451 | -- which wont work with multiple files in multiple directories. WTF? |
@@ -671,21 +465,21 @@ local postDownload = function(host, r, k) | |||
671 | " && [ ! -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then cp -a" .. | 465 | " && [ ! -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then cp -a" .. |
672 | " results/" .. host .. "/merged/dists/" .. r .. k .. ".old" .. | 466 | " results/" .. host .. "/merged/dists/" .. r .. k .. ".old" .. |
673 | " results/" .. host .. "/merged/dists/" .. r .. k .. "; fi") | 467 | " results/" .. host .. "/merged/dists/" .. r .. k .. "; fi") |
674 | if ".gz" == k:sub(-3, -1) then execute("ionice -c3 nice -n 19 gzip -dfk results/" .. host .. "/merged/dists/" .. r .. k) end | 468 | if ".gz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 gzip -dfk results/" .. host .. "/merged/dists/" .. r .. k) end |
675 | if ".xz" == k:sub(-3, -1) then execute("ionice -c3 nice -n 19 xz -dfk results/" .. host .. "/merged/dists/" .. r .. k .. " 2>/dev/null") end | 469 | if ".xz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 xz -dfk results/" .. host .. "/merged/dists/" .. r .. k .. " 2>/dev/null") end |
676 | if testing("Integrity") then | 470 | if APT.testing("Integrity") then |
677 | if ".gpg" == k:sub(-4, -1) then | 471 | if ".gpg" == k:sub(-4, -1) then |
678 | local status, out = execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. r .. k .. | 472 | local status, out = APT.execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. r .. k .. |
679 | " results/" .. host .. "/merged/dists/" .. r .. k:sub(1, -5) .. " 2>/dev/null") | 473 | " results/" .. host .. "/merged/dists/" .. r .. k:sub(1, -5) .. " 2>/dev/null") |
680 | if "0" ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. r .. k, "http", "Integrity", host) end | 474 | if "0" ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. r .. k, "http", "Integrity", host) end |
681 | end | 475 | end |
682 | -- TODO - should check the PGP sig of InRelease as well. | 476 | -- TODO - should check the PGP sig of InRelease as well. |
683 | end | 477 | end |
684 | if testing("Integrity") or testing("Updated") then | 478 | if APT.testing("Integrity") or APT.testing("Updated") then |
685 | if "Packages." == file:sub(1, 9) then | 479 | if "Packages." == file:sub(1, 9) then |
686 | -- TODO - compare the SHA256 sums in pkgmaster's Release for both the packed and unpacked versions. | 480 | -- TODO - compare the SHA256 sums in pkgmaster's Release for both the packed and unpacked versions. |
687 | -- Also note that this might get only a partial download due to maxtime. | 481 | -- Also note that this might get only a partial download due to maxtime. |
688 | if options.referenceSite.value == host then | 482 | if APT.options.referenceSite.value == host then |
689 | local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed', "w+") | 483 | local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed', "w+") |
690 | if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed' .. ' file - ' .. e) else | 484 | if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed' .. ' file - ' .. e) else |
691 | local pp = {} | 485 | local pp = {} |
@@ -711,7 +505,7 @@ local postDownload = function(host, r, k) | |||
711 | end | 505 | end |
712 | Pp:close() | 506 | Pp:close() |
713 | os.execute('sort results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed >results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages_parsed-sorted') | 507 | os.execute('sort results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed >results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages_parsed-sorted') |
714 | if checkFile('Packages/' .. r .. dir .. 'Packages_parsed-sorted') then | 508 | if APT.checkFile('Packages/' .. r .. dir .. 'Packages_parsed-sorted') then |
715 | os.execute('diff -U 0 Packages/' .. r .. dir .. 'Packages_parsed-sorted ' .. | 509 | os.execute('diff -U 0 Packages/' .. r .. dir .. 'Packages_parsed-sorted ' .. |
716 | 'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages_parsed-sorted ' .. | 510 | 'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages_parsed-sorted ' .. |
717 | ' | grep -E "^-" | grep -Ev "^\\+\\+\\+|^---" >>results/OLD_PACKAGES_' .. r .. '.txt') | 511 | ' | grep -E "^-" | grep -Ev "^\\+\\+\\+|^---" >>results/OLD_PACKAGES_' .. r .. '.txt') |
@@ -737,14 +531,14 @@ local postDownload = function(host, r, k) | |||
737 | end | 531 | end |
738 | 532 | ||
739 | local downloadLock = "flock -n results/curl-" | 533 | local downloadLock = "flock -n results/curl-" |
740 | local download = "curl --connect-timeout " .. options.timeout.value .. " --create-dirs -f -L --max-time " .. options.maxtime.value .. " -z 'results/stamp.old' -v -R " | 534 | local download = "curl --connect-timeout " .. APT.options.timeout.value .. " --create-dirs -f -L --max-time " .. APT.options.maxtime.value .. " -z 'results/stamp.old' -v -R " |
741 | local downloads = function(host, URL, release, list) | 535 | local downloads = function(host, URL, release, list) |
742 | if nil == URL then URL = "" end | 536 | if nil == URL then URL = "" end |
743 | local lock = "META-" .. host .. ".lock" | 537 | local lock = "META-" .. host .. ".lock" |
744 | local log = " --stderr results/curl-META-" .. host .. ".log" | 538 | local log = " --stderr results/curl-META-" .. host .. ".log" |
745 | local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock .. " " .. download .. log .. " -K results/" .. host .. ".curl" | 539 | local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock .. " " .. download .. log .. " -K results/" .. host .. ".curl" |
746 | if testing("IPv4") and (not testing("IPv6")) then cm = cm .. ' -4' end | 540 | if APT.testing("IPv4") and (not APT.testing("IPv6")) then cm = cm .. ' -4' end |
747 | if (not testing("IPv4")) and testing("IPv6") then cm = cm .. ' -6' end | 541 | if (not APT.testing("IPv4")) and APT.testing("IPv6") then cm = cm .. ' -6' end |
748 | f, e = io.open("results/" .. host .. ".curl", "a+") | 542 | f, e = io.open("results/" .. host .. ".curl", "a+") |
749 | if nil == f then C("opening curl file - " .. e); return end | 543 | if nil == f then C("opening curl file - " .. e); return end |
750 | 544 | ||
@@ -772,7 +566,7 @@ local downloads = function(host, URL, release, list) | |||
772 | end | 566 | end |
773 | end | 567 | end |
774 | f:close() | 568 | f:close() |
775 | fork(cm) | 569 | APT.fork(cm) |
776 | end | 570 | end |
777 | 571 | ||
778 | 572 | ||
@@ -781,7 +575,7 @@ local getMirrors = function () | |||
781 | local host = "" | 575 | local host = "" |
782 | local m = {} | 576 | local m = {} |
783 | local active = true | 577 | local active = true |
784 | local URL = "http://" .. options.referenceSite.value .. "/mirror_list.txt" | 578 | local URL = "http://" .. APT.options.referenceSite.value .. "/mirror_list.txt" |
785 | I("getting mirrors.") | 579 | I("getting mirrors.") |
786 | local p, c, h = http.request(URL) | 580 | local p, c, h = http.request(URL) |
787 | if nil == p then E(c .. " fetching " .. URL) else | 581 | if nil == p then E(c .. " fetching " .. URL) else |
@@ -818,95 +612,20 @@ local getMirrors = function () | |||
818 | mirrors[host] = m | 612 | mirrors[host] = m |
819 | end | 613 | end |
820 | end | 614 | end |
821 | if testing("DNSRR") then | 615 | if APT.testing("DNSRR") then |
822 | mirrors[options.roundRobin.value] = { ["Protocols"] = { ["http"] = true; ["https"] = true; }; ["FQDN"] = 'deb.devuan.org'; ["Active"] = 'yes'; ["BaseURL"] = 'deb.devuan.org'; } | 616 | mirrors[APT.options.roundRobin.value] = { ["Protocols"] = { ["http"] = true; ["https"] = true; }; ["FQDN"] = 'deb.devuan.org'; ["Active"] = 'yes'; ["BaseURL"] = 'deb.devuan.org'; } |
823 | gatherIPs(options.roundRobin.value) | 617 | gatherIPs(APT.options.roundRobin.value) |
824 | mirrors[options.roundRobin.value].IPs = IP[options.roundRobin.value] | 618 | mirrors[APT.options.roundRobin.value].IPs = IP[APT.options.roundRobin.value] |
825 | end | 619 | end |
826 | local file, e = io.open("results/mirrors.lua", "w+") | 620 | local file, e = io.open("results/mirrors.lua", "w+") |
827 | if nil == file then C("opening mirrors file - " .. e) else | 621 | if nil == file then C("opening mirrors file - " .. e) else |
828 | file:write(dumpTable(mirrors, "", "mirrors") .. "\nreturn mirrors\n") | 622 | file:write(APT.dumpTable(mirrors, "", "mirrors") .. "\nreturn mirrors\n") |
829 | file:close() | 623 | file:close() |
830 | end | 624 | end |
831 | return mirrors | 625 | return mirrors |
832 | end | 626 | end |
833 | 627 | ||
834 | 628 | ||
835 | if 0 ~= #args then | ||
836 | local option = "" | ||
837 | for i, a in pairs(args) do | ||
838 | if ("--help" == a) or ("-h" == a) then | ||
839 | print("I should write some docs, huh? Read README.md for instructions.") | ||
840 | elseif "--version" == a then | ||
841 | print("apt-panopticon version 0.1 WIP development version") | ||
842 | elseif "-v" == a then | ||
843 | verbosity = verbosity + 1 | ||
844 | sendArgs = sendArgs .. a .. " " | ||
845 | elseif "-q" == a then | ||
846 | verbosity = -1 | ||
847 | sendArgs = sendArgs .. a .. " " | ||
848 | elseif "-k" == a then | ||
849 | keep = true | ||
850 | elseif "-n" == a then | ||
851 | fork = false | ||
852 | elseif "-o" == a then | ||
853 | origin = true | ||
854 | elseif "--" == a:sub(1, 2) then | ||
855 | local s, e = a:find("=") | ||
856 | if nil == s then e = -1 end | ||
857 | option = a:sub(3, e - 1) | ||
858 | local o = options[option] | ||
859 | if nil == o then | ||
860 | print("Unknown option --" .. option) | ||
861 | option = "" | ||
862 | else | ||
863 | option = a | ||
864 | sendArgs = sendArgs .. a .. " " | ||
865 | local s, e = a:find("=") | ||
866 | if nil == s then e = 0 end | ||
867 | option = a:sub(3, e - 1) | ||
868 | if "table" == options[option].typ then | ||
869 | local result = {} | ||
870 | for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do | ||
871 | local f = t:sub(1, 1) | ||
872 | local n = t:sub(2, -1) | ||
873 | if ("+" ~= f) and ("-" ~= f) then | ||
874 | table.insert(result, t) | ||
875 | end | ||
876 | end | ||
877 | if 0 ~= #result then | ||
878 | options[option].value = result | ||
879 | else | ||
880 | for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do | ||
881 | local f = t:sub(1, 1) | ||
882 | local n = t:sub(2, -1) | ||
883 | if "+" == f then | ||
884 | table.insert(options[option].value, n) | ||
885 | elseif "-" == f then | ||
886 | local r = {} | ||
887 | for i, k in pairs(options[option].value) do | ||
888 | if k ~= n then table.insert(r, k) end | ||
889 | end | ||
890 | options[option].value = r | ||
891 | end | ||
892 | end | ||
893 | end | ||
894 | else | ||
895 | options[option].value = a:sub(e + 1, -1) | ||
896 | end | ||
897 | option = "" | ||
898 | end | ||
899 | elseif "-" == a:sub(1, 1) then | ||
900 | print("Unknown option " .. a) | ||
901 | else | ||
902 | table.insert(arg, a) | ||
903 | end | ||
904 | end | ||
905 | end | ||
906 | |||
907 | --print(dumpTable(options.tests.value, "", "tests")) | ||
908 | |||
909 | |||
910 | if 0 < #arg then | 629 | if 0 < #arg then |
911 | if "/" == arg[1]:sub(-1, -1) then | 630 | if "/" == arg[1]:sub(-1, -1) then |
912 | W("slash at end of path! " .. arg[1]) | 631 | W("slash at end of path! " .. arg[1]) |
@@ -918,37 +637,37 @@ if 0 < #arg then | |||
918 | end | 637 | end |
919 | local pu = url.parse("http://" .. arg[1]) | 638 | local pu = url.parse("http://" .. arg[1]) |
920 | 639 | ||
921 | if testing("Integrity") or testing("Updated") then | 640 | if APT.testing("Integrity") or APT.testing("Updated") then |
922 | if origin and options.referenceSite.value == pu.host then | 641 | if APT.origin and APT.options.referenceSite.value == pu.host then |
923 | -- if not keep then execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end | 642 | -- if not APT.keep then os.execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end |
924 | end | 643 | end |
925 | end | 644 | end |
926 | 645 | ||
927 | if nil ~= arg[2] then | 646 | if nil ~= arg[2] then |
928 | logFile, e = io.open("results/LOG_" .. pu.host .. "_" .. arg[2] .. ".html", "a+") | 647 | APT.logFile, e = io.open("results/LOG_" .. pu.host .. "_" .. arg[2] .. ".html", "a+") |
929 | else | 648 | else |
930 | logFile, e = io.open("results/LOG_" .. pu.host .. ".html", "a+") | 649 | APT.logFile, e = io.open("results/LOG_" .. pu.host .. ".html", "a+") |
931 | end | 650 | end |
932 | if nil == logFile then C("opening log file - " .. e); return end | 651 | if nil == APT.logFile then C("opening log file - " .. e); return end |
933 | logPre() | 652 | APT.logPre() |
934 | I("Starting tests for " ..arg[1] .. " with these tests - " .. table.concat(options.tests.value, ", ")) | 653 | I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", ")) |
935 | mirrors = loadfile("results/mirrors.lua")() | 654 | APT.mirrors = loadfile("results/mirrors.lua")() |
936 | if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end | 655 | if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end |
937 | if nil ~= arg[3] then I(" Using file " .. arg[3]); end | 656 | if nil ~= arg[3] then I(" Using file " .. arg[3]); end |
938 | 657 | ||
939 | for k, v in pairs{"ftp", "http", "https", "rsync"} do | 658 | for k, v in pairs{"ftp", "http", "https", "rsync"} do |
940 | if testing(v) then | 659 | if APT.testing(v) then |
941 | local tests = {errors = 0; warnings = 0} | 660 | local tests = {errors = 0; warnings = 0} |
942 | if testing("Integrity") then tests.Integrity = {errors = 0; warnings = 0} end | 661 | if APT.testing("Integrity") then tests.Integrity = {errors = 0; warnings = 0} end |
943 | if testing("Protocol") then tests.Protocol = {errors = 0; warnings = 0} end | 662 | if APT.testing("Protocol") then tests.Protocol = {errors = 0; warnings = 0} end |
944 | if testing("Updated") then tests.Updated = {errors = 0; warnings = 0} end | 663 | if APT.testing("Updated") then tests.Updated = {errors = 0; warnings = 0} end |
945 | if testing("URLSanity") then tests.URLSanity = {errors = 0; warnings = 0} end | 664 | if APT.testing("URLSanity") then tests.URLSanity = {errors = 0; warnings = 0} end |
946 | results[v] = tests | 665 | APT.results[v] = tests |
947 | end | 666 | end |
948 | end | 667 | end |
949 | if origin then | 668 | if APT.origin then |
950 | if testing("Integrity") or testing("Updated") then | 669 | if APT.testing("Integrity") or APT.testing("Updated") then |
951 | if origin and (options.roundRobin.value ~= pu.host) then | 670 | if APT.origin and (APT.options.roundRobin.value ~= pu.host) then |
952 | I("Starting file downloads for " .. pu.host) | 671 | I("Starting file downloads for " .. pu.host) |
953 | downloads(pu.host, pu.path) | 672 | downloads(pu.host, pu.path) |
954 | end | 673 | end |
@@ -958,10 +677,10 @@ if 0 < #arg then | |||
958 | checkHost(pu.host, pu.host, pu.path, arg[2], arg[3]) | 677 | checkHost(pu.host, pu.host, pu.path, arg[2], arg[3]) |
959 | end | 678 | end |
960 | 679 | ||
961 | if testing("Integrity") or testing("Updated") then | 680 | if APT.testing("Integrity") or APT.testing("Updated") then |
962 | if 4 > (totalTimeouts) then | 681 | if 4 > (totalTimeouts) then |
963 | if origin and (options.roundRobin.value ~= pu.host) then | 682 | if APT.origin and (APT.options.roundRobin.value ~= pu.host) then |
964 | while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end | 683 | while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end |
965 | os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " .. | 684 | os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " .. |
966 | "mv results/curl-META-" .. pu.host .. ".log results/curl-Release-" .. pu.host .. ".log") | 685 | "mv results/curl-META-" .. pu.host .. ".log results/curl-Release-" .. pu.host .. ".log") |
967 | for i, n in pairs(releases) do | 686 | for i, n in pairs(releases) do |
@@ -971,24 +690,25 @@ if 0 < #arg then | |||
971 | end | 690 | end |
972 | end | 691 | end |
973 | 692 | ||
974 | if checkFile('results/' .. pu.host .. '/merged/dists/' .. n .. '/Release') and | 693 | if APT.checkFile('results/' .. pu.host .. '/merged/dists/' .. n .. '/Release') then |
975 | checkFile('results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED') then | ||
976 | os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED') | 694 | os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED') |
977 | if options.referenceSite.value == pu.host then | 695 | if APT.checkFile('results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED') then |
978 | os.execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. | 696 | if APT.options.referenceSite.value == pu.host then |
979 | 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. | 697 | os.execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. |
980 | '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt') | 698 | 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. |
981 | os.execute('rm -f results/' .. pu.host .. '/merged/dists/' .. n .. '/Release 2>/dev/null') | 699 | '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt') |
982 | -- TODO - Maybe check the date in Release, though since they are updated daily, is there any point? Perhaps it's for checking amprolla got run? | 700 | -- TODO - Maybe check the date in Release, though since they are updated daily, is there any point? Perhaps it's for checking amprolla got run? |
983 | else | 701 | os.execute('rm -f results/' .. pu.host .. '/merged/dists/' .. n .. '/Release 2>/dev/null') |
702 | else | ||
984 | -- TODO - compare to the pkgmaster copy. | 703 | -- TODO - compare to the pkgmaster copy. |
985 | end | 704 | end |
986 | 705 | ||
987 | local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") | 706 | local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") |
988 | if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else | 707 | if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else |
989 | local diff = dfile:read("*a") | 708 | local diff = dfile:read("*a") |
990 | if "" ~= diff then | 709 | if "" ~= diff then |
991 | downloads(pu.host, pu.path, n, diff) | 710 | downloads(pu.host, pu.path, n, diff) |
711 | end | ||
992 | end | 712 | end |
993 | end | 713 | end |
994 | end | 714 | end |
@@ -996,7 +716,7 @@ if 0 < #arg then | |||
996 | end | 716 | end |
997 | 717 | ||
998 | downloads(pu.host, pu.path, "", "") | 718 | downloads(pu.host, pu.path, "", "") |
999 | while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end | 719 | while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end |
1000 | os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " .. | 720 | os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " .. |
1001 | "mv results/curl-META-" .. pu.host .. ".log results/curl-Packages-" .. pu.host .. ".log") | 721 | "mv results/curl-META-" .. pu.host .. ".log results/curl-Packages-" .. pu.host .. ".log") |
1002 | 722 | ||
@@ -1008,7 +728,7 @@ if 0 < #arg then | |||
1008 | postDownload(pu.host, n, "/" .. l) | 728 | postDownload(pu.host, n, "/" .. l) |
1009 | end | 729 | end |
1010 | end | 730 | end |
1011 | if options.referenceSite.value == pu.host then | 731 | if APT.options.referenceSite.value == pu.host then |
1012 | -- In case it wasn't dealt with already. | 732 | -- In case it wasn't dealt with already. |
1013 | os.execute('touch results/NEW_Packages_' .. n .. '.test.txt') | 733 | os.execute('touch results/NEW_Packages_' .. n .. '.test.txt') |
1014 | end | 734 | end |
@@ -1027,26 +747,26 @@ if 0 < #arg then | |||
1027 | end | 747 | end |
1028 | end | 748 | end |
1029 | downloads(pu.host, pu.path, nil, "") | 749 | downloads(pu.host, pu.path, nil, "") |
1030 | while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end | 750 | while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end |
1031 | for i, n in pairs(releases) do | 751 | for i, n in pairs(releases) do |
1032 | local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") | 752 | local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") |
1033 | if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else | 753 | if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else |
1034 | for l in nfile:lines() do | 754 | for l in nfile:lines() do |
1035 | local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |') | 755 | local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |') |
1036 | if nil ~= p then | 756 | if nil ~= p then |
1037 | if checkFile('results/' .. pu.host .. "/merged/" .. p) then | 757 | if APT.checkFile('results/' .. pu.host .. "/merged/" .. p) then |
1038 | local status, fsz = execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5') | 758 | local status, fsz = APT.execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5') |
1039 | if testing("Integrity") then | 759 | if APT.testing("Integrity") then |
1040 | if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. | 760 | if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. |
1041 | E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) | 761 | E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) |
1042 | print('|' .. sz .. '~=' .. fsz:sub(2, -2) .. '|') | 762 | print('|' .. sz .. '~=' .. fsz:sub(2, -2) .. '|') |
1043 | else | 763 | else |
1044 | local status, fsha = execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') | 764 | local status, fsha = APT.execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') |
1045 | if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end | 765 | if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end |
1046 | -- TODO - maybe check the PGP key, though packages are mostly not signed. | 766 | -- TODO - maybe check the PGP key, though packages are mostly not signed. |
1047 | end | 767 | end |
1048 | end | 768 | end |
1049 | if testing("Updated") then | 769 | if APT.testing("Updated") then |
1050 | if sz ~= fsz:sub(2, -2) then | 770 | if sz ~= fsz:sub(2, -2) then |
1051 | E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) | 771 | E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) |
1052 | end | 772 | end |
@@ -1060,20 +780,20 @@ if 0 < #arg then | |||
1060 | end | 780 | end |
1061 | end | 781 | end |
1062 | 782 | ||
1063 | results["timeout"] = false | 783 | APT.results["timeout"] = false |
1064 | else | 784 | else |
1065 | results["timeout"] = true | 785 | APT.results["timeout"] = true |
1066 | end | 786 | end |
1067 | end | 787 | end |
1068 | 788 | ||
1069 | if origin and options.referenceSite.value ~= pu.host then | 789 | if APT.origin and APT.options.referenceSite.value ~= pu.host then |
1070 | if not keep then os.execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end | 790 | if not APT.keep then os.execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end |
1071 | os.execute('rm STATUS_' .. pu.host .. '* 2>/dev/null') | 791 | os.execute('rm STATUS_' .. pu.host .. '* 2>/dev/null') |
1072 | end | 792 | end |
1073 | 793 | ||
1074 | local min, max, spd = 999999999999, 0 | 794 | local min, max, spd = 999999999999, 0 |
1075 | for i, mt in pairs({'Release', 'Packages', 'META'}) do | 795 | for i, mt in pairs({'Release', 'Packages', 'META'}) do |
1076 | if checkFile("results/curl-" .. mt .. "-" .. pu.host .. ".log") then | 796 | if APT.checkFile("results/curl-" .. mt .. "-" .. pu.host .. ".log") then |
1077 | for l in io.lines("results/curl-" .. mt .. "-" .. pu.host .. ".log") do | 797 | for l in io.lines("results/curl-" .. mt .. "-" .. pu.host .. ".log") do |
1078 | local speed, crrnt = l:match('^%c *%d+ +%d+k? +%d+ +%d+k? +%d+ +%d+ +(%d+k?) +%d+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +(%d+k?)') | 798 | local speed, crrnt = l:match('^%c *%d+ +%d+k? +%d+ +%d+k? +%d+ +%d+ +(%d+k?) +%d+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +(%d+k?)') |
1079 | if nil ~= speed then | 799 | if nil ~= speed then |
@@ -1089,17 +809,17 @@ if 0 < #arg then | |||
1089 | end | 809 | end |
1090 | end | 810 | end |
1091 | end | 811 | end |
1092 | results["speed"] = {min = min, max = max} | 812 | APT.results["speed"] = {min = min, max = max} |
1093 | 813 | ||
1094 | local f = pu.host | 814 | local f = pu.host |
1095 | if "" ~= ip then f = f .. "_" .. ip end | 815 | if "" ~= ip then f = f .. "_" .. ip end |
1096 | local rfile, e = io.open("results/" .. f .. ".lua", "w+") | 816 | local rfile, e = io.open("results/" .. f .. ".lua", "w+") |
1097 | if nil == rfile then C("opening results file - " .. e) else | 817 | if nil == rfile then C("opening results file - " .. e) else |
1098 | rfile:write(dumpTable(results, "", "results") .. "\nreturn results\n") | 818 | rfile:write(APT.dumpTable(APT.results, "", "results") .. "\nreturn results\n") |
1099 | rfile:close() | 819 | rfile:close() |
1100 | end | 820 | end |
1101 | logPost() | 821 | APT.logPost() |
1102 | logFile:close() | 822 | APT.logFile:close() |
1103 | else | 823 | else |
1104 | local fadt = io.popen("ls -dl results_old 2>/dev/null | cut -d '>' -f 2 | cut -d ' ' -f 2") | 824 | local fadt = io.popen("ls -dl results_old 2>/dev/null | cut -d '>' -f 2 | cut -d ' ' -f 2") |
1105 | local adt = fadt:read('*l') | 825 | local adt = fadt:read('*l') |
@@ -1113,25 +833,25 @@ else | |||
1113 | if nil ~= dt then os.execute('mkdir -p results_' .. dt .. '; rm -f results; ln -s results_' .. dt .. ' results 2>/dev/null') end | 833 | if nil ~= dt then os.execute('mkdir -p results_' .. dt .. '; rm -f results; ln -s results_' .. dt .. ' results 2>/dev/null') end |
1114 | os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp') | 834 | os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp') |
1115 | os.execute("rm -f results/*.check 2>/dev/null") | 835 | os.execute("rm -f results/*.check 2>/dev/null") |
1116 | if not keep then | 836 | if not APT.keep then |
1117 | os.execute("rm -f results/*.curl 2>/dev/null") | 837 | os.execute("rm -f results/*.curl 2>/dev/null") |
1118 | os.execute("rm -f results/*.log 2>/dev/null") | 838 | os.execute("rm -f results/*.log 2>/dev/null") |
1119 | os.execute("rm -f results/*.html 2>/dev/null") | 839 | os.execute("rm -f results/*.html 2>/dev/null") |
1120 | os.execute("rm -f results/*.txt 2>/dev/null") | 840 | os.execute("rm -f results/*.txt 2>/dev/null") |
1121 | end | 841 | end |
1122 | 842 | ||
1123 | logFile, e = io.open("results/LOG_apt-panopticon.html", "a+") | 843 | APT.logFile, e = io.open("results/LOG_apt-panopticon.html", "a+") |
1124 | if nil == logFile then C("opening log file - " .. e); return end | 844 | if nil == APT.logFile then C("opening log file - " .. e); return end |
1125 | logPre() | 845 | APT.logPre() |
1126 | I("Starting tests " .. table.concat(options.tests.value, ", ")) | 846 | I("Starting tests " .. table.concat(APT.options.tests.value, ", ")) |
1127 | os.execute("mkdir -p results") | 847 | os.execute("mkdir -p results") |
1128 | mirrors = getMirrors() | 848 | APT.mirrors = getMirrors() |
1129 | checkHost(options.referenceSite.value) | 849 | checkHost(APT.options.referenceSite.value) |
1130 | for i, n in pairs(releases) do | 850 | for i, n in pairs(releases) do |
1131 | while not checkFile('results/NEW_Packages_' .. n .. '.test.txt') do os.execute("sleep 10") end | 851 | while not APT.checkFile('results/NEW_Packages_' .. n .. '.test.txt') do os.execute("sleep 10") end |
1132 | end | 852 | end |
1133 | 853 | ||
1134 | for k, m in pairs(mirrors) do | 854 | for k, m in pairs(APT.mirrors) do |
1135 | if "/" == m.BaseURL:sub(-1, -1) then | 855 | if "/" == m.BaseURL:sub(-1, -1) then |
1136 | W("slash at end of BaseURL in mirror_list.txt! " .. m.BaseURL, "", "", m.FQDN) | 856 | W("slash at end of BaseURL in mirror_list.txt! " .. m.BaseURL, "", "", m.FQDN) |
1137 | m.BaseURL = m.BaseURL:sub(1, -2) | 857 | m.BaseURL = m.BaseURL:sub(1, -2) |
@@ -1141,30 +861,27 @@ else | |||
1141 | m.BaseURL = m.BaseURL:sub(1, -2) | 861 | m.BaseURL = m.BaseURL:sub(1, -2) |
1142 | end | 862 | end |
1143 | local pu = url.parse("http://" .. m.BaseURL) | 863 | local pu = url.parse("http://" .. m.BaseURL) |
1144 | if options.referenceSite.value ~= pu.host then | 864 | if APT.options.referenceSite.value ~= pu.host then |
1145 | checkHost(m.BaseURL) | 865 | checkHost(m.BaseURL) |
1146 | checkExes("apt-panopticon.lua " .. sendArgs) | 866 | APT.checkExes("apt-panopticon.lua " .. sendArgs) |
1147 | if testing("Integrity") or testing("Updated") then checkExes(downloadLock) end | 867 | if APT.testing("Integrity") or APT.testing("Updated") then APT.checkExes(downloadLock) end |
1148 | end | 868 | end |
1149 | end | 869 | end |
1150 | 870 | ||
1151 | while 1 <= checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end | 871 | while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end |
1152 | 872 | ||
1153 | os.execute("rm -f results/*.check; rm -f results/*.lock 2>/dev/null") | 873 | os.execute("rm -f results/*.check; rm -f results/*.lock 2>/dev/null") |
1154 | 874 | ||
1155 | -- Create the reports. | 875 | -- Create the reports. |
1156 | for n, r in pairs(options.reports.value) do | 876 | for n, r in pairs(APT.options.reports.value) do |
1157 | local report = "apt-panopticon-report-" .. r .. ".lua" | 877 | if APT.checkFile("apt-panopticon-report-" .. r .. ".lua") then |
1158 | local rfile, e = io.open(report, "r") | 878 | I("Creating " .. r .. " report.") |
1159 | if nil == rfile then C("opening " .. report .. " file - " .. e) else | 879 | APT.execute("./apt-panopticon-report-" .. r .. ".lua") |
1160 | rfile:close() | ||
1161 | I("Creating " .. report .. " report.") | ||
1162 | execute("./" .. report .. " ") | ||
1163 | end | 880 | end |
1164 | end | 881 | end |
1165 | 882 | ||
1166 | if nil ~= adt then os.execute('rm -fr ' .. adt .. ' 2>/dev/null') end | 883 | if nil ~= adt then os.execute('rm -fr ' .. adt .. ' 2>/dev/null') end |
1167 | 884 | ||
1168 | logPost() | 885 | APT.logPost() |
1169 | logFile:close() | 886 | APT.logFile:close() |
1170 | end | 887 | end |