aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--apt-panopticommon.lua563
-rwxr-xr-xapt-panopticon-report-email-web.lua250
-rwxr-xr-xapt-panopticon.lua507
3 files changed, 722 insertions, 598 deletions
diff --git a/apt-panopticommon.lua b/apt-panopticommon.lua
new file mode 100644
index 0000000..b38595c
--- /dev/null
+++ b/apt-panopticommon.lua
@@ -0,0 +1,563 @@
1local APT = {}
2
3-- https://oss.oetiker.ch/rrdtool/prog/rrdlua.en.html
4APT.rrd = require 'rrd'
5
6verbosity = -1
7APT.origin = false
8APT.keep = false
9-- TODO - Should actually implement this.
10APT.fork = true
11
12APT.options =
13{
14 referenceSite =
15 {
16 typ = "string",
17 help = "",
18 value = "pkgmaster.devuan.org",
19 },
20 roundRobin =
21 {
22 typ = "string",
23 help = "",
24 value = "deb.devuan.org",
25 },
26 tests =
27 {
28 typ = "table",
29 help = "",
30 value =
31 {
32 "IPv4",
33 "IPv6",
34-- "ftp",
35 "http",
36 "https",
37-- "rsync",
38 "DNSRR",
39 "Protocol",
40 "URLSanity",
41 "Integrity",
42 "Updated",
43 },
44 },
45 maxtime =
46 {
47 typ = "number",
48 help = "",
49 value = 300,
50 },
51 timeout =
52 {
53 typ = "number",
54 help = "",
55 value = 15,
56 },
57 reports =
58 {
59 typ = "table",
60 help = "",
61 value =
62 {
63 "RRD", -- RRD has to be before web, coz web creates a graph from the RRD data.
64 "email-web",
65-- "Nagios",
66-- "Prometheus",
67 },
68 },
69}
70
71APT.parseArgs = function(args)
72 local arg = {}
73 local sendArgs = ""
74 if 0 ~= #(args) then
75 local option = ""
76 for i, a in pairs(args) do
77 if ("--help" == a) or ("-h" == a) then
78 print("I should write some docs, huh? Read README.md for instructions.")
79 elseif "--version" == a then
80 print("apt-panopticon version 0.1 WIP development version")
81 elseif "-v" == a then
82 verbosity = verbosity + 1
83 sendArgs = sendArgs .. a .. " "
84 elseif "-q" == a then
85 verbosity = -1
86 sendArgs = sendArgs .. a .. " "
87 elseif "-k" == a then
88 APT.keep = true
89 elseif "-n" == a then
90 APT.fork = false
91 elseif "-o" == a then
92 APT.origin = true
93 elseif "--" == a:sub(1, 2) then
94 local s, e = a:find("=")
95 if nil == s then e = -1 end
96 option = a:sub(3, e - 1)
97 local o = APT.options[option]
98 if nil == o then
99 print("Unknown option --" .. option)
100 option = ""
101 else
102 option = a
103 sendArgs = sendArgs .. a .. " "
104 local s, e = a:find("=")
105 if nil == s then e = 0 end
106 option = a:sub(3, e - 1)
107 if "table" == APT.options[option].typ then
108 local result = {}
109 for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do
110 local f = t:sub(1, 1)
111 local n = t:sub(2, -1)
112 if ("+" ~= f) and ("-" ~= f) then
113 table.insert(result, t)
114 end
115 end
116 if 0 ~= #result then
117 APT.options[option].value = result
118 else
119 for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do
120 local f = t:sub(1, 1)
121 local n = t:sub(2, -1)
122 if "+" == f then
123 table.insert(APT.options[option].value, n)
124 elseif "-" == f then
125 local r = {}
126 for i, k in pairs(APT.options[option].value) do
127 if k ~= n then table.insert(r, k) end
128 end
129 APT.options[option].value = r
130 end
131 end
132 end
133 else
134 APT.options[option].value = a:sub(e + 1, -1)
135 end
136 option = ""
137 end
138 elseif "-" == a:sub(1, 1) then
139 print("Unknown option " .. a)
140 else
141 table.insert(arg, a)
142 end
143 end
144 end
145 return arg, sendArgs
146end
147
148--print(APT.dumpTable(APT.options, "", "options"))
149
150
151
152--[[ Ordered table iterator, allow to iterate on the natural order of the keys of a table.
153 From http://lua-users.org/wiki/SortedIteration
154 ]]
155function __genOrderedIndex( t )
156 local orderedIndex = {}
157 for key in pairs(t) do
158 table.insert( orderedIndex, key )
159 end
160 table.sort( orderedIndex )
161 return orderedIndex
162end
163function orderedNext(t, state)
164 -- Equivalent of the next function, but returns the keys in the alphabetic
165 -- order. We use a temporary ordered key table that is stored in the
166 -- table being iterated.
167
168 local key = nil
169 --print("orderedNext: state = "..tostring(state) )
170 if state == nil then
171 -- the first time, generate the index
172 t.__orderedIndex = __genOrderedIndex( t )
173 key = t.__orderedIndex[1]
174 else
175 -- fetch the next value
176 for i = 1,table.getn(t.__orderedIndex) do
177 if t.__orderedIndex[i] == state then
178 key = t.__orderedIndex[i+1]
179 end
180 end
181 end
182
183 if key then
184 return key, t[key]
185 end
186
187 -- no more value to return, cleanup
188 t.__orderedIndex = nil
189 return
190end
191function APT.orderedPairs(t)
192 -- Equivalent of the pairs() function on tables. Allows to iterate
193 -- in order
194 return orderedNext, t, nil
195end
196
197-- Use this to dump a table to a string, with HTML.
198APT.dumpTableHTML = function (table, space, name)
199 local r = name .. "\n"
200 r = r .. dumpTableHTMLSub(table, space .. " ")
201 r = r .. space .. ""
202 return r
203end
204dumpTableHTMLSub = function (table, space)
205 local r = ""
206 for k, v in APT.orderedPairs(table) do
207 if type(v) == "table" then
208 if " " == space then
209 r = r .. space .. APT.dumpTableHTML(v, space, k .. "<ul>") .. "</ul>\n"
210 else
211 r = r .. "<li>" .. space .. APT.dumpTableHTML(v, space, k .. "<ul>") .. "</ul></li>\n"
212 end
213 else
214 r = r .. space .. "<li>" .. k .. "</li>\n"
215 end
216 end
217 return r
218end
219
220-- Use this to dump a table to a string.
221APT.dumpTable = function (table, space, name)
222 local r = ""
223 if "" == space then r = r .. space .. name .. " =\n" else r = r .. space .. "[" .. name .. "] =\n" end
224 r = r .. space .. "{\n"
225 r = r .. dumpTableSub(table, space .. " ")
226 if "" == space then r = r .. space .. "}\n" else r = r .. space .. "},\n" end
227 return r
228end
229dumpTableSub = function (table, space)
230 local r = ""
231 for k, v in pairs(table) do
232 if type(k) == "string" then k = '"' .. k .. '"' end
233 if type(v) == "table" then
234 r = r .. APT.dumpTable(v, space, k)
235 elseif type(v) == "string" then
236 r = r .. space .. "[" .. k .. "] = '" .. v .. "';\n"
237 elseif type(v) == "function" then
238 r = r .. space .. "[" .. k .. "] = function ();\n"
239 elseif type(v) == "userdata" then
240 r = r .. space .. "userdata " .. "[" .. k .. "];\n"
241 elseif type(v) == "boolean" then
242 if (v) then
243 r = r .. space .. "[" .. k .. "] = true;\n"
244 else
245 r = r .. space .. "[" .. k .. "] = false;\n"
246 end
247 else
248 r = r .. space .. "[" .. k .. "] = " .. v .. ";\n"
249 end
250 end
251 return r
252end
253
254APT.results = {}
255APT.logFile = nil
256APT.html = false
257
258APT.logPre = function()
259 if nil ~= APT.logFile then
260 APT.logFile:write("<html><head>\n")
261 APT.logFile:write("</head><body bgcolor='black' text='white' alink='red' link='blue' vlink='purple'>\n")
262 end
263end
264APT.logPost = function()
265 if nil ~= APT.logFile then
266 APT.logFile:write("</body></html> \n")
267 end
268end
269
270local log = function(v, t, s, prot, test, host)
271 local x = ""
272 if nil == prot then prot = "" end
273 if nil == test then test = "" end
274 x = x .. prot
275 if "" ~= test then
276 if #x > 0 then x = x .. " " end
277 x = x .. test
278 end
279 if nil ~= host then
280 if #x > 0 then x = x .. " " end
281 x = x .. host
282 end
283 if #x > 0 then
284 t = t .. "(" .. x .. ")"
285 if "" ~= prot then
286 if "" == test then
287 if nil == APT.results[prot] then APT.results[prot] = {errors = 0; warnings = 0} end
288 if v == 0 then APT.results[prot].errors = APT.results[prot].errors + 1 end
289 if v == 1 then APT.results[prot].warnings = APT.results[prot].warnings + 1 end
290 else
291 if nil == APT.results[prot] then APT.results[prot] = {errors = 0; warnings = 0} end
292 if nil == APT.results[prot][test] then APT.results[prot][test] = {errors = 0; warnings = 0} end
293 if v == 0 then APT.results[prot][test].errors = APT.results[prot][test].errors + 1 end
294 if v == 1 then APT.results[prot][test].warnings = APT.results[prot][test].warnings + 1 end
295 end
296 end
297 end
298 if v <= verbosity then
299 if 3 <= verbosity then t = os.date() .. " " .. t end
300 print(t .. ": " .. s)
301 end
302 if nil ~= APT.logFile then
303 if APT.html then
304 local colour = "white"
305 if -1 == v then colour = "fuchsia" end -- CRITICAL
306 if 0 == v then colour = "red " end -- ERROR
307 if 1 == v then colour = "yellow " end -- WARNING
308 if 2 == v then colour = "white " end -- INFO
309 if 3 == v then colour = "gray " end -- DEBUG
310 APT.logFile:write(os.date() .. " <font color='" .. colour .. "'><b>" .. t .. "</b></font>: " .. s .. "</br>\n")
311 else
312 APT.logFile:write(os.date() .. " " .. t .. ": " .. s .. "\n")
313 end
314 APT.logFile:flush()
315 end
316end
317APT.D = function(s) log(3, "DEBUG ", s) end
318APT.I = function(s) log(2, "INFO ", s) end
319APT.W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end
320APT.E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end
321APT.C = function(s) log(-1, "CRITICAL", s) end
322local D = APT.D
323local I = APT.I
324local W = APT.W
325local E = APT.E
326local C = APT.C
327
328
329APT.mirrors = {}
330
331APT.testing = function(t, host)
332 for i, v in pairs(APT.options.tests.value) do
333 if t == v then
334 local h = APT.mirrors[host]
335 if nil == h then return true end
336 if true == h["Protocols"][t] then return true else D("Skipping " .. t .. " checks for " .. host) end
337 end
338 end
339 return false
340end
341
342APT.execute = function (s)
343 D(" executing <pre><code>" .. s .. "</code></pre>")
344 --[[ Damn os.execute()
345 Lua 5.1 says it returns "a status code, which is system-dependent"
346 Lua 5.2 says it returns true/nil, "exit"/"signal", the status code.
347 I'm getting 7168 or 0. No idea what the fuck that is.
348 local ok, rslt, status = os.execute(s)
349 ]]
350 local f = io.popen(s .. ' ; echo "$?"', 'r')
351 local status = ""
352 local result = ""
353 -- The last line will be the command's returned status, collect everything else in result.
354 for l in f:lines() do
355 result = result .. status .. "\n"
356 status = l
357 end
358 return status, result
359end
360
361APT.fork = function(s)
362 D(" forking <pre><code>" .. s .. "</code></pre>")
363 os.execute(s .. " &")
364end
365
366APT.checkExes = function (exe)
367 local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | grep -v "flock -n apt-panopticon.lock " | wc -l'):read("*l")
368 D(count .. " " .. exe .. " commands still running.")
369 return tonumber(count)
370end
371
372APT.checkFile = function(f)
373 local h, e = io.open(f, "r")
374 if nil == h then return false else h:close(); return true end
375end
376
377APT.plurals = function(e, w)
378 local result = ""
379 if 1 == e then
380 result = e .. " error"
381 elseif e ~= 0 then
382 result = e .. " errors"
383 end
384 if ("" ~= result) and APT.html then result = "<font color='red'><b>" .. result .. "</b></font>" end
385-- result = " " .. result
386 if 0 < w then
387 if 0 < e then result = result .. ", " end
388 if 1 == w then
389 result = result .. w .. " warning"
390 else
391 result = result .. w .. " warnings"
392 end
393 if ("" ~= result) and APT.html then result = "<font color='yellow'><b>" .. result .. "</b></font>" end
394-- result = " " .. result
395 end
396 if "" ~= result then result = " (" .. result .. ")" end
397 return result
398end
399
400APT.collate = function(host, ip, results)
401 local f = "results/" .. host .. "_" .. ip .. ".lua"
402 local rfile, e = io.open(f, "r")
403 if nil == rfile then I("opening " .. f .. " file - " .. e) else
404 rfile:close()
405 local rs = loadfile(f)()
406 for k, v in pairs(rs) do
407 if "table" == type(v) then
408 if ("speed" == k) and (nil ~= results.speed) then
409 if v.min < results.speed.min then results.speed.min = v.min end
410 if v.max > results.speed.max then results.speed.max = v.max end
411 else
412 for i, u in pairs(v) do
413 if "table" == type(u) then
414 for h, t in pairs(u) do
415 local a = results[k][i][h]
416 if nil == a then a = 0 end
417 results[k][i][h] = a + t
418 end
419 else
420 local a = results[k]
421 if nil == a then a = 0; results[k] = {} else a = a[i] end
422 if nil == a then a = 0 end
423 results[k][i] = a + u
424 end
425 end
426 end
427 elseif "timeout" ~= k then
428 local a = results[k]
429 if nil == a then a = 0 end
430 results[k] = a + v
431 end
432 end
433 end
434 return results
435end
436
437
438
439
440APT.now = 0
441local status
442status, APT.now = APT.execute('TZ="GMT" ls -l --time-style="+%s" results/stamp | cut -d " " -f 6-6')
443APT.now = tonumber(APT.now)
444APT.protocols = {'ftp', 'http', 'https', 'rsync'}
445APT.tests = {'raw', 'Integrity', 'Protocol', 'Updated', 'URLSanity', 'Speed'}
446local start = 'now-2week'
447local step = '10min'
448local hb = '150min'
449local DSIe = 'DS:IntegrityErrors:GAUGE:' .. hb .. ':0:U'
450local DSIw = 'DS:IntegrityWarnings:GAUGE:' .. hb .. ':0:U'
451local DSPe = 'DS:ProtocolErrors:GAUGE:' .. hb .. ':0:U'
452local DSPw = 'DS:ProtocolWarnings:GAUGE:' .. hb .. ':0:U'
453local DSUe = 'DS:UpdatedErrors:GAUGE:' .. hb .. ':0:U'
454local DSUw = 'DS:UpdatedWarnings:GAUGE:' .. hb .. ':0:U'
455local DSSe = 'DS:URLSanityErrors:GAUGE:' .. hb .. ':0:U'
456local DSSw = 'DS:URLSanityWarnings:GAUGE:' .. hb .. ':0:U'
457local DSx = 'DS:max:GAUGE:' .. hb .. ':0:U'
458local DSn = 'DS:min:GAUGE:' .. hb .. ':0:U'
459
460-- What Collectd uses.
461local RRAc0 = 'RRA:AVERAGE:0.9:1:1200'
462local RRAc1 = 'RRA:MIN:0.9:1:1200'
463local RRAc2 = 'RRA:MAX:0.9:1:1200'
464local RRAc3 = 'RRA:AVERAGE:0.9:7:1235'
465local RRAc4 = 'RRA:MIN:0.9:7:1235'
466local RRAc5 = 'RRA:MAX:0.9:7:1235'
467local RRAc6 = 'RRA:AVERAGE:0.9:50:1210'
468local RRAc7 = 'RRA:MIN:0.9:50:1210'
469local RRAc8 = 'RRA:MAX:0.9:50:1210'
470local RRAc9 = 'RRA:AVERAGE:0.9:223:1202'
471local RRAc10 = 'RRA:MIN:0.9:223:1202'
472local RRAc11 = 'RRA:MAX:0.9:223:1202'
473local RRAc12 = 'RRA:AVERAGE:0.9:2635:1201'
474local RRAc13 = 'RRA:MIN:0.9:2635:1201'
475local RRAc14 = 'RRA:MAX:0.9:2635:1201'
476
477-- Try LAST.
478local RRAl0 = 'RRA:LAST:0.9:1:1200'
479local RRAl1 = 'RRA:LAST:0.9:7:1235'
480local RRAl2 = 'RRA:LAST:0.9:50:1210'
481local RRAl3 = 'RRA:LAST:0.9:223:1202'
482local RRAl4 = 'RRA:LAST:0.9:2635:1201'
483
484--[[
485/var/lib/collectd/rrd/onefang/interface-eth0/if_packets.rrd
486
487 rrd/host.IP/protocol.rrd test-error count
488 rrd/host.IP/protocol.rrd test-warning count
489
490 rrh/host.IP/protocol-speed.rrd min
491 rrh/host.IP/protocol-speed.rrd max
492]]
493
494APT.createRRD = function(host, ip)
495 if nil ~= ip then host = host .. '_' .. ip end
496 for i, p in pairs(APT.protocols) do
497-- for j, t in pairs(tests) do
498 os.execute( 'mkdir -p rrd/' .. host .. '/' .. p:upper())
499 if not APT.checkFile('rrd/' .. host .. '/' .. p:upper() .. '/Speed.rrd') then
500 D('Creating ' .. 'rrd/' .. host .. '/' .. p:upper() .. '/Speed.rrd')
501-- if 'Speed' == t then
502 APT.rrd.create( 'rrd/' .. host .. '/' .. p:upper() .. '/Speed.rrd', '--start', start, '--step', step, DSx, DSn,
503 RRAc0, RRAc1, RRAc2, RRAl0, RRAc3, RRAc4, RRAc5, RRAl1, RRAc6, RRAc7, RRAc8, RRAl2, RRAc9, RRAc10, RRAc11, RRAl3, RRAc12, RRAc13, RRAc14, RRAl4)
504-- else
505 end
506 if not APT.checkFile('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd') then
507 D('Creating ' .. 'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd')
508 APT.rrd.create( 'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', '--start', start, '--step', step, DSIe, DSIw, DSPe, DSPw, DSUe, DSUw, DSSe, DSSw,
509 RRAc0, RRAc1, RRAc2, RRAl0, RRAc3, RRAc4, RRAc5, RRAl1, RRAc6, RRAc7, RRAc8, RRAl2, RRAc9, RRAc10, RRAc11, RRAl3, RRAc12, RRAc13, RRAc14, RRAl4)
510-- end
511 -- Start them at 0 so the average has something to work on.
512 APT.rrd.update( 'rrd/' .. host .. '/' .. p:upper() .. '/Speed.rrd', (APT.now - 600) .. ':0:0')
513 APT.rrd.update( 'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', (APT.now - 600) .. ':0:0:0:0:0:0:0:0')
514 end
515-- end
516 end
517end
518
519APT.updateRRD = function(results, host, ip)
520 if nil ~= ip then host = host .. '_' .. ip end
521 for i, p in pairs(APT.protocols) do
522 if nil ~= results.speed then
523 APT.rrd.update( 'rrd/' .. host .. '/' .. p:upper() .. '/Speed.rrd', APT.now .. ':' .. results.speed.max .. ':' .. results.speed.min)
524 end
525 if nil ~= results[p] then
526 APT.rrd.update('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', APT.now .. ':' ..
527 results[p]['Integrity'].errors .. ':' .. results[p]['Integrity'].warnings .. ':' ..
528 results[p]['Protocol'].errors .. ':' .. results[p]['Protocol'].warnings .. ':' ..
529 results[p]['Updated'].errors .. ':' .. results[p]['Updated'].warnings .. ':' ..
530 results[p]['URLSanity'].errors .. ':' .. results[p]['URLSanity'].warnings)
531 else
532 APT.rrd.update('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', APT.now .. ':U:U:U:U:U:U:U:U')
533 end
534 end
535end
536
537
538APT.doRRD = function(l, k, v)
539 if APT.checkFile(l .. "/" .. k .. ".lua") then
540 local results = loadfile(l .. "/" .. k .. ".lua")()
541 APT.createRRD(k)
542 APT.updateRRD(results, k)
543 if til ~= v then
544 local IPs = v.IPs
545 for i, u in pairs(IPs) do
546 if "table" == type(u) then
547 for h, t in pairs(u) do
548 APT.createRRD(k, h)
549 results = APT.collate(k, h, results)
550 APT.updateRRD(results, k, h)
551 end
552 else
553 APT.createRRD(k, i)
554 results = APT.collate(k, i, results)
555 APT.updateRRD(results, k, i)
556 end
557 end
558 end
559 end
560end
561
562
563return APT
diff --git a/apt-panopticon-report-email-web.lua b/apt-panopticon-report-email-web.lua
index 1144550..ef61183 100755
--- a/apt-panopticon-report-email-web.lua
+++ b/apt-panopticon-report-email-web.lua
@@ -1,143 +1,22 @@
1#!/usr/bin/env luajit 1#!/usr/bin/env luajit
2 2
3local args = {...} 3local APT = require 'apt-panopticommon'
4local D = APT.D
5local I = APT.I
6local W = APT.W
7local E = APT.E
8local C = APT.C
9local arg, sendArgs = APT.parseArgs({...})
4 10
5verbosity = -1
6local logFile
7local html = false
8
9
10--[[ Ordered table iterator, allow to iterate on the natural order of the keys of a table.
11 From http://lua-users.org/wiki/SortedIteration
12 ]]
13function __genOrderedIndex( t )
14 local orderedIndex = {}
15 for key in pairs(t) do
16 table.insert( orderedIndex, key )
17 end
18 table.sort( orderedIndex )
19 return orderedIndex
20end
21function orderedNext(t, state)
22 -- Equivalent of the next function, but returns the keys in the alphabetic
23 -- order. We use a temporary ordered key table that is stored in the
24 -- table being iterated.
25
26 local key = nil
27 --print("orderedNext: state = "..tostring(state) )
28 if state == nil then
29 -- the first time, generate the index
30 t.__orderedIndex = __genOrderedIndex( t )
31 key = t.__orderedIndex[1]
32 else
33 -- fetch the next value
34 for i = 1,table.getn(t.__orderedIndex) do
35 if t.__orderedIndex[i] == state then
36 key = t.__orderedIndex[i+1]
37 end
38 end
39 end
40
41 if key then
42 return key, t[key]
43 end
44
45 -- no more value to return, cleanup
46 t.__orderedIndex = nil
47 return
48end
49function orderedPairs(t)
50 -- Equivalent of the pairs() function on tables. Allows to iterate
51 -- in order
52 return orderedNext, t, nil
53end
54
55-- Use this to dump a table to a string, with HTML.
56dumpTableHTML = function (table, space, name)
57 local r = name .. "\n"
58 r = r .. dumpTableHTMLSub(table, space .. " ")
59 r = r .. space .. ""
60 return r
61end
62dumpTableHTMLSub = function (table, space)
63 local r = ""
64 for k, v in orderedPairs(table) do
65 if type(v) == "table" then
66 if " " == space then
67 r = r .. space .. dumpTableHTML(v, space, k .. "<ul>") .. "</ul>\n"
68 else
69 r = r .. "<li>" .. space .. dumpTableHTML(v, space, k .. "<ul>") .. "</ul></li>\n"
70 end
71 else
72 r = r .. space .. "<li>" .. k .. "</li>\n"
73 end
74 end
75 return r
76end
77
78local checkFile = function(f)
79 local h, e = io.open(f, "r")
80 if nil == h then return false else h:close(); return true end
81end
82
83local plurals = function(e, w)
84 local result = ""
85 if 1 == e then
86 result = e .. " error"
87 elseif e ~= 0 then
88 result = e .. " errors"
89 end
90 if ("" ~= result) and html then result = "<font color='red'><b>" .. result .. "</b></font>" end
91-- result = " " .. result
92 if 0 < w then
93 if 0 < e then result = result .. ", " end
94 if 1 == w then
95 result = result .. w .. " warning"
96 else
97 result = result .. w .. " warnings"
98 end
99 if ("" ~= result) and html then result = "<font color='yellow'><b>" .. result .. "</b></font>" end
100-- result = " " .. result
101 end
102 if "" ~= result then result = " (" .. result .. ")" end
103 return result
104end
105 11
106local results = {} 12local results = {}
107 13APT.mirrors = loadfile("results/mirrors.lua")()
108local log = function(v, t, s, prot, test, host)
109 local x = ""
110 if nil == prot then prot = "" end
111 if nil ~= test then x = x .. test else test = "" end
112 if nil ~= host then
113 if #x > 0 then x = x .. " " end
114 x = x .. host
115 end
116 if #x > 0 then
117 t = t .. "(" .. x .. ")"
118 end
119 if v <= verbosity then
120 if 3 <= verbosity then t = os.date() .. " " .. t end
121 print(t .. ": " .. s)
122 end
123 if nil ~= logFile then
124 logFile:write(os.date() .. " " .. t .. ": " .. s .. "\n")
125 logFile:flush()
126 end
127end
128local D = function(s) log(3, "DEBUG ", s) end
129local I = function(s) log(2, "INFO ", s) end
130local W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end
131local E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end
132local C = function(s) log(-1, "CRITICAL", s) end
133
134local mirrors = loadfile("results/mirrors.lua")()
135 14
136local revDNS = function(dom, IP) 15local revDNS = function(dom, IP)
137 if "deb.devuan.org" ~= dom then 16 if "deb.devuan.org" ~= dom then
138 if nil ~= mirrors["deb.devuan.org"] then 17 if nil ~= APT.mirrors["deb.devuan.org"] then
139 if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][IP] then 18 if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][IP] then
140 if html then 19 if APT.html then
141 return "<font color='purple'><b>DNS-RR</b></font>" 20 return "<font color='purple'><b>DNS-RR</b></font>"
142 else 21 else
143 return "DNS-RR" 22 return "DNS-RR"
@@ -145,7 +24,7 @@ local revDNS = function(dom, IP)
145 end 24 end
146 end 25 end
147 else 26 else
148 for k, v in pairs(mirrors) do 27 for k, v in pairs(APT.mirrors) do
149 if "deb.devuan.org" ~= k then 28 if "deb.devuan.org" ~= k then
150 local IPs = v.IPs 29 local IPs = v.IPs
151 for i, u in pairs(IPs) do 30 for i, u in pairs(IPs) do
@@ -168,7 +47,7 @@ local status = function(host, results, typ)
168 local result = "" 47 local result = ""
169 local e = 0 48 local e = 0
170 local w = 0 49 local w = 0
171 local s = nil ~= mirrors[host].Protocols[typ] 50 local s = nil ~= APT.mirrors[host].Protocols[typ]
172 local to = results.timeout 51 local to = results.timeout
173 if ('http' ~= typ) and ('https' ~= typ) and ('ftp' ~= typ) and ('rsync' ~= typ) then s = true end 52 if ('http' ~= typ) and ('https' ~= typ) and ('ftp' ~= typ) and ('rsync' ~= typ) then s = true end
174 if nil ~= results[typ] then 53 if nil ~= results[typ] then
@@ -198,7 +77,7 @@ local status = function(host, results, typ)
198 if to then 77 if to then
199 result = "[TIMEOUT" 78 result = "[TIMEOUT"
200 if not s then result = result .. "*" end 79 if not s then result = result .. "*" end
201 if html then 80 if APT.html then
202 if s then 81 if s then
203 result = "[<font color='blue'><b>TIMEOUT</b></font>" 82 result = "[<font color='blue'><b>TIMEOUT</b></font>"
204 else 83 else
@@ -208,14 +87,14 @@ local status = function(host, results, typ)
208 elseif 0 < e then 87 elseif 0 < e then
209 result = "[FAILED" 88 result = "[FAILED"
210 if not s then result = result .. "*" end 89 if not s then result = result .. "*" end
211 if html then 90 if APT.html then
212 if s then 91 if s then
213 result = "[<font color='red'><b>FAILED</b></font>" 92 result = "[<font color='red'><b>FAILED</b></font>"
214 else 93 else
215 result = "[<font color='darkred'><b>FAILED*</b></font>" 94 result = "[<font color='darkred'><b>FAILED*</b></font>"
216 end 95 end
217 end 96 end
218 if html then 97 if APT.html then
219 faulty = faulty .. host .. " (" .. typ .. ")<br>\n" 98 faulty = faulty .. host .. " (" .. typ .. ")<br>\n"
220 else 99 else
221 faulty = faulty .. host .. " (" .. typ .. ")\n" 100 faulty = faulty .. host .. " (" .. typ .. ")\n"
@@ -223,7 +102,7 @@ local status = function(host, results, typ)
223 else 102 else
224 result = "[OK" 103 result = "[OK"
225 if not s then result = result .. "*" end 104 if not s then result = result .. "*" end
226 if html then 105 if APT.html then
227 if s then 106 if s then
228 result = "[<font color='lime'><b>OK</b></font>" 107 result = "[<font color='lime'><b>OK</b></font>"
229 else 108 else
@@ -231,43 +110,7 @@ local status = function(host, results, typ)
231 end 110 end
232 end 111 end
233 end 112 end
234 return result .. plurals(e, w) .. "]" 113 return result .. APT.plurals(e, w) .. "]"
235end
236
237local collate = function(host, ip, results)
238 local f = "results/" .. host .. "_" .. ip .. ".lua"
239 local rfile, e = io.open(f, "r")
240 if nil == rfile then I("opening " .. f .. " file - " .. e) else
241 rfile:close()
242 local rs = loadfile(f)()
243 for k, v in pairs(rs) do
244 if "table" == type(v) then
245 if "speed" == k then
246 if v.min < results.speed.min then results.speed.min = v.min end
247 if v.max > results.speed.max then results.speed.max = v.max end
248 else
249 for i, u in pairs(v) do
250 if "table" == type(u) then
251 for h, t in pairs(u) do
252 local a = results[k][i][h]
253 if nil == a then a = 0 end
254 results[k][i][h] = a + t
255 end
256 else
257 local a = results[k][i]
258 if nil == a then a = 0 end
259 results[k][i] = a + u
260 end
261 end
262 end
263 elseif "timeout" ~= k then
264 local a = results[k]
265 if nil == a then a = 0 end
266 results[k] = a + v
267 end
268 end
269 end
270 return results
271end 114end
272 115
273local m = {} 116local m = {}
@@ -287,20 +130,20 @@ local logCount = function(domain, ip)
287 if nil ~= l:match("><b>WARNING ") then warnings = warnings + 1 end 130 if nil ~= l:match("><b>WARNING ") then warnings = warnings + 1 end
288 end 131 end
289 rfile:close() 132 rfile:close()
290 if html then 133 if APT.html then
291 if nil == ip then 134 if nil == ip then
292 log = "<a href='" .. nm .. "'>" .. domain .. "</a>" 135 log = "<a href='" .. nm .. "'>" .. domain .. "</a>"
293 else 136 else
294 log = "<a href='" .. nm .. "'>" .. ip .. "</a>" 137 log = "<a href='" .. nm .. "'>" .. ip .. "</a>"
295 end 138 end
296 end 139 end
297 log = log .. plurals(errors, warnings) 140 log = log .. APT.plurals(errors, warnings)
298 end 141 end
299 return log 142 return log
300end 143end
301 144
302 145
303html = false 146APT.html = false
304local email, e = io.open("results/Report-email.txt", "w+") 147local email, e = io.open("results/Report-email.txt", "w+")
305if nil == email then C("opening mirrors file - " .. e) else 148if nil == email then C("opening mirrors file - " .. e) else
306 email:write( "Dear Mirror Admins,\n\n" .. 149 email:write( "Dear Mirror Admins,\n\n" ..
@@ -316,17 +159,17 @@ if nil == email then C("opening mirrors file - " .. e) else
316 "Please see below the current status of the Devuan Package Mirror \nnetwork:\n\n" .. 159 "Please see below the current status of the Devuan Package Mirror \nnetwork:\n\n" ..
317 "==== package mirror status " .. os.date("!%Y-%m-%d %H:%M") .. " GMT ====\n" .. 160 "==== package mirror status " .. os.date("!%Y-%m-%d %H:%M") .. " GMT ====\n" ..
318 "[skip] means that the test hasn't been written yet.\n\n") 161 "[skip] means that the test hasn't been written yet.\n\n")
319 for k, v in orderedPairs(mirrors) do 162 for k, v in APT.orderedPairs(APT.mirrors) do
320 local results = loadfile("results/" .. k .. ".lua")() 163 local results = loadfile("results/" .. k .. ".lua")()
321 email:write(k .. "....\n") 164 email:write(k .. "....\n")
322 local IPs = v.IPs 165 local IPs = v.IPs
323 for i, u in pairs(IPs) do 166 for i, u in pairs(IPs) do
324 if "table" == type(u) then 167 if "table" == type(u) then
325 for h, t in pairs(u) do 168 for h, t in pairs(u) do
326 results = collate(k, h, results) 169 results = APT.collate(k, h, results)
327 end 170 end
328 else 171 else
329 results = collate(k, i, results) 172 results = APT.collate(k, i, results)
330 end 173 end
331 end 174 end
332 local ftp = "[skip]" 175 local ftp = "[skip]"
@@ -340,11 +183,11 @@ if nil == email then C("opening mirrors file - " .. e) else
340 local updated = status(k, results, "Updated") 183 local updated = status(k, results, "Updated")
341 184
342 -- DNS-RR test. 185 -- DNS-RR test.
343 if ("deb.devuan.org" ~= k) and (nil ~= mirrors["deb.devuan.org"]) then 186 if ("deb.devuan.org" ~= k) and (nil ~= APT.mirrors["deb.devuan.org"]) then
344 for l, w in pairs(mirrors[k].IPs) do 187 for l, w in pairs(APT.mirrors[k].IPs) do
345 if type(w) == "table" then 188 if type(w) == "table" then
346 for i, u in pairs(w) do 189 for i, u in pairs(w) do
347 if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then 190 if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then
348 local log = logCount("deb.devuan.org", i) 191 local log = logCount("deb.devuan.org", i)
349 if "" ~= log then 192 if "" ~= log then
350 if "" == dns then dns = " " else dns = dns .. " " end 193 if "" == dns then dns = " " else dns = dns .. " " end
@@ -356,7 +199,7 @@ if nil == email then C("opening mirrors file - " .. e) else
356 end 199 end
357 end 200 end
358 else 201 else
359 if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then 202 if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then
360 local log = logCount("deb.devuan.org", l) 203 local log = logCount("deb.devuan.org", l)
361 if "" ~= log then 204 if "" ~= log then
362 if "" == dns then dns = " " else dns = dns .. " " end 205 if "" == dns then dns = " " else dns = dns .. " " end
@@ -391,7 +234,7 @@ end
391results = {} 234results = {}
392m = {} 235m = {}
393faulty = "" 236faulty = ""
394html = true 237APT.html = true
395local web, e = io.open("results/Report-web.html", "w+") 238local web, e = io.open("results/Report-web.html", "w+")
396if nil == web then C("opening mirrors file - " .. e) else 239if nil == web then C("opening mirrors file - " .. e) else
397 web:write( "<html><head><title>apt-panopticon results</title>\n" .. 240 web:write( "<html><head><title>apt-panopticon results</title>\n" ..
@@ -420,7 +263,7 @@ if nil == web then C("opening mirrors file - " .. e) else
420 "<table>\n<tr><th></th><th>FTP</th><th>HTTP</th><th>HTTPS</th><th>RSYNC</th><th>DNS round robin</th>" .. 263 "<table>\n<tr><th></th><th>FTP</th><th>HTTP</th><th>HTTPS</th><th>RSYNC</th><th>DNS round robin</th>" ..
421 "<th>Protocol</th><th>URL sanity</th><th>Integrity</th><th>Updated</th><th colspan='2'>Speed range</th></tr>\n" 264 "<th>Protocol</th><th>URL sanity</th><th>Integrity</th><th>Updated</th><th colspan='2'>Speed range</th></tr>\n"
422 ) 265 )
423 for k, v in orderedPairs(mirrors) do 266 for k, v in APT.orderedPairs(APT.mirrors) do
424 local results = loadfile("results/" .. k .. ".lua")() 267 local results = loadfile("results/" .. k .. ".lua")()
425 local active = "" 268 local active = ""
426 if "yes" == v.Active then 269 if "yes" == v.Active then
@@ -433,10 +276,10 @@ if nil == web then C("opening mirrors file - " .. e) else
433 for i, u in pairs(IPs) do 276 for i, u in pairs(IPs) do
434 if "table" == type(u) then 277 if "table" == type(u) then
435 for h, t in pairs(u) do 278 for h, t in pairs(u) do
436 results = collate(k, h, results) 279 results = APT.collate(k, h, results)
437 end 280 end
438 else 281 else
439 results = collate(k, i, results) 282 results = APT.collate(k, i, results)
440 end 283 end
441 end 284 end
442 local ftp = "[<font color='grey'><b>skip</b></font>]" 285 local ftp = "[<font color='grey'><b>skip</b></font>]"
@@ -455,11 +298,11 @@ if nil == web then C("opening mirrors file - " .. e) else
455 local spd = '' 298 local spd = ''
456 299
457 -- DNS-RR test. 300 -- DNS-RR test.
458 if ("deb.devuan.org" ~= k) and (nil ~= mirrors["deb.devuan.org"]) then 301 if ("deb.devuan.org" ~= k) and (nil ~= APT.mirrors["deb.devuan.org"]) then
459 for l, w in pairs(mirrors[k].IPs) do 302 for l, w in pairs(APT.mirrors[k].IPs) do
460 if type(w) == "table" then 303 if type(w) == "table" then
461 for i, u in pairs(w) do 304 for i, u in pairs(w) do
462 if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then 305 if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then
463 local log = logCount("deb.devuan.org", i) 306 local log = logCount("deb.devuan.org", i)
464 if "" ~= log then 307 if "" ~= log then
465 if "" == dns then dns = " " else dns = dns .. " &nbsp; " end 308 if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
@@ -471,7 +314,7 @@ if nil == web then C("opening mirrors file - " .. e) else
471 end 314 end
472 end 315 end
473 else 316 else
474 if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then 317 if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then
475 local log = logCount("deb.devuan.org", l) 318 local log = logCount("deb.devuan.org", l)
476 if "" ~= log then 319 if "" ~= log then
477 if "" == dns then dns = " " else dns = dns .. " &nbsp; " end 320 if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
@@ -502,18 +345,18 @@ if nil == web then C("opening mirrors file - " .. e) else
502 web:write( "</table>\n<br>\n<h2>==== faulty mirrors: ====</h2>\n" .. faulty) 345 web:write( "</table>\n<br>\n<h2>==== faulty mirrors: ====</h2>\n" .. faulty)
503 web:write( "<br>\n<br>\n<h2>==== DNS and logs: ====</h2>\n") 346 web:write( "<br>\n<br>\n<h2>==== DNS and logs: ====</h2>\n")
504 347
505 for k, v in pairs(mirrors) do 348 for k, v in pairs(APT.mirrors) do
506 local log = k 349 local log = k
507 local n = {} 350 local n = {}
508 log = logCount(k) 351 log = logCount(k)
509 mirrors[k].Protocols = nil 352 APT.mirrors[k].Protocols = nil
510 mirrors[k].FQDN = nil 353 APT.mirrors[k].FQDN = nil
511 mirrors[k].Active = nil 354 APT.mirrors[k].Active = nil
512 mirrors[k].Rate = nil 355 APT.mirrors[k].Rate = nil
513 mirrors[k].BaseURL = nil 356 APT.mirrors[k].BaseURL = nil
514 mirrors[k].Country = nil 357 APT.mirrors[k].Country = nil
515 mirrors[k].Bandwidth = nil 358 APT.mirrors[k].Bandwidth = nil
516 for l, w in pairs(mirrors[k].IPs) do 359 for l, w in pairs(APT.mirrors[k].IPs) do
517 if type(w) == "table" then 360 if type(w) == "table" then
518 n[l] = {} 361 n[l] = {}
519 for i, u in pairs(w) do 362 for i, u in pairs(w) do
@@ -535,11 +378,12 @@ if nil == web then C("opening mirrors file - " .. e) else
535 "pkgmaster.devuan.org is the master mirror, all the others sync to it. &nbsp; " .. 378 "pkgmaster.devuan.org is the master mirror, all the others sync to it. &nbsp; " ..
536 "</p>\n" 379 "</p>\n"
537 ) 380 )
538 web:write(dumpTableHTML(m, "", ""))
539 web:write( "\n<br>\n<hr>\n\n" .. 381 web:write( "\n<br>\n<hr>\n\n" ..
382 web:write(APT.dumpTableHTML(m, "", ""))
540 "<p>The <a href='Report-email.txt'>email report</a>. &nbsp; " .. 383 "<p>The <a href='Report-email.txt'>email report</a>. &nbsp; " ..
541 "All <a href='../results'>the logs and other output</a>. &nbsp; " .. 384 "All <a href='../results'>the logs and other output</a>. &nbsp; " ..
542 "You can get the <a href='https://sledjhamr.org/cgit/apt-panopticon/about/'>source code here</a>.</p>" .. 385 "You can get the <a href='https://sledjhamr.org/cgit/apt-panopticon/about/'>source code here</a>.</p>" ..
543 "</body></html>\n") 386 "</body></html>\n")
544 web:close() 387 web:close()
545end 388end
389
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index 37778ac..e454740 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -1,7 +1,13 @@
1#!/usr/bin/env luajit 1#!/usr/bin/env luajit
2 2
3 3local APT = require 'apt-panopticommon'
4local args = {...} 4local D = APT.D
5local I = APT.I
6local W = APT.W
7local E = APT.E
8local C = APT.C
9local arg, sendArgs = APT.parseArgs({...})
10APT.html = true
5 11
6--[[ TODO - What to do about HTTPS://deb.devuan.org/ redirects. 12--[[ TODO - What to do about HTTPS://deb.devuan.org/ redirects.
7 Some mirrors give a 404. 13 Some mirrors give a 404.
@@ -10,70 +16,6 @@ local args = {...}
10 They shouldn't have the proper certificate, but are giving a result anyway. 16 They shouldn't have the proper certificate, but are giving a result anyway.
11]] 17]]
12 18
13origin = false
14verbosity = -1
15keep = false
16-- TODO - Should actually implement this.
17fork = true
18options =
19{
20 referenceSite =
21 {
22 typ = "string",
23 help = "",
24 value = "pkgmaster.devuan.org",
25 },
26 roundRobin =
27 {
28 typ = "string",
29 help = "",
30 value = "deb.devuan.org",
31 },
32 tests =
33 {
34 typ = "table",
35 help = "",
36 value =
37 {
38 "IPv4",
39 "IPv6",
40-- "ftp",
41 "http",
42 "https",
43-- "rsync",
44 "DNSRR",
45 "Protocol",
46 "URLSanity",
47 "Integrity",
48 "Updated",
49 },
50 },
51 maxtime =
52 {
53 typ = "number",
54 help = "",
55 value = 300,
56 },
57 timeout =
58 {
59 typ = "number",
60 help = "",
61 value = 15,
62 },
63 reports =
64 {
65 typ = "table",
66 help = "",
67 value =
68 {
69 "email-web",
70-- "Nagios",
71-- "Prometheus",
72-- "RRD",
73 },
74 },
75}
76
77local defaultURL = {scheme = "http"} 19local defaultURL = {scheme = "http"}
78local releases = {"jessie", "ascii", "beowulf", "ceres"} 20local releases = {"jessie", "ascii", "beowulf", "ceres"}
79local releaseFiles = 21local releaseFiles =
@@ -108,9 +50,6 @@ local referenceDevs =
108 "merged/pool/DEVUAN/main/d/desktop-base/desktop-base_3.0_all.deb", 50 "merged/pool/DEVUAN/main/d/desktop-base/desktop-base_3.0_all.deb",
109 "merged/pool/DEVUAN/main/u/util-linux/util-linux_2.32.1-0.1+devuan2.1_amd64.deb", 51 "merged/pool/DEVUAN/main/u/util-linux/util-linux_2.32.1-0.1+devuan2.1_amd64.deb",
110} 52}
111local arg = {}
112local sendArgs = ""
113local logFile
114 53
115local curlStatus = 54local curlStatus =
116{ 55{
@@ -218,151 +157,8 @@ local http = require 'socket.http'
218local url = require 'socket.url' 157local url = require 'socket.url'
219 158
220 159
221-- Use this to dump a table to a string.
222dumpTable = function (table, space, name)
223 local r = ""
224 if "" == space then r = r .. space .. name .. " =\n" else r = r .. space .. "[" .. name .. "] =\n" end
225 r = r .. space .. "{\n"
226 r = r .. dumpTableSub(table, space .. " ")
227 if "" == space then r = r .. space .. "}\n" else r = r .. space .. "},\n" end
228 return r
229end
230dumpTableSub = function (table, space)
231 local r = ""
232 for k, v in pairs(table) do
233 if type(k) == "string" then k = '"' .. k .. '"' end
234 if type(v) == "table" then
235 r = r .. dumpTable(v, space, k)
236 elseif type(v) == "string" then
237 r = r .. space .. "[" .. k .. "] = '" .. v .. "';\n"
238 elseif type(v) == "function" then
239 r = r .. space .. "[" .. k .. "] = function ();\n"
240 elseif type(v) == "userdata" then
241 r = r .. space .. "userdata " .. "[" .. k .. "];\n"
242 elseif type(v) == "boolean" then
243 if (v) then
244 r = r .. space .. "[" .. k .. "] = true;\n"
245 else
246 r = r .. space .. "[" .. k .. "] = false;\n"
247 end
248 else
249 r = r .. space .. "[" .. k .. "] = " .. v .. ";\n"
250 end
251 end
252 return r
253end
254
255local ip = "" 160local ip = ""
256local results = {}
257
258local logPre = function()
259 if nil ~= logFile then
260 logFile:write("<html><head>\n")
261 logFile:write("</head><body bgcolor='black' text='white' alink='red' link='blue' vlink='purple'>\n")
262 end
263end
264local logPost = function()
265 if nil ~= logFile then
266 logFile:write("</body></html> \n")
267 end
268end
269
270local log = function(v, t, s, prot, test, host)
271 local x = ""
272 if nil == prot then prot = "" end
273 if nil == test then test = "" end
274 x = x .. prot
275 if "" ~= test then
276 if #x > 0 then x = x .. " " end
277 x = x .. test
278 end
279 if nil ~= host then
280 if #x > 0 then x = x .. " " end
281 x = x .. host
282 end
283 if #x > 0 then
284 t = t .. "(" .. x .. ")"
285 if "" ~= prot then
286 if "" == test then
287 if nil == results[prot] then results[prot] = {errors = 0; warnings = 0} end
288 if v == 0 then results[prot].errors = results[prot].errors + 1 end
289 if v == 1 then results[prot].warnings = results[prot].warnings + 1 end
290 else
291 if nil == results[prot] then results[prot] = {errors = 0; warnings = 0} end
292 if nil == results[prot][test] then results[prot][test] = {errors = 0; warnings = 0} end
293 if v == 0 then results[prot][test].errors = results[prot][test].errors + 1 end
294 if v == 1 then results[prot][test].warnings = results[prot][test].warnings + 1 end
295 end
296 end
297 end
298 if v <= verbosity then
299 if 3 <= verbosity then t = os.date() .. " " .. t end
300 print(t .. ": " .. s)
301 end
302 if nil ~= logFile then
303 local colour = "white"
304 if -1 == v then colour = "fuchsia" end -- CRITICAL
305 if 0 == v then colour = "red " end -- ERROR
306 if 1 == v then colour = "yellow " end -- WARNING
307 if 2 == v then colour = "white " end -- INFO
308 if 3 == v then colour = "gray " end -- DEBUG
309 logFile:write(os.date() .. " <font color='" .. colour .. "'><b>" .. t .. "</b></font>: " .. s .. "</br>\n")
310 logFile:flush()
311 end
312end
313local D = function(s) log(3, "DEBUG ", s) end
314local I = function(s) log(2, "INFO ", s) end
315local W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end
316local E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end
317local C = function(s) log(-1, "CRITICAL", s) end
318
319local mirrors = {}
320
321local testing = function(t, host)
322 for i, v in pairs(options.tests.value) do
323 if t == v then
324 local h = mirrors[host]
325 if nil == h then return true end
326 if true == h["Protocols"][t] then return true else D("Skipping " .. t .. " checks for " .. host) end
327 end
328 end
329 return false
330end
331
332local execute = function (s)
333 D(" executing <pre><code>" .. s .. "</code></pre>")
334 --[[ Damn os.execute()
335 Lua 5.1 says it returns "a status code, which is system-dependent"
336 Lua 5.2 says it returns true/nil, "exit"/"signal", the status code.
337 I'm getting 7168 or 0. No idea what the fuck that is.
338 local ok, rslt, status = os.execute(s)
339 ]]
340 local f = io.popen(s .. ' ; echo "$?"', 'r')
341 local status = ""
342 local result = ""
343 -- The last line will be the command's returned status, collect everything else in result.
344 for l in f:lines() do
345 result = result .. status .. "\n"
346 status = l
347 end
348 return status, result
349end
350
351local fork = function(s)
352 D(" forking <pre><code>" .. s .. "</code></pre>")
353 os.execute(s .. " &")
354end
355 161
356local checkExes = function (exe)
357 local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | grep -v "flock -n apt-panopticon.lock " | wc -l'):read("*l")
358 D(count .. " " .. exe .. " commands still running.")
359 return tonumber(count)
360end
361
362local checkFile = function(f)
363 local h, e = io.open(f, "r")
364 if nil == h then return false else h:close(); return true end
365end
366 162
367local repoExists = function (r) 163local repoExists = function (r)
368 r = r:match("([%a-]*)") 164 r = r:match("([%a-]*)")
@@ -441,9 +237,9 @@ checkHEAD = function (host, URL, r, retry, sanity)
441 return 237 return
442 end 238 end
443 D(PU.scheme .. " :// " .. check .. " " .. host .. " -> " .. URL) 239 D(PU.scheme .. " :// " .. check .. " " .. host .. " -> " .. URL)
444 if not testing(PU.scheme, host) then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL); return end 240 if not APT.testing(PU.scheme, host) then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL); return end
445 -- TODO - Perhaps we should try it anyway, and mark it as a warning if it DOES work? 241 -- TODO - Perhaps we should try it anyway, and mark it as a warning if it DOES work?
446 if "https" == PU.scheme and options.roundRobin.value == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert."); return end 242 if "https" == PU.scheme and APT.options.roundRobin.value == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert."); return end
447 243
448 --[[ Using curl command line - 244 --[[ Using curl command line -
449 -I - HEAD 245 -I - HEAD
@@ -468,10 +264,10 @@ checkHEAD = function (host, URL, r, retry, sanity)
468 end 264 end
469 IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"' 265 IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"'
470 end 266 end
471 local cmd = 'ionice -c3 nice -n 19 curl -I --retry 0 -s --path-as-is --connect-timeout ' .. options.timeout.value .. ' --max-redirs 0 ' .. 267 local cmd = 'ionice -c3 nice -n 19 curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' ..
472 IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' .. 268 IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' ..
473 hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"' 269 hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"'
474 local status, result = execute(cmd) 270 local status, result = APT.execute(cmd)
475 os.execute('cat results/"HEADERS_' .. fname .. '" >>results/"STATUS_' .. fname .. '" 2>/dev/null; rm -f results/"HEADERS_' .. fname .. '" 2>/dev/null') 271 os.execute('cat results/"HEADERS_' .. fname .. '" >>results/"STATUS_' .. fname .. '" 2>/dev/null; rm -f results/"HEADERS_' .. fname .. '" 2>/dev/null')
476 if "0" ~= status then 272 if "0" ~= status then
477 local msg = curlStatus[0 + status] 273 local msg = curlStatus[0 + status]
@@ -522,7 +318,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
522 if nil ~= location then 318 if nil ~= location then
523 pu = url.parse(location, defaultURL) 319 pu = url.parse(location, defaultURL)
524 if ('http' == location:sub(1, 4)) and (pu.scheme ~= PU.scheme) then -- Sometimes a location sans scheme is returned, this is not a protocol change. 320 if ('http' == location:sub(1, 4)) and (pu.scheme ~= PU.scheme) then -- Sometimes a location sans scheme is returned, this is not a protocol change.
525 if testing("Protocol") then W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host) end 321 if APT.testing("Protocol") then W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host) end
526 if (pu.host == host) and pu.path == PU.path then D("Not testing protocol change " .. URL .. " -> " .. location); return end 322 if (pu.host == host) and pu.path == PU.path then D("Not testing protocol change " .. URL .. " -> " .. location); return end
527 end 323 end
528 324
@@ -555,7 +351,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
555end 351end
556 352
557local checkTimeouts = function(host, scheme, URL) 353local checkTimeouts = function(host, scheme, URL)
558 if testing(scheme) then 354 if APT.testing(scheme) then
559 totalTimeouts = totalTimeouts + timeouts; timeouts = 0 355 totalTimeouts = totalTimeouts + timeouts; timeouts = 0
560 checkHEAD(host, scheme .. "://" .. URL) 356 checkHEAD(host, scheme .. "://" .. URL)
561 if 4 <= (totalTimeouts) then 357 if 4 <= (totalTimeouts) then
@@ -563,7 +359,7 @@ local checkTimeouts = function(host, scheme, URL)
563 return true 359 return true
564 end 360 end
565 end 361 end
566 if testing("URLSanity") then 362 if APT.testing("URLSanity") then
567 URL = URL:gsub("merged/", "merged///") 363 URL = URL:gsub("merged/", "merged///")
568 totalTimeouts = totalTimeouts + timeouts; timeouts = 0 364 totalTimeouts = totalTimeouts + timeouts; timeouts = 0
569 checkHEAD(host, scheme .. "://" .. URL, 0, 0, true) 365 checkHEAD(host, scheme .. "://" .. URL, 0, 0, true)
@@ -617,24 +413,24 @@ checkHost = function (orig, host, path, ip, file)
617 else 413 else
618 if orig == host then 414 if orig == host then
619 D("checkHost " .. orig .. "" .. file) 415 D("checkHost " .. orig .. "" .. file)
620 if testing("IPv4") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file) end 416 if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file) end
621 else D("checkHost " .. orig .. " -> " .. host) end 417 else D("checkHost " .. orig .. " -> " .. host) end
622 local h = mirrors[ph.host] 418 local h = APT.mirrors[ph.host]
623 if nil == h then return end 419 if nil == h then return end
624 for k, v in pairs(h.IPs) do 420 for k, v in pairs(h.IPs) do
625 if "table" == type(v) then 421 if "table" == type(v) then
626 for k1, v1 in pairs(v) do 422 for k1, v1 in pairs(v) do
627 if v1 == "A" then 423 if v1 == "A" then
628 if testing("IPv4") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end 424 if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end
629 elseif v1 == "AAAA" then 425 elseif v1 == "AAAA" then
630 if testing("IPv6") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end 426 if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end
631 end 427 end
632 end 428 end
633 else 429 else
634 if v == "A" then 430 if v == "A" then
635 if testing("IPv4") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end 431 if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end
636 elseif v == "AAAA" then 432 elseif v == "AAAA" then
637 if testing("IPv6") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end 433 if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end
638 end 434 end
639 end 435 end
640 end 436 end
@@ -644,14 +440,12 @@ end
644 440
645local addDownload = function(host, URL, f, r, k) 441local addDownload = function(host, URL, f, r, k)
646 local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename. 442 local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename.
647 local o, e = io.open("results/" .. host .. "/merged/dists/" .. r .. k, "r") 443 if APT.checkFile("results/" .. host .. "/merged/dists/" .. r .. k) then
648 if nil ~= o then
649 o:close()
650 -- Curls "check timestamp and overwrite file" stuff sucks. 444 -- Curls "check timestamp and overwrite file" stuff sucks.
651 -- -R means the destination file gets the timestamp of the remote file. 445 -- -R means the destination file gets the timestamp of the remote file.
652 -- Can only do ONE timestamp check per command. 446 -- Can only do ONE timestamp check per command.
653 -- This doesn't work either. All downloads get all these headers. Pffft 447 -- This doesn't work either. All downloads get all these headers. Pffft
654-- local status, ts = execute('TZ="GMT" ls -l --time-style="+%a, %d %b %Y %T %Z" results/' .. host .. "/merged/dists/" .. r .. k .. ' | cut -d " " -f 6-11') 448-- local status, ts = APT.execute('TZ="GMT" ls -l --time-style="+%a, %d %b %Y %T %Z" results/' .. host .. "/merged/dists/" .. r .. k .. ' | cut -d " " -f 6-11')
655-- f:write('header "If-Modified-Since: ' .. ts:sub(2, -2) .. '"\n') 449-- f:write('header "If-Modified-Since: ' .. ts:sub(2, -2) .. '"\n')
656 -- Curl will DELETE the existing file if the timestamp fails to download a new one, unless we change directory first, 450 -- Curl will DELETE the existing file if the timestamp fails to download a new one, unless we change directory first,
657 -- which wont work with multiple files in multiple directories. WTF? 451 -- which wont work with multiple files in multiple directories. WTF?
@@ -671,21 +465,21 @@ local postDownload = function(host, r, k)
671 " && [ ! -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then cp -a" .. 465 " && [ ! -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then cp -a" ..
672 " results/" .. host .. "/merged/dists/" .. r .. k .. ".old" .. 466 " results/" .. host .. "/merged/dists/" .. r .. k .. ".old" ..
673 " results/" .. host .. "/merged/dists/" .. r .. k .. "; fi") 467 " results/" .. host .. "/merged/dists/" .. r .. k .. "; fi")
674 if ".gz" == k:sub(-3, -1) then execute("ionice -c3 nice -n 19 gzip -dfk results/" .. host .. "/merged/dists/" .. r .. k) end 468 if ".gz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 gzip -dfk results/" .. host .. "/merged/dists/" .. r .. k) end
675 if ".xz" == k:sub(-3, -1) then execute("ionice -c3 nice -n 19 xz -dfk results/" .. host .. "/merged/dists/" .. r .. k .. " 2>/dev/null") end 469 if ".xz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 xz -dfk results/" .. host .. "/merged/dists/" .. r .. k .. " 2>/dev/null") end
676 if testing("Integrity") then 470 if APT.testing("Integrity") then
677 if ".gpg" == k:sub(-4, -1) then 471 if ".gpg" == k:sub(-4, -1) then
678 local status, out = execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. r .. k .. 472 local status, out = APT.execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. r .. k ..
679 " results/" .. host .. "/merged/dists/" .. r .. k:sub(1, -5) .. " 2>/dev/null") 473 " results/" .. host .. "/merged/dists/" .. r .. k:sub(1, -5) .. " 2>/dev/null")
680 if "0" ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. r .. k, "http", "Integrity", host) end 474 if "0" ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. r .. k, "http", "Integrity", host) end
681 end 475 end
682-- TODO - should check the PGP sig of InRelease as well. 476-- TODO - should check the PGP sig of InRelease as well.
683 end 477 end
684 if testing("Integrity") or testing("Updated") then 478 if APT.testing("Integrity") or APT.testing("Updated") then
685 if "Packages." == file:sub(1, 9) then 479 if "Packages." == file:sub(1, 9) then
686-- TODO - compare the SHA256 sums in pkgmaster's Release for both the packed and unpacked versions. 480-- TODO - compare the SHA256 sums in pkgmaster's Release for both the packed and unpacked versions.
687-- Also note that this might get only a partial download due to maxtime. 481-- Also note that this might get only a partial download due to maxtime.
688 if options.referenceSite.value == host then 482 if APT.options.referenceSite.value == host then
689 local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed', "w+") 483 local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed', "w+")
690 if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed' .. ' file - ' .. e) else 484 if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed' .. ' file - ' .. e) else
691 local pp = {} 485 local pp = {}
@@ -711,7 +505,7 @@ local postDownload = function(host, r, k)
711 end 505 end
712 Pp:close() 506 Pp:close()
713 os.execute('sort results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed >results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages_parsed-sorted') 507 os.execute('sort results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed >results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages_parsed-sorted')
714 if checkFile('Packages/' .. r .. dir .. 'Packages_parsed-sorted') then 508 if APT.checkFile('Packages/' .. r .. dir .. 'Packages_parsed-sorted') then
715 os.execute('diff -U 0 Packages/' .. r .. dir .. 'Packages_parsed-sorted ' .. 509 os.execute('diff -U 0 Packages/' .. r .. dir .. 'Packages_parsed-sorted ' ..
716 'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages_parsed-sorted ' .. 510 'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages_parsed-sorted ' ..
717 ' | grep -E "^-" | grep -Ev "^\\+\\+\\+|^---" >>results/OLD_PACKAGES_' .. r .. '.txt') 511 ' | grep -E "^-" | grep -Ev "^\\+\\+\\+|^---" >>results/OLD_PACKAGES_' .. r .. '.txt')
@@ -737,14 +531,14 @@ local postDownload = function(host, r, k)
737end 531end
738 532
739local downloadLock = "flock -n results/curl-" 533local downloadLock = "flock -n results/curl-"
740local download = "curl --connect-timeout " .. options.timeout.value .. " --create-dirs -f -L --max-time " .. options.maxtime.value .. " -z 'results/stamp.old' -v -R " 534local download = "curl --connect-timeout " .. APT.options.timeout.value .. " --create-dirs -f -L --max-time " .. APT.options.maxtime.value .. " -z 'results/stamp.old' -v -R "
741local downloads = function(host, URL, release, list) 535local downloads = function(host, URL, release, list)
742 if nil == URL then URL = "" end 536 if nil == URL then URL = "" end
743 local lock = "META-" .. host .. ".lock" 537 local lock = "META-" .. host .. ".lock"
744 local log = " --stderr results/curl-META-" .. host .. ".log" 538 local log = " --stderr results/curl-META-" .. host .. ".log"
745 local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock .. " " .. download .. log .. " -K results/" .. host .. ".curl" 539 local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock .. " " .. download .. log .. " -K results/" .. host .. ".curl"
746 if testing("IPv4") and (not testing("IPv6")) then cm = cm .. ' -4' end 540 if APT.testing("IPv4") and (not APT.testing("IPv6")) then cm = cm .. ' -4' end
747 if (not testing("IPv4")) and testing("IPv6") then cm = cm .. ' -6' end 541 if (not APT.testing("IPv4")) and APT.testing("IPv6") then cm = cm .. ' -6' end
748 f, e = io.open("results/" .. host .. ".curl", "a+") 542 f, e = io.open("results/" .. host .. ".curl", "a+")
749 if nil == f then C("opening curl file - " .. e); return end 543 if nil == f then C("opening curl file - " .. e); return end
750 544
@@ -772,7 +566,7 @@ local downloads = function(host, URL, release, list)
772 end 566 end
773 end 567 end
774 f:close() 568 f:close()
775 fork(cm) 569 APT.fork(cm)
776end 570end
777 571
778 572
@@ -781,7 +575,7 @@ local getMirrors = function ()
781 local host = "" 575 local host = ""
782 local m = {} 576 local m = {}
783 local active = true 577 local active = true
784 local URL = "http://" .. options.referenceSite.value .. "/mirror_list.txt" 578 local URL = "http://" .. APT.options.referenceSite.value .. "/mirror_list.txt"
785 I("getting mirrors.") 579 I("getting mirrors.")
786 local p, c, h = http.request(URL) 580 local p, c, h = http.request(URL)
787 if nil == p then E(c .. " fetching " .. URL) else 581 if nil == p then E(c .. " fetching " .. URL) else
@@ -818,95 +612,20 @@ local getMirrors = function ()
818 mirrors[host] = m 612 mirrors[host] = m
819 end 613 end
820 end 614 end
821 if testing("DNSRR") then 615 if APT.testing("DNSRR") then
822 mirrors[options.roundRobin.value] = { ["Protocols"] = { ["http"] = true; ["https"] = true; }; ["FQDN"] = 'deb.devuan.org'; ["Active"] = 'yes'; ["BaseURL"] = 'deb.devuan.org'; } 616 mirrors[APT.options.roundRobin.value] = { ["Protocols"] = { ["http"] = true; ["https"] = true; }; ["FQDN"] = 'deb.devuan.org'; ["Active"] = 'yes'; ["BaseURL"] = 'deb.devuan.org'; }
823 gatherIPs(options.roundRobin.value) 617 gatherIPs(APT.options.roundRobin.value)
824 mirrors[options.roundRobin.value].IPs = IP[options.roundRobin.value] 618 mirrors[APT.options.roundRobin.value].IPs = IP[APT.options.roundRobin.value]
825 end 619 end
826 local file, e = io.open("results/mirrors.lua", "w+") 620 local file, e = io.open("results/mirrors.lua", "w+")
827 if nil == file then C("opening mirrors file - " .. e) else 621 if nil == file then C("opening mirrors file - " .. e) else
828 file:write(dumpTable(mirrors, "", "mirrors") .. "\nreturn mirrors\n") 622 file:write(APT.dumpTable(mirrors, "", "mirrors") .. "\nreturn mirrors\n")
829 file:close() 623 file:close()
830 end 624 end
831 return mirrors 625 return mirrors
832end 626end
833 627
834 628
835if 0 ~= #args then
836 local option = ""
837 for i, a in pairs(args) do
838 if ("--help" == a) or ("-h" == a) then
839 print("I should write some docs, huh? Read README.md for instructions.")
840 elseif "--version" == a then
841 print("apt-panopticon version 0.1 WIP development version")
842 elseif "-v" == a then
843 verbosity = verbosity + 1
844 sendArgs = sendArgs .. a .. " "
845 elseif "-q" == a then
846 verbosity = -1
847 sendArgs = sendArgs .. a .. " "
848 elseif "-k" == a then
849 keep = true
850 elseif "-n" == a then
851 fork = false
852 elseif "-o" == a then
853 origin = true
854 elseif "--" == a:sub(1, 2) then
855 local s, e = a:find("=")
856 if nil == s then e = -1 end
857 option = a:sub(3, e - 1)
858 local o = options[option]
859 if nil == o then
860 print("Unknown option --" .. option)
861 option = ""
862 else
863 option = a
864 sendArgs = sendArgs .. a .. " "
865 local s, e = a:find("=")
866 if nil == s then e = 0 end
867 option = a:sub(3, e - 1)
868 if "table" == options[option].typ then
869 local result = {}
870 for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do
871 local f = t:sub(1, 1)
872 local n = t:sub(2, -1)
873 if ("+" ~= f) and ("-" ~= f) then
874 table.insert(result, t)
875 end
876 end
877 if 0 ~= #result then
878 options[option].value = result
879 else
880 for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do
881 local f = t:sub(1, 1)
882 local n = t:sub(2, -1)
883 if "+" == f then
884 table.insert(options[option].value, n)
885 elseif "-" == f then
886 local r = {}
887 for i, k in pairs(options[option].value) do
888 if k ~= n then table.insert(r, k) end
889 end
890 options[option].value = r
891 end
892 end
893 end
894 else
895 options[option].value = a:sub(e + 1, -1)
896 end
897 option = ""
898 end
899 elseif "-" == a:sub(1, 1) then
900 print("Unknown option " .. a)
901 else
902 table.insert(arg, a)
903 end
904 end
905end
906
907--print(dumpTable(options.tests.value, "", "tests"))
908
909
910if 0 < #arg then 629if 0 < #arg then
911 if "/" == arg[1]:sub(-1, -1) then 630 if "/" == arg[1]:sub(-1, -1) then
912 W("slash at end of path! " .. arg[1]) 631 W("slash at end of path! " .. arg[1])
@@ -918,37 +637,37 @@ if 0 < #arg then
918 end 637 end
919 local pu = url.parse("http://" .. arg[1]) 638 local pu = url.parse("http://" .. arg[1])
920 639
921 if testing("Integrity") or testing("Updated") then 640 if APT.testing("Integrity") or APT.testing("Updated") then
922 if origin and options.referenceSite.value == pu.host then 641 if APT.origin and APT.options.referenceSite.value == pu.host then
923-- if not keep then execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end 642-- if not APT.keep then os.execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end
924 end 643 end
925 end 644 end
926 645
927 if nil ~= arg[2] then 646 if nil ~= arg[2] then
928 logFile, e = io.open("results/LOG_" .. pu.host .. "_" .. arg[2] .. ".html", "a+") 647 APT.logFile, e = io.open("results/LOG_" .. pu.host .. "_" .. arg[2] .. ".html", "a+")
929 else 648 else
930 logFile, e = io.open("results/LOG_" .. pu.host .. ".html", "a+") 649 APT.logFile, e = io.open("results/LOG_" .. pu.host .. ".html", "a+")
931 end 650 end
932 if nil == logFile then C("opening log file - " .. e); return end 651 if nil == APT.logFile then C("opening log file - " .. e); return end
933 logPre() 652 APT.logPre()
934 I("Starting tests for " ..arg[1] .. " with these tests - " .. table.concat(options.tests.value, ", ")) 653 I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", "))
935 mirrors = loadfile("results/mirrors.lua")() 654 APT.mirrors = loadfile("results/mirrors.lua")()
936 if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end 655 if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end
937 if nil ~= arg[3] then I(" Using file " .. arg[3]); end 656 if nil ~= arg[3] then I(" Using file " .. arg[3]); end
938 657
939 for k, v in pairs{"ftp", "http", "https", "rsync"} do 658 for k, v in pairs{"ftp", "http", "https", "rsync"} do
940 if testing(v) then 659 if APT.testing(v) then
941 local tests = {errors = 0; warnings = 0} 660 local tests = {errors = 0; warnings = 0}
942 if testing("Integrity") then tests.Integrity = {errors = 0; warnings = 0} end 661 if APT.testing("Integrity") then tests.Integrity = {errors = 0; warnings = 0} end
943 if testing("Protocol") then tests.Protocol = {errors = 0; warnings = 0} end 662 if APT.testing("Protocol") then tests.Protocol = {errors = 0; warnings = 0} end
944 if testing("Updated") then tests.Updated = {errors = 0; warnings = 0} end 663 if APT.testing("Updated") then tests.Updated = {errors = 0; warnings = 0} end
945 if testing("URLSanity") then tests.URLSanity = {errors = 0; warnings = 0} end 664 if APT.testing("URLSanity") then tests.URLSanity = {errors = 0; warnings = 0} end
946 results[v] = tests 665 APT.results[v] = tests
947 end 666 end
948 end 667 end
949 if origin then 668 if APT.origin then
950 if testing("Integrity") or testing("Updated") then 669 if APT.testing("Integrity") or APT.testing("Updated") then
951 if origin and (options.roundRobin.value ~= pu.host) then 670 if APT.origin and (APT.options.roundRobin.value ~= pu.host) then
952 I("Starting file downloads for " .. pu.host) 671 I("Starting file downloads for " .. pu.host)
953 downloads(pu.host, pu.path) 672 downloads(pu.host, pu.path)
954 end 673 end
@@ -958,10 +677,10 @@ if 0 < #arg then
958 checkHost(pu.host, pu.host, pu.path, arg[2], arg[3]) 677 checkHost(pu.host, pu.host, pu.path, arg[2], arg[3])
959 end 678 end
960 679
961 if testing("Integrity") or testing("Updated") then 680 if APT.testing("Integrity") or APT.testing("Updated") then
962 if 4 > (totalTimeouts) then 681 if 4 > (totalTimeouts) then
963 if origin and (options.roundRobin.value ~= pu.host) then 682 if APT.origin and (APT.options.roundRobin.value ~= pu.host) then
964 while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end 683 while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
965 os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " .. 684 os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " ..
966 "mv results/curl-META-" .. pu.host .. ".log results/curl-Release-" .. pu.host .. ".log") 685 "mv results/curl-META-" .. pu.host .. ".log results/curl-Release-" .. pu.host .. ".log")
967 for i, n in pairs(releases) do 686 for i, n in pairs(releases) do
@@ -971,24 +690,25 @@ if 0 < #arg then
971 end 690 end
972 end 691 end
973 692
974 if checkFile('results/' .. pu.host .. '/merged/dists/' .. n .. '/Release') and 693 if APT.checkFile('results/' .. pu.host .. '/merged/dists/' .. n .. '/Release') then
975 checkFile('results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED') then
976 os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED') 694 os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED')
977 if options.referenceSite.value == pu.host then 695 if APT.checkFile('results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED') then
978 os.execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. 696 if APT.options.referenceSite.value == pu.host then
979 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. 697 os.execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
980 '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt') 698 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
981 os.execute('rm -f results/' .. pu.host .. '/merged/dists/' .. n .. '/Release 2>/dev/null') 699 '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt')
982-- TODO - Maybe check the date in Release, though since they are updated daily, is there any point? Perhaps it's for checking amprolla got run? 700-- TODO - Maybe check the date in Release, though since they are updated daily, is there any point? Perhaps it's for checking amprolla got run?
983 else 701 os.execute('rm -f results/' .. pu.host .. '/merged/dists/' .. n .. '/Release 2>/dev/null')
702 else
984-- TODO - compare to the pkgmaster copy. 703-- TODO - compare to the pkgmaster copy.
985 end 704 end
986 705
987 local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") 706 local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r")
988 if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else 707 if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else
989 local diff = dfile:read("*a") 708 local diff = dfile:read("*a")
990 if "" ~= diff then 709 if "" ~= diff then
991 downloads(pu.host, pu.path, n, diff) 710 downloads(pu.host, pu.path, n, diff)
711 end
992 end 712 end
993 end 713 end
994 end 714 end
@@ -996,7 +716,7 @@ if 0 < #arg then
996 end 716 end
997 717
998 downloads(pu.host, pu.path, "", "") 718 downloads(pu.host, pu.path, "", "")
999 while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end 719 while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
1000 os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " .. 720 os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " ..
1001 "mv results/curl-META-" .. pu.host .. ".log results/curl-Packages-" .. pu.host .. ".log") 721 "mv results/curl-META-" .. pu.host .. ".log results/curl-Packages-" .. pu.host .. ".log")
1002 722
@@ -1008,7 +728,7 @@ if 0 < #arg then
1008 postDownload(pu.host, n, "/" .. l) 728 postDownload(pu.host, n, "/" .. l)
1009 end 729 end
1010 end 730 end
1011 if options.referenceSite.value == pu.host then 731 if APT.options.referenceSite.value == pu.host then
1012 -- In case it wasn't dealt with already. 732 -- In case it wasn't dealt with already.
1013 os.execute('touch results/NEW_Packages_' .. n .. '.test.txt') 733 os.execute('touch results/NEW_Packages_' .. n .. '.test.txt')
1014 end 734 end
@@ -1027,26 +747,26 @@ if 0 < #arg then
1027 end 747 end
1028 end 748 end
1029 downloads(pu.host, pu.path, nil, "") 749 downloads(pu.host, pu.path, nil, "")
1030 while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end 750 while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
1031 for i, n in pairs(releases) do 751 for i, n in pairs(releases) do
1032 local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") 752 local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
1033 if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else 753 if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
1034 for l in nfile:lines() do 754 for l in nfile:lines() do
1035 local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |') 755 local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |')
1036 if nil ~= p then 756 if nil ~= p then
1037 if checkFile('results/' .. pu.host .. "/merged/" .. p) then 757 if APT.checkFile('results/' .. pu.host .. "/merged/" .. p) then
1038 local status, fsz = execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5') 758 local status, fsz = APT.execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5')
1039 if testing("Integrity") then 759 if APT.testing("Integrity") then
1040 if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. 760 if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end.
1041 E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) 761 E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host)
1042 print('|' .. sz .. '~=' .. fsz:sub(2, -2) .. '|') 762 print('|' .. sz .. '~=' .. fsz:sub(2, -2) .. '|')
1043 else 763 else
1044 local status, fsha = execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') 764 local status, fsha = APT.execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1')
1045 if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end 765 if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end
1046-- TODO - maybe check the PGP key, though packages are mostly not signed. 766-- TODO - maybe check the PGP key, though packages are mostly not signed.
1047 end 767 end
1048 end 768 end
1049 if testing("Updated") then 769 if APT.testing("Updated") then
1050 if sz ~= fsz:sub(2, -2) then 770 if sz ~= fsz:sub(2, -2) then
1051 E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) 771 E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host)
1052 end 772 end
@@ -1060,20 +780,20 @@ if 0 < #arg then
1060 end 780 end
1061 end 781 end
1062 782
1063 results["timeout"] = false 783 APT.results["timeout"] = false
1064 else 784 else
1065 results["timeout"] = true 785 APT.results["timeout"] = true
1066 end 786 end
1067 end 787 end
1068 788
1069 if origin and options.referenceSite.value ~= pu.host then 789 if APT.origin and APT.options.referenceSite.value ~= pu.host then
1070 if not keep then os.execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end 790 if not APT.keep then os.execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end
1071 os.execute('rm STATUS_' .. pu.host .. '* 2>/dev/null') 791 os.execute('rm STATUS_' .. pu.host .. '* 2>/dev/null')
1072 end 792 end
1073 793
1074 local min, max, spd = 999999999999, 0 794 local min, max, spd = 999999999999, 0
1075 for i, mt in pairs({'Release', 'Packages', 'META'}) do 795 for i, mt in pairs({'Release', 'Packages', 'META'}) do
1076 if checkFile("results/curl-" .. mt .. "-" .. pu.host .. ".log") then 796 if APT.checkFile("results/curl-" .. mt .. "-" .. pu.host .. ".log") then
1077 for l in io.lines("results/curl-" .. mt .. "-" .. pu.host .. ".log") do 797 for l in io.lines("results/curl-" .. mt .. "-" .. pu.host .. ".log") do
1078 local speed, crrnt = l:match('^%c *%d+ +%d+k? +%d+ +%d+k? +%d+ +%d+ +(%d+k?) +%d+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +(%d+k?)') 798 local speed, crrnt = l:match('^%c *%d+ +%d+k? +%d+ +%d+k? +%d+ +%d+ +(%d+k?) +%d+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +(%d+k?)')
1079 if nil ~= speed then 799 if nil ~= speed then
@@ -1089,17 +809,17 @@ if 0 < #arg then
1089 end 809 end
1090 end 810 end
1091 end 811 end
1092 results["speed"] = {min = min, max = max} 812 APT.results["speed"] = {min = min, max = max}
1093 813
1094 local f = pu.host 814 local f = pu.host
1095 if "" ~= ip then f = f .. "_" .. ip end 815 if "" ~= ip then f = f .. "_" .. ip end
1096 local rfile, e = io.open("results/" .. f .. ".lua", "w+") 816 local rfile, e = io.open("results/" .. f .. ".lua", "w+")
1097 if nil == rfile then C("opening results file - " .. e) else 817 if nil == rfile then C("opening results file - " .. e) else
1098 rfile:write(dumpTable(results, "", "results") .. "\nreturn results\n") 818 rfile:write(APT.dumpTable(APT.results, "", "results") .. "\nreturn results\n")
1099 rfile:close() 819 rfile:close()
1100 end 820 end
1101 logPost() 821 APT.logPost()
1102 logFile:close() 822 APT.logFile:close()
1103else 823else
1104 local fadt = io.popen("ls -dl results_old 2>/dev/null | cut -d '>' -f 2 | cut -d ' ' -f 2") 824 local fadt = io.popen("ls -dl results_old 2>/dev/null | cut -d '>' -f 2 | cut -d ' ' -f 2")
1105 local adt = fadt:read('*l') 825 local adt = fadt:read('*l')
@@ -1113,25 +833,25 @@ else
1113 if nil ~= dt then os.execute('mkdir -p results_' .. dt .. '; rm -f results; ln -s results_' .. dt .. ' results 2>/dev/null') end 833 if nil ~= dt then os.execute('mkdir -p results_' .. dt .. '; rm -f results; ln -s results_' .. dt .. ' results 2>/dev/null') end
1114 os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp') 834 os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp')
1115 os.execute("rm -f results/*.check 2>/dev/null") 835 os.execute("rm -f results/*.check 2>/dev/null")
1116 if not keep then 836 if not APT.keep then
1117 os.execute("rm -f results/*.curl 2>/dev/null") 837 os.execute("rm -f results/*.curl 2>/dev/null")
1118 os.execute("rm -f results/*.log 2>/dev/null") 838 os.execute("rm -f results/*.log 2>/dev/null")
1119 os.execute("rm -f results/*.html 2>/dev/null") 839 os.execute("rm -f results/*.html 2>/dev/null")
1120 os.execute("rm -f results/*.txt 2>/dev/null") 840 os.execute("rm -f results/*.txt 2>/dev/null")
1121 end 841 end
1122 842
1123 logFile, e = io.open("results/LOG_apt-panopticon.html", "a+") 843 APT.logFile, e = io.open("results/LOG_apt-panopticon.html", "a+")
1124 if nil == logFile then C("opening log file - " .. e); return end 844 if nil == APT.logFile then C("opening log file - " .. e); return end
1125 logPre() 845 APT.logPre()
1126 I("Starting tests " .. table.concat(options.tests.value, ", ")) 846 I("Starting tests " .. table.concat(APT.options.tests.value, ", "))
1127 os.execute("mkdir -p results") 847 os.execute("mkdir -p results")
1128 mirrors = getMirrors() 848 APT.mirrors = getMirrors()
1129 checkHost(options.referenceSite.value) 849 checkHost(APT.options.referenceSite.value)
1130 for i, n in pairs(releases) do 850 for i, n in pairs(releases) do
1131 while not checkFile('results/NEW_Packages_' .. n .. '.test.txt') do os.execute("sleep 10") end 851 while not APT.checkFile('results/NEW_Packages_' .. n .. '.test.txt') do os.execute("sleep 10") end
1132 end 852 end
1133 853
1134 for k, m in pairs(mirrors) do 854 for k, m in pairs(APT.mirrors) do
1135 if "/" == m.BaseURL:sub(-1, -1) then 855 if "/" == m.BaseURL:sub(-1, -1) then
1136 W("slash at end of BaseURL in mirror_list.txt! " .. m.BaseURL, "", "", m.FQDN) 856 W("slash at end of BaseURL in mirror_list.txt! " .. m.BaseURL, "", "", m.FQDN)
1137 m.BaseURL = m.BaseURL:sub(1, -2) 857 m.BaseURL = m.BaseURL:sub(1, -2)
@@ -1141,30 +861,27 @@ else
1141 m.BaseURL = m.BaseURL:sub(1, -2) 861 m.BaseURL = m.BaseURL:sub(1, -2)
1142 end 862 end
1143 local pu = url.parse("http://" .. m.BaseURL) 863 local pu = url.parse("http://" .. m.BaseURL)
1144 if options.referenceSite.value ~= pu.host then 864 if APT.options.referenceSite.value ~= pu.host then
1145 checkHost(m.BaseURL) 865 checkHost(m.BaseURL)
1146 checkExes("apt-panopticon.lua " .. sendArgs) 866 APT.checkExes("apt-panopticon.lua " .. sendArgs)
1147 if testing("Integrity") or testing("Updated") then checkExes(downloadLock) end 867 if APT.testing("Integrity") or APT.testing("Updated") then APT.checkExes(downloadLock) end
1148 end 868 end
1149 end 869 end
1150 870
1151 while 1 <= checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end 871 while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end
1152 872
1153 os.execute("rm -f results/*.check; rm -f results/*.lock 2>/dev/null") 873 os.execute("rm -f results/*.check; rm -f results/*.lock 2>/dev/null")
1154 874
1155 -- Create the reports. 875 -- Create the reports.
1156 for n, r in pairs(options.reports.value) do 876 for n, r in pairs(APT.options.reports.value) do
1157 local report = "apt-panopticon-report-" .. r .. ".lua" 877 if APT.checkFile("apt-panopticon-report-" .. r .. ".lua") then
1158 local rfile, e = io.open(report, "r") 878 I("Creating " .. r .. " report.")
1159 if nil == rfile then C("opening " .. report .. " file - " .. e) else 879 APT.execute("./apt-panopticon-report-" .. r .. ".lua")
1160 rfile:close()
1161 I("Creating " .. report .. " report.")
1162 execute("./" .. report .. " ")
1163 end 880 end
1164 end 881 end
1165 882
1166 if nil ~= adt then os.execute('rm -fr ' .. adt .. ' 2>/dev/null') end 883 if nil ~= adt then os.execute('rm -fr ' .. adt .. ' 2>/dev/null') end
1167 884
1168 logPost() 885 APT.logPost()
1169 logFile:close() 886 APT.logFile:close()
1170end 887end