diff options
author | onefang | 2019-12-10 15:45:11 +1000 |
---|---|---|
committer | onefang | 2019-12-10 15:45:11 +1000 |
commit | 4bb51520521d9ab612d50dd2bcf904d70cf15735 (patch) | |
tree | 8d9fde355e1ecc1b06f400f99b2285ca6492023d | |
parent | Add freedom -1. (diff) | |
download | apt-panopticon-4bb51520521d9ab612d50dd2bcf904d70cf15735.zip apt-panopticon-4bb51520521d9ab612d50dd2bcf904d70cf15735.tar.gz apt-panopticon-4bb51520521d9ab612d50dd2bcf904d70cf15735.tar.bz2 apt-panopticon-4bb51520521d9ab612d50dd2bcf904d70cf15735.tar.xz |
Move common code to it's own module.
Some minor clean ups and test tweaks likely came for the ride.
-rw-r--r-- | apt-panopticommon.lua | 563 | ||||
-rwxr-xr-x | apt-panopticon-report-email-web.lua | 250 | ||||
-rwxr-xr-x | apt-panopticon.lua | 507 |
3 files changed, 722 insertions, 598 deletions
diff --git a/apt-panopticommon.lua b/apt-panopticommon.lua new file mode 100644 index 0000000..b38595c --- /dev/null +++ b/apt-panopticommon.lua | |||
@@ -0,0 +1,563 @@ | |||
1 | local APT = {} | ||
2 | |||
3 | -- https://oss.oetiker.ch/rrdtool/prog/rrdlua.en.html | ||
4 | APT.rrd = require 'rrd' | ||
5 | |||
6 | verbosity = -1 | ||
7 | APT.origin = false | ||
8 | APT.keep = false | ||
9 | -- TODO - Should actually implement this. | ||
10 | APT.fork = true | ||
11 | |||
12 | APT.options = | ||
13 | { | ||
14 | referenceSite = | ||
15 | { | ||
16 | typ = "string", | ||
17 | help = "", | ||
18 | value = "pkgmaster.devuan.org", | ||
19 | }, | ||
20 | roundRobin = | ||
21 | { | ||
22 | typ = "string", | ||
23 | help = "", | ||
24 | value = "deb.devuan.org", | ||
25 | }, | ||
26 | tests = | ||
27 | { | ||
28 | typ = "table", | ||
29 | help = "", | ||
30 | value = | ||
31 | { | ||
32 | "IPv4", | ||
33 | "IPv6", | ||
34 | -- "ftp", | ||
35 | "http", | ||
36 | "https", | ||
37 | -- "rsync", | ||
38 | "DNSRR", | ||
39 | "Protocol", | ||
40 | "URLSanity", | ||
41 | "Integrity", | ||
42 | "Updated", | ||
43 | }, | ||
44 | }, | ||
45 | maxtime = | ||
46 | { | ||
47 | typ = "number", | ||
48 | help = "", | ||
49 | value = 300, | ||
50 | }, | ||
51 | timeout = | ||
52 | { | ||
53 | typ = "number", | ||
54 | help = "", | ||
55 | value = 15, | ||
56 | }, | ||
57 | reports = | ||
58 | { | ||
59 | typ = "table", | ||
60 | help = "", | ||
61 | value = | ||
62 | { | ||
63 | "RRD", -- RRD has to be before web, coz web creates a graph from the RRD data. | ||
64 | "email-web", | ||
65 | -- "Nagios", | ||
66 | -- "Prometheus", | ||
67 | }, | ||
68 | }, | ||
69 | } | ||
70 | |||
71 | APT.parseArgs = function(args) | ||
72 | local arg = {} | ||
73 | local sendArgs = "" | ||
74 | if 0 ~= #(args) then | ||
75 | local option = "" | ||
76 | for i, a in pairs(args) do | ||
77 | if ("--help" == a) or ("-h" == a) then | ||
78 | print("I should write some docs, huh? Read README.md for instructions.") | ||
79 | elseif "--version" == a then | ||
80 | print("apt-panopticon version 0.1 WIP development version") | ||
81 | elseif "-v" == a then | ||
82 | verbosity = verbosity + 1 | ||
83 | sendArgs = sendArgs .. a .. " " | ||
84 | elseif "-q" == a then | ||
85 | verbosity = -1 | ||
86 | sendArgs = sendArgs .. a .. " " | ||
87 | elseif "-k" == a then | ||
88 | APT.keep = true | ||
89 | elseif "-n" == a then | ||
90 | APT.fork = false | ||
91 | elseif "-o" == a then | ||
92 | APT.origin = true | ||
93 | elseif "--" == a:sub(1, 2) then | ||
94 | local s, e = a:find("=") | ||
95 | if nil == s then e = -1 end | ||
96 | option = a:sub(3, e - 1) | ||
97 | local o = APT.options[option] | ||
98 | if nil == o then | ||
99 | print("Unknown option --" .. option) | ||
100 | option = "" | ||
101 | else | ||
102 | option = a | ||
103 | sendArgs = sendArgs .. a .. " " | ||
104 | local s, e = a:find("=") | ||
105 | if nil == s then e = 0 end | ||
106 | option = a:sub(3, e - 1) | ||
107 | if "table" == APT.options[option].typ then | ||
108 | local result = {} | ||
109 | for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do | ||
110 | local f = t:sub(1, 1) | ||
111 | local n = t:sub(2, -1) | ||
112 | if ("+" ~= f) and ("-" ~= f) then | ||
113 | table.insert(result, t) | ||
114 | end | ||
115 | end | ||
116 | if 0 ~= #result then | ||
117 | APT.options[option].value = result | ||
118 | else | ||
119 | for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do | ||
120 | local f = t:sub(1, 1) | ||
121 | local n = t:sub(2, -1) | ||
122 | if "+" == f then | ||
123 | table.insert(APT.options[option].value, n) | ||
124 | elseif "-" == f then | ||
125 | local r = {} | ||
126 | for i, k in pairs(APT.options[option].value) do | ||
127 | if k ~= n then table.insert(r, k) end | ||
128 | end | ||
129 | APT.options[option].value = r | ||
130 | end | ||
131 | end | ||
132 | end | ||
133 | else | ||
134 | APT.options[option].value = a:sub(e + 1, -1) | ||
135 | end | ||
136 | option = "" | ||
137 | end | ||
138 | elseif "-" == a:sub(1, 1) then | ||
139 | print("Unknown option " .. a) | ||
140 | else | ||
141 | table.insert(arg, a) | ||
142 | end | ||
143 | end | ||
144 | end | ||
145 | return arg, sendArgs | ||
146 | end | ||
147 | |||
148 | --print(APT.dumpTable(APT.options, "", "options")) | ||
149 | |||
150 | |||
151 | |||
152 | --[[ Ordered table iterator, allow to iterate on the natural order of the keys of a table. | ||
153 | From http://lua-users.org/wiki/SortedIteration | ||
154 | ]] | ||
155 | function __genOrderedIndex( t ) | ||
156 | local orderedIndex = {} | ||
157 | for key in pairs(t) do | ||
158 | table.insert( orderedIndex, key ) | ||
159 | end | ||
160 | table.sort( orderedIndex ) | ||
161 | return orderedIndex | ||
162 | end | ||
163 | function orderedNext(t, state) | ||
164 | -- Equivalent of the next function, but returns the keys in the alphabetic | ||
165 | -- order. We use a temporary ordered key table that is stored in the | ||
166 | -- table being iterated. | ||
167 | |||
168 | local key = nil | ||
169 | --print("orderedNext: state = "..tostring(state) ) | ||
170 | if state == nil then | ||
171 | -- the first time, generate the index | ||
172 | t.__orderedIndex = __genOrderedIndex( t ) | ||
173 | key = t.__orderedIndex[1] | ||
174 | else | ||
175 | -- fetch the next value | ||
176 | for i = 1,table.getn(t.__orderedIndex) do | ||
177 | if t.__orderedIndex[i] == state then | ||
178 | key = t.__orderedIndex[i+1] | ||
179 | end | ||
180 | end | ||
181 | end | ||
182 | |||
183 | if key then | ||
184 | return key, t[key] | ||
185 | end | ||
186 | |||
187 | -- no more value to return, cleanup | ||
188 | t.__orderedIndex = nil | ||
189 | return | ||
190 | end | ||
191 | function APT.orderedPairs(t) | ||
192 | -- Equivalent of the pairs() function on tables. Allows to iterate | ||
193 | -- in order | ||
194 | return orderedNext, t, nil | ||
195 | end | ||
196 | |||
197 | -- Use this to dump a table to a string, with HTML. | ||
198 | APT.dumpTableHTML = function (table, space, name) | ||
199 | local r = name .. "\n" | ||
200 | r = r .. dumpTableHTMLSub(table, space .. " ") | ||
201 | r = r .. space .. "" | ||
202 | return r | ||
203 | end | ||
204 | dumpTableHTMLSub = function (table, space) | ||
205 | local r = "" | ||
206 | for k, v in APT.orderedPairs(table) do | ||
207 | if type(v) == "table" then | ||
208 | if " " == space then | ||
209 | r = r .. space .. APT.dumpTableHTML(v, space, k .. "<ul>") .. "</ul>\n" | ||
210 | else | ||
211 | r = r .. "<li>" .. space .. APT.dumpTableHTML(v, space, k .. "<ul>") .. "</ul></li>\n" | ||
212 | end | ||
213 | else | ||
214 | r = r .. space .. "<li>" .. k .. "</li>\n" | ||
215 | end | ||
216 | end | ||
217 | return r | ||
218 | end | ||
219 | |||
220 | -- Use this to dump a table to a string. | ||
221 | APT.dumpTable = function (table, space, name) | ||
222 | local r = "" | ||
223 | if "" == space then r = r .. space .. name .. " =\n" else r = r .. space .. "[" .. name .. "] =\n" end | ||
224 | r = r .. space .. "{\n" | ||
225 | r = r .. dumpTableSub(table, space .. " ") | ||
226 | if "" == space then r = r .. space .. "}\n" else r = r .. space .. "},\n" end | ||
227 | return r | ||
228 | end | ||
229 | dumpTableSub = function (table, space) | ||
230 | local r = "" | ||
231 | for k, v in pairs(table) do | ||
232 | if type(k) == "string" then k = '"' .. k .. '"' end | ||
233 | if type(v) == "table" then | ||
234 | r = r .. APT.dumpTable(v, space, k) | ||
235 | elseif type(v) == "string" then | ||
236 | r = r .. space .. "[" .. k .. "] = '" .. v .. "';\n" | ||
237 | elseif type(v) == "function" then | ||
238 | r = r .. space .. "[" .. k .. "] = function ();\n" | ||
239 | elseif type(v) == "userdata" then | ||
240 | r = r .. space .. "userdata " .. "[" .. k .. "];\n" | ||
241 | elseif type(v) == "boolean" then | ||
242 | if (v) then | ||
243 | r = r .. space .. "[" .. k .. "] = true;\n" | ||
244 | else | ||
245 | r = r .. space .. "[" .. k .. "] = false;\n" | ||
246 | end | ||
247 | else | ||
248 | r = r .. space .. "[" .. k .. "] = " .. v .. ";\n" | ||
249 | end | ||
250 | end | ||
251 | return r | ||
252 | end | ||
253 | |||
254 | APT.results = {} | ||
255 | APT.logFile = nil | ||
256 | APT.html = false | ||
257 | |||
258 | APT.logPre = function() | ||
259 | if nil ~= APT.logFile then | ||
260 | APT.logFile:write("<html><head>\n") | ||
261 | APT.logFile:write("</head><body bgcolor='black' text='white' alink='red' link='blue' vlink='purple'>\n") | ||
262 | end | ||
263 | end | ||
264 | APT.logPost = function() | ||
265 | if nil ~= APT.logFile then | ||
266 | APT.logFile:write("</body></html> \n") | ||
267 | end | ||
268 | end | ||
269 | |||
270 | local log = function(v, t, s, prot, test, host) | ||
271 | local x = "" | ||
272 | if nil == prot then prot = "" end | ||
273 | if nil == test then test = "" end | ||
274 | x = x .. prot | ||
275 | if "" ~= test then | ||
276 | if #x > 0 then x = x .. " " end | ||
277 | x = x .. test | ||
278 | end | ||
279 | if nil ~= host then | ||
280 | if #x > 0 then x = x .. " " end | ||
281 | x = x .. host | ||
282 | end | ||
283 | if #x > 0 then | ||
284 | t = t .. "(" .. x .. ")" | ||
285 | if "" ~= prot then | ||
286 | if "" == test then | ||
287 | if nil == APT.results[prot] then APT.results[prot] = {errors = 0; warnings = 0} end | ||
288 | if v == 0 then APT.results[prot].errors = APT.results[prot].errors + 1 end | ||
289 | if v == 1 then APT.results[prot].warnings = APT.results[prot].warnings + 1 end | ||
290 | else | ||
291 | if nil == APT.results[prot] then APT.results[prot] = {errors = 0; warnings = 0} end | ||
292 | if nil == APT.results[prot][test] then APT.results[prot][test] = {errors = 0; warnings = 0} end | ||
293 | if v == 0 then APT.results[prot][test].errors = APT.results[prot][test].errors + 1 end | ||
294 | if v == 1 then APT.results[prot][test].warnings = APT.results[prot][test].warnings + 1 end | ||
295 | end | ||
296 | end | ||
297 | end | ||
298 | if v <= verbosity then | ||
299 | if 3 <= verbosity then t = os.date() .. " " .. t end | ||
300 | print(t .. ": " .. s) | ||
301 | end | ||
302 | if nil ~= APT.logFile then | ||
303 | if APT.html then | ||
304 | local colour = "white" | ||
305 | if -1 == v then colour = "fuchsia" end -- CRITICAL | ||
306 | if 0 == v then colour = "red " end -- ERROR | ||
307 | if 1 == v then colour = "yellow " end -- WARNING | ||
308 | if 2 == v then colour = "white " end -- INFO | ||
309 | if 3 == v then colour = "gray " end -- DEBUG | ||
310 | APT.logFile:write(os.date() .. " <font color='" .. colour .. "'><b>" .. t .. "</b></font>: " .. s .. "</br>\n") | ||
311 | else | ||
312 | APT.logFile:write(os.date() .. " " .. t .. ": " .. s .. "\n") | ||
313 | end | ||
314 | APT.logFile:flush() | ||
315 | end | ||
316 | end | ||
317 | APT.D = function(s) log(3, "DEBUG ", s) end | ||
318 | APT.I = function(s) log(2, "INFO ", s) end | ||
319 | APT.W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end | ||
320 | APT.E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end | ||
321 | APT.C = function(s) log(-1, "CRITICAL", s) end | ||
322 | local D = APT.D | ||
323 | local I = APT.I | ||
324 | local W = APT.W | ||
325 | local E = APT.E | ||
326 | local C = APT.C | ||
327 | |||
328 | |||
329 | APT.mirrors = {} | ||
330 | |||
331 | APT.testing = function(t, host) | ||
332 | for i, v in pairs(APT.options.tests.value) do | ||
333 | if t == v then | ||
334 | local h = APT.mirrors[host] | ||
335 | if nil == h then return true end | ||
336 | if true == h["Protocols"][t] then return true else D("Skipping " .. t .. " checks for " .. host) end | ||
337 | end | ||
338 | end | ||
339 | return false | ||
340 | end | ||
341 | |||
342 | APT.execute = function (s) | ||
343 | D(" executing <pre><code>" .. s .. "</code></pre>") | ||
344 | --[[ Damn os.execute() | ||
345 | Lua 5.1 says it returns "a status code, which is system-dependent" | ||
346 | Lua 5.2 says it returns true/nil, "exit"/"signal", the status code. | ||
347 | I'm getting 7168 or 0. No idea what the fuck that is. | ||
348 | local ok, rslt, status = os.execute(s) | ||
349 | ]] | ||
350 | local f = io.popen(s .. ' ; echo "$?"', 'r') | ||
351 | local status = "" | ||
352 | local result = "" | ||
353 | -- The last line will be the command's returned status, collect everything else in result. | ||
354 | for l in f:lines() do | ||
355 | result = result .. status .. "\n" | ||
356 | status = l | ||
357 | end | ||
358 | return status, result | ||
359 | end | ||
360 | |||
361 | APT.fork = function(s) | ||
362 | D(" forking <pre><code>" .. s .. "</code></pre>") | ||
363 | os.execute(s .. " &") | ||
364 | end | ||
365 | |||
366 | APT.checkExes = function (exe) | ||
367 | local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | grep -v "flock -n apt-panopticon.lock " | wc -l'):read("*l") | ||
368 | D(count .. " " .. exe .. " commands still running.") | ||
369 | return tonumber(count) | ||
370 | end | ||
371 | |||
372 | APT.checkFile = function(f) | ||
373 | local h, e = io.open(f, "r") | ||
374 | if nil == h then return false else h:close(); return true end | ||
375 | end | ||
376 | |||
377 | APT.plurals = function(e, w) | ||
378 | local result = "" | ||
379 | if 1 == e then | ||
380 | result = e .. " error" | ||
381 | elseif e ~= 0 then | ||
382 | result = e .. " errors" | ||
383 | end | ||
384 | if ("" ~= result) and APT.html then result = "<font color='red'><b>" .. result .. "</b></font>" end | ||
385 | -- result = " " .. result | ||
386 | if 0 < w then | ||
387 | if 0 < e then result = result .. ", " end | ||
388 | if 1 == w then | ||
389 | result = result .. w .. " warning" | ||
390 | else | ||
391 | result = result .. w .. " warnings" | ||
392 | end | ||
393 | if ("" ~= result) and APT.html then result = "<font color='yellow'><b>" .. result .. "</b></font>" end | ||
394 | -- result = " " .. result | ||
395 | end | ||
396 | if "" ~= result then result = " (" .. result .. ")" end | ||
397 | return result | ||
398 | end | ||
399 | |||
400 | APT.collate = function(host, ip, results) | ||
401 | local f = "results/" .. host .. "_" .. ip .. ".lua" | ||
402 | local rfile, e = io.open(f, "r") | ||
403 | if nil == rfile then I("opening " .. f .. " file - " .. e) else | ||
404 | rfile:close() | ||
405 | local rs = loadfile(f)() | ||
406 | for k, v in pairs(rs) do | ||
407 | if "table" == type(v) then | ||
408 | if ("speed" == k) and (nil ~= results.speed) then | ||
409 | if v.min < results.speed.min then results.speed.min = v.min end | ||
410 | if v.max > results.speed.max then results.speed.max = v.max end | ||
411 | else | ||
412 | for i, u in pairs(v) do | ||
413 | if "table" == type(u) then | ||
414 | for h, t in pairs(u) do | ||
415 | local a = results[k][i][h] | ||
416 | if nil == a then a = 0 end | ||
417 | results[k][i][h] = a + t | ||
418 | end | ||
419 | else | ||
420 | local a = results[k] | ||
421 | if nil == a then a = 0; results[k] = {} else a = a[i] end | ||
422 | if nil == a then a = 0 end | ||
423 | results[k][i] = a + u | ||
424 | end | ||
425 | end | ||
426 | end | ||
427 | elseif "timeout" ~= k then | ||
428 | local a = results[k] | ||
429 | if nil == a then a = 0 end | ||
430 | results[k] = a + v | ||
431 | end | ||
432 | end | ||
433 | end | ||
434 | return results | ||
435 | end | ||
436 | |||
437 | |||
438 | |||
439 | |||
440 | APT.now = 0 | ||
441 | local status | ||
442 | status, APT.now = APT.execute('TZ="GMT" ls -l --time-style="+%s" results/stamp | cut -d " " -f 6-6') | ||
443 | APT.now = tonumber(APT.now) | ||
444 | APT.protocols = {'ftp', 'http', 'https', 'rsync'} | ||
445 | APT.tests = {'raw', 'Integrity', 'Protocol', 'Updated', 'URLSanity', 'Speed'} | ||
446 | local start = 'now-2week' | ||
447 | local step = '10min' | ||
448 | local hb = '150min' | ||
449 | local DSIe = 'DS:IntegrityErrors:GAUGE:' .. hb .. ':0:U' | ||
450 | local DSIw = 'DS:IntegrityWarnings:GAUGE:' .. hb .. ':0:U' | ||
451 | local DSPe = 'DS:ProtocolErrors:GAUGE:' .. hb .. ':0:U' | ||
452 | local DSPw = 'DS:ProtocolWarnings:GAUGE:' .. hb .. ':0:U' | ||
453 | local DSUe = 'DS:UpdatedErrors:GAUGE:' .. hb .. ':0:U' | ||
454 | local DSUw = 'DS:UpdatedWarnings:GAUGE:' .. hb .. ':0:U' | ||
455 | local DSSe = 'DS:URLSanityErrors:GAUGE:' .. hb .. ':0:U' | ||
456 | local DSSw = 'DS:URLSanityWarnings:GAUGE:' .. hb .. ':0:U' | ||
457 | local DSx = 'DS:max:GAUGE:' .. hb .. ':0:U' | ||
458 | local DSn = 'DS:min:GAUGE:' .. hb .. ':0:U' | ||
459 | |||
460 | -- What Collectd uses. | ||
461 | local RRAc0 = 'RRA:AVERAGE:0.9:1:1200' | ||
462 | local RRAc1 = 'RRA:MIN:0.9:1:1200' | ||
463 | local RRAc2 = 'RRA:MAX:0.9:1:1200' | ||
464 | local RRAc3 = 'RRA:AVERAGE:0.9:7:1235' | ||
465 | local RRAc4 = 'RRA:MIN:0.9:7:1235' | ||
466 | local RRAc5 = 'RRA:MAX:0.9:7:1235' | ||
467 | local RRAc6 = 'RRA:AVERAGE:0.9:50:1210' | ||
468 | local RRAc7 = 'RRA:MIN:0.9:50:1210' | ||
469 | local RRAc8 = 'RRA:MAX:0.9:50:1210' | ||
470 | local RRAc9 = 'RRA:AVERAGE:0.9:223:1202' | ||
471 | local RRAc10 = 'RRA:MIN:0.9:223:1202' | ||
472 | local RRAc11 = 'RRA:MAX:0.9:223:1202' | ||
473 | local RRAc12 = 'RRA:AVERAGE:0.9:2635:1201' | ||
474 | local RRAc13 = 'RRA:MIN:0.9:2635:1201' | ||
475 | local RRAc14 = 'RRA:MAX:0.9:2635:1201' | ||
476 | |||
477 | -- Try LAST. | ||
478 | local RRAl0 = 'RRA:LAST:0.9:1:1200' | ||
479 | local RRAl1 = 'RRA:LAST:0.9:7:1235' | ||
480 | local RRAl2 = 'RRA:LAST:0.9:50:1210' | ||
481 | local RRAl3 = 'RRA:LAST:0.9:223:1202' | ||
482 | local RRAl4 = 'RRA:LAST:0.9:2635:1201' | ||
483 | |||
484 | --[[ | ||
485 | /var/lib/collectd/rrd/onefang/interface-eth0/if_packets.rrd | ||
486 | |||
487 | rrd/host.IP/protocol.rrd test-error count | ||
488 | rrd/host.IP/protocol.rrd test-warning count | ||
489 | |||
490 | rrh/host.IP/protocol-speed.rrd min | ||
491 | rrh/host.IP/protocol-speed.rrd max | ||
492 | ]] | ||
493 | |||
494 | APT.createRRD = function(host, ip) | ||
495 | if nil ~= ip then host = host .. '_' .. ip end | ||
496 | for i, p in pairs(APT.protocols) do | ||
497 | -- for j, t in pairs(tests) do | ||
498 | os.execute( 'mkdir -p rrd/' .. host .. '/' .. p:upper()) | ||
499 | if not APT.checkFile('rrd/' .. host .. '/' .. p:upper() .. '/Speed.rrd') then | ||
500 | D('Creating ' .. 'rrd/' .. host .. '/' .. p:upper() .. '/Speed.rrd') | ||
501 | -- if 'Speed' == t then | ||
502 | APT.rrd.create( 'rrd/' .. host .. '/' .. p:upper() .. '/Speed.rrd', '--start', start, '--step', step, DSx, DSn, | ||
503 | RRAc0, RRAc1, RRAc2, RRAl0, RRAc3, RRAc4, RRAc5, RRAl1, RRAc6, RRAc7, RRAc8, RRAl2, RRAc9, RRAc10, RRAc11, RRAl3, RRAc12, RRAc13, RRAc14, RRAl4) | ||
504 | -- else | ||
505 | end | ||
506 | if not APT.checkFile('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd') then | ||
507 | D('Creating ' .. 'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd') | ||
508 | APT.rrd.create( 'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', '--start', start, '--step', step, DSIe, DSIw, DSPe, DSPw, DSUe, DSUw, DSSe, DSSw, | ||
509 | RRAc0, RRAc1, RRAc2, RRAl0, RRAc3, RRAc4, RRAc5, RRAl1, RRAc6, RRAc7, RRAc8, RRAl2, RRAc9, RRAc10, RRAc11, RRAl3, RRAc12, RRAc13, RRAc14, RRAl4) | ||
510 | -- end | ||
511 | -- Start them at 0 so the average has something to work on. | ||
512 | APT.rrd.update( 'rrd/' .. host .. '/' .. p:upper() .. '/Speed.rrd', (APT.now - 600) .. ':0:0') | ||
513 | APT.rrd.update( 'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', (APT.now - 600) .. ':0:0:0:0:0:0:0:0') | ||
514 | end | ||
515 | -- end | ||
516 | end | ||
517 | end | ||
518 | |||
519 | APT.updateRRD = function(results, host, ip) | ||
520 | if nil ~= ip then host = host .. '_' .. ip end | ||
521 | for i, p in pairs(APT.protocols) do | ||
522 | if nil ~= results.speed then | ||
523 | APT.rrd.update( 'rrd/' .. host .. '/' .. p:upper() .. '/Speed.rrd', APT.now .. ':' .. results.speed.max .. ':' .. results.speed.min) | ||
524 | end | ||
525 | if nil ~= results[p] then | ||
526 | APT.rrd.update('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', APT.now .. ':' .. | ||
527 | results[p]['Integrity'].errors .. ':' .. results[p]['Integrity'].warnings .. ':' .. | ||
528 | results[p]['Protocol'].errors .. ':' .. results[p]['Protocol'].warnings .. ':' .. | ||
529 | results[p]['Updated'].errors .. ':' .. results[p]['Updated'].warnings .. ':' .. | ||
530 | results[p]['URLSanity'].errors .. ':' .. results[p]['URLSanity'].warnings) | ||
531 | else | ||
532 | APT.rrd.update('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', APT.now .. ':U:U:U:U:U:U:U:U') | ||
533 | end | ||
534 | end | ||
535 | end | ||
536 | |||
537 | |||
538 | APT.doRRD = function(l, k, v) | ||
539 | if APT.checkFile(l .. "/" .. k .. ".lua") then | ||
540 | local results = loadfile(l .. "/" .. k .. ".lua")() | ||
541 | APT.createRRD(k) | ||
542 | APT.updateRRD(results, k) | ||
543 | if til ~= v then | ||
544 | local IPs = v.IPs | ||
545 | for i, u in pairs(IPs) do | ||
546 | if "table" == type(u) then | ||
547 | for h, t in pairs(u) do | ||
548 | APT.createRRD(k, h) | ||
549 | results = APT.collate(k, h, results) | ||
550 | APT.updateRRD(results, k, h) | ||
551 | end | ||
552 | else | ||
553 | APT.createRRD(k, i) | ||
554 | results = APT.collate(k, i, results) | ||
555 | APT.updateRRD(results, k, i) | ||
556 | end | ||
557 | end | ||
558 | end | ||
559 | end | ||
560 | end | ||
561 | |||
562 | |||
563 | return APT | ||
diff --git a/apt-panopticon-report-email-web.lua b/apt-panopticon-report-email-web.lua index 1144550..ef61183 100755 --- a/apt-panopticon-report-email-web.lua +++ b/apt-panopticon-report-email-web.lua | |||
@@ -1,143 +1,22 @@ | |||
1 | #!/usr/bin/env luajit | 1 | #!/usr/bin/env luajit |
2 | 2 | ||
3 | local args = {...} | 3 | local APT = require 'apt-panopticommon' |
4 | local D = APT.D | ||
5 | local I = APT.I | ||
6 | local W = APT.W | ||
7 | local E = APT.E | ||
8 | local C = APT.C | ||
9 | local arg, sendArgs = APT.parseArgs({...}) | ||
4 | 10 | ||
5 | verbosity = -1 | ||
6 | local logFile | ||
7 | local html = false | ||
8 | |||
9 | |||
10 | --[[ Ordered table iterator, allow to iterate on the natural order of the keys of a table. | ||
11 | From http://lua-users.org/wiki/SortedIteration | ||
12 | ]] | ||
13 | function __genOrderedIndex( t ) | ||
14 | local orderedIndex = {} | ||
15 | for key in pairs(t) do | ||
16 | table.insert( orderedIndex, key ) | ||
17 | end | ||
18 | table.sort( orderedIndex ) | ||
19 | return orderedIndex | ||
20 | end | ||
21 | function orderedNext(t, state) | ||
22 | -- Equivalent of the next function, but returns the keys in the alphabetic | ||
23 | -- order. We use a temporary ordered key table that is stored in the | ||
24 | -- table being iterated. | ||
25 | |||
26 | local key = nil | ||
27 | --print("orderedNext: state = "..tostring(state) ) | ||
28 | if state == nil then | ||
29 | -- the first time, generate the index | ||
30 | t.__orderedIndex = __genOrderedIndex( t ) | ||
31 | key = t.__orderedIndex[1] | ||
32 | else | ||
33 | -- fetch the next value | ||
34 | for i = 1,table.getn(t.__orderedIndex) do | ||
35 | if t.__orderedIndex[i] == state then | ||
36 | key = t.__orderedIndex[i+1] | ||
37 | end | ||
38 | end | ||
39 | end | ||
40 | |||
41 | if key then | ||
42 | return key, t[key] | ||
43 | end | ||
44 | |||
45 | -- no more value to return, cleanup | ||
46 | t.__orderedIndex = nil | ||
47 | return | ||
48 | end | ||
49 | function orderedPairs(t) | ||
50 | -- Equivalent of the pairs() function on tables. Allows to iterate | ||
51 | -- in order | ||
52 | return orderedNext, t, nil | ||
53 | end | ||
54 | |||
55 | -- Use this to dump a table to a string, with HTML. | ||
56 | dumpTableHTML = function (table, space, name) | ||
57 | local r = name .. "\n" | ||
58 | r = r .. dumpTableHTMLSub(table, space .. " ") | ||
59 | r = r .. space .. "" | ||
60 | return r | ||
61 | end | ||
62 | dumpTableHTMLSub = function (table, space) | ||
63 | local r = "" | ||
64 | for k, v in orderedPairs(table) do | ||
65 | if type(v) == "table" then | ||
66 | if " " == space then | ||
67 | r = r .. space .. dumpTableHTML(v, space, k .. "<ul>") .. "</ul>\n" | ||
68 | else | ||
69 | r = r .. "<li>" .. space .. dumpTableHTML(v, space, k .. "<ul>") .. "</ul></li>\n" | ||
70 | end | ||
71 | else | ||
72 | r = r .. space .. "<li>" .. k .. "</li>\n" | ||
73 | end | ||
74 | end | ||
75 | return r | ||
76 | end | ||
77 | |||
78 | local checkFile = function(f) | ||
79 | local h, e = io.open(f, "r") | ||
80 | if nil == h then return false else h:close(); return true end | ||
81 | end | ||
82 | |||
83 | local plurals = function(e, w) | ||
84 | local result = "" | ||
85 | if 1 == e then | ||
86 | result = e .. " error" | ||
87 | elseif e ~= 0 then | ||
88 | result = e .. " errors" | ||
89 | end | ||
90 | if ("" ~= result) and html then result = "<font color='red'><b>" .. result .. "</b></font>" end | ||
91 | -- result = " " .. result | ||
92 | if 0 < w then | ||
93 | if 0 < e then result = result .. ", " end | ||
94 | if 1 == w then | ||
95 | result = result .. w .. " warning" | ||
96 | else | ||
97 | result = result .. w .. " warnings" | ||
98 | end | ||
99 | if ("" ~= result) and html then result = "<font color='yellow'><b>" .. result .. "</b></font>" end | ||
100 | -- result = " " .. result | ||
101 | end | ||
102 | if "" ~= result then result = " (" .. result .. ")" end | ||
103 | return result | ||
104 | end | ||
105 | 11 | ||
106 | local results = {} | 12 | local results = {} |
107 | 13 | APT.mirrors = loadfile("results/mirrors.lua")() | |
108 | local log = function(v, t, s, prot, test, host) | ||
109 | local x = "" | ||
110 | if nil == prot then prot = "" end | ||
111 | if nil ~= test then x = x .. test else test = "" end | ||
112 | if nil ~= host then | ||
113 | if #x > 0 then x = x .. " " end | ||
114 | x = x .. host | ||
115 | end | ||
116 | if #x > 0 then | ||
117 | t = t .. "(" .. x .. ")" | ||
118 | end | ||
119 | if v <= verbosity then | ||
120 | if 3 <= verbosity then t = os.date() .. " " .. t end | ||
121 | print(t .. ": " .. s) | ||
122 | end | ||
123 | if nil ~= logFile then | ||
124 | logFile:write(os.date() .. " " .. t .. ": " .. s .. "\n") | ||
125 | logFile:flush() | ||
126 | end | ||
127 | end | ||
128 | local D = function(s) log(3, "DEBUG ", s) end | ||
129 | local I = function(s) log(2, "INFO ", s) end | ||
130 | local W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end | ||
131 | local E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end | ||
132 | local C = function(s) log(-1, "CRITICAL", s) end | ||
133 | |||
134 | local mirrors = loadfile("results/mirrors.lua")() | ||
135 | 14 | ||
136 | local revDNS = function(dom, IP) | 15 | local revDNS = function(dom, IP) |
137 | if "deb.devuan.org" ~= dom then | 16 | if "deb.devuan.org" ~= dom then |
138 | if nil ~= mirrors["deb.devuan.org"] then | 17 | if nil ~= APT.mirrors["deb.devuan.org"] then |
139 | if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][IP] then | 18 | if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][IP] then |
140 | if html then | 19 | if APT.html then |
141 | return "<font color='purple'><b>DNS-RR</b></font>" | 20 | return "<font color='purple'><b>DNS-RR</b></font>" |
142 | else | 21 | else |
143 | return "DNS-RR" | 22 | return "DNS-RR" |
@@ -145,7 +24,7 @@ local revDNS = function(dom, IP) | |||
145 | end | 24 | end |
146 | end | 25 | end |
147 | else | 26 | else |
148 | for k, v in pairs(mirrors) do | 27 | for k, v in pairs(APT.mirrors) do |
149 | if "deb.devuan.org" ~= k then | 28 | if "deb.devuan.org" ~= k then |
150 | local IPs = v.IPs | 29 | local IPs = v.IPs |
151 | for i, u in pairs(IPs) do | 30 | for i, u in pairs(IPs) do |
@@ -168,7 +47,7 @@ local status = function(host, results, typ) | |||
168 | local result = "" | 47 | local result = "" |
169 | local e = 0 | 48 | local e = 0 |
170 | local w = 0 | 49 | local w = 0 |
171 | local s = nil ~= mirrors[host].Protocols[typ] | 50 | local s = nil ~= APT.mirrors[host].Protocols[typ] |
172 | local to = results.timeout | 51 | local to = results.timeout |
173 | if ('http' ~= typ) and ('https' ~= typ) and ('ftp' ~= typ) and ('rsync' ~= typ) then s = true end | 52 | if ('http' ~= typ) and ('https' ~= typ) and ('ftp' ~= typ) and ('rsync' ~= typ) then s = true end |
174 | if nil ~= results[typ] then | 53 | if nil ~= results[typ] then |
@@ -198,7 +77,7 @@ local status = function(host, results, typ) | |||
198 | if to then | 77 | if to then |
199 | result = "[TIMEOUT" | 78 | result = "[TIMEOUT" |
200 | if not s then result = result .. "*" end | 79 | if not s then result = result .. "*" end |
201 | if html then | 80 | if APT.html then |
202 | if s then | 81 | if s then |
203 | result = "[<font color='blue'><b>TIMEOUT</b></font>" | 82 | result = "[<font color='blue'><b>TIMEOUT</b></font>" |
204 | else | 83 | else |
@@ -208,14 +87,14 @@ local status = function(host, results, typ) | |||
208 | elseif 0 < e then | 87 | elseif 0 < e then |
209 | result = "[FAILED" | 88 | result = "[FAILED" |
210 | if not s then result = result .. "*" end | 89 | if not s then result = result .. "*" end |
211 | if html then | 90 | if APT.html then |
212 | if s then | 91 | if s then |
213 | result = "[<font color='red'><b>FAILED</b></font>" | 92 | result = "[<font color='red'><b>FAILED</b></font>" |
214 | else | 93 | else |
215 | result = "[<font color='darkred'><b>FAILED*</b></font>" | 94 | result = "[<font color='darkred'><b>FAILED*</b></font>" |
216 | end | 95 | end |
217 | end | 96 | end |
218 | if html then | 97 | if APT.html then |
219 | faulty = faulty .. host .. " (" .. typ .. ")<br>\n" | 98 | faulty = faulty .. host .. " (" .. typ .. ")<br>\n" |
220 | else | 99 | else |
221 | faulty = faulty .. host .. " (" .. typ .. ")\n" | 100 | faulty = faulty .. host .. " (" .. typ .. ")\n" |
@@ -223,7 +102,7 @@ local status = function(host, results, typ) | |||
223 | else | 102 | else |
224 | result = "[OK" | 103 | result = "[OK" |
225 | if not s then result = result .. "*" end | 104 | if not s then result = result .. "*" end |
226 | if html then | 105 | if APT.html then |
227 | if s then | 106 | if s then |
228 | result = "[<font color='lime'><b>OK</b></font>" | 107 | result = "[<font color='lime'><b>OK</b></font>" |
229 | else | 108 | else |
@@ -231,43 +110,7 @@ local status = function(host, results, typ) | |||
231 | end | 110 | end |
232 | end | 111 | end |
233 | end | 112 | end |
234 | return result .. plurals(e, w) .. "]" | 113 | return result .. APT.plurals(e, w) .. "]" |
235 | end | ||
236 | |||
237 | local collate = function(host, ip, results) | ||
238 | local f = "results/" .. host .. "_" .. ip .. ".lua" | ||
239 | local rfile, e = io.open(f, "r") | ||
240 | if nil == rfile then I("opening " .. f .. " file - " .. e) else | ||
241 | rfile:close() | ||
242 | local rs = loadfile(f)() | ||
243 | for k, v in pairs(rs) do | ||
244 | if "table" == type(v) then | ||
245 | if "speed" == k then | ||
246 | if v.min < results.speed.min then results.speed.min = v.min end | ||
247 | if v.max > results.speed.max then results.speed.max = v.max end | ||
248 | else | ||
249 | for i, u in pairs(v) do | ||
250 | if "table" == type(u) then | ||
251 | for h, t in pairs(u) do | ||
252 | local a = results[k][i][h] | ||
253 | if nil == a then a = 0 end | ||
254 | results[k][i][h] = a + t | ||
255 | end | ||
256 | else | ||
257 | local a = results[k][i] | ||
258 | if nil == a then a = 0 end | ||
259 | results[k][i] = a + u | ||
260 | end | ||
261 | end | ||
262 | end | ||
263 | elseif "timeout" ~= k then | ||
264 | local a = results[k] | ||
265 | if nil == a then a = 0 end | ||
266 | results[k] = a + v | ||
267 | end | ||
268 | end | ||
269 | end | ||
270 | return results | ||
271 | end | 114 | end |
272 | 115 | ||
273 | local m = {} | 116 | local m = {} |
@@ -287,20 +130,20 @@ local logCount = function(domain, ip) | |||
287 | if nil ~= l:match("><b>WARNING ") then warnings = warnings + 1 end | 130 | if nil ~= l:match("><b>WARNING ") then warnings = warnings + 1 end |
288 | end | 131 | end |
289 | rfile:close() | 132 | rfile:close() |
290 | if html then | 133 | if APT.html then |
291 | if nil == ip then | 134 | if nil == ip then |
292 | log = "<a href='" .. nm .. "'>" .. domain .. "</a>" | 135 | log = "<a href='" .. nm .. "'>" .. domain .. "</a>" |
293 | else | 136 | else |
294 | log = "<a href='" .. nm .. "'>" .. ip .. "</a>" | 137 | log = "<a href='" .. nm .. "'>" .. ip .. "</a>" |
295 | end | 138 | end |
296 | end | 139 | end |
297 | log = log .. plurals(errors, warnings) | 140 | log = log .. APT.plurals(errors, warnings) |
298 | end | 141 | end |
299 | return log | 142 | return log |
300 | end | 143 | end |
301 | 144 | ||
302 | 145 | ||
303 | html = false | 146 | APT.html = false |
304 | local email, e = io.open("results/Report-email.txt", "w+") | 147 | local email, e = io.open("results/Report-email.txt", "w+") |
305 | if nil == email then C("opening mirrors file - " .. e) else | 148 | if nil == email then C("opening mirrors file - " .. e) else |
306 | email:write( "Dear Mirror Admins,\n\n" .. | 149 | email:write( "Dear Mirror Admins,\n\n" .. |
@@ -316,17 +159,17 @@ if nil == email then C("opening mirrors file - " .. e) else | |||
316 | "Please see below the current status of the Devuan Package Mirror \nnetwork:\n\n" .. | 159 | "Please see below the current status of the Devuan Package Mirror \nnetwork:\n\n" .. |
317 | "==== package mirror status " .. os.date("!%Y-%m-%d %H:%M") .. " GMT ====\n" .. | 160 | "==== package mirror status " .. os.date("!%Y-%m-%d %H:%M") .. " GMT ====\n" .. |
318 | "[skip] means that the test hasn't been written yet.\n\n") | 161 | "[skip] means that the test hasn't been written yet.\n\n") |
319 | for k, v in orderedPairs(mirrors) do | 162 | for k, v in APT.orderedPairs(APT.mirrors) do |
320 | local results = loadfile("results/" .. k .. ".lua")() | 163 | local results = loadfile("results/" .. k .. ".lua")() |
321 | email:write(k .. "....\n") | 164 | email:write(k .. "....\n") |
322 | local IPs = v.IPs | 165 | local IPs = v.IPs |
323 | for i, u in pairs(IPs) do | 166 | for i, u in pairs(IPs) do |
324 | if "table" == type(u) then | 167 | if "table" == type(u) then |
325 | for h, t in pairs(u) do | 168 | for h, t in pairs(u) do |
326 | results = collate(k, h, results) | 169 | results = APT.collate(k, h, results) |
327 | end | 170 | end |
328 | else | 171 | else |
329 | results = collate(k, i, results) | 172 | results = APT.collate(k, i, results) |
330 | end | 173 | end |
331 | end | 174 | end |
332 | local ftp = "[skip]" | 175 | local ftp = "[skip]" |
@@ -340,11 +183,11 @@ if nil == email then C("opening mirrors file - " .. e) else | |||
340 | local updated = status(k, results, "Updated") | 183 | local updated = status(k, results, "Updated") |
341 | 184 | ||
342 | -- DNS-RR test. | 185 | -- DNS-RR test. |
343 | if ("deb.devuan.org" ~= k) and (nil ~= mirrors["deb.devuan.org"]) then | 186 | if ("deb.devuan.org" ~= k) and (nil ~= APT.mirrors["deb.devuan.org"]) then |
344 | for l, w in pairs(mirrors[k].IPs) do | 187 | for l, w in pairs(APT.mirrors[k].IPs) do |
345 | if type(w) == "table" then | 188 | if type(w) == "table" then |
346 | for i, u in pairs(w) do | 189 | for i, u in pairs(w) do |
347 | if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then | 190 | if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then |
348 | local log = logCount("deb.devuan.org", i) | 191 | local log = logCount("deb.devuan.org", i) |
349 | if "" ~= log then | 192 | if "" ~= log then |
350 | if "" == dns then dns = " " else dns = dns .. " " end | 193 | if "" == dns then dns = " " else dns = dns .. " " end |
@@ -356,7 +199,7 @@ if nil == email then C("opening mirrors file - " .. e) else | |||
356 | end | 199 | end |
357 | end | 200 | end |
358 | else | 201 | else |
359 | if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then | 202 | if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then |
360 | local log = logCount("deb.devuan.org", l) | 203 | local log = logCount("deb.devuan.org", l) |
361 | if "" ~= log then | 204 | if "" ~= log then |
362 | if "" == dns then dns = " " else dns = dns .. " " end | 205 | if "" == dns then dns = " " else dns = dns .. " " end |
@@ -391,7 +234,7 @@ end | |||
391 | results = {} | 234 | results = {} |
392 | m = {} | 235 | m = {} |
393 | faulty = "" | 236 | faulty = "" |
394 | html = true | 237 | APT.html = true |
395 | local web, e = io.open("results/Report-web.html", "w+") | 238 | local web, e = io.open("results/Report-web.html", "w+") |
396 | if nil == web then C("opening mirrors file - " .. e) else | 239 | if nil == web then C("opening mirrors file - " .. e) else |
397 | web:write( "<html><head><title>apt-panopticon results</title>\n" .. | 240 | web:write( "<html><head><title>apt-panopticon results</title>\n" .. |
@@ -420,7 +263,7 @@ if nil == web then C("opening mirrors file - " .. e) else | |||
420 | "<table>\n<tr><th></th><th>FTP</th><th>HTTP</th><th>HTTPS</th><th>RSYNC</th><th>DNS round robin</th>" .. | 263 | "<table>\n<tr><th></th><th>FTP</th><th>HTTP</th><th>HTTPS</th><th>RSYNC</th><th>DNS round robin</th>" .. |
421 | "<th>Protocol</th><th>URL sanity</th><th>Integrity</th><th>Updated</th><th colspan='2'>Speed range</th></tr>\n" | 264 | "<th>Protocol</th><th>URL sanity</th><th>Integrity</th><th>Updated</th><th colspan='2'>Speed range</th></tr>\n" |
422 | ) | 265 | ) |
423 | for k, v in orderedPairs(mirrors) do | 266 | for k, v in APT.orderedPairs(APT.mirrors) do |
424 | local results = loadfile("results/" .. k .. ".lua")() | 267 | local results = loadfile("results/" .. k .. ".lua")() |
425 | local active = "" | 268 | local active = "" |
426 | if "yes" == v.Active then | 269 | if "yes" == v.Active then |
@@ -433,10 +276,10 @@ if nil == web then C("opening mirrors file - " .. e) else | |||
433 | for i, u in pairs(IPs) do | 276 | for i, u in pairs(IPs) do |
434 | if "table" == type(u) then | 277 | if "table" == type(u) then |
435 | for h, t in pairs(u) do | 278 | for h, t in pairs(u) do |
436 | results = collate(k, h, results) | 279 | results = APT.collate(k, h, results) |
437 | end | 280 | end |
438 | else | 281 | else |
439 | results = collate(k, i, results) | 282 | results = APT.collate(k, i, results) |
440 | end | 283 | end |
441 | end | 284 | end |
442 | local ftp = "[<font color='grey'><b>skip</b></font>]" | 285 | local ftp = "[<font color='grey'><b>skip</b></font>]" |
@@ -455,11 +298,11 @@ if nil == web then C("opening mirrors file - " .. e) else | |||
455 | local spd = '' | 298 | local spd = '' |
456 | 299 | ||
457 | -- DNS-RR test. | 300 | -- DNS-RR test. |
458 | if ("deb.devuan.org" ~= k) and (nil ~= mirrors["deb.devuan.org"]) then | 301 | if ("deb.devuan.org" ~= k) and (nil ~= APT.mirrors["deb.devuan.org"]) then |
459 | for l, w in pairs(mirrors[k].IPs) do | 302 | for l, w in pairs(APT.mirrors[k].IPs) do |
460 | if type(w) == "table" then | 303 | if type(w) == "table" then |
461 | for i, u in pairs(w) do | 304 | for i, u in pairs(w) do |
462 | if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then | 305 | if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then |
463 | local log = logCount("deb.devuan.org", i) | 306 | local log = logCount("deb.devuan.org", i) |
464 | if "" ~= log then | 307 | if "" ~= log then |
465 | if "" == dns then dns = " " else dns = dns .. " " end | 308 | if "" == dns then dns = " " else dns = dns .. " " end |
@@ -471,7 +314,7 @@ if nil == web then C("opening mirrors file - " .. e) else | |||
471 | end | 314 | end |
472 | end | 315 | end |
473 | else | 316 | else |
474 | if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then | 317 | if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then |
475 | local log = logCount("deb.devuan.org", l) | 318 | local log = logCount("deb.devuan.org", l) |
476 | if "" ~= log then | 319 | if "" ~= log then |
477 | if "" == dns then dns = " " else dns = dns .. " " end | 320 | if "" == dns then dns = " " else dns = dns .. " " end |
@@ -502,18 +345,18 @@ if nil == web then C("opening mirrors file - " .. e) else | |||
502 | web:write( "</table>\n<br>\n<h2>==== faulty mirrors: ====</h2>\n" .. faulty) | 345 | web:write( "</table>\n<br>\n<h2>==== faulty mirrors: ====</h2>\n" .. faulty) |
503 | web:write( "<br>\n<br>\n<h2>==== DNS and logs: ====</h2>\n") | 346 | web:write( "<br>\n<br>\n<h2>==== DNS and logs: ====</h2>\n") |
504 | 347 | ||
505 | for k, v in pairs(mirrors) do | 348 | for k, v in pairs(APT.mirrors) do |
506 | local log = k | 349 | local log = k |
507 | local n = {} | 350 | local n = {} |
508 | log = logCount(k) | 351 | log = logCount(k) |
509 | mirrors[k].Protocols = nil | 352 | APT.mirrors[k].Protocols = nil |
510 | mirrors[k].FQDN = nil | 353 | APT.mirrors[k].FQDN = nil |
511 | mirrors[k].Active = nil | 354 | APT.mirrors[k].Active = nil |
512 | mirrors[k].Rate = nil | 355 | APT.mirrors[k].Rate = nil |
513 | mirrors[k].BaseURL = nil | 356 | APT.mirrors[k].BaseURL = nil |
514 | mirrors[k].Country = nil | 357 | APT.mirrors[k].Country = nil |
515 | mirrors[k].Bandwidth = nil | 358 | APT.mirrors[k].Bandwidth = nil |
516 | for l, w in pairs(mirrors[k].IPs) do | 359 | for l, w in pairs(APT.mirrors[k].IPs) do |
517 | if type(w) == "table" then | 360 | if type(w) == "table" then |
518 | n[l] = {} | 361 | n[l] = {} |
519 | for i, u in pairs(w) do | 362 | for i, u in pairs(w) do |
@@ -535,11 +378,12 @@ if nil == web then C("opening mirrors file - " .. e) else | |||
535 | "pkgmaster.devuan.org is the master mirror, all the others sync to it. " .. | 378 | "pkgmaster.devuan.org is the master mirror, all the others sync to it. " .. |
536 | "</p>\n" | 379 | "</p>\n" |
537 | ) | 380 | ) |
538 | web:write(dumpTableHTML(m, "", "")) | ||
539 | web:write( "\n<br>\n<hr>\n\n" .. | 381 | web:write( "\n<br>\n<hr>\n\n" .. |
382 | web:write(APT.dumpTableHTML(m, "", "")) | ||
540 | "<p>The <a href='Report-email.txt'>email report</a>. " .. | 383 | "<p>The <a href='Report-email.txt'>email report</a>. " .. |
541 | "All <a href='../results'>the logs and other output</a>. " .. | 384 | "All <a href='../results'>the logs and other output</a>. " .. |
542 | "You can get the <a href='https://sledjhamr.org/cgit/apt-panopticon/about/'>source code here</a>.</p>" .. | 385 | "You can get the <a href='https://sledjhamr.org/cgit/apt-panopticon/about/'>source code here</a>.</p>" .. |
543 | "</body></html>\n") | 386 | "</body></html>\n") |
544 | web:close() | 387 | web:close() |
545 | end | 388 | end |
389 | |||
diff --git a/apt-panopticon.lua b/apt-panopticon.lua index 37778ac..e454740 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua | |||
@@ -1,7 +1,13 @@ | |||
1 | #!/usr/bin/env luajit | 1 | #!/usr/bin/env luajit |
2 | 2 | ||
3 | 3 | local APT = require 'apt-panopticommon' | |
4 | local args = {...} | 4 | local D = APT.D |
5 | local I = APT.I | ||
6 | local W = APT.W | ||
7 | local E = APT.E | ||
8 | local C = APT.C | ||
9 | local arg, sendArgs = APT.parseArgs({...}) | ||
10 | APT.html = true | ||
5 | 11 | ||
6 | --[[ TODO - What to do about HTTPS://deb.devuan.org/ redirects. | 12 | --[[ TODO - What to do about HTTPS://deb.devuan.org/ redirects. |
7 | Some mirrors give a 404. | 13 | Some mirrors give a 404. |
@@ -10,70 +16,6 @@ local args = {...} | |||
10 | They shouldn't have the proper certificate, but are giving a result anyway. | 16 | They shouldn't have the proper certificate, but are giving a result anyway. |
11 | ]] | 17 | ]] |
12 | 18 | ||
13 | origin = false | ||
14 | verbosity = -1 | ||
15 | keep = false | ||
16 | -- TODO - Should actually implement this. | ||
17 | fork = true | ||
18 | options = | ||
19 | { | ||
20 | referenceSite = | ||
21 | { | ||
22 | typ = "string", | ||
23 | help = "", | ||
24 | value = "pkgmaster.devuan.org", | ||
25 | }, | ||
26 | roundRobin = | ||
27 | { | ||
28 | typ = "string", | ||
29 | help = "", | ||
30 | value = "deb.devuan.org", | ||
31 | }, | ||
32 | tests = | ||
33 | { | ||
34 | typ = "table", | ||
35 | help = "", | ||
36 | value = | ||
37 | { | ||
38 | "IPv4", | ||
39 | "IPv6", | ||
40 | -- "ftp", | ||
41 | "http", | ||
42 | "https", | ||
43 | -- "rsync", | ||
44 | "DNSRR", | ||
45 | "Protocol", | ||
46 | "URLSanity", | ||
47 | "Integrity", | ||
48 | "Updated", | ||
49 | }, | ||
50 | }, | ||
51 | maxtime = | ||
52 | { | ||
53 | typ = "number", | ||
54 | help = "", | ||
55 | value = 300, | ||
56 | }, | ||
57 | timeout = | ||
58 | { | ||
59 | typ = "number", | ||
60 | help = "", | ||
61 | value = 15, | ||
62 | }, | ||
63 | reports = | ||
64 | { | ||
65 | typ = "table", | ||
66 | help = "", | ||
67 | value = | ||
68 | { | ||
69 | "email-web", | ||
70 | -- "Nagios", | ||
71 | -- "Prometheus", | ||
72 | -- "RRD", | ||
73 | }, | ||
74 | }, | ||
75 | } | ||
76 | |||
77 | local defaultURL = {scheme = "http"} | 19 | local defaultURL = {scheme = "http"} |
78 | local releases = {"jessie", "ascii", "beowulf", "ceres"} | 20 | local releases = {"jessie", "ascii", "beowulf", "ceres"} |
79 | local releaseFiles = | 21 | local releaseFiles = |
@@ -108,9 +50,6 @@ local referenceDevs = | |||
108 | "merged/pool/DEVUAN/main/d/desktop-base/desktop-base_3.0_all.deb", | 50 | "merged/pool/DEVUAN/main/d/desktop-base/desktop-base_3.0_all.deb", |
109 | "merged/pool/DEVUAN/main/u/util-linux/util-linux_2.32.1-0.1+devuan2.1_amd64.deb", | 51 | "merged/pool/DEVUAN/main/u/util-linux/util-linux_2.32.1-0.1+devuan2.1_amd64.deb", |
110 | } | 52 | } |
111 | local arg = {} | ||
112 | local sendArgs = "" | ||
113 | local logFile | ||
114 | 53 | ||
115 | local curlStatus = | 54 | local curlStatus = |
116 | { | 55 | { |
@@ -218,151 +157,8 @@ local http = require 'socket.http' | |||
218 | local url = require 'socket.url' | 157 | local url = require 'socket.url' |
219 | 158 | ||
220 | 159 | ||
221 | -- Use this to dump a table to a string. | ||
222 | dumpTable = function (table, space, name) | ||
223 | local r = "" | ||
224 | if "" == space then r = r .. space .. name .. " =\n" else r = r .. space .. "[" .. name .. "] =\n" end | ||
225 | r = r .. space .. "{\n" | ||
226 | r = r .. dumpTableSub(table, space .. " ") | ||
227 | if "" == space then r = r .. space .. "}\n" else r = r .. space .. "},\n" end | ||
228 | return r | ||
229 | end | ||
230 | dumpTableSub = function (table, space) | ||
231 | local r = "" | ||
232 | for k, v in pairs(table) do | ||
233 | if type(k) == "string" then k = '"' .. k .. '"' end | ||
234 | if type(v) == "table" then | ||
235 | r = r .. dumpTable(v, space, k) | ||
236 | elseif type(v) == "string" then | ||
237 | r = r .. space .. "[" .. k .. "] = '" .. v .. "';\n" | ||
238 | elseif type(v) == "function" then | ||
239 | r = r .. space .. "[" .. k .. "] = function ();\n" | ||
240 | elseif type(v) == "userdata" then | ||
241 | r = r .. space .. "userdata " .. "[" .. k .. "];\n" | ||
242 | elseif type(v) == "boolean" then | ||
243 | if (v) then | ||
244 | r = r .. space .. "[" .. k .. "] = true;\n" | ||
245 | else | ||
246 | r = r .. space .. "[" .. k .. "] = false;\n" | ||
247 | end | ||
248 | else | ||
249 | r = r .. space .. "[" .. k .. "] = " .. v .. ";\n" | ||
250 | end | ||
251 | end | ||
252 | return r | ||
253 | end | ||
254 | |||
255 | local ip = "" | 160 | local ip = "" |
256 | local results = {} | ||
257 | |||
258 | local logPre = function() | ||
259 | if nil ~= logFile then | ||
260 | logFile:write("<html><head>\n") | ||
261 | logFile:write("</head><body bgcolor='black' text='white' alink='red' link='blue' vlink='purple'>\n") | ||
262 | end | ||
263 | end | ||
264 | local logPost = function() | ||
265 | if nil ~= logFile then | ||
266 | logFile:write("</body></html> \n") | ||
267 | end | ||
268 | end | ||
269 | |||
270 | local log = function(v, t, s, prot, test, host) | ||
271 | local x = "" | ||
272 | if nil == prot then prot = "" end | ||
273 | if nil == test then test = "" end | ||
274 | x = x .. prot | ||
275 | if "" ~= test then | ||
276 | if #x > 0 then x = x .. " " end | ||
277 | x = x .. test | ||
278 | end | ||
279 | if nil ~= host then | ||
280 | if #x > 0 then x = x .. " " end | ||
281 | x = x .. host | ||
282 | end | ||
283 | if #x > 0 then | ||
284 | t = t .. "(" .. x .. ")" | ||
285 | if "" ~= prot then | ||
286 | if "" == test then | ||
287 | if nil == results[prot] then results[prot] = {errors = 0; warnings = 0} end | ||
288 | if v == 0 then results[prot].errors = results[prot].errors + 1 end | ||
289 | if v == 1 then results[prot].warnings = results[prot].warnings + 1 end | ||
290 | else | ||
291 | if nil == results[prot] then results[prot] = {errors = 0; warnings = 0} end | ||
292 | if nil == results[prot][test] then results[prot][test] = {errors = 0; warnings = 0} end | ||
293 | if v == 0 then results[prot][test].errors = results[prot][test].errors + 1 end | ||
294 | if v == 1 then results[prot][test].warnings = results[prot][test].warnings + 1 end | ||
295 | end | ||
296 | end | ||
297 | end | ||
298 | if v <= verbosity then | ||
299 | if 3 <= verbosity then t = os.date() .. " " .. t end | ||
300 | print(t .. ": " .. s) | ||
301 | end | ||
302 | if nil ~= logFile then | ||
303 | local colour = "white" | ||
304 | if -1 == v then colour = "fuchsia" end -- CRITICAL | ||
305 | if 0 == v then colour = "red " end -- ERROR | ||
306 | if 1 == v then colour = "yellow " end -- WARNING | ||
307 | if 2 == v then colour = "white " end -- INFO | ||
308 | if 3 == v then colour = "gray " end -- DEBUG | ||
309 | logFile:write(os.date() .. " <font color='" .. colour .. "'><b>" .. t .. "</b></font>: " .. s .. "</br>\n") | ||
310 | logFile:flush() | ||
311 | end | ||
312 | end | ||
313 | local D = function(s) log(3, "DEBUG ", s) end | ||
314 | local I = function(s) log(2, "INFO ", s) end | ||
315 | local W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end | ||
316 | local E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end | ||
317 | local C = function(s) log(-1, "CRITICAL", s) end | ||
318 | |||
319 | local mirrors = {} | ||
320 | |||
321 | local testing = function(t, host) | ||
322 | for i, v in pairs(options.tests.value) do | ||
323 | if t == v then | ||
324 | local h = mirrors[host] | ||
325 | if nil == h then return true end | ||
326 | if true == h["Protocols"][t] then return true else D("Skipping " .. t .. " checks for " .. host) end | ||
327 | end | ||
328 | end | ||
329 | return false | ||
330 | end | ||
331 | |||
332 | local execute = function (s) | ||
333 | D(" executing <pre><code>" .. s .. "</code></pre>") | ||
334 | --[[ Damn os.execute() | ||
335 | Lua 5.1 says it returns "a status code, which is system-dependent" | ||
336 | Lua 5.2 says it returns true/nil, "exit"/"signal", the status code. | ||
337 | I'm getting 7168 or 0. No idea what the fuck that is. | ||
338 | local ok, rslt, status = os.execute(s) | ||
339 | ]] | ||
340 | local f = io.popen(s .. ' ; echo "$?"', 'r') | ||
341 | local status = "" | ||
342 | local result = "" | ||
343 | -- The last line will be the command's returned status, collect everything else in result. | ||
344 | for l in f:lines() do | ||
345 | result = result .. status .. "\n" | ||
346 | status = l | ||
347 | end | ||
348 | return status, result | ||
349 | end | ||
350 | |||
351 | local fork = function(s) | ||
352 | D(" forking <pre><code>" .. s .. "</code></pre>") | ||
353 | os.execute(s .. " &") | ||
354 | end | ||
355 | 161 | ||
356 | local checkExes = function (exe) | ||
357 | local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | grep -v "flock -n apt-panopticon.lock " | wc -l'):read("*l") | ||
358 | D(count .. " " .. exe .. " commands still running.") | ||
359 | return tonumber(count) | ||
360 | end | ||
361 | |||
362 | local checkFile = function(f) | ||
363 | local h, e = io.open(f, "r") | ||
364 | if nil == h then return false else h:close(); return true end | ||
365 | end | ||
366 | 162 | ||
367 | local repoExists = function (r) | 163 | local repoExists = function (r) |
368 | r = r:match("([%a-]*)") | 164 | r = r:match("([%a-]*)") |
@@ -441,9 +237,9 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
441 | return | 237 | return |
442 | end | 238 | end |
443 | D(PU.scheme .. " :// " .. check .. " " .. host .. " -> " .. URL) | 239 | D(PU.scheme .. " :// " .. check .. " " .. host .. " -> " .. URL) |
444 | if not testing(PU.scheme, host) then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL); return end | 240 | if not APT.testing(PU.scheme, host) then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL); return end |
445 | -- TODO - Perhaps we should try it anyway, and mark it as a warning if it DOES work? | 241 | -- TODO - Perhaps we should try it anyway, and mark it as a warning if it DOES work? |
446 | if "https" == PU.scheme and options.roundRobin.value == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert."); return end | 242 | if "https" == PU.scheme and APT.options.roundRobin.value == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert."); return end |
447 | 243 | ||
448 | --[[ Using curl command line - | 244 | --[[ Using curl command line - |
449 | -I - HEAD | 245 | -I - HEAD |
@@ -468,10 +264,10 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
468 | end | 264 | end |
469 | IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"' | 265 | IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"' |
470 | end | 266 | end |
471 | local cmd = 'ionice -c3 nice -n 19 curl -I --retry 0 -s --path-as-is --connect-timeout ' .. options.timeout.value .. ' --max-redirs 0 ' .. | 267 | local cmd = 'ionice -c3 nice -n 19 curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' .. |
472 | IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' .. | 268 | IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' .. |
473 | hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"' | 269 | hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"' |
474 | local status, result = execute(cmd) | 270 | local status, result = APT.execute(cmd) |
475 | os.execute('cat results/"HEADERS_' .. fname .. '" >>results/"STATUS_' .. fname .. '" 2>/dev/null; rm -f results/"HEADERS_' .. fname .. '" 2>/dev/null') | 271 | os.execute('cat results/"HEADERS_' .. fname .. '" >>results/"STATUS_' .. fname .. '" 2>/dev/null; rm -f results/"HEADERS_' .. fname .. '" 2>/dev/null') |
476 | if "0" ~= status then | 272 | if "0" ~= status then |
477 | local msg = curlStatus[0 + status] | 273 | local msg = curlStatus[0 + status] |
@@ -522,7 +318,7 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
522 | if nil ~= location then | 318 | if nil ~= location then |
523 | pu = url.parse(location, defaultURL) | 319 | pu = url.parse(location, defaultURL) |
524 | if ('http' == location:sub(1, 4)) and (pu.scheme ~= PU.scheme) then -- Sometimes a location sans scheme is returned, this is not a protocol change. | 320 | if ('http' == location:sub(1, 4)) and (pu.scheme ~= PU.scheme) then -- Sometimes a location sans scheme is returned, this is not a protocol change. |
525 | if testing("Protocol") then W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host) end | 321 | if APT.testing("Protocol") then W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host) end |
526 | if (pu.host == host) and pu.path == PU.path then D("Not testing protocol change " .. URL .. " -> " .. location); return end | 322 | if (pu.host == host) and pu.path == PU.path then D("Not testing protocol change " .. URL .. " -> " .. location); return end |
527 | end | 323 | end |
528 | 324 | ||
@@ -555,7 +351,7 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
555 | end | 351 | end |
556 | 352 | ||
557 | local checkTimeouts = function(host, scheme, URL) | 353 | local checkTimeouts = function(host, scheme, URL) |
558 | if testing(scheme) then | 354 | if APT.testing(scheme) then |
559 | totalTimeouts = totalTimeouts + timeouts; timeouts = 0 | 355 | totalTimeouts = totalTimeouts + timeouts; timeouts = 0 |
560 | checkHEAD(host, scheme .. "://" .. URL) | 356 | checkHEAD(host, scheme .. "://" .. URL) |
561 | if 4 <= (totalTimeouts) then | 357 | if 4 <= (totalTimeouts) then |
@@ -563,7 +359,7 @@ local checkTimeouts = function(host, scheme, URL) | |||
563 | return true | 359 | return true |
564 | end | 360 | end |
565 | end | 361 | end |
566 | if testing("URLSanity") then | 362 | if APT.testing("URLSanity") then |
567 | URL = URL:gsub("merged/", "merged///") | 363 | URL = URL:gsub("merged/", "merged///") |
568 | totalTimeouts = totalTimeouts + timeouts; timeouts = 0 | 364 | totalTimeouts = totalTimeouts + timeouts; timeouts = 0 |
569 | checkHEAD(host, scheme .. "://" .. URL, 0, 0, true) | 365 | checkHEAD(host, scheme .. "://" .. URL, 0, 0, true) |
@@ -617,24 +413,24 @@ checkHost = function (orig, host, path, ip, file) | |||
617 | else | 413 | else |
618 | if orig == host then | 414 | if orig == host then |
619 | D("checkHost " .. orig .. "" .. file) | 415 | D("checkHost " .. orig .. "" .. file) |
620 | if testing("IPv4") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file) end | 416 | if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file) end |
621 | else D("checkHost " .. orig .. " -> " .. host) end | 417 | else D("checkHost " .. orig .. " -> " .. host) end |
622 | local h = mirrors[ph.host] | 418 | local h = APT.mirrors[ph.host] |
623 | if nil == h then return end | 419 | if nil == h then return end |
624 | for k, v in pairs(h.IPs) do | 420 | for k, v in pairs(h.IPs) do |
625 | if "table" == type(v) then | 421 | if "table" == type(v) then |
626 | for k1, v1 in pairs(v) do | 422 | for k1, v1 in pairs(v) do |
627 | if v1 == "A" then | 423 | if v1 == "A" then |
628 | if testing("IPv4") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end | 424 | if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end |
629 | elseif v1 == "AAAA" then | 425 | elseif v1 == "AAAA" then |
630 | if testing("IPv6") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end | 426 | if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end |
631 | end | 427 | end |
632 | end | 428 | end |
633 | else | 429 | else |
634 | if v == "A" then | 430 | if v == "A" then |
635 | if testing("IPv4") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end | 431 | if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end |
636 | elseif v == "AAAA" then | 432 | elseif v == "AAAA" then |
637 | if testing("IPv6") then fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end | 433 | if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end |
638 | end | 434 | end |
639 | end | 435 | end |
640 | end | 436 | end |
@@ -644,14 +440,12 @@ end | |||
644 | 440 | ||
645 | local addDownload = function(host, URL, f, r, k) | 441 | local addDownload = function(host, URL, f, r, k) |
646 | local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename. | 442 | local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename. |
647 | local o, e = io.open("results/" .. host .. "/merged/dists/" .. r .. k, "r") | 443 | if APT.checkFile("results/" .. host .. "/merged/dists/" .. r .. k) then |
648 | if nil ~= o then | ||
649 | o:close() | ||
650 | -- Curls "check timestamp and overwrite file" stuff sucks. | 444 | -- Curls "check timestamp and overwrite file" stuff sucks. |
651 | -- -R means the destination file gets the timestamp of the remote file. | 445 | -- -R means the destination file gets the timestamp of the remote file. |
652 | -- Can only do ONE timestamp check per command. | 446 | -- Can only do ONE timestamp check per command. |
653 | -- This doesn't work either. All downloads get all these headers. Pffft | 447 | -- This doesn't work either. All downloads get all these headers. Pffft |
654 | -- local status, ts = execute('TZ="GMT" ls -l --time-style="+%a, %d %b %Y %T %Z" results/' .. host .. "/merged/dists/" .. r .. k .. ' | cut -d " " -f 6-11') | 448 | -- local status, ts = APT.execute('TZ="GMT" ls -l --time-style="+%a, %d %b %Y %T %Z" results/' .. host .. "/merged/dists/" .. r .. k .. ' | cut -d " " -f 6-11') |
655 | -- f:write('header "If-Modified-Since: ' .. ts:sub(2, -2) .. '"\n') | 449 | -- f:write('header "If-Modified-Since: ' .. ts:sub(2, -2) .. '"\n') |
656 | -- Curl will DELETE the existing file if the timestamp fails to download a new one, unless we change directory first, | 450 | -- Curl will DELETE the existing file if the timestamp fails to download a new one, unless we change directory first, |
657 | -- which wont work with multiple files in multiple directories. WTF? | 451 | -- which wont work with multiple files in multiple directories. WTF? |
@@ -671,21 +465,21 @@ local postDownload = function(host, r, k) | |||
671 | " && [ ! -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then cp -a" .. | 465 | " && [ ! -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then cp -a" .. |
672 | " results/" .. host .. "/merged/dists/" .. r .. k .. ".old" .. | 466 | " results/" .. host .. "/merged/dists/" .. r .. k .. ".old" .. |
673 | " results/" .. host .. "/merged/dists/" .. r .. k .. "; fi") | 467 | " results/" .. host .. "/merged/dists/" .. r .. k .. "; fi") |
674 | if ".gz" == k:sub(-3, -1) then execute("ionice -c3 nice -n 19 gzip -dfk results/" .. host .. "/merged/dists/" .. r .. k) end | 468 | if ".gz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 gzip -dfk results/" .. host .. "/merged/dists/" .. r .. k) end |
675 | if ".xz" == k:sub(-3, -1) then execute("ionice -c3 nice -n 19 xz -dfk results/" .. host .. "/merged/dists/" .. r .. k .. " 2>/dev/null") end | 469 | if ".xz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 xz -dfk results/" .. host .. "/merged/dists/" .. r .. k .. " 2>/dev/null") end |
676 | if testing("Integrity") then | 470 | if APT.testing("Integrity") then |
677 | if ".gpg" == k:sub(-4, -1) then | 471 | if ".gpg" == k:sub(-4, -1) then |
678 | local status, out = execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. r .. k .. | 472 | local status, out = APT.execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. r .. k .. |
679 | " results/" .. host .. "/merged/dists/" .. r .. k:sub(1, -5) .. " 2>/dev/null") | 473 | " results/" .. host .. "/merged/dists/" .. r .. k:sub(1, -5) .. " 2>/dev/null") |
680 | if "0" ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. r .. k, "http", "Integrity", host) end | 474 | if "0" ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. r .. k, "http", "Integrity", host) end |
681 | end | 475 | end |
682 | -- TODO - should check the PGP sig of InRelease as well. | 476 | -- TODO - should check the PGP sig of InRelease as well. |
683 | end | 477 | end |
684 | if testing("Integrity") or testing("Updated") then | 478 | if APT.testing("Integrity") or APT.testing("Updated") then |
685 | if "Packages." == file:sub(1, 9) then | 479 | if "Packages." == file:sub(1, 9) then |
686 | -- TODO - compare the SHA256 sums in pkgmaster's Release for both the packed and unpacked versions. | 480 | -- TODO - compare the SHA256 sums in pkgmaster's Release for both the packed and unpacked versions. |
687 | -- Also note that this might get only a partial download due to maxtime. | 481 | -- Also note that this might get only a partial download due to maxtime. |
688 | if options.referenceSite.value == host then | 482 | if APT.options.referenceSite.value == host then |
689 | local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed', "w+") | 483 | local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed', "w+") |
690 | if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed' .. ' file - ' .. e) else | 484 | if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed' .. ' file - ' .. e) else |
691 | local pp = {} | 485 | local pp = {} |
@@ -711,7 +505,7 @@ local postDownload = function(host, r, k) | |||
711 | end | 505 | end |
712 | Pp:close() | 506 | Pp:close() |
713 | os.execute('sort results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed >results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages_parsed-sorted') | 507 | os.execute('sort results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed >results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages_parsed-sorted') |
714 | if checkFile('Packages/' .. r .. dir .. 'Packages_parsed-sorted') then | 508 | if APT.checkFile('Packages/' .. r .. dir .. 'Packages_parsed-sorted') then |
715 | os.execute('diff -U 0 Packages/' .. r .. dir .. 'Packages_parsed-sorted ' .. | 509 | os.execute('diff -U 0 Packages/' .. r .. dir .. 'Packages_parsed-sorted ' .. |
716 | 'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages_parsed-sorted ' .. | 510 | 'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages_parsed-sorted ' .. |
717 | ' | grep -E "^-" | grep -Ev "^\\+\\+\\+|^---" >>results/OLD_PACKAGES_' .. r .. '.txt') | 511 | ' | grep -E "^-" | grep -Ev "^\\+\\+\\+|^---" >>results/OLD_PACKAGES_' .. r .. '.txt') |
@@ -737,14 +531,14 @@ local postDownload = function(host, r, k) | |||
737 | end | 531 | end |
738 | 532 | ||
739 | local downloadLock = "flock -n results/curl-" | 533 | local downloadLock = "flock -n results/curl-" |
740 | local download = "curl --connect-timeout " .. options.timeout.value .. " --create-dirs -f -L --max-time " .. options.maxtime.value .. " -z 'results/stamp.old' -v -R " | 534 | local download = "curl --connect-timeout " .. APT.options.timeout.value .. " --create-dirs -f -L --max-time " .. APT.options.maxtime.value .. " -z 'results/stamp.old' -v -R " |
741 | local downloads = function(host, URL, release, list) | 535 | local downloads = function(host, URL, release, list) |
742 | if nil == URL then URL = "" end | 536 | if nil == URL then URL = "" end |
743 | local lock = "META-" .. host .. ".lock" | 537 | local lock = "META-" .. host .. ".lock" |
744 | local log = " --stderr results/curl-META-" .. host .. ".log" | 538 | local log = " --stderr results/curl-META-" .. host .. ".log" |
745 | local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock .. " " .. download .. log .. " -K results/" .. host .. ".curl" | 539 | local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock .. " " .. download .. log .. " -K results/" .. host .. ".curl" |
746 | if testing("IPv4") and (not testing("IPv6")) then cm = cm .. ' -4' end | 540 | if APT.testing("IPv4") and (not APT.testing("IPv6")) then cm = cm .. ' -4' end |
747 | if (not testing("IPv4")) and testing("IPv6") then cm = cm .. ' -6' end | 541 | if (not APT.testing("IPv4")) and APT.testing("IPv6") then cm = cm .. ' -6' end |
748 | f, e = io.open("results/" .. host .. ".curl", "a+") | 542 | f, e = io.open("results/" .. host .. ".curl", "a+") |
749 | if nil == f then C("opening curl file - " .. e); return end | 543 | if nil == f then C("opening curl file - " .. e); return end |
750 | 544 | ||
@@ -772,7 +566,7 @@ local downloads = function(host, URL, release, list) | |||
772 | end | 566 | end |
773 | end | 567 | end |
774 | f:close() | 568 | f:close() |
775 | fork(cm) | 569 | APT.fork(cm) |
776 | end | 570 | end |
777 | 571 | ||
778 | 572 | ||
@@ -781,7 +575,7 @@ local getMirrors = function () | |||
781 | local host = "" | 575 | local host = "" |
782 | local m = {} | 576 | local m = {} |
783 | local active = true | 577 | local active = true |
784 | local URL = "http://" .. options.referenceSite.value .. "/mirror_list.txt" | 578 | local URL = "http://" .. APT.options.referenceSite.value .. "/mirror_list.txt" |
785 | I("getting mirrors.") | 579 | I("getting mirrors.") |
786 | local p, c, h = http.request(URL) | 580 | local p, c, h = http.request(URL) |
787 | if nil == p then E(c .. " fetching " .. URL) else | 581 | if nil == p then E(c .. " fetching " .. URL) else |
@@ -818,95 +612,20 @@ local getMirrors = function () | |||
818 | mirrors[host] = m | 612 | mirrors[host] = m |
819 | end | 613 | end |
820 | end | 614 | end |
821 | if testing("DNSRR") then | 615 | if APT.testing("DNSRR") then |
822 | mirrors[options.roundRobin.value] = { ["Protocols"] = { ["http"] = true; ["https"] = true; }; ["FQDN"] = 'deb.devuan.org'; ["Active"] = 'yes'; ["BaseURL"] = 'deb.devuan.org'; } | 616 | mirrors[APT.options.roundRobin.value] = { ["Protocols"] = { ["http"] = true; ["https"] = true; }; ["FQDN"] = 'deb.devuan.org'; ["Active"] = 'yes'; ["BaseURL"] = 'deb.devuan.org'; } |
823 | gatherIPs(options.roundRobin.value) | 617 | gatherIPs(APT.options.roundRobin.value) |
824 | mirrors[options.roundRobin.value].IPs = IP[options.roundRobin.value] | 618 | mirrors[APT.options.roundRobin.value].IPs = IP[APT.options.roundRobin.value] |
825 | end | 619 | end |
826 | local file, e = io.open("results/mirrors.lua", "w+") | 620 | local file, e = io.open("results/mirrors.lua", "w+") |
827 | if nil == file then C("opening mirrors file - " .. e) else | 621 | if nil == file then C("opening mirrors file - " .. e) else |
828 | file:write(dumpTable(mirrors, "", "mirrors") .. "\nreturn mirrors\n") | 622 | file:write(APT.dumpTable(mirrors, "", "mirrors") .. "\nreturn mirrors\n") |
829 | file:close() | 623 | file:close() |
830 | end | 624 | end |
831 | return mirrors | 625 | return mirrors |
832 | end | 626 | end |
833 | 627 | ||
834 | 628 | ||
835 | if 0 ~= #args then | ||
836 | local option = "" | ||
837 | for i, a in pairs(args) do | ||
838 | if ("--help" == a) or ("-h" == a) then | ||
839 | print("I should write some docs, huh? Read README.md for instructions.") | ||
840 | elseif "--version" == a then | ||
841 | print("apt-panopticon version 0.1 WIP development version") | ||
842 | elseif "-v" == a then | ||
843 | verbosity = verbosity + 1 | ||
844 | sendArgs = sendArgs .. a .. " " | ||
845 | elseif "-q" == a then | ||
846 | verbosity = -1 | ||
847 | sendArgs = sendArgs .. a .. " " | ||
848 | elseif "-k" == a then | ||
849 | keep = true | ||
850 | elseif "-n" == a then | ||
851 | fork = false | ||
852 | elseif "-o" == a then | ||
853 | origin = true | ||
854 | elseif "--" == a:sub(1, 2) then | ||
855 | local s, e = a:find("=") | ||
856 | if nil == s then e = -1 end | ||
857 | option = a:sub(3, e - 1) | ||
858 | local o = options[option] | ||
859 | if nil == o then | ||
860 | print("Unknown option --" .. option) | ||
861 | option = "" | ||
862 | else | ||
863 | option = a | ||
864 | sendArgs = sendArgs .. a .. " " | ||
865 | local s, e = a:find("=") | ||
866 | if nil == s then e = 0 end | ||
867 | option = a:sub(3, e - 1) | ||
868 | if "table" == options[option].typ then | ||
869 | local result = {} | ||
870 | for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do | ||
871 | local f = t:sub(1, 1) | ||
872 | local n = t:sub(2, -1) | ||
873 | if ("+" ~= f) and ("-" ~= f) then | ||
874 | table.insert(result, t) | ||
875 | end | ||
876 | end | ||
877 | if 0 ~= #result then | ||
878 | options[option].value = result | ||
879 | else | ||
880 | for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do | ||
881 | local f = t:sub(1, 1) | ||
882 | local n = t:sub(2, -1) | ||
883 | if "+" == f then | ||
884 | table.insert(options[option].value, n) | ||
885 | elseif "-" == f then | ||
886 | local r = {} | ||
887 | for i, k in pairs(options[option].value) do | ||
888 | if k ~= n then table.insert(r, k) end | ||
889 | end | ||
890 | options[option].value = r | ||
891 | end | ||
892 | end | ||
893 | end | ||
894 | else | ||
895 | options[option].value = a:sub(e + 1, -1) | ||
896 | end | ||
897 | option = "" | ||
898 | end | ||
899 | elseif "-" == a:sub(1, 1) then | ||
900 | print("Unknown option " .. a) | ||
901 | else | ||
902 | table.insert(arg, a) | ||
903 | end | ||
904 | end | ||
905 | end | ||
906 | |||
907 | --print(dumpTable(options.tests.value, "", "tests")) | ||
908 | |||
909 | |||
910 | if 0 < #arg then | 629 | if 0 < #arg then |
911 | if "/" == arg[1]:sub(-1, -1) then | 630 | if "/" == arg[1]:sub(-1, -1) then |
912 | W("slash at end of path! " .. arg[1]) | 631 | W("slash at end of path! " .. arg[1]) |
@@ -918,37 +637,37 @@ if 0 < #arg then | |||
918 | end | 637 | end |
919 | local pu = url.parse("http://" .. arg[1]) | 638 | local pu = url.parse("http://" .. arg[1]) |
920 | 639 | ||
921 | if testing("Integrity") or testing("Updated") then | 640 | if APT.testing("Integrity") or APT.testing("Updated") then |
922 | if origin and options.referenceSite.value == pu.host then | 641 | if APT.origin and APT.options.referenceSite.value == pu.host then |
923 | -- if not keep then execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end | 642 | -- if not APT.keep then os.execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end |
924 | end | 643 | end |
925 | end | 644 | end |
926 | 645 | ||
927 | if nil ~= arg[2] then | 646 | if nil ~= arg[2] then |
928 | logFile, e = io.open("results/LOG_" .. pu.host .. "_" .. arg[2] .. ".html", "a+") | 647 | APT.logFile, e = io.open("results/LOG_" .. pu.host .. "_" .. arg[2] .. ".html", "a+") |
929 | else | 648 | else |
930 | logFile, e = io.open("results/LOG_" .. pu.host .. ".html", "a+") | 649 | APT.logFile, e = io.open("results/LOG_" .. pu.host .. ".html", "a+") |
931 | end | 650 | end |
932 | if nil == logFile then C("opening log file - " .. e); return end | 651 | if nil == APT.logFile then C("opening log file - " .. e); return end |
933 | logPre() | 652 | APT.logPre() |
934 | I("Starting tests for " ..arg[1] .. " with these tests - " .. table.concat(options.tests.value, ", ")) | 653 | I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", ")) |
935 | mirrors = loadfile("results/mirrors.lua")() | 654 | APT.mirrors = loadfile("results/mirrors.lua")() |
936 | if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end | 655 | if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end |
937 | if nil ~= arg[3] then I(" Using file " .. arg[3]); end | 656 | if nil ~= arg[3] then I(" Using file " .. arg[3]); end |
938 | 657 | ||
939 | for k, v in pairs{"ftp", "http", "https", "rsync"} do | 658 | for k, v in pairs{"ftp", "http", "https", "rsync"} do |
940 | if testing(v) then | 659 | if APT.testing(v) then |
941 | local tests = {errors = 0; warnings = 0} | 660 | local tests = {errors = 0; warnings = 0} |
942 | if testing("Integrity") then tests.Integrity = {errors = 0; warnings = 0} end | 661 | if APT.testing("Integrity") then tests.Integrity = {errors = 0; warnings = 0} end |
943 | if testing("Protocol") then tests.Protocol = {errors = 0; warnings = 0} end | 662 | if APT.testing("Protocol") then tests.Protocol = {errors = 0; warnings = 0} end |
944 | if testing("Updated") then tests.Updated = {errors = 0; warnings = 0} end | 663 | if APT.testing("Updated") then tests.Updated = {errors = 0; warnings = 0} end |
945 | if testing("URLSanity") then tests.URLSanity = {errors = 0; warnings = 0} end | 664 | if APT.testing("URLSanity") then tests.URLSanity = {errors = 0; warnings = 0} end |
946 | results[v] = tests | 665 | APT.results[v] = tests |
947 | end | 666 | end |
948 | end | 667 | end |
949 | if origin then | 668 | if APT.origin then |
950 | if testing("Integrity") or testing("Updated") then | 669 | if APT.testing("Integrity") or APT.testing("Updated") then |
951 | if origin and (options.roundRobin.value ~= pu.host) then | 670 | if APT.origin and (APT.options.roundRobin.value ~= pu.host) then |
952 | I("Starting file downloads for " .. pu.host) | 671 | I("Starting file downloads for " .. pu.host) |
953 | downloads(pu.host, pu.path) | 672 | downloads(pu.host, pu.path) |
954 | end | 673 | end |
@@ -958,10 +677,10 @@ if 0 < #arg then | |||
958 | checkHost(pu.host, pu.host, pu.path, arg[2], arg[3]) | 677 | checkHost(pu.host, pu.host, pu.path, arg[2], arg[3]) |
959 | end | 678 | end |
960 | 679 | ||
961 | if testing("Integrity") or testing("Updated") then | 680 | if APT.testing("Integrity") or APT.testing("Updated") then |
962 | if 4 > (totalTimeouts) then | 681 | if 4 > (totalTimeouts) then |
963 | if origin and (options.roundRobin.value ~= pu.host) then | 682 | if APT.origin and (APT.options.roundRobin.value ~= pu.host) then |
964 | while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end | 683 | while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end |
965 | os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " .. | 684 | os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " .. |
966 | "mv results/curl-META-" .. pu.host .. ".log results/curl-Release-" .. pu.host .. ".log") | 685 | "mv results/curl-META-" .. pu.host .. ".log results/curl-Release-" .. pu.host .. ".log") |
967 | for i, n in pairs(releases) do | 686 | for i, n in pairs(releases) do |
@@ -971,24 +690,25 @@ if 0 < #arg then | |||
971 | end | 690 | end |
972 | end | 691 | end |
973 | 692 | ||
974 | if checkFile('results/' .. pu.host .. '/merged/dists/' .. n .. '/Release') and | 693 | if APT.checkFile('results/' .. pu.host .. '/merged/dists/' .. n .. '/Release') then |
975 | checkFile('results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED') then | ||
976 | os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED') | 694 | os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED') |
977 | if options.referenceSite.value == pu.host then | 695 | if APT.checkFile('results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED') then |
978 | os.execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. | 696 | if APT.options.referenceSite.value == pu.host then |
979 | 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. | 697 | os.execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. |
980 | '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt') | 698 | 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. |
981 | os.execute('rm -f results/' .. pu.host .. '/merged/dists/' .. n .. '/Release 2>/dev/null') | 699 | '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt') |
982 | -- TODO - Maybe check the date in Release, though since they are updated daily, is there any point? Perhaps it's for checking amprolla got run? | 700 | -- TODO - Maybe check the date in Release, though since they are updated daily, is there any point? Perhaps it's for checking amprolla got run? |
983 | else | 701 | os.execute('rm -f results/' .. pu.host .. '/merged/dists/' .. n .. '/Release 2>/dev/null') |
702 | else | ||
984 | -- TODO - compare to the pkgmaster copy. | 703 | -- TODO - compare to the pkgmaster copy. |
985 | end | 704 | end |
986 | 705 | ||
987 | local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") | 706 | local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") |
988 | if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else | 707 | if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else |
989 | local diff = dfile:read("*a") | 708 | local diff = dfile:read("*a") |
990 | if "" ~= diff then | 709 | if "" ~= diff then |
991 | downloads(pu.host, pu.path, n, diff) | 710 | downloads(pu.host, pu.path, n, diff) |
711 | end | ||
992 | end | 712 | end |
993 | end | 713 | end |
994 | end | 714 | end |
@@ -996,7 +716,7 @@ if 0 < #arg then | |||
996 | end | 716 | end |
997 | 717 | ||
998 | downloads(pu.host, pu.path, "", "") | 718 | downloads(pu.host, pu.path, "", "") |
999 | while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end | 719 | while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end |
1000 | os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " .. | 720 | os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " .. |
1001 | "mv results/curl-META-" .. pu.host .. ".log results/curl-Packages-" .. pu.host .. ".log") | 721 | "mv results/curl-META-" .. pu.host .. ".log results/curl-Packages-" .. pu.host .. ".log") |
1002 | 722 | ||
@@ -1008,7 +728,7 @@ if 0 < #arg then | |||
1008 | postDownload(pu.host, n, "/" .. l) | 728 | postDownload(pu.host, n, "/" .. l) |
1009 | end | 729 | end |
1010 | end | 730 | end |
1011 | if options.referenceSite.value == pu.host then | 731 | if APT.options.referenceSite.value == pu.host then |
1012 | -- In case it wasn't dealt with already. | 732 | -- In case it wasn't dealt with already. |
1013 | os.execute('touch results/NEW_Packages_' .. n .. '.test.txt') | 733 | os.execute('touch results/NEW_Packages_' .. n .. '.test.txt') |
1014 | end | 734 | end |
@@ -1027,26 +747,26 @@ if 0 < #arg then | |||
1027 | end | 747 | end |
1028 | end | 748 | end |
1029 | downloads(pu.host, pu.path, nil, "") | 749 | downloads(pu.host, pu.path, nil, "") |
1030 | while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end | 750 | while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end |
1031 | for i, n in pairs(releases) do | 751 | for i, n in pairs(releases) do |
1032 | local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") | 752 | local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") |
1033 | if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else | 753 | if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else |
1034 | for l in nfile:lines() do | 754 | for l in nfile:lines() do |
1035 | local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |') | 755 | local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |') |
1036 | if nil ~= p then | 756 | if nil ~= p then |
1037 | if checkFile('results/' .. pu.host .. "/merged/" .. p) then | 757 | if APT.checkFile('results/' .. pu.host .. "/merged/" .. p) then |
1038 | local status, fsz = execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5') | 758 | local status, fsz = APT.execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5') |
1039 | if testing("Integrity") then | 759 | if APT.testing("Integrity") then |
1040 | if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. | 760 | if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. |
1041 | E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) | 761 | E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) |
1042 | print('|' .. sz .. '~=' .. fsz:sub(2, -2) .. '|') | 762 | print('|' .. sz .. '~=' .. fsz:sub(2, -2) .. '|') |
1043 | else | 763 | else |
1044 | local status, fsha = execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') | 764 | local status, fsha = APT.execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') |
1045 | if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end | 765 | if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end |
1046 | -- TODO - maybe check the PGP key, though packages are mostly not signed. | 766 | -- TODO - maybe check the PGP key, though packages are mostly not signed. |
1047 | end | 767 | end |
1048 | end | 768 | end |
1049 | if testing("Updated") then | 769 | if APT.testing("Updated") then |
1050 | if sz ~= fsz:sub(2, -2) then | 770 | if sz ~= fsz:sub(2, -2) then |
1051 | E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) | 771 | E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) |
1052 | end | 772 | end |
@@ -1060,20 +780,20 @@ if 0 < #arg then | |||
1060 | end | 780 | end |
1061 | end | 781 | end |
1062 | 782 | ||
1063 | results["timeout"] = false | 783 | APT.results["timeout"] = false |
1064 | else | 784 | else |
1065 | results["timeout"] = true | 785 | APT.results["timeout"] = true |
1066 | end | 786 | end |
1067 | end | 787 | end |
1068 | 788 | ||
1069 | if origin and options.referenceSite.value ~= pu.host then | 789 | if APT.origin and APT.options.referenceSite.value ~= pu.host then |
1070 | if not keep then os.execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end | 790 | if not APT.keep then os.execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end |
1071 | os.execute('rm STATUS_' .. pu.host .. '* 2>/dev/null') | 791 | os.execute('rm STATUS_' .. pu.host .. '* 2>/dev/null') |
1072 | end | 792 | end |
1073 | 793 | ||
1074 | local min, max, spd = 999999999999, 0 | 794 | local min, max, spd = 999999999999, 0 |
1075 | for i, mt in pairs({'Release', 'Packages', 'META'}) do | 795 | for i, mt in pairs({'Release', 'Packages', 'META'}) do |
1076 | if checkFile("results/curl-" .. mt .. "-" .. pu.host .. ".log") then | 796 | if APT.checkFile("results/curl-" .. mt .. "-" .. pu.host .. ".log") then |
1077 | for l in io.lines("results/curl-" .. mt .. "-" .. pu.host .. ".log") do | 797 | for l in io.lines("results/curl-" .. mt .. "-" .. pu.host .. ".log") do |
1078 | local speed, crrnt = l:match('^%c *%d+ +%d+k? +%d+ +%d+k? +%d+ +%d+ +(%d+k?) +%d+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +(%d+k?)') | 798 | local speed, crrnt = l:match('^%c *%d+ +%d+k? +%d+ +%d+k? +%d+ +%d+ +(%d+k?) +%d+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +(%d+k?)') |
1079 | if nil ~= speed then | 799 | if nil ~= speed then |
@@ -1089,17 +809,17 @@ if 0 < #arg then | |||
1089 | end | 809 | end |
1090 | end | 810 | end |
1091 | end | 811 | end |
1092 | results["speed"] = {min = min, max = max} | 812 | APT.results["speed"] = {min = min, max = max} |
1093 | 813 | ||
1094 | local f = pu.host | 814 | local f = pu.host |
1095 | if "" ~= ip then f = f .. "_" .. ip end | 815 | if "" ~= ip then f = f .. "_" .. ip end |
1096 | local rfile, e = io.open("results/" .. f .. ".lua", "w+") | 816 | local rfile, e = io.open("results/" .. f .. ".lua", "w+") |
1097 | if nil == rfile then C("opening results file - " .. e) else | 817 | if nil == rfile then C("opening results file - " .. e) else |
1098 | rfile:write(dumpTable(results, "", "results") .. "\nreturn results\n") | 818 | rfile:write(APT.dumpTable(APT.results, "", "results") .. "\nreturn results\n") |
1099 | rfile:close() | 819 | rfile:close() |
1100 | end | 820 | end |
1101 | logPost() | 821 | APT.logPost() |
1102 | logFile:close() | 822 | APT.logFile:close() |
1103 | else | 823 | else |
1104 | local fadt = io.popen("ls -dl results_old 2>/dev/null | cut -d '>' -f 2 | cut -d ' ' -f 2") | 824 | local fadt = io.popen("ls -dl results_old 2>/dev/null | cut -d '>' -f 2 | cut -d ' ' -f 2") |
1105 | local adt = fadt:read('*l') | 825 | local adt = fadt:read('*l') |
@@ -1113,25 +833,25 @@ else | |||
1113 | if nil ~= dt then os.execute('mkdir -p results_' .. dt .. '; rm -f results; ln -s results_' .. dt .. ' results 2>/dev/null') end | 833 | if nil ~= dt then os.execute('mkdir -p results_' .. dt .. '; rm -f results; ln -s results_' .. dt .. ' results 2>/dev/null') end |
1114 | os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp') | 834 | os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp') |
1115 | os.execute("rm -f results/*.check 2>/dev/null") | 835 | os.execute("rm -f results/*.check 2>/dev/null") |
1116 | if not keep then | 836 | if not APT.keep then |
1117 | os.execute("rm -f results/*.curl 2>/dev/null") | 837 | os.execute("rm -f results/*.curl 2>/dev/null") |
1118 | os.execute("rm -f results/*.log 2>/dev/null") | 838 | os.execute("rm -f results/*.log 2>/dev/null") |
1119 | os.execute("rm -f results/*.html 2>/dev/null") | 839 | os.execute("rm -f results/*.html 2>/dev/null") |
1120 | os.execute("rm -f results/*.txt 2>/dev/null") | 840 | os.execute("rm -f results/*.txt 2>/dev/null") |
1121 | end | 841 | end |
1122 | 842 | ||
1123 | logFile, e = io.open("results/LOG_apt-panopticon.html", "a+") | 843 | APT.logFile, e = io.open("results/LOG_apt-panopticon.html", "a+") |
1124 | if nil == logFile then C("opening log file - " .. e); return end | 844 | if nil == APT.logFile then C("opening log file - " .. e); return end |
1125 | logPre() | 845 | APT.logPre() |
1126 | I("Starting tests " .. table.concat(options.tests.value, ", ")) | 846 | I("Starting tests " .. table.concat(APT.options.tests.value, ", ")) |
1127 | os.execute("mkdir -p results") | 847 | os.execute("mkdir -p results") |
1128 | mirrors = getMirrors() | 848 | APT.mirrors = getMirrors() |
1129 | checkHost(options.referenceSite.value) | 849 | checkHost(APT.options.referenceSite.value) |
1130 | for i, n in pairs(releases) do | 850 | for i, n in pairs(releases) do |
1131 | while not checkFile('results/NEW_Packages_' .. n .. '.test.txt') do os.execute("sleep 10") end | 851 | while not APT.checkFile('results/NEW_Packages_' .. n .. '.test.txt') do os.execute("sleep 10") end |
1132 | end | 852 | end |
1133 | 853 | ||
1134 | for k, m in pairs(mirrors) do | 854 | for k, m in pairs(APT.mirrors) do |
1135 | if "/" == m.BaseURL:sub(-1, -1) then | 855 | if "/" == m.BaseURL:sub(-1, -1) then |
1136 | W("slash at end of BaseURL in mirror_list.txt! " .. m.BaseURL, "", "", m.FQDN) | 856 | W("slash at end of BaseURL in mirror_list.txt! " .. m.BaseURL, "", "", m.FQDN) |
1137 | m.BaseURL = m.BaseURL:sub(1, -2) | 857 | m.BaseURL = m.BaseURL:sub(1, -2) |
@@ -1141,30 +861,27 @@ else | |||
1141 | m.BaseURL = m.BaseURL:sub(1, -2) | 861 | m.BaseURL = m.BaseURL:sub(1, -2) |
1142 | end | 862 | end |
1143 | local pu = url.parse("http://" .. m.BaseURL) | 863 | local pu = url.parse("http://" .. m.BaseURL) |
1144 | if options.referenceSite.value ~= pu.host then | 864 | if APT.options.referenceSite.value ~= pu.host then |
1145 | checkHost(m.BaseURL) | 865 | checkHost(m.BaseURL) |
1146 | checkExes("apt-panopticon.lua " .. sendArgs) | 866 | APT.checkExes("apt-panopticon.lua " .. sendArgs) |
1147 | if testing("Integrity") or testing("Updated") then checkExes(downloadLock) end | 867 | if APT.testing("Integrity") or APT.testing("Updated") then APT.checkExes(downloadLock) end |
1148 | end | 868 | end |
1149 | end | 869 | end |
1150 | 870 | ||
1151 | while 1 <= checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end | 871 | while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end |
1152 | 872 | ||
1153 | os.execute("rm -f results/*.check; rm -f results/*.lock 2>/dev/null") | 873 | os.execute("rm -f results/*.check; rm -f results/*.lock 2>/dev/null") |
1154 | 874 | ||
1155 | -- Create the reports. | 875 | -- Create the reports. |
1156 | for n, r in pairs(options.reports.value) do | 876 | for n, r in pairs(APT.options.reports.value) do |
1157 | local report = "apt-panopticon-report-" .. r .. ".lua" | 877 | if APT.checkFile("apt-panopticon-report-" .. r .. ".lua") then |
1158 | local rfile, e = io.open(report, "r") | 878 | I("Creating " .. r .. " report.") |
1159 | if nil == rfile then C("opening " .. report .. " file - " .. e) else | 879 | APT.execute("./apt-panopticon-report-" .. r .. ".lua") |
1160 | rfile:close() | ||
1161 | I("Creating " .. report .. " report.") | ||
1162 | execute("./" .. report .. " ") | ||
1163 | end | 880 | end |
1164 | end | 881 | end |
1165 | 882 | ||
1166 | if nil ~= adt then os.execute('rm -fr ' .. adt .. ' 2>/dev/null') end | 883 | if nil ~= adt then os.execute('rm -fr ' .. adt .. ' 2>/dev/null') end |
1167 | 884 | ||
1168 | logPost() | 885 | APT.logPost() |
1169 | logFile:close() | 886 | APT.logFile:close() |
1170 | end | 887 | end |