diff options
Diffstat (limited to 'apt-panopticon.lua')
-rwxr-xr-x | apt-panopticon.lua | 634 |
1 files changed, 634 insertions, 0 deletions
diff --git a/apt-panopticon.lua b/apt-panopticon.lua new file mode 100755 index 0000000..c68ec48 --- /dev/null +++ b/apt-panopticon.lua | |||
@@ -0,0 +1,634 @@ | |||
1 | #!/usr/bin/env luajit | ||
2 | |||
3 | |||
4 | local args = {...} | ||
5 | |||
6 | --[[ TODO - What to do about HTTPS://deb.devuan.org/ redirects. | ||
7 | Some mirrors give a 404. | ||
8 | Sledjhamr gives a 404, coz it's not listening on 443 for deb.devuan.org. | ||
9 | Some mirrors give a 200. | ||
10 | They shouldn't have the proper certificate, but are giving a result anyway. | ||
11 | ]] | ||
12 | |||
13 | origin = false | ||
14 | verbosity = -1 | ||
15 | keep = false | ||
16 | -- TODO - Should actually implement this. | ||
17 | fork = true | ||
18 | options = | ||
19 | { | ||
20 | referenceSite = | ||
21 | { | ||
22 | typ = "string", | ||
23 | help = "", | ||
24 | value = "pkgmaster.devuan.org", | ||
25 | }, | ||
26 | roundRobin = | ||
27 | { | ||
28 | typ = "string", | ||
29 | help = "", | ||
30 | value = "deb.devuan.org", | ||
31 | }, | ||
32 | tests = | ||
33 | { | ||
34 | typ = "table", | ||
35 | help = "", | ||
36 | value = | ||
37 | { | ||
38 | "IPv4", | ||
39 | "IPv6", | ||
40 | -- "ftp", | ||
41 | "http", | ||
42 | "https", | ||
43 | -- "rsync", | ||
44 | -- "DNS-RR", | ||
45 | "Protocol", | ||
46 | -- "URL-Sanity", | ||
47 | -- "Integrity", | ||
48 | -- "Updated", | ||
49 | }, | ||
50 | }, | ||
51 | } | ||
52 | |||
53 | local defaultURL = {scheme = "http"} | ||
54 | local downloadLock = "flock -n results/wget-" | ||
55 | local download = "wget --timeout=300 -np -N -r -P results " -- Note wget has a default read timeout of 900 seconds (15 minutes). | ||
56 | local releases = {"jessie", "ascii", "beowulf", "ceres"} | ||
57 | local releaseFiles = | ||
58 | { | ||
59 | -- Release file. | ||
60 | "/Release", | ||
61 | "/InRelease", | ||
62 | "/main/binary-all/Packages.gz", | ||
63 | -- Contents files. | ||
64 | "/main/Contents-all.gz", | ||
65 | "/main/Contents-amd64.gz", | ||
66 | "/main/Contents-arm64.gz", | ||
67 | "-security/main/Contents-all.gz", | ||
68 | "-security/main/Contents-amd64.gz", | ||
69 | "-security/main/Contents-arm64.gz", | ||
70 | } | ||
71 | local notExist = | ||
72 | { | ||
73 | "ceres-security" -- This will never exist, it's our code name for the testing suite. | ||
74 | } | ||
75 | local referenceDebs = | ||
76 | { | ||
77 | -- Debian package. | ||
78 | "merged/pool/DEBIAN/main/d/dash/dash_0.5.8-2.4_amd64.deb", | ||
79 | -- Debian security package. NOTE this one should always be redirected? | ||
80 | "merged/pool/DEBIAN-SECURITY/updates/main/a/apt/apt-transport-https_1.4.9_amd64.deb", | ||
81 | } | ||
82 | local referenceDevs = | ||
83 | { | ||
84 | -- Devuan package. NOTE this one should not get redirected, but that's more a warning than an error. | ||
85 | "merged/pool/DEVUAN/main/d/desktop-base/desktop-base_2.0.3_all.deb", | ||
86 | -- "merged/pool/DEVUAN/main/u/util-linux/util-linux_2.32.1-0.1+devuan2.1_amd64.deb", | ||
87 | } | ||
88 | local arg = {} | ||
89 | local sendArgs = "" | ||
90 | local logFile | ||
91 | |||
92 | |||
93 | local socket = require 'socket' | ||
94 | local ftp = require 'socket.ftp' | ||
95 | local http = require 'socket.http' | ||
96 | local https = require 'ssl.https' -- See https://github.com/brunoos/luasec/wiki/LuaSec-0.6 for docs. | ||
97 | local url = require 'socket.url' | ||
98 | |||
99 | |||
100 | -- Use this to dump a table to a string. | ||
101 | dumpTable = function (table, space, name) | ||
102 | local r = "" | ||
103 | if "" == space then r = r .. space .. name .. " =\n" else r = r .. space .. "[" .. name .. "] =\n" end | ||
104 | r = r .. space .. "{\n" | ||
105 | r = r .. dumpTableSub(table, space .. " ") | ||
106 | if "" == space then r = r .. space .. "}\n" else r = r .. space .. "},\n" end | ||
107 | return r | ||
108 | end | ||
109 | dumpTableSub = function (table, space) | ||
110 | local r = "" | ||
111 | for k, v in pairs(table) do | ||
112 | if type(k) == "string" then k = '"' .. k .. '"' end | ||
113 | if type(v) == "table" then | ||
114 | r = r .. dumpTable(v, space, k) | ||
115 | elseif type(v) == "string" then | ||
116 | r = r .. space .. "[" .. k .. "] = '" .. v .. "';\n" | ||
117 | elseif type(v) == "function" then | ||
118 | r = r .. space .. "[" .. k .. "] = function ();\n" | ||
119 | elseif type(v) == "userdata" then | ||
120 | r = r .. space .. "userdata " .. "[" .. k .. "];\n" | ||
121 | elseif type(v) == "boolean" then | ||
122 | if (v) then | ||
123 | r = r .. space .. "[" .. k .. "] = true;\n" | ||
124 | else | ||
125 | r = r .. space .. "[" .. k .. "] = false;\n" | ||
126 | end | ||
127 | else | ||
128 | r = r .. space .. "[" .. k .. "] = " .. v .. ";\n" | ||
129 | end | ||
130 | end | ||
131 | return r | ||
132 | end | ||
133 | |||
134 | local ip = "" | ||
135 | local results = {} | ||
136 | |||
137 | local log = function(v, t, s, prot, test, host) | ||
138 | local x = "" | ||
139 | if nil == prot then prot = "" end | ||
140 | if nil ~= test then x = x .. test else test = "" end | ||
141 | if nil ~= host then | ||
142 | if #x > 0 then x = x .. " " end | ||
143 | x = x .. host | ||
144 | end | ||
145 | if #x > 0 then | ||
146 | t = t .. "(" .. x .. ")" | ||
147 | if "" == test then | ||
148 | if v == 0 then results[prot].errors = results[prot].errors + 1 end | ||
149 | if v == 1 then results[prot].warnings = results[prot].warnings + 1 end | ||
150 | else | ||
151 | if v == 0 then results[prot][test].errors = results[prot][test].errors + 1 end | ||
152 | if v == 1 then results[prot][test].warnings = results[prot][test].warnings + 1 end | ||
153 | end | ||
154 | end | ||
155 | if v <= verbosity then | ||
156 | if 3 <= verbosity then t = os.date() .. " " .. t end | ||
157 | print(t .. ": " .. s) | ||
158 | end | ||
159 | if nil ~= logFile then | ||
160 | logFile:write(os.date() .. " " .. t .. ": " .. s .. "\n") | ||
161 | logFile:flush() | ||
162 | end | ||
163 | end | ||
164 | local D = function(s) log(3, "DEBUG ", s) end | ||
165 | local I = function(s) log(2, "INFO ", s) end | ||
166 | local W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end | ||
167 | local E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end | ||
168 | local C = function(s) log(-1, "CRITICAL", s) end | ||
169 | |||
170 | local mirrors = {} | ||
171 | |||
172 | local testing = function(t, host) | ||
173 | for i, v in pairs(options.tests.value) do | ||
174 | if t == v then | ||
175 | local h = mirrors[host] | ||
176 | if nil == h then return true end | ||
177 | if true == h["Protocols"][t] then return true else D("Skipping " .. t .. " checks for " .. host) end | ||
178 | end | ||
179 | end | ||
180 | return false | ||
181 | end | ||
182 | |||
183 | local checkExes = function (exe) | ||
184 | local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | wc -l'):read("*l") | ||
185 | D(count .. " " .. exe .. " commands still running.") | ||
186 | return tonumber(count) | ||
187 | end | ||
188 | |||
189 | local repoExists = function (r) | ||
190 | r = r:match("([%a-]*)") | ||
191 | if nil == r then return false end | ||
192 | for k, v in pairs(notExist) do | ||
193 | if v == r then return false end | ||
194 | end | ||
195 | return true | ||
196 | end | ||
197 | |||
198 | local IP = {} | ||
199 | gatherIPs = function (host) | ||
200 | if nil == IP[host] then | ||
201 | local IPs | ||
202 | local dig = io.popen('dig +keepopen +noall +nottlid +answer ' .. host .. ' A ' .. host .. ' AAAA ' .. host .. ' CNAME ' .. host .. ' SRV | sort -r | uniq') | ||
203 | repeat | ||
204 | IPs = dig:read("*l") | ||
205 | if nil ~= IPs then | ||
206 | for k, t, v in IPs:gmatch("([%w_%-%.]*)%.%s*IN%s*(%a*)%s*(.*)") do | ||
207 | if "." == v:sub(-1, -1) then v = v:sub(1, -2) end | ||
208 | if nil == IP[k] then IP[k] = {} end | ||
209 | IP[k][v] = t | ||
210 | D(" DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v) | ||
211 | if t == "CNAME" then | ||
212 | gatherIPs(v) | ||
213 | IP[k][v] = IP[v] | ||
214 | elseif v == "SRV" then | ||
215 | print("SVR record found, now what do we do?") | ||
216 | end | ||
217 | end | ||
218 | end | ||
219 | until nil == IPs | ||
220 | end | ||
221 | end | ||
222 | |||
223 | -- Returns FTP directory listing | ||
224 | local nlst = function (u) | ||
225 | local t = {} | ||
226 | local p = url.parse(u) | ||
227 | p.command = "nlst" | ||
228 | p.sink = ltn12.sink.table(t) | ||
229 | local r, e = ftp.get(p) | ||
230 | return r and table.concat(t), e | ||
231 | end | ||
232 | |||
233 | local timeouts = 0; | ||
234 | checkHEAD = function (host, URL, r, retry) | ||
235 | if nil == r then r = 0 end | ||
236 | if nil == retry then retry = 0 end | ||
237 | local check = "Checking file" | ||
238 | local PU = url.parse(URL, defaultURL) | ||
239 | local pu = url.parse(PU.scheme .. "://" .. host, defaultURL) | ||
240 | if 0 < r then | ||
241 | check = "Redirecting to" | ||
242 | end | ||
243 | if 0 < retry then | ||
244 | os.execute("sleep " .. math.random(1, 4)) | ||
245 | check = "Retry " .. retry .. " " .. check | ||
246 | end | ||
247 | if 3 < timeouts then | ||
248 | E("too many timeouts! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) | ||
249 | return | ||
250 | end | ||
251 | if 20 < r then | ||
252 | E("too many redirects! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) | ||
253 | return | ||
254 | end | ||
255 | if 4 < retry then | ||
256 | E("too many retries! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) | ||
257 | return | ||
258 | end | ||
259 | D(PU.scheme .. " :// " .. check .. " " .. host .. " -> " .. URL) | ||
260 | if not testing(PU.scheme, host) then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL); return end | ||
261 | -- TODO - Perhaps we should try it anyway, and mark it as a warning if it DOES work? | ||
262 | if "https" == PU.scheme and options.roundRobin.value == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert."); return end | ||
263 | local hd = {} | ||
264 | if pu.host ~= PU.host then hd = {Host = host} end | ||
265 | local htp = http; | ||
266 | if PU.scheme == "https" then htp = https end | ||
267 | -- NOTE - the docs for lua-sec say that redirect isn't supported is version 0.6, no idea if that means it ignores redirections like we want. | ||
268 | -- TODO - find out! | ||
269 | -- The protocol and options are lua-sec specific arguments. | ||
270 | local p, c, h, s = htp.request{method = "HEAD", redirect = false, url = URL, headers = hd, protocol = "any", options = "all"} | ||
271 | if nil == s then s = "" end | ||
272 | if nil == p then | ||
273 | E(" " .. c .. " " .. s .. "! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) | ||
274 | -- So far the only errors are "timeout", "Network is unreachable", and "closed", and I suspect "closed" is due to saturating my bandwidth. | ||
275 | if "timeout" == c then timeouts = timeouts + 1 end | ||
276 | if ("closed" == c) or ("Network is unreachable" == c) or ("timeout" == c) then checkHEAD(host, URL, r, retry + 1, timeouts) end | ||
277 | else | ||
278 | if ("4" == tostring(c):sub(1, 1)) or ("5" == tostring(c):sub(1, 1)) then | ||
279 | E(" " .. c .. " " .. s .. ". " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) | ||
280 | else | ||
281 | I(" " .. c .. " " .. s .. ". " .. check .. " " .. host .. " -> " .. URL) | ||
282 | timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through. | ||
283 | end | ||
284 | l = h.location | ||
285 | if nil ~= l then | ||
286 | pu = url.parse(l, defaultURL) | ||
287 | if (pu.scheme ~= PU.scheme) then | ||
288 | if testing("Protocol") then W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l, PU.scheme, "Protocol", host) end | ||
289 | if (pu.host == host) and pu.path == PU.path then D("Not testing protocol change " .. URL .. " -> " .. l); return end | ||
290 | end | ||
291 | |||
292 | if l == URL then | ||
293 | E(" redirect loop! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, PU.scheme, host) | ||
294 | elseif nil == pu.host then | ||
295 | I(" relative redirect. " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l) | ||
296 | checkHEAD(host, PU.scheme .. "://" .. PU.host .. l, r + 1) | ||
297 | elseif (PU.host == pu.host) or (host == pu.host) then | ||
298 | checkHEAD(pu.host, l, r + 1) | ||
299 | else | ||
300 | --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location. | ||
301 | Not good for deb.debian.org, which we should only be throwing .debs at. | ||
302 | What we do is loop through the DNS entries, and only test the specific protocol & file being tested here. | ||
303 | |||
304 | This is what I came up with for checking if we are already testing a specific URL. | ||
305 | Still duplicates a tiny bit, but much less than the previous find based method. | ||
306 | ]] | ||
307 | local file = pu.host .. "://" .. pu.path | ||
308 | local f = io.popen(string.format('if [ ! -f results/%s.check ] ; then touch results/%s.check; echo -n "check"; fi', file:gsub("/", "_"), file:gsub("/", "_") )):read("*a") | ||
309 | if (nil == f) or ("check" == f) then | ||
310 | I(" Now checking redirected host " .. file) | ||
311 | checkHost(pu.host, pu.host, nil, "redir", pu.path) | ||
312 | else | ||
313 | D(" Already checking " .. file) | ||
314 | end | ||
315 | end | ||
316 | end | ||
317 | end | ||
318 | end | ||
319 | |||
320 | local checkFiles = function (host, ip, path, file) | ||
321 | if nil == path then path = "" end | ||
322 | if nil ~= file then | ||
323 | if "redir" == ip then ip = host end | ||
324 | I(" Checking IP for file " .. host .. " -> " .. ip .. " " .. path .. " " .. file) | ||
325 | if testing("http", host) then checkHEAD(host, "http://" .. ip .. path .. "/" .. file) end | ||
326 | if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/" .. file) end | ||
327 | else | ||
328 | I(" Checking IP " .. host .. " -> " .. ip .. " " .. path) | ||
329 | for i, s in pairs(referenceDevs) do | ||
330 | if testing("http", host) then checkHEAD(host, "http://" .. ip .. path .. "/" .. s) end | ||
331 | if 3 < timeouts then return end | ||
332 | if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/" .. s) end | ||
333 | if 3 < timeouts then return end | ||
334 | end | ||
335 | |||
336 | for i, s in pairs(releases) do | ||
337 | for j, k in pairs(releaseFiles) do | ||
338 | if repoExists(s .. k) then | ||
339 | if testing("http", host) then checkHEAD(host, "http://" .. ip .. path .. "/merged/dists/" .. s .. k) end | ||
340 | if 3 < timeouts then return end | ||
341 | if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/merged/dists/" .. s .. k) end | ||
342 | if 3 < timeouts then return end | ||
343 | end | ||
344 | end | ||
345 | end | ||
346 | end | ||
347 | end | ||
348 | |||
349 | local execute = function (s) | ||
350 | D(" executing " .. s) | ||
351 | os.execute(s) | ||
352 | end | ||
353 | |||
354 | checkHost = function (orig, host, path, ip, file) | ||
355 | if nil == host then host = orig end | ||
356 | if nil == path then path = "" end | ||
357 | if nil == file then file = "" end | ||
358 | local ph = url.parse("http://" .. host) | ||
359 | if (nil ~= ip) and ("redir" ~= ip) then | ||
360 | local po = url.parse("http://" .. orig) | ||
361 | if "" ~= file then | ||
362 | D("checking redirected file " .. po.host .. " " .. file) | ||
363 | checkFiles(po.host, ip, path, file) | ||
364 | else | ||
365 | checkFiles(po.host, ip, path) | ||
366 | end | ||
367 | else | ||
368 | if orig == host then | ||
369 | D("checkHost " .. orig .. "" .. file) | ||
370 | if testing("IPv4") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file .." &") end | ||
371 | else D("checkHost " .. orig .. " -> " .. host) end | ||
372 | local h = mirrors[ph.host] | ||
373 | if nil == h then return end | ||
374 | for k, v in pairs(h.IPs) do | ||
375 | if "table" == type(v) then | ||
376 | for k1, v1 in pairs(v) do | ||
377 | if v1 == "A" then | ||
378 | if testing("IPv4") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file .." &") end | ||
379 | elseif v1 == "AAAA" then | ||
380 | if testing("IPv6") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k1 .. "] " .. file .. " &") end | ||
381 | end | ||
382 | end | ||
383 | else | ||
384 | if v == "A" then | ||
385 | if testing("IPv4") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file .." &") end | ||
386 | elseif v == "AAAA" then | ||
387 | if testing("IPv6") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k .. "] " .. file .. " &") end | ||
388 | end | ||
389 | end | ||
390 | end | ||
391 | end | ||
392 | end | ||
393 | |||
394 | local downloads = function (cut, host, URL) | ||
395 | if 0 ~= cut then cd = " --cut-dirs=" .. cut .. " " else cd = "" end | ||
396 | if nil == URL then URL = "/" end | ||
397 | local lock = "%s-" .. host .. ".log " | ||
398 | local log = " --rejected-log=results/wget-%s_REJECTS-" .. host .. ".log -a results/wget-%s-" .. host .. ".log " | ||
399 | I("starting file download commands for " .. host .. " " .. URL) | ||
400 | local cm = "ionice -c3 " .. downloadLock .. lock:format("debs") .. download .. log:format("debs", "debs") .. cd | ||
401 | for i, s in pairs(referenceDevs) do | ||
402 | cm = cm .. " https://" .. host .. URL .. "/" .. s | ||
403 | end | ||
404 | for i, s in pairs(referenceDebs) do | ||
405 | cm = cm .. " https://" .. host .. URL .. "/" .. s | ||
406 | end | ||
407 | for i, s in pairs(releases) do | ||
408 | execute(cm .. " &") | ||
409 | cm = "ionice -c3 " .. downloadLock .. lock:format(s) .. download .. log:format(s, s) .. cd | ||
410 | if repoExists(s .. k) then | ||
411 | for j, k in pairs(releaseFiles) do | ||
412 | cm = cm .. " https://" .. host .. URL .. "/merged/dists/" .. s .. k | ||
413 | end | ||
414 | end | ||
415 | end | ||
416 | execute(cm .. " &") | ||
417 | end | ||
418 | |||
419 | local getMirrors = function () | ||
420 | local mirrors = {} | ||
421 | local host = "" | ||
422 | local m = {} | ||
423 | local active = true | ||
424 | local URL = "https://" .. options.referenceSite.value .. "/mirror_list.txt" | ||
425 | I("getting mirrors.") | ||
426 | local p, c, h = http.request(URL) | ||
427 | if nil == p then E(c .. " fetching " .. URL) else | ||
428 | for l in p:gmatch("\n*([^\n]+)\n*") do | ||
429 | local t, d = l:match("(%a*):%s*(.*)") | ||
430 | d = string.lower(d) | ||
431 | if "FQDN" == t then | ||
432 | if "" ~= host then | ||
433 | if active then mirrors[host] = m end | ||
434 | m = {} | ||
435 | active = true | ||
436 | end | ||
437 | host = d | ||
438 | m[t] = d | ||
439 | gatherIPs(host) | ||
440 | m["IPs"] = IP[host] | ||
441 | elseif "Protocols" == t then | ||
442 | local prot = {} | ||
443 | for w in d:gmatch("(%w+)") do | ||
444 | prot[w] = true; | ||
445 | end | ||
446 | m[t] = prot | ||
447 | elseif "Active" == t and nil == d:find("yes", 1, true) then | ||
448 | W("Mirror " .. host .. " is not active - " .. d) | ||
449 | active = false | ||
450 | -- TODO - Should do some input validation on BaseURL, and everything else. | ||
451 | else | ||
452 | m[t] = d | ||
453 | end | ||
454 | end | ||
455 | if "" ~= host and active then | ||
456 | mirrors[host] = m | ||
457 | end | ||
458 | end | ||
459 | mirrors[options.roundRobin.value] = { ["Protocols"] = { ["http"] = true; ["https"] = true; }; ["FQDN"] = 'deb.devuan.org'; ["Active"] = 'yes'; ["BaseURL"] = 'deb.devuan.org'; } | ||
460 | gatherIPs(options.roundRobin.value) | ||
461 | mirrors[options.roundRobin.value].IPs = IP[options.roundRobin.value] | ||
462 | local file, e = io.open("results/mirrors.lua", "w+") | ||
463 | if nil == file then C("opening mirrors file - " .. e) else | ||
464 | file:write(dumpTable(mirrors, "", "mirrors") .. "\nreturn mirrors\n") | ||
465 | file:close() | ||
466 | end | ||
467 | return mirrors | ||
468 | end | ||
469 | |||
470 | |||
471 | if 0 ~= #args then | ||
472 | local option = "" | ||
473 | for i, a in pairs(args) do | ||
474 | if ("--help" == a) or ("-h" == a) then | ||
475 | print("I should write some docs, huh? Read README.md for instructions.") | ||
476 | elseif "--version" == a then | ||
477 | print("apt-panopticon version 0.1 WIP development version") | ||
478 | elseif "-v" == a then | ||
479 | verbosity = verbosity + 1 | ||
480 | sendArgs = sendArgs .. a .. " " | ||
481 | elseif "-q" == a then | ||
482 | verbosity = -1 | ||
483 | sendArgs = sendArgs .. a .. " " | ||
484 | elseif "-k" == a then | ||
485 | keep = true | ||
486 | elseif "-n" == a then | ||
487 | fork = false | ||
488 | elseif "-o" == a then | ||
489 | origin = true | ||
490 | elseif "--" == a:sub(1, 2) then | ||
491 | local s, e = a:find("=") | ||
492 | if nil == s then e = -1 end | ||
493 | option = a:sub(3, e - 1) | ||
494 | local o = options[option] | ||
495 | if nil == o then | ||
496 | print("Unknown option --" .. option) | ||
497 | option = "" | ||
498 | else | ||
499 | option = a | ||
500 | sendArgs = sendArgs .. a .. " " | ||
501 | local s, e = a:find("=") | ||
502 | if nil == s then e = 0 end | ||
503 | option = a:sub(3, e - 1) | ||
504 | if "table" == options[option].typ then | ||
505 | local result = {} | ||
506 | for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do | ||
507 | local f = t:sub(1, 1) | ||
508 | local n = t:sub(2, -1) | ||
509 | if ("+" ~= f) and ("-" ~= f) then | ||
510 | table.insert(result, t) | ||
511 | end | ||
512 | end | ||
513 | if 0 ~= #result then | ||
514 | options[option].value = result | ||
515 | else | ||
516 | for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do | ||
517 | local f = t:sub(1, 1) | ||
518 | local n = t:sub(2, -1) | ||
519 | if "+" == f then | ||
520 | table.insert(options[option].value, n) | ||
521 | elseif "-" == f then | ||
522 | local r = {} | ||
523 | for i, k in pairs(options[option].value) do | ||
524 | if k ~= n then table.insert(r, k) end | ||
525 | end | ||
526 | options[option].value = r | ||
527 | end | ||
528 | end | ||
529 | end | ||
530 | else | ||
531 | options[option].value = a | ||
532 | end | ||
533 | option = "" | ||
534 | end | ||
535 | elseif "-" == a:sub(1, 1) then | ||
536 | print("Unknown option " .. a) | ||
537 | else | ||
538 | table.insert(arg, a) | ||
539 | end | ||
540 | end | ||
541 | end | ||
542 | |||
543 | --print(dumpTable(options.tests.value, "", "tests")) | ||
544 | |||
545 | execute("mkdir -p results") | ||
546 | |||
547 | if 0 < #arg then | ||
548 | if "/" == arg[1]:sub(-1, -1) then | ||
549 | W("slash at end of path! " .. arg[1]) | ||
550 | arg[1] = arg[1]:sub(1, -2) | ||
551 | end | ||
552 | if " " == arg[1]:sub(-1, -1) then | ||
553 | W("space at end of path! " .. arg[1]) | ||
554 | arg[1] = arg[1]:sub(1, -2) | ||
555 | end | ||
556 | local pu = url.parse("http://" .. arg[1]) | ||
557 | if nil ~= arg[2] then | ||
558 | logFile, e = io.open("results/" .. pu.host .. "_" .. arg[2] .. ".log", "a+") | ||
559 | else | ||
560 | logFile, e = io.open("results/" .. pu.host .. ".log", "a+") | ||
561 | end | ||
562 | if nil == logFile then C("opening log file - " .. e); return end | ||
563 | I("Starting tests for " ..arg[1] .. " with these tests - " .. table.concat(options.tests.value, ", ")) | ||
564 | mirrors = loadfile("results/mirrors.lua")() | ||
565 | if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end | ||
566 | if nil ~= arg[3] then I(" Using file " .. arg[3]); end | ||
567 | |||
568 | for k, v in pairs{"ftp", "http", "https", "rsync"} do | ||
569 | if testing(v) then | ||
570 | local tests = {errors = 0; warnings = 0} | ||
571 | if testing("Integrity") then tests.Integrity = {errors = 0; warnings = 0} end | ||
572 | if testing("Protocol") then tests.Protocol = {errors = 0; warnings = 0} end | ||
573 | if testing("Updated") then tests.Updated = {errors = 0; warnings = 0} end | ||
574 | if testing("URL-Sanity") then tests.URL_Sanity = {errors = 0; warnings = 0} end | ||
575 | results[v] = tests | ||
576 | end | ||
577 | end | ||
578 | |||
579 | if testing("Integrity") or testing("Updated") then | ||
580 | if nil == arg[3] then | ||
581 | if not keep then execute("rm -fr results/" .. pu.host) end | ||
582 | cut = 0 | ||
583 | for t in arg[1]:gmatch("(/)") do | ||
584 | cut = cut + 1 | ||
585 | end | ||
586 | downloads(cut, pu.host, pu.path) | ||
587 | checkExes("apt-panopticon.lua " .. sendArgs) | ||
588 | checkExes(downloadLock) | ||
589 | end | ||
590 | end | ||
591 | if origin then | ||
592 | checkFiles(pu.host, pu.host, pu.path); | ||
593 | else | ||
594 | checkHost(pu.host, pu.host, pu.path, arg[2], arg[3]) | ||
595 | end | ||
596 | logFile:close() | ||
597 | local rfile, e = io.open("results/" .. pu.host .. "_" .. ip .. ".lua", "w+") | ||
598 | if nil == rfile then C("opening results file - " .. e) else | ||
599 | rfile:write(dumpTable(results, "", "results") .. "\nreturn results\n") | ||
600 | rfile:close() | ||
601 | end | ||
602 | else | ||
603 | if not keep then os.execute("rm -f results/*.log") end | ||
604 | os.execute("rm -f results/*.check") | ||
605 | os.execute("mkdir -p results; touch results/stamp") | ||
606 | logFile, e = io.open("results/apt-panopticon.log", "a+") | ||
607 | if nil == logFile then C("opening log file - " .. e); return end | ||
608 | I("Starting tests " .. table.concat(options.tests.value, ", ")) | ||
609 | execute("mkdir -p results") | ||
610 | mirrors = getMirrors() | ||
611 | checkHost(options.referenceSite.value) | ||
612 | for k, m in pairs(mirrors) do | ||
613 | if "/" == m.BaseURL:sub(-1, -1) then | ||
614 | W("slash at end of BaseURL in mirror_list.txt! " .. m.BaseURL) | ||
615 | m.BaseURL = m.BaseURL:sub(1, -2) | ||
616 | end | ||
617 | if " " == m.BaseURL:sub(-1, -1) then | ||
618 | W("space at end of BaseURL in mirror_list.txt! " .. m.BaseURL) | ||
619 | m.BaseURL = m.BaseURL:sub(1, -2) | ||
620 | end | ||
621 | local pu = url.parse("http://" .. m.BaseURL) | ||
622 | if options.referenceSite.value ~= pu.host then | ||
623 | checkHost(m.BaseURL) | ||
624 | checkExes("apt-panopticon.lua " .. sendArgs) | ||
625 | if testing("Integrity") or testing("Updated") then checkExes(downloadLock) end | ||
626 | end | ||
627 | end | ||
628 | while 1 <= checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end | ||
629 | if testing("Integrity") or testing("Updated") then | ||
630 | while 0 < checkExes(downloadLock) do os.execute("sleep 10") end | ||
631 | end | ||
632 | os.execute("rm -f results/*.check") | ||
633 | logFile:close() | ||
634 | end | ||