diff options
author | onefang | 2019-12-30 17:05:18 +1000 |
---|---|---|
committer | onefang | 2019-12-30 17:05:18 +1000 |
commit | f02b222005f7d656a6da8d25dca5a5fef56857c2 (patch) | |
tree | d8746d182a6b11fd590d94199c31312d369c2996 | |
parent | TODO++ (diff) | |
download | apt-panopticon-f02b222005f7d656a6da8d25dca5a5fef56857c2.zip apt-panopticon-f02b222005f7d656a6da8d25dca5a5fef56857c2.tar.gz apt-panopticon-f02b222005f7d656a6da8d25dca5a5fef56857c2.tar.bz2 apt-panopticon-f02b222005f7d656a6da8d25dca5a5fef56857c2.tar.xz |
A lot of tweaking of the logging and reporting, making it easier to read.
-rw-r--r-- | apt-panopticommon.lua | 19 | ||||
-rwxr-xr-x | apt-panopticon-report-email-web.lua | 18 | ||||
-rwxr-xr-x | apt-panopticon.lua | 114 |
3 files changed, 72 insertions, 79 deletions
diff --git a/apt-panopticommon.lua b/apt-panopticommon.lua index 4e75f3f..ca952d3 100644 --- a/apt-panopticommon.lua +++ b/apt-panopticommon.lua | |||
@@ -348,6 +348,11 @@ APT.allpairs = function(tbl, func) | |||
348 | end | 348 | end |
349 | end | 349 | end |
350 | 350 | ||
351 | APT.lnk = function(URL) | ||
352 | return '<a href="' .. URL .. '">' .. URL .. '</a>' | ||
353 | end | ||
354 | |||
355 | |||
351 | APT.search = function(t, s) | 356 | APT.search = function(t, s) |
352 | for i, v in pairs(t) do | 357 | for i, v in pairs(t) do |
353 | if v == s then return true end | 358 | if v == s then return true end |
@@ -441,12 +446,12 @@ local log = function(v, t, s, prot, test, host) | |||
441 | APT.logFile:flush() | 446 | APT.logFile:flush() |
442 | end | 447 | end |
443 | end | 448 | end |
444 | APT.D = function(s) log(4, "DEBUG ", s) end | 449 | APT.D = function(s, h) log(4, "DEBUG ", s, nil, nil, h) end |
445 | APT.I = function(s) log(3, "INFO ", s) end | 450 | APT.I = function(s, h) log(3, "INFO ", s, nil, nil, h) end |
446 | APT.T = function(s, p, t, h) log(2, "TIMEOUT ", s, p, t, h) end | 451 | APT.T = function(s, p, t, h) log(2, "TIMEOUT ", s, p, t, h) end |
447 | APT.W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end | 452 | APT.W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end |
448 | APT.E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end | 453 | APT.E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end |
449 | APT.C = function(s) log(-1, "CRITICAL", s) end | 454 | APT.C = function(s) log(-1, "CRITICAL", s) end |
450 | local D = APT.D | 455 | local D = APT.D |
451 | local I = APT.I | 456 | local I = APT.I |
452 | local T = APT.T | 457 | local T = APT.T |
@@ -463,7 +468,7 @@ APT.testing = function(t, host) | |||
463 | if t == v then | 468 | if t == v then |
464 | local h = APT.mirrors[host] | 469 | local h = APT.mirrors[host] |
465 | if nil == h then return true end | 470 | if nil == h then return true end |
466 | if true == h["Protocols"][t] then return true else D("Skipping " .. t .. " checks for " .. host) end | 471 | if true == h["Protocols"][t] then return true else I("Skipping " .. t .. " checks for " .. host) end |
467 | end | 472 | end |
468 | end | 473 | end |
469 | return false | 474 | return false |
diff --git a/apt-panopticon-report-email-web.lua b/apt-panopticon-report-email-web.lua index 97f7cef..05699ee 100755 --- a/apt-panopticon-report-email-web.lua +++ b/apt-panopticon-report-email-web.lua | |||
@@ -213,7 +213,7 @@ end | |||
213 | local makeTable = function(web, hosts) | 213 | local makeTable = function(web, hosts) |
214 | web:write("<table>\n<tr><th></th><th>" .. lnk('FTP') .. "</th><th>" .. lnk('HTTP') .. "</th><th>" .. lnk('HTTPS') .. "</th><th>" .. lnk('RSYNC') .. "</th>" .. | 214 | web:write("<table>\n<tr><th></th><th>" .. lnk('FTP') .. "</th><th>" .. lnk('HTTP') .. "</th><th>" .. lnk('HTTPS') .. "</th><th>" .. lnk('RSYNC') .. "</th>" .. |
215 | "<th>" .. lnk('DNS round robin', 'DNS-RR') .. "</th><th>" .. lnk('Protocol') .. "</th><th>" .. lnk('URL sanity', 'URL-Sanity') .. "</th><th>" .. lnk('Integrity') .. "</th>" .. | 215 | "<th>" .. lnk('DNS round robin', 'DNS-RR') .. "</th><th>" .. lnk('Protocol') .. "</th><th>" .. lnk('URL sanity', 'URL-Sanity') .. "</th><th>" .. lnk('Integrity') .. "</th>" .. |
216 | "<th>" .. lnk('Updated') .. "</th><th colspan='2'>" .. lnk('Speed range', 'Speed') .. "</th>" .. | 216 | "<th colspan='2'>" .. lnk('Updated') .. "</th><th colspan='2'>" .. lnk('Speed range', 'Speed') .. "</th>" .. |
217 | "<th colspan='2'>" .. lnk('Weekly statistics', 'Weekly') .. "</th><th>" .. lnk('Graphs') .. "</th></tr>\n") | 217 | "<th colspan='2'>" .. lnk('Weekly statistics', 'Weekly') .. "</th><th>" .. lnk('Graphs') .. "</th></tr>\n") |
218 | local bg = '' | 218 | local bg = '' |
219 | for k, v in APT.orderedPairs(hosts) do | 219 | for k, v in APT.orderedPairs(hosts) do |
@@ -235,13 +235,13 @@ local makeTable = function(web, hosts) | |||
235 | local sanity = status(hosts, k, results, "URLSanity") | 235 | local sanity = status(hosts, k, results, "URLSanity") |
236 | local integrity = status(hosts, k, results, "Integrity") | 236 | local integrity = status(hosts, k, results, "Integrity") |
237 | local updated = status(hosts, k, results, "Updated") | 237 | local updated = status(hosts, k, results, "Updated") |
238 | local rate = v.Rate | 238 | local rate = v.Rate |
239 | if nil ~= rate then updated = updated .. ' ' .. rate end | ||
240 | local min = tonumber(results.speed.min) | 239 | local min = tonumber(results.speed.min) |
241 | local max = tonumber(results.speed.max) | 240 | local max = tonumber(results.speed.max) |
242 | local spd = '<td></td><td></td>' | 241 | local spd = '<td></td><td></td>' |
243 | local week = '<td></td><td></td>' | 242 | local week = '<td> </td><td> </td>' |
244 | local graph = '<a href="../apt-panopticon_cgp/host.php?h=' .. k .. '">graphs</a>' | 243 | local graph = '<a href="../apt-panopticon_cgp/host.php?h=' .. k .. '">graphs</a>' |
244 | if nil == rate then rate = '' end | ||
245 | 245 | ||
246 | -- DNS-RR test. | 246 | -- DNS-RR test. |
247 | if (APT.options.roundRobin.value ~= k) and (nil ~= hosts[APT.options.roundRobin.value]) then | 247 | if (APT.options.roundRobin.value ~= k) and (nil ~= hosts[APT.options.roundRobin.value]) then |
@@ -291,12 +291,12 @@ local makeTable = function(web, hosts) | |||
291 | if '0.00' == percentUp then percentUpdated = '??' end -- We are counting errors, and you can't get an error if you can't check anything. | 291 | if '0.00' == percentUp then percentUpdated = '??' end -- We are counting errors, and you can't get an error if you can't check anything. |
292 | -- TODO - try to account for this better, this is just a quick hack. | 292 | -- TODO - try to account for this better, this is just a quick hack. |
293 | end | 293 | end |
294 | week = '<td>' .. percentUp .. '% up</td><td>' .. percentUpdated .. '% updated</td>' | 294 | week = '<td> ' .. percentUp .. '% up</td><td> ' .. percentUpdated .. '% updated</td>' |
295 | end | 295 | end |
296 | 296 | ||
297 | web:write("<td>" .. ftp .. " </td><td>" .. http .. " </td><td>" .. https .. " </td><td>" .. rsync .. " </td><td>" .. dns .. | 297 | web:write("<td>" .. ftp .. " </td><td>" .. http .. " </td><td>" .. https .. " </td><td>" .. rsync .. " </td><td>" .. dns .. |
298 | " </td><td>" .. protocol .. " </td><td>" .. sanity .. " </td><td>" .. integrity .. " </td><td>" .. updated .. | 298 | " </td><td>" .. protocol .. " </td><td>" .. sanity .. " </td><td>" .. integrity .. " </td>" .. '<td>' .. rate .. |
299 | " " .. spd .. " " .. week .." <td>" .. graph .. "</td></tr>\n") | 299 | '</td><td>' .. updated .. '</td> ' .. spd .. " " .. week .." <td>" .. graph .. "</td></tr>\n") |
300 | if "" ~= active then | 300 | if "" ~= active then |
301 | web:write("<tr><td style='background-color:dimgrey'>" .. active .. "</td></tr>\n") | 301 | web:write("<tr><td style='background-color:dimgrey'>" .. active .. "</td></tr>\n") |
302 | end | 302 | end |
@@ -482,7 +482,7 @@ if nil == web then C("opening mirrors file - " .. e) else | |||
482 | " means the mirror had some timeouts, and tests where not yet aborted. The darker colour means unsupported by the mirror, but tested anyway.</p>" .. | 482 | " means the mirror had some timeouts, and tests where not yet aborted. The darker colour means unsupported by the mirror, but tested anyway.</p>" .. |
483 | "<p><font color='blue'><b>TIMEOUT</b></font> or <font color='darkblue'><b>TIMEOUT</b></font>" .. | 483 | "<p><font color='blue'><b>TIMEOUT</b></font> or <font color='darkblue'><b>TIMEOUT</b></font>" .. |
484 | " means the mirror had too many timeouts, and tests where aborted, so there is no result for this test.</p>" .. | 484 | " means the mirror had too many timeouts, and tests where aborted, so there is no result for this test.</p>" .. |
485 | "<p>NOTE: timeouts may be due to a problem on the testing computer.</p>" .. | 485 | "<p>NOTE: timeouts may be due to a problem on the testing computer, it might be busy with other things, or be having it's own network problems..</p>" .. |
486 | "<p>The DNS round robin (DNS-RR) column shows the IPs for that mirror, or <font color='grey'><b>no</b></font> if it isn't part of the DNS-RR. " .. | 486 | "<p>The DNS round robin (DNS-RR) column shows the IPs for that mirror, or <font color='grey'><b>no</b></font> if it isn't part of the DNS-RR. " .. |
487 | "The IPs link to the testing log for that IP accessed via the DNS-RR. " .. | 487 | "The IPs link to the testing log for that IP accessed via the DNS-RR. " .. |
488 | APT.options.roundRobin.value .. " is the DNS-RR itself, so it doesn't get tested directly.</p>\n" .. | 488 | APT.options.roundRobin.value .. " is the DNS-RR itself, so it doesn't get tested directly.</p>\n" .. |
@@ -497,7 +497,7 @@ if nil == web then C("opening mirrors file - " .. e) else | |||
497 | 497 | ||
498 | m = makeIPlist(APT.mirrors) | 498 | m = makeIPlist(APT.mirrors) |
499 | web:write( "<p>This lists each mirror, and the DNS entries for that mirror. " .. | 499 | web:write( "<p>This lists each mirror, and the DNS entries for that mirror. " .. |
500 | "The links point to the testing log files for " .. logCount("apt-panopticon") .. " for each domain name / IP combination that was tested. " .. | 500 | "The links point to the testing log files (" .. logCount("apt-panopticon") .. ") for each domain name / IP combination that was tested. " .. |
501 | "If a mirror has a CNAME, that CNAME is listed along with that CNAMEs DNS entries. " .. | 501 | "If a mirror has a CNAME, that CNAME is listed along with that CNAMEs DNS entries. " .. |
502 | APT.options.roundRobin.value .. " is the DNS round robin, which points to the mirrors that are part of the DNS-RR. " .. | 502 | APT.options.roundRobin.value .. " is the DNS round robin, which points to the mirrors that are part of the DNS-RR. " .. |
503 | "If an IP is part of the DNS-RR, it is marked with '<font color='purple'><b>DNS-RR</b></font>' " .. | 503 | "If an IP is part of the DNS-RR, it is marked with '<font color='purple'><b>DNS-RR</b></font>' " .. |
diff --git a/apt-panopticon.lua b/apt-panopticon.lua index b140831..889058b 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua | |||
@@ -163,7 +163,7 @@ local ip = "" | |||
163 | local cor = nil | 163 | local cor = nil |
164 | 164 | ||
165 | local downloadLock = "flock -n results/curl-" | 165 | local downloadLock = "flock -n results/curl-" |
166 | 166 | local arw = ' <font color="magenta"><b>-></b></font> ' | |
167 | 167 | ||
168 | local repoExists = function (r) | 168 | local repoExists = function (r) |
169 | r = r:match("([%a-]*)") | 169 | r = r:match("([%a-]*)") |
@@ -215,14 +215,15 @@ end | |||
215 | 215 | ||
216 | local timeouts = 0; | 216 | local timeouts = 0; |
217 | local totalTimeouts = 0 | 217 | local totalTimeouts = 0 |
218 | local spcd = ' ' | ||
218 | checkHEAD = function (host, URL, r, retry, sanity) | 219 | checkHEAD = function (host, URL, r, retry, sanity) |
219 | if nil == r then r = 0 end | 220 | if nil == r then r = 0 end |
220 | if nil == retry then retry = 0 end | 221 | if nil == retry then retry = 0 end |
221 | if nil == sanity then sanity = false end | 222 | if true == sanity then sanity = 'URLSanity' else sanity = '' end |
222 | local check = "Checking file" | 223 | local check = "HEAD testing file" |
223 | local PU = url.parse(URL, defaultURL) | 224 | local PU = url.parse(URL, defaultURL) |
224 | local pu = url.parse(PU.scheme .. "://" .. host, defaultURL) | 225 | local pu = url.parse(PU.scheme .. "://" .. host, defaultURL) |
225 | if not APT.testing(PU.scheme, host) and APT.redir then I("Not supported, not tested " .. PU.scheme .. " " .. host .. " -> " .. URL, PU.scheme, "", host); return end | 226 | if not APT.testing(PU.scheme, host) and APT.redir then I(spcd .. string.upper(PU.scheme) .. " not supported, not tested. " .. URL, host); return end |
226 | if 0 < r then | 227 | if 0 < r then |
227 | check = "Redirecting to" | 228 | check = "Redirecting to" |
228 | end | 229 | end |
@@ -231,23 +232,27 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
231 | check = "Retry " .. retry .. " " .. check | 232 | check = "Retry " .. retry .. " " .. check |
232 | end | 233 | end |
233 | if 2 <= timeouts then | 234 | if 2 <= timeouts then |
234 | E("too many timeouts! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) | 235 | E(spcd .. spcd .. "too many timeouts! " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host) |
235 | return | 236 | return |
236 | end | 237 | end |
237 | if APT.options.timeouts.value <= (totalTimeouts) then | 238 | if APT.options.timeouts.value <= (totalTimeouts) then |
238 | E("Way too many timeouts!", PU.scheme, "", host) | 239 | E(spcd .. spcd .. "Way too many timeouts!", PU.scheme, "", host) |
239 | return | 240 | return |
240 | end | 241 | end |
241 | if 20 <= r then | 242 | if 20 <= r then |
242 | E("too many redirects! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) | 243 | E(spcd .. spcd .. "too many redirects! " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host) |
243 | return | 244 | return |
244 | end | 245 | end |
245 | if APT.options.retries.value <= retry then | 246 | if APT.options.retries.value <= retry then |
246 | E("too many retries! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) | 247 | E(spcd .. spcd .. "too many retries! " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host) |
248 | return | ||
249 | end | ||
250 | if "https" == PU.scheme and APT.options.roundRobin.value == host then | ||
251 | I(spcd .. "Not testing " .. APT.lnk(URL) .. " mirrors wont have the correct HTTPS certificate for the round robin.", host) | ||
247 | return | 252 | return |
253 | else | ||
254 | I(spcd .. check .. " " .. APT.lnk(URL), host) | ||
248 | end | 255 | end |
249 | D(PU.scheme .. " :// " .. check .. " " .. host .. " -> " .. URL) | ||
250 | if "https" == PU.scheme and APT.options.roundRobin.value == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert."); return end | ||
251 | 256 | ||
252 | --[[ Using curl command line - | 257 | --[[ Using curl command line - |
253 | -I - HEAD | 258 | -I - HEAD |
@@ -282,20 +287,13 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
282 | local msg = curlStatus[status] | 287 | local msg = curlStatus[status] |
283 | if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end | 288 | if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end |
284 | if (28 == status) or (7 == status) then | 289 | if (28 == status) or (7 == status) then |
285 | if sanity then | 290 | T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host) |
286 | T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "URLSanity", host) | ||
287 | else | ||
288 | T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "", host) | ||
289 | end | ||
290 | timeouts = timeouts + 1 | 291 | timeouts = timeouts + 1 |
291 | else | 292 | else |
292 | if sanity then | 293 | E(spcd .. spcd .. "The curl command return an error code of " .. status .. " - " .. msg .. ' for '.. APT.lnk(URL), PU.scheme, sanity, host) |
293 | E(" The curl command return an error code of " .. status .. " - " .. msg, PU.scheme, "URLSanity", host) | ||
294 | else | ||
295 | E(" The curl command return an error code of " .. status .. " - " .. msg, PU.scheme, "", host) | ||
296 | end | ||
297 | end | 294 | end |
298 | checkHEAD(host, URL, r, retry + 1, sanity) | 295 | if 60 == status then return end -- Certificate is invalid, don't bother retrying. |
296 | checkHEAD(host, URL, r, retry + 1, '' ~= sanity) | ||
299 | return | 297 | return |
300 | end | 298 | end |
301 | local code = "???" | 299 | local code = "???" |
@@ -303,17 +301,13 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
303 | local location = nil | 301 | local location = nil |
304 | local tmot = 1 | 302 | local tmot = 1 |
305 | while not APT.checkFile('results/STATUS_' .. fname) do | 303 | while not APT.checkFile('results/STATUS_' .. fname) do |
306 | I('Waiting for results/STATUS_' .. fname .. ' file.') | 304 | D(spcd .. spcd .. 'Waiting for results/STATUS_' .. fname .. ' file.') |
307 | os.execute('sleep ' .. tmot) | 305 | os.execute('sleep ' .. tmot) |
308 | tmot = tmot * 2 | 306 | tmot = tmot * 2 |
309 | if 8 < tmot then | 307 | if 8 < tmot then |
310 | if sanity then | 308 | T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host) |
311 | T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "URLSanity", host) | ||
312 | else | ||
313 | T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "", host) | ||
314 | end | ||
315 | timeouts = timeouts + 1 | 309 | timeouts = timeouts + 1 |
316 | checkHEAD(host, URL, r, retry + 1, sanity) | 310 | checkHEAD(host, URL, r, retry + 1, '' ~= sanity) |
317 | return | 311 | return |
318 | end | 312 | end |
319 | end | 313 | end |
@@ -323,10 +317,10 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
323 | if "#" == line:sub(1, 1) then | 317 | if "#" == line:sub(1, 1) then |
324 | code = line:sub(2, 4) | 318 | code = line:sub(2, 4) |
325 | if ("https" == PU.scheme) and ("0" ~= line:sub(6, 6)) then | 319 | if ("https" == PU.scheme) and ("0" ~= line:sub(6, 6)) then |
326 | if sanity then | 320 | if '' ~= sanity then |
327 | E(" The certificate is invalid.", PU.scheme, "URLSanity", host) | 321 | E(spcd .. spcd .. "The certificate is invalid.", PU.scheme, sanity, host) |
328 | else | 322 | else |
329 | E(" The certificate is invalid.", PU.scheme, "https", host) | 323 | E(spcd .. spcd .. "The certificate is invalid.", PU.scheme, "https", host) |
330 | end | 324 | end |
331 | end | 325 | end |
332 | elseif "http" == line:sub(1, 4):lower() then | 326 | elseif "http" == line:sub(1, 4):lower() then |
@@ -339,39 +333,35 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
339 | os.execute('rm -f results/STATUS_' .. fname .. ' 2>/dev/null') | 333 | os.execute('rm -f results/STATUS_' .. fname .. ' 2>/dev/null') |
340 | end | 334 | end |
341 | if ("4" == tostring(code):sub(1, 1)) or ("5" == tostring(code):sub(1, 1)) then | 335 | if ("4" == tostring(code):sub(1, 1)) or ("5" == tostring(code):sub(1, 1)) then |
342 | if sanity then | 336 | E(spcd .. spcd .. code .. " " .. cstr .. ". " .. check .. " " .. APT.lnk(URL), PU.scheme, sanity, host) |
343 | E(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "URLSanity", host) | ||
344 | else | ||
345 | E(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) | ||
346 | end | ||
347 | else | 337 | else |
348 | if not APT.testing(PU.scheme, host) then | 338 | if not APT.testing(PU.scheme, host) then |
349 | W("Not supported, but works " .. PU.scheme .. " " .. host .. " -> " .. URL, PU.scheme, "", host) | 339 | W(spcd .. spcd .. "Not supported, but works " .. PU.scheme .. " " .. APT.lnk(URL), PU.scheme, "", host) |
350 | end | 340 | end |
351 | I(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL) | 341 | I(spcd .. spcd .. code .. " " .. cstr .. ". " .. check .. " " .. APT.lnk(URL), host) |
352 | -- timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through. | 342 | -- timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through. |
353 | if (1 <= APT.options.bandwidth.value) and (nil ~= location) then | 343 | if (1 <= APT.options.bandwidth.value) and (nil ~= location) then |
354 | pu = url.parse(location, defaultURL) | 344 | pu = url.parse(location, defaultURL) |
355 | if ('http' == location:sub(1, 4)) and (pu.scheme ~= PU.scheme) then -- Sometimes a location sans scheme is returned, this is not a protocol change. | 345 | if ('http' == location:sub(1, 4)) and (pu.scheme ~= PU.scheme) then -- Sometimes a location sans scheme is returned, this is not a protocol change. |
356 | if APT.testing("Protocol") then | 346 | if APT.testing("Protocol") then |
357 | if APT.options.roundRobin.value == host then -- Coz HTTPS shouldn't happen via the round robin. | 347 | if APT.options.roundRobin.value == host then -- Coz HTTPS shouldn't happen via the round robin. |
358 | E(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host) | 348 | E(spcd .. spcd .. "Protocol changed during redirect! " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "Protocol", host) |
359 | end | 349 | end |
360 | W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host) | 350 | W(spcd .. spcd .. "Protocol changed during redirect! " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "Protocol", host) |
361 | else | 351 | else |
362 | end | 352 | end |
363 | end | 353 | end |
364 | 354 | ||
365 | if location == URL then | 355 | if location == URL then |
366 | E(" redirect loop! " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location, PU.scheme, "", host) | 356 | E(spcd .. spcd .. "Redirect loop! " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "", host) |
367 | elseif nil == pu.host then | 357 | elseif nil == pu.host then |
368 | I(" relative redirect. " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location) | 358 | I(spcd .. spcd .. "Relative redirect. " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host) |
369 | checkHEAD(host, PU.scheme .. "://" .. PU.host .. location, r + 1, retry, sanity) | 359 | checkHEAD(host, PU.scheme .. "://" .. PU.host .. location, r + 1, retry, '' ~= sanity) |
370 | elseif (PU.host == pu.host) or (host == pu.host) then | 360 | elseif (PU.host == pu.host) or (host == pu.host) then |
371 | I(" redirect to same host. " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location) | 361 | I(spcd .. spcd .. "Redirect to same host. " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host) |
372 | checkHEAD(pu.host, location, r + 1, retry, sanity) | 362 | checkHEAD(pu.host, location, r + 1, retry, '' ~= sanity) |
373 | else | 363 | else |
374 | I(" redirect to different host. " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location) | 364 | I(spcd .. spcd .. "Redirect to different host. " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host) |
375 | --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location. | 365 | --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location. |
376 | Not good for deb.debian.org, which we should only be throwing .debs at. | 366 | Not good for deb.debian.org, which we should only be throwing .debs at. |
377 | What we do is loop through the DNS entries, and only test the specific protocol & file being tested here. | 367 | What we do is loop through the DNS entries, and only test the specific protocol & file being tested here. |
@@ -384,10 +374,10 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
384 | if 'https' == pu.scheme then extraArgs = extraArgs .. ' --tests=-http' end | 374 | if 'https' == pu.scheme then extraArgs = extraArgs .. ' --tests=-http' end |
385 | if 'http' == pu.scheme then extraArgs = extraArgs .. ' --tests=-https' end | 375 | if 'http' == pu.scheme then extraArgs = extraArgs .. ' --tests=-https' end |
386 | local pth = path:match('^(.*/pool/).*$') | 376 | local pth = path:match('^(.*/pool/).*$') |
387 | if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E('Odd redirect path ' .. path) end | 377 | if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E(spcd .. spcd .. 'Odd redirect path ' .. path) end |
388 | I(" Now checking redirected host " .. u) | 378 | I(spcd .. spcd .. "Now checking redirected host " .. u .. ' for ' .. APT.lnk(URL) .. arw .. APT.lnk(location), host) |
389 | APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork() | 379 | APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork() |
390 | D('logging to ' .. APT.logName(pu.host, nil, file)[2]) | 380 | D(spcd .. 'logging to ' .. APT.logName(pu.host, nil, file)[2]) |
391 | end | 381 | end |
392 | end | 382 | end |
393 | end | 383 | end |
@@ -418,11 +408,10 @@ local checkFiles = function (host, ip, path, file) | |||
418 | if nil == path then path = "" end | 408 | if nil == path then path = "" end |
419 | if nil ~= file then | 409 | if nil ~= file then |
420 | if "redir" == ip then ip = host end | 410 | if "redir" == ip then ip = host end |
421 | I(" Checking IP for file " .. host .. " -> " .. ip .. " " .. path .. " " .. file) | ||
422 | if checkTimeouts(host, "http", ip .. path .. "/" .. file) then return end | 411 | if checkTimeouts(host, "http", ip .. path .. "/" .. file) then return end |
423 | if checkTimeouts(host, "https", ip .. path .. "/" .. file) then return end | 412 | if checkTimeouts(host, "https", ip .. path .. "/" .. file) then return end |
424 | else | 413 | else |
425 | I(" Checking IP " .. host .. " -> " .. ip .. " " .. path) | 414 | I(" HEAD testing files for " .. host .. arw .. ip .. " " .. path, host) |
426 | if 1 <= APT.options.bandwidth.value then | 415 | if 1 <= APT.options.bandwidth.value then |
427 | -- Do these first, coz they are likely to fork off a different server. | 416 | -- Do these first, coz they are likely to fork off a different server. |
428 | for i, s in pairs(referenceDebs) do | 417 | for i, s in pairs(referenceDebs) do |
@@ -467,10 +456,10 @@ checkHost = function (orig, host, path, ip, file) | |||
467 | end | 456 | end |
468 | else | 457 | else |
469 | if orig == host then | 458 | if orig == host then |
470 | D("checkHost " .. orig .. "" .. file) | 459 | I("Testing mirror " .. orig .. "" .. file) |
471 | APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork() | 460 | APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork() |
472 | D('logging to ' .. APT.logName(ph.host, nil, file)[2]) | 461 | D('logging to ' .. APT.logName(ph.host, nil, file)[2]) |
473 | else D("checkHost " .. orig .. " -> " .. host) end | 462 | else D("checkHost " .. orig .. arw .. host) end |
474 | end | 463 | end |
475 | end | 464 | end |
476 | 465 | ||
@@ -534,7 +523,7 @@ local downloads = function(host, URL, meta, release, list) | |||
534 | addDownload(host, URL, f, release, "/" .. l) | 523 | addDownload(host, URL, f, release, "/" .. l) |
535 | end | 524 | end |
536 | else | 525 | else |
537 | D('Downloading http://' .. host .. URL .. '/merged/' .. list) | 526 | I('Downloading ' .. APT.lnk('http://' .. host .. URL .. '/merged/' .. list)) |
538 | f:write('url "' .. 'http://' .. host .. URL .. '/merged/' .. list .. '"\n') | 527 | f:write('url "' .. 'http://' .. host .. URL .. '/merged/' .. list .. '"\n') |
539 | f:write('output "results/' .. host .. '/merged/' .. list .. '"\n') | 528 | f:write('output "results/' .. host .. '/merged/' .. list .. '"\n') |
540 | end | 529 | end |
@@ -582,8 +571,8 @@ local getMirrors = function () | |||
582 | local host = "" | 571 | local host = "" |
583 | local m = {} | 572 | local m = {} |
584 | local active = true | 573 | local active = true |
585 | local URL = "http://" .. APT.options.referenceSite.value .. "/mirror_list.txt" | 574 | local URL = 'http://' .. APT.options.referenceSite.value .. '/mirror_list.txt' |
586 | I("getting mirrors.") | 575 | I('Downloadin gand parsing http://' .. APT.options.referenceSite.value .. '/mirror_list.txt') |
587 | local p, c, h = http.request(URL) | 576 | local p, c, h = http.request(URL) |
588 | if nil == p then E(c .. " fetching " .. URL) else | 577 | if nil == p then E(c .. " fetching " .. URL) else |
589 | 578 | ||
@@ -683,12 +672,12 @@ local parseDebs = function(host) | |||
683 | end | 672 | end |
684 | if APT.testing("Updated") then | 673 | if APT.testing("Updated") then |
685 | if sz ~= fsz:sub(2, -2) then | 674 | if sz ~= fsz:sub(2, -2) then |
686 | E('Package size mismatch - results/' .. host .. "/merged/" .. p, 'http', 'Updated', host) | 675 | E('Package size mismatch for ' .. host .. "/merged/" .. p, 'http', 'Updated', host) |
687 | end | 676 | end |
688 | end | 677 | end |
689 | os.execute('rm -f results/' .. host .. "/merged/" .. p) | 678 | os.execute('rm -f results/' .. host .. "/merged/" .. p) |
690 | else | 679 | else |
691 | E('Failed to download - results/' .. host .. "/merged/" .. p, 'http', 'Updated', host) | 680 | E('Failed to download ' .. host .. "/merged/" .. p, 'http', 'Updated', host) |
692 | end | 681 | end |
693 | end | 682 | end |
694 | end | 683 | end |
@@ -803,7 +792,7 @@ local parseRelease = function(host) | |||
803 | if APT.testing("Integrity") then | 792 | if APT.testing("Integrity") then |
804 | local status = APT.exe( "gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. | 793 | local status = APT.exe( "gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. |
805 | " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status | 794 | " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status |
806 | if 0 ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end | 795 | if 0 ~= status then E("GPG check failed for " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end |
807 | -- TODO - should check the PGP sig of InRelease as well. | 796 | -- TODO - should check the PGP sig of InRelease as well. |
808 | end | 797 | end |
809 | os.execute('rm results/' .. host .. '/merged/dists/' .. n .. '/' .. o) | 798 | os.execute('rm results/' .. host .. '/merged/dists/' .. n .. '/' .. o) |
@@ -992,12 +981,11 @@ if 0 < #arg then | |||
992 | end | 981 | end |
993 | end | 982 | end |
994 | 983 | ||
995 | if APT.origin or APT.redir then APT.results["IPs"] = gatherIPs(pu.host) end | ||
996 | |||
997 | if not APT.logOpen(pu.host, arg[2], arg[3]) then return end | 984 | if not APT.logOpen(pu.host, arg[2], arg[3]) then return end |
998 | I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", ")) | 985 | I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", ")) |
999 | if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end | 986 | if APT.origin or APT.redir then APT.results["IPs"] = gatherIPs(pu.host) end |
1000 | if nil ~= arg[3] then I(" Using file " .. arg[3]); end | 987 | if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end |
988 | if nil ~= arg[3] then I(" Using file " .. arg[3]); end | ||
1001 | APT.mirrors = loadfile("results/mirrors.lua")() | 989 | APT.mirrors = loadfile("results/mirrors.lua")() |
1002 | APT.results = APT.padResults(APT.results) | 990 | APT.results = APT.padResults(APT.results) |
1003 | 991 | ||