aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
authoronefang2019-12-30 17:05:18 +1000
committeronefang2019-12-30 17:05:18 +1000
commitf02b222005f7d656a6da8d25dca5a5fef56857c2 (patch)
treed8746d182a6b11fd590d94199c31312d369c2996
parentTODO++ (diff)
downloadapt-panopticon-f02b222005f7d656a6da8d25dca5a5fef56857c2.zip
apt-panopticon-f02b222005f7d656a6da8d25dca5a5fef56857c2.tar.gz
apt-panopticon-f02b222005f7d656a6da8d25dca5a5fef56857c2.tar.bz2
apt-panopticon-f02b222005f7d656a6da8d25dca5a5fef56857c2.tar.xz
A lot of tweaking of the logging and reporting, making it easier to read.
Diffstat (limited to '')
-rw-r--r--apt-panopticommon.lua19
-rwxr-xr-xapt-panopticon-report-email-web.lua18
-rwxr-xr-xapt-panopticon.lua114
3 files changed, 72 insertions, 79 deletions
diff --git a/apt-panopticommon.lua b/apt-panopticommon.lua
index 4e75f3f..ca952d3 100644
--- a/apt-panopticommon.lua
+++ b/apt-panopticommon.lua
@@ -348,6 +348,11 @@ APT.allpairs = function(tbl, func)
348 end 348 end
349end 349end
350 350
351APT.lnk = function(URL)
352 return '<a href="' .. URL .. '">' .. URL .. '</a>'
353end
354
355
351APT.search = function(t, s) 356APT.search = function(t, s)
352 for i, v in pairs(t) do 357 for i, v in pairs(t) do
353 if v == s then return true end 358 if v == s then return true end
@@ -441,12 +446,12 @@ local log = function(v, t, s, prot, test, host)
441 APT.logFile:flush() 446 APT.logFile:flush()
442 end 447 end
443end 448end
444APT.D = function(s) log(4, "DEBUG ", s) end 449APT.D = function(s, h) log(4, "DEBUG ", s, nil, nil, h) end
445APT.I = function(s) log(3, "INFO ", s) end 450APT.I = function(s, h) log(3, "INFO ", s, nil, nil, h) end
446APT.T = function(s, p, t, h) log(2, "TIMEOUT ", s, p, t, h) end 451APT.T = function(s, p, t, h) log(2, "TIMEOUT ", s, p, t, h) end
447APT.W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end 452APT.W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end
448APT.E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end 453APT.E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end
449APT.C = function(s) log(-1, "CRITICAL", s) end 454APT.C = function(s) log(-1, "CRITICAL", s) end
450local D = APT.D 455local D = APT.D
451local I = APT.I 456local I = APT.I
452local T = APT.T 457local T = APT.T
@@ -463,7 +468,7 @@ APT.testing = function(t, host)
463 if t == v then 468 if t == v then
464 local h = APT.mirrors[host] 469 local h = APT.mirrors[host]
465 if nil == h then return true end 470 if nil == h then return true end
466 if true == h["Protocols"][t] then return true else D("Skipping " .. t .. " checks for " .. host) end 471 if true == h["Protocols"][t] then return true else I("Skipping " .. t .. " checks for " .. host) end
467 end 472 end
468 end 473 end
469 return false 474 return false
diff --git a/apt-panopticon-report-email-web.lua b/apt-panopticon-report-email-web.lua
index 97f7cef..05699ee 100755
--- a/apt-panopticon-report-email-web.lua
+++ b/apt-panopticon-report-email-web.lua
@@ -213,7 +213,7 @@ end
213local makeTable = function(web, hosts) 213local makeTable = function(web, hosts)
214 web:write("<table>\n<tr><th></th><th>" .. lnk('FTP') .. "</th><th>" .. lnk('HTTP') .. "</th><th>" .. lnk('HTTPS') .. "</th><th>" .. lnk('RSYNC') .. "</th>" .. 214 web:write("<table>\n<tr><th></th><th>" .. lnk('FTP') .. "</th><th>" .. lnk('HTTP') .. "</th><th>" .. lnk('HTTPS') .. "</th><th>" .. lnk('RSYNC') .. "</th>" ..
215 "<th>" .. lnk('DNS round robin', 'DNS-RR') .. "</th><th>" .. lnk('Protocol') .. "</th><th>" .. lnk('URL sanity', 'URL-Sanity') .. "</th><th>" .. lnk('Integrity') .. "</th>" .. 215 "<th>" .. lnk('DNS round robin', 'DNS-RR') .. "</th><th>" .. lnk('Protocol') .. "</th><th>" .. lnk('URL sanity', 'URL-Sanity') .. "</th><th>" .. lnk('Integrity') .. "</th>" ..
216 "<th>" .. lnk('Updated') .. "</th><th colspan='2'>" .. lnk('Speed range', 'Speed') .. "</th>" .. 216 "<th colspan='2'>" .. lnk('Updated') .. "</th><th colspan='2'>" .. lnk('Speed range', 'Speed') .. "</th>" ..
217 "<th colspan='2'>" .. lnk('Weekly statistics', 'Weekly') .. "</th><th>" .. lnk('Graphs') .. "</th></tr>\n") 217 "<th colspan='2'>" .. lnk('Weekly statistics', 'Weekly') .. "</th><th>" .. lnk('Graphs') .. "</th></tr>\n")
218 local bg = '' 218 local bg = ''
219 for k, v in APT.orderedPairs(hosts) do 219 for k, v in APT.orderedPairs(hosts) do
@@ -235,13 +235,13 @@ local makeTable = function(web, hosts)
235 local sanity = status(hosts, k, results, "URLSanity") 235 local sanity = status(hosts, k, results, "URLSanity")
236 local integrity = status(hosts, k, results, "Integrity") 236 local integrity = status(hosts, k, results, "Integrity")
237 local updated = status(hosts, k, results, "Updated") 237 local updated = status(hosts, k, results, "Updated")
238 local rate = v.Rate 238 local rate = v.Rate
239 if nil ~= rate then updated = updated .. ' ' .. rate end
240 local min = tonumber(results.speed.min) 239 local min = tonumber(results.speed.min)
241 local max = tonumber(results.speed.max) 240 local max = tonumber(results.speed.max)
242 local spd = '<td></td><td></td>' 241 local spd = '<td></td><td></td>'
243 local week = '<td></td><td></td>' 242 local week = '<td>&nbsp;</td><td>&nbsp;</td>'
244 local graph = '<a href="../apt-panopticon_cgp/host.php?h=' .. k .. '">graphs</a>' 243 local graph = '<a href="../apt-panopticon_cgp/host.php?h=' .. k .. '">graphs</a>'
244 if nil == rate then rate = '' end
245 245
246 -- DNS-RR test. 246 -- DNS-RR test.
247 if (APT.options.roundRobin.value ~= k) and (nil ~= hosts[APT.options.roundRobin.value]) then 247 if (APT.options.roundRobin.value ~= k) and (nil ~= hosts[APT.options.roundRobin.value]) then
@@ -291,12 +291,12 @@ local makeTable = function(web, hosts)
291 if '0.00' == percentUp then percentUpdated = '??' end -- We are counting errors, and you can't get an error if you can't check anything. 291 if '0.00' == percentUp then percentUpdated = '??' end -- We are counting errors, and you can't get an error if you can't check anything.
292 -- TODO - try to account for this better, this is just a quick hack. 292 -- TODO - try to account for this better, this is just a quick hack.
293 end 293 end
294 week = '<td>' .. percentUp .. '% up</td><td>' .. percentUpdated .. '% updated</td>' 294 week = '<td>&nbsp;' .. percentUp .. '% up</td><td>&nbsp;' .. percentUpdated .. '% updated</td>'
295 end 295 end
296 296
297 web:write("<td>" .. ftp .. "&nbsp;</td><td>" .. http .. "&nbsp;</td><td>" .. https .. "&nbsp;</td><td>" .. rsync .. "&nbsp;</td><td>" .. dns .. 297 web:write("<td>" .. ftp .. "&nbsp;</td><td>" .. http .. "&nbsp;</td><td>" .. https .. "&nbsp;</td><td>" .. rsync .. "&nbsp;</td><td>" .. dns ..
298 "&nbsp;</td><td>" .. protocol .. "&nbsp;</td><td>" .. sanity .. "&nbsp;</td><td>" .. integrity .. "&nbsp;</td><td>" .. updated .. 298 "&nbsp;</td><td>" .. protocol .. "&nbsp;</td><td>" .. sanity .. "&nbsp;</td><td>" .. integrity .. "&nbsp;</td>" .. '<td>' .. rate ..
299 "&nbsp;" .. spd .. "&nbsp;" .. week .."&nbsp;<td>" .. graph .. "</td></tr>\n") 299 '</td><td>' .. updated .. '</td>&nbsp;' .. spd .. "&nbsp;" .. week .."&nbsp;<td>" .. graph .. "</td></tr>\n")
300 if "" ~= active then 300 if "" ~= active then
301 web:write("<tr><td style='background-color:dimgrey'>" .. active .. "</td></tr>\n") 301 web:write("<tr><td style='background-color:dimgrey'>" .. active .. "</td></tr>\n")
302 end 302 end
@@ -482,7 +482,7 @@ if nil == web then C("opening mirrors file - " .. e) else
482 " means the mirror had some timeouts, and tests where not yet aborted. The darker colour means unsupported by the mirror, but tested anyway.</p>" .. 482 " means the mirror had some timeouts, and tests where not yet aborted. The darker colour means unsupported by the mirror, but tested anyway.</p>" ..
483 "<p><font color='blue'><b>TIMEOUT</b></font> or <font color='darkblue'><b>TIMEOUT</b></font>" .. 483 "<p><font color='blue'><b>TIMEOUT</b></font> or <font color='darkblue'><b>TIMEOUT</b></font>" ..
484 " means the mirror had too many timeouts, and tests where aborted, so there is no result for this test.</p>" .. 484 " means the mirror had too many timeouts, and tests where aborted, so there is no result for this test.</p>" ..
485 "<p>NOTE: timeouts may be due to a problem on the testing computer.</p>" .. 485 "<p>NOTE: timeouts may be due to a problem on the testing computer, it might be busy with other things, or be having it's own network problems..</p>" ..
486 "<p>The DNS round robin (DNS-RR) column shows the IPs for that mirror, or <font color='grey'><b>no</b></font> if it isn't part of the DNS-RR. &nbsp; " .. 486 "<p>The DNS round robin (DNS-RR) column shows the IPs for that mirror, or <font color='grey'><b>no</b></font> if it isn't part of the DNS-RR. &nbsp; " ..
487 "The IPs link to the testing log for that IP accessed via the DNS-RR. &nbsp; " .. 487 "The IPs link to the testing log for that IP accessed via the DNS-RR. &nbsp; " ..
488 APT.options.roundRobin.value .. " is the DNS-RR itself, so it doesn't get tested directly.</p>\n" .. 488 APT.options.roundRobin.value .. " is the DNS-RR itself, so it doesn't get tested directly.</p>\n" ..
@@ -497,7 +497,7 @@ if nil == web then C("opening mirrors file - " .. e) else
497 497
498 m = makeIPlist(APT.mirrors) 498 m = makeIPlist(APT.mirrors)
499 web:write( "<p>This lists each mirror, and the DNS entries for that mirror. &nbsp; " .. 499 web:write( "<p>This lists each mirror, and the DNS entries for that mirror. &nbsp; " ..
500 "The links point to the testing log files for " .. logCount("apt-panopticon") .. " for each domain name / IP combination that was tested. &nbsp; " .. 500 "The links point to the testing log files (" .. logCount("apt-panopticon") .. ") for each domain name / IP combination that was tested. &nbsp; " ..
501 "If a mirror has a CNAME, that CNAME is listed along with that CNAMEs DNS entries. &nbsp; " .. 501 "If a mirror has a CNAME, that CNAME is listed along with that CNAMEs DNS entries. &nbsp; " ..
502 APT.options.roundRobin.value .. " is the DNS round robin, which points to the mirrors that are part of the DNS-RR. &nbsp; " .. 502 APT.options.roundRobin.value .. " is the DNS round robin, which points to the mirrors that are part of the DNS-RR. &nbsp; " ..
503 "If an IP is part of the DNS-RR, it is marked with '<font color='purple'><b>DNS-RR</b></font>' &nbsp; " .. 503 "If an IP is part of the DNS-RR, it is marked with '<font color='purple'><b>DNS-RR</b></font>' &nbsp; " ..
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index b140831..889058b 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -163,7 +163,7 @@ local ip = ""
163local cor = nil 163local cor = nil
164 164
165local downloadLock = "flock -n results/curl-" 165local downloadLock = "flock -n results/curl-"
166 166local arw = ' &nbsp; <font color="magenta"><b>-></b></font> &nbsp; '
167 167
168local repoExists = function (r) 168local repoExists = function (r)
169 r = r:match("([%a-]*)") 169 r = r:match("([%a-]*)")
@@ -215,14 +215,15 @@ end
215 215
216local timeouts = 0; 216local timeouts = 0;
217local totalTimeouts = 0 217local totalTimeouts = 0
218local spcd = ' &nbsp; '
218checkHEAD = function (host, URL, r, retry, sanity) 219checkHEAD = function (host, URL, r, retry, sanity)
219 if nil == r then r = 0 end 220 if nil == r then r = 0 end
220 if nil == retry then retry = 0 end 221 if nil == retry then retry = 0 end
221 if nil == sanity then sanity = false end 222 if true == sanity then sanity = 'URLSanity' else sanity = '' end
222 local check = "Checking file" 223 local check = "HEAD testing file"
223 local PU = url.parse(URL, defaultURL) 224 local PU = url.parse(URL, defaultURL)
224 local pu = url.parse(PU.scheme .. "://" .. host, defaultURL) 225 local pu = url.parse(PU.scheme .. "://" .. host, defaultURL)
225 if not APT.testing(PU.scheme, host) and APT.redir then I("Not supported, not tested " .. PU.scheme .. " " .. host .. " -> " .. URL, PU.scheme, "", host); return end 226 if not APT.testing(PU.scheme, host) and APT.redir then I(spcd .. string.upper(PU.scheme) .. " not supported, not tested. &nbsp; " .. URL, host); return end
226 if 0 < r then 227 if 0 < r then
227 check = "Redirecting to" 228 check = "Redirecting to"
228 end 229 end
@@ -231,23 +232,27 @@ checkHEAD = function (host, URL, r, retry, sanity)
231 check = "Retry " .. retry .. " " .. check 232 check = "Retry " .. retry .. " " .. check
232 end 233 end
233 if 2 <= timeouts then 234 if 2 <= timeouts then
234 E("too many timeouts! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) 235 E(spcd .. spcd .. "too many timeouts! " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host)
235 return 236 return
236 end 237 end
237 if APT.options.timeouts.value <= (totalTimeouts) then 238 if APT.options.timeouts.value <= (totalTimeouts) then
238 E("Way too many timeouts!", PU.scheme, "", host) 239 E(spcd .. spcd .. "Way too many timeouts!", PU.scheme, "", host)
239 return 240 return
240 end 241 end
241 if 20 <= r then 242 if 20 <= r then
242 E("too many redirects! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) 243 E(spcd .. spcd .. "too many redirects! " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host)
243 return 244 return
244 end 245 end
245 if APT.options.retries.value <= retry then 246 if APT.options.retries.value <= retry then
246 E("too many retries! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) 247 E(spcd .. spcd .. "too many retries! " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host)
248 return
249 end
250 if "https" == PU.scheme and APT.options.roundRobin.value == host then
251 I(spcd .. "Not testing " .. APT.lnk(URL) .. " mirrors wont have the correct HTTPS certificate for the round robin.", host)
247 return 252 return
253 else
254 I(spcd .. check .. " " .. APT.lnk(URL), host)
248 end 255 end
249 D(PU.scheme .. " :// " .. check .. " " .. host .. " -> " .. URL)
250 if "https" == PU.scheme and APT.options.roundRobin.value == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert."); return end
251 256
252 --[[ Using curl command line - 257 --[[ Using curl command line -
253 -I - HEAD 258 -I - HEAD
@@ -282,20 +287,13 @@ checkHEAD = function (host, URL, r, retry, sanity)
282 local msg = curlStatus[status] 287 local msg = curlStatus[status]
283 if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end 288 if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end
284 if (28 == status) or (7 == status) then 289 if (28 == status) or (7 == status) then
285 if sanity then 290 T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host)
286 T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "URLSanity", host)
287 else
288 T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "", host)
289 end
290 timeouts = timeouts + 1 291 timeouts = timeouts + 1
291 else 292 else
292 if sanity then 293 E(spcd .. spcd .. "The curl command return an error code of " .. status .. " - " .. msg .. ' for '.. APT.lnk(URL), PU.scheme, sanity, host)
293 E(" The curl command return an error code of " .. status .. " - " .. msg, PU.scheme, "URLSanity", host)
294 else
295 E(" The curl command return an error code of " .. status .. " - " .. msg, PU.scheme, "", host)
296 end
297 end 294 end
298 checkHEAD(host, URL, r, retry + 1, sanity) 295 if 60 == status then return end -- Certificate is invalid, don't bother retrying.
296 checkHEAD(host, URL, r, retry + 1, '' ~= sanity)
299 return 297 return
300 end 298 end
301 local code = "???" 299 local code = "???"
@@ -303,17 +301,13 @@ checkHEAD = function (host, URL, r, retry, sanity)
303 local location = nil 301 local location = nil
304 local tmot = 1 302 local tmot = 1
305 while not APT.checkFile('results/STATUS_' .. fname) do 303 while not APT.checkFile('results/STATUS_' .. fname) do
306 I('Waiting for results/STATUS_' .. fname .. ' file.') 304 D(spcd .. spcd .. 'Waiting for results/STATUS_' .. fname .. ' file.')
307 os.execute('sleep ' .. tmot) 305 os.execute('sleep ' .. tmot)
308 tmot = tmot * 2 306 tmot = tmot * 2
309 if 8 < tmot then 307 if 8 < tmot then
310 if sanity then 308 T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host)
311 T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "URLSanity", host)
312 else
313 T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "", host)
314 end
315 timeouts = timeouts + 1 309 timeouts = timeouts + 1
316 checkHEAD(host, URL, r, retry + 1, sanity) 310 checkHEAD(host, URL, r, retry + 1, '' ~= sanity)
317 return 311 return
318 end 312 end
319 end 313 end
@@ -323,10 +317,10 @@ checkHEAD = function (host, URL, r, retry, sanity)
323 if "#" == line:sub(1, 1) then 317 if "#" == line:sub(1, 1) then
324 code = line:sub(2, 4) 318 code = line:sub(2, 4)
325 if ("https" == PU.scheme) and ("0" ~= line:sub(6, 6)) then 319 if ("https" == PU.scheme) and ("0" ~= line:sub(6, 6)) then
326 if sanity then 320 if '' ~= sanity then
327 E(" The certificate is invalid.", PU.scheme, "URLSanity", host) 321 E(spcd .. spcd .. "The certificate is invalid.", PU.scheme, sanity, host)
328 else 322 else
329 E(" The certificate is invalid.", PU.scheme, "https", host) 323 E(spcd .. spcd .. "The certificate is invalid.", PU.scheme, "https", host)
330 end 324 end
331 end 325 end
332 elseif "http" == line:sub(1, 4):lower() then 326 elseif "http" == line:sub(1, 4):lower() then
@@ -339,39 +333,35 @@ checkHEAD = function (host, URL, r, retry, sanity)
339 os.execute('rm -f results/STATUS_' .. fname .. ' 2>/dev/null') 333 os.execute('rm -f results/STATUS_' .. fname .. ' 2>/dev/null')
340 end 334 end
341 if ("4" == tostring(code):sub(1, 1)) or ("5" == tostring(code):sub(1, 1)) then 335 if ("4" == tostring(code):sub(1, 1)) or ("5" == tostring(code):sub(1, 1)) then
342 if sanity then 336 E(spcd .. spcd .. code .. " " .. cstr .. ". " .. check .. " " .. APT.lnk(URL), PU.scheme, sanity, host)
343 E(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "URLSanity", host)
344 else
345 E(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
346 end
347 else 337 else
348 if not APT.testing(PU.scheme, host) then 338 if not APT.testing(PU.scheme, host) then
349 W("Not supported, but works " .. PU.scheme .. " " .. host .. " -> " .. URL, PU.scheme, "", host) 339 W(spcd .. spcd .. "Not supported, but works " .. PU.scheme .. " " .. APT.lnk(URL), PU.scheme, "", host)
350 end 340 end
351 I(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL) 341 I(spcd .. spcd .. code .. " " .. cstr .. ". " .. check .. " " .. APT.lnk(URL), host)
352-- timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through. 342-- timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through.
353 if (1 <= APT.options.bandwidth.value) and (nil ~= location) then 343 if (1 <= APT.options.bandwidth.value) and (nil ~= location) then
354 pu = url.parse(location, defaultURL) 344 pu = url.parse(location, defaultURL)
355 if ('http' == location:sub(1, 4)) and (pu.scheme ~= PU.scheme) then -- Sometimes a location sans scheme is returned, this is not a protocol change. 345 if ('http' == location:sub(1, 4)) and (pu.scheme ~= PU.scheme) then -- Sometimes a location sans scheme is returned, this is not a protocol change.
356 if APT.testing("Protocol") then 346 if APT.testing("Protocol") then
357 if APT.options.roundRobin.value == host then -- Coz HTTPS shouldn't happen via the round robin. 347 if APT.options.roundRobin.value == host then -- Coz HTTPS shouldn't happen via the round robin.
358 E(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host) 348 E(spcd .. spcd .. "Protocol changed during redirect! " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "Protocol", host)
359 end 349 end
360 W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host) 350 W(spcd .. spcd .. "Protocol changed during redirect! " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "Protocol", host)
361 else 351 else
362 end 352 end
363 end 353 end
364 354
365 if location == URL then 355 if location == URL then
366 E(" redirect loop! " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location, PU.scheme, "", host) 356 E(spcd .. spcd .. "Redirect loop! " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "", host)
367 elseif nil == pu.host then 357 elseif nil == pu.host then
368 I(" relative redirect. " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location) 358 I(spcd .. spcd .. "Relative redirect. " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
369 checkHEAD(host, PU.scheme .. "://" .. PU.host .. location, r + 1, retry, sanity) 359 checkHEAD(host, PU.scheme .. "://" .. PU.host .. location, r + 1, retry, '' ~= sanity)
370 elseif (PU.host == pu.host) or (host == pu.host) then 360 elseif (PU.host == pu.host) or (host == pu.host) then
371 I(" redirect to same host. " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location) 361 I(spcd .. spcd .. "Redirect to same host. " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
372 checkHEAD(pu.host, location, r + 1, retry, sanity) 362 checkHEAD(pu.host, location, r + 1, retry, '' ~= sanity)
373 else 363 else
374 I(" redirect to different host. " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location) 364 I(spcd .. spcd .. "Redirect to different host. " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
375 --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location. 365 --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location.
376 Not good for deb.debian.org, which we should only be throwing .debs at. 366 Not good for deb.debian.org, which we should only be throwing .debs at.
377 What we do is loop through the DNS entries, and only test the specific protocol & file being tested here. 367 What we do is loop through the DNS entries, and only test the specific protocol & file being tested here.
@@ -384,10 +374,10 @@ checkHEAD = function (host, URL, r, retry, sanity)
384 if 'https' == pu.scheme then extraArgs = extraArgs .. ' --tests=-http' end 374 if 'https' == pu.scheme then extraArgs = extraArgs .. ' --tests=-http' end
385 if 'http' == pu.scheme then extraArgs = extraArgs .. ' --tests=-https' end 375 if 'http' == pu.scheme then extraArgs = extraArgs .. ' --tests=-https' end
386 local pth = path:match('^(.*/pool/).*$') 376 local pth = path:match('^(.*/pool/).*$')
387 if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E('Odd redirect path ' .. path) end 377 if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E(spcd .. spcd .. 'Odd redirect path ' .. path) end
388 I(" Now checking redirected host " .. u) 378 I(spcd .. spcd .. "Now checking redirected host " .. u .. ' &nbsp; for &nbsp; ' .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
389 APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork() 379 APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork()
390 D('logging to ' .. APT.logName(pu.host, nil, file)[2]) 380 D(spcd .. 'logging to ' .. APT.logName(pu.host, nil, file)[2])
391 end 381 end
392 end 382 end
393 end 383 end
@@ -418,11 +408,10 @@ local checkFiles = function (host, ip, path, file)
418 if nil == path then path = "" end 408 if nil == path then path = "" end
419 if nil ~= file then 409 if nil ~= file then
420 if "redir" == ip then ip = host end 410 if "redir" == ip then ip = host end
421 I(" Checking IP for file " .. host .. " -> " .. ip .. " " .. path .. " " .. file)
422 if checkTimeouts(host, "http", ip .. path .. "/" .. file) then return end 411 if checkTimeouts(host, "http", ip .. path .. "/" .. file) then return end
423 if checkTimeouts(host, "https", ip .. path .. "/" .. file) then return end 412 if checkTimeouts(host, "https", ip .. path .. "/" .. file) then return end
424 else 413 else
425 I(" Checking IP " .. host .. " -> " .. ip .. " " .. path) 414 I(" HEAD testing files for " .. host .. arw .. ip .. " " .. path, host)
426 if 1 <= APT.options.bandwidth.value then 415 if 1 <= APT.options.bandwidth.value then
427 -- Do these first, coz they are likely to fork off a different server. 416 -- Do these first, coz they are likely to fork off a different server.
428 for i, s in pairs(referenceDebs) do 417 for i, s in pairs(referenceDebs) do
@@ -467,10 +456,10 @@ checkHost = function (orig, host, path, ip, file)
467 end 456 end
468 else 457 else
469 if orig == host then 458 if orig == host then
470 D("checkHost " .. orig .. "" .. file) 459 I("Testing mirror " .. orig .. "" .. file)
471 APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork() 460 APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork()
472 D('logging to ' .. APT.logName(ph.host, nil, file)[2]) 461 D('logging to ' .. APT.logName(ph.host, nil, file)[2])
473 else D("checkHost " .. orig .. " -> " .. host) end 462 else D("checkHost " .. orig .. arw .. host) end
474 end 463 end
475end 464end
476 465
@@ -534,7 +523,7 @@ local downloads = function(host, URL, meta, release, list)
534 addDownload(host, URL, f, release, "/" .. l) 523 addDownload(host, URL, f, release, "/" .. l)
535 end 524 end
536 else 525 else
537 D('Downloading http://' .. host .. URL .. '/merged/' .. list) 526 I('Downloading ' .. APT.lnk('http://' .. host .. URL .. '/merged/' .. list))
538 f:write('url "' .. 'http://' .. host .. URL .. '/merged/' .. list .. '"\n') 527 f:write('url "' .. 'http://' .. host .. URL .. '/merged/' .. list .. '"\n')
539 f:write('output "results/' .. host .. '/merged/' .. list .. '"\n') 528 f:write('output "results/' .. host .. '/merged/' .. list .. '"\n')
540 end 529 end
@@ -582,8 +571,8 @@ local getMirrors = function ()
582 local host = "" 571 local host = ""
583 local m = {} 572 local m = {}
584 local active = true 573 local active = true
585 local URL = "http://" .. APT.options.referenceSite.value .. "/mirror_list.txt" 574 local URL = 'http://' .. APT.options.referenceSite.value .. '/mirror_list.txt'
586 I("getting mirrors.") 575 I('Downloadin gand parsing http://' .. APT.options.referenceSite.value .. '/mirror_list.txt')
587 local p, c, h = http.request(URL) 576 local p, c, h = http.request(URL)
588 if nil == p then E(c .. " fetching " .. URL) else 577 if nil == p then E(c .. " fetching " .. URL) else
589 578
@@ -683,12 +672,12 @@ local parseDebs = function(host)
683 end 672 end
684 if APT.testing("Updated") then 673 if APT.testing("Updated") then
685 if sz ~= fsz:sub(2, -2) then 674 if sz ~= fsz:sub(2, -2) then
686 E('Package size mismatch - results/' .. host .. "/merged/" .. p, 'http', 'Updated', host) 675 E('Package size mismatch for ' .. host .. "/merged/" .. p, 'http', 'Updated', host)
687 end 676 end
688 end 677 end
689 os.execute('rm -f results/' .. host .. "/merged/" .. p) 678 os.execute('rm -f results/' .. host .. "/merged/" .. p)
690 else 679 else
691 E('Failed to download - results/' .. host .. "/merged/" .. p, 'http', 'Updated', host) 680 E('Failed to download ' .. host .. "/merged/" .. p, 'http', 'Updated', host)
692 end 681 end
693 end 682 end
694 end 683 end
@@ -803,7 +792,7 @@ local parseRelease = function(host)
803 if APT.testing("Integrity") then 792 if APT.testing("Integrity") then
804 local status = APT.exe( "gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. 793 local status = APT.exe( "gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o ..
805 " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status 794 " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status
806 if 0 ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end 795 if 0 ~= status then E("GPG check failed for " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end
807-- TODO - should check the PGP sig of InRelease as well. 796-- TODO - should check the PGP sig of InRelease as well.
808 end 797 end
809 os.execute('rm results/' .. host .. '/merged/dists/' .. n .. '/' .. o) 798 os.execute('rm results/' .. host .. '/merged/dists/' .. n .. '/' .. o)
@@ -992,12 +981,11 @@ if 0 < #arg then
992 end 981 end
993 end 982 end
994 983
995 if APT.origin or APT.redir then APT.results["IPs"] = gatherIPs(pu.host) end
996
997 if not APT.logOpen(pu.host, arg[2], arg[3]) then return end 984 if not APT.logOpen(pu.host, arg[2], arg[3]) then return end
998 I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", ")) 985 I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", "))
999 if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end 986 if APT.origin or APT.redir then APT.results["IPs"] = gatherIPs(pu.host) end
1000 if nil ~= arg[3] then I(" Using file " .. arg[3]); end 987 if nil ~= arg[2] then I(" &nbsp; Using IP " .. arg[2]); ip = arg[2] end
988 if nil ~= arg[3] then I(" &nbsp; Using file " .. arg[3]); end
1001 APT.mirrors = loadfile("results/mirrors.lua")() 989 APT.mirrors = loadfile("results/mirrors.lua")()
1002 APT.results = APT.padResults(APT.results) 990 APT.results = APT.padResults(APT.results)
1003 991