aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/apt-panopticon.lua
diff options
context:
space:
mode:
Diffstat (limited to 'apt-panopticon.lua')
-rwxr-xr-xapt-panopticon.lua114
1 files changed, 51 insertions, 63 deletions
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index b140831..889058b 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -163,7 +163,7 @@ local ip = ""
163local cor = nil 163local cor = nil
164 164
165local downloadLock = "flock -n results/curl-" 165local downloadLock = "flock -n results/curl-"
166 166local arw = ' &nbsp; <font color="magenta"><b>-></b></font> &nbsp; '
167 167
168local repoExists = function (r) 168local repoExists = function (r)
169 r = r:match("([%a-]*)") 169 r = r:match("([%a-]*)")
@@ -215,14 +215,15 @@ end
215 215
216local timeouts = 0; 216local timeouts = 0;
217local totalTimeouts = 0 217local totalTimeouts = 0
218local spcd = ' &nbsp; '
218checkHEAD = function (host, URL, r, retry, sanity) 219checkHEAD = function (host, URL, r, retry, sanity)
219 if nil == r then r = 0 end 220 if nil == r then r = 0 end
220 if nil == retry then retry = 0 end 221 if nil == retry then retry = 0 end
221 if nil == sanity then sanity = false end 222 if true == sanity then sanity = 'URLSanity' else sanity = '' end
222 local check = "Checking file" 223 local check = "HEAD testing file"
223 local PU = url.parse(URL, defaultURL) 224 local PU = url.parse(URL, defaultURL)
224 local pu = url.parse(PU.scheme .. "://" .. host, defaultURL) 225 local pu = url.parse(PU.scheme .. "://" .. host, defaultURL)
225 if not APT.testing(PU.scheme, host) and APT.redir then I("Not supported, not tested " .. PU.scheme .. " " .. host .. " -> " .. URL, PU.scheme, "", host); return end 226 if not APT.testing(PU.scheme, host) and APT.redir then I(spcd .. string.upper(PU.scheme) .. " not supported, not tested. &nbsp; " .. URL, host); return end
226 if 0 < r then 227 if 0 < r then
227 check = "Redirecting to" 228 check = "Redirecting to"
228 end 229 end
@@ -231,23 +232,27 @@ checkHEAD = function (host, URL, r, retry, sanity)
231 check = "Retry " .. retry .. " " .. check 232 check = "Retry " .. retry .. " " .. check
232 end 233 end
233 if 2 <= timeouts then 234 if 2 <= timeouts then
234 E("too many timeouts! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) 235 E(spcd .. spcd .. "too many timeouts! " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host)
235 return 236 return
236 end 237 end
237 if APT.options.timeouts.value <= (totalTimeouts) then 238 if APT.options.timeouts.value <= (totalTimeouts) then
238 E("Way too many timeouts!", PU.scheme, "", host) 239 E(spcd .. spcd .. "Way too many timeouts!", PU.scheme, "", host)
239 return 240 return
240 end 241 end
241 if 20 <= r then 242 if 20 <= r then
242 E("too many redirects! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) 243 E(spcd .. spcd .. "too many redirects! " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host)
243 return 244 return
244 end 245 end
245 if APT.options.retries.value <= retry then 246 if APT.options.retries.value <= retry then
246 E("too many retries! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) 247 E(spcd .. spcd .. "too many retries! " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host)
248 return
249 end
250 if "https" == PU.scheme and APT.options.roundRobin.value == host then
251 I(spcd .. "Not testing " .. APT.lnk(URL) .. " mirrors wont have the correct HTTPS certificate for the round robin.", host)
247 return 252 return
253 else
254 I(spcd .. check .. " " .. APT.lnk(URL), host)
248 end 255 end
249 D(PU.scheme .. " :// " .. check .. " " .. host .. " -> " .. URL)
250 if "https" == PU.scheme and APT.options.roundRobin.value == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert."); return end
251 256
252 --[[ Using curl command line - 257 --[[ Using curl command line -
253 -I - HEAD 258 -I - HEAD
@@ -282,20 +287,13 @@ checkHEAD = function (host, URL, r, retry, sanity)
282 local msg = curlStatus[status] 287 local msg = curlStatus[status]
283 if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end 288 if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end
284 if (28 == status) or (7 == status) then 289 if (28 == status) or (7 == status) then
285 if sanity then 290 T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host)
286 T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "URLSanity", host)
287 else
288 T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "", host)
289 end
290 timeouts = timeouts + 1 291 timeouts = timeouts + 1
291 else 292 else
292 if sanity then 293 E(spcd .. spcd .. "The curl command return an error code of " .. status .. " - " .. msg .. ' for '.. APT.lnk(URL), PU.scheme, sanity, host)
293 E(" The curl command return an error code of " .. status .. " - " .. msg, PU.scheme, "URLSanity", host)
294 else
295 E(" The curl command return an error code of " .. status .. " - " .. msg, PU.scheme, "", host)
296 end
297 end 294 end
298 checkHEAD(host, URL, r, retry + 1, sanity) 295 if 60 == status then return end -- Certificate is invalid, don't bother retrying.
296 checkHEAD(host, URL, r, retry + 1, '' ~= sanity)
299 return 297 return
300 end 298 end
301 local code = "???" 299 local code = "???"
@@ -303,17 +301,13 @@ checkHEAD = function (host, URL, r, retry, sanity)
303 local location = nil 301 local location = nil
304 local tmot = 1 302 local tmot = 1
305 while not APT.checkFile('results/STATUS_' .. fname) do 303 while not APT.checkFile('results/STATUS_' .. fname) do
306 I('Waiting for results/STATUS_' .. fname .. ' file.') 304 D(spcd .. spcd .. 'Waiting for results/STATUS_' .. fname .. ' file.')
307 os.execute('sleep ' .. tmot) 305 os.execute('sleep ' .. tmot)
308 tmot = tmot * 2 306 tmot = tmot * 2
309 if 8 < tmot then 307 if 8 < tmot then
310 if sanity then 308 T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host)
311 T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "URLSanity", host)
312 else
313 T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "", host)
314 end
315 timeouts = timeouts + 1 309 timeouts = timeouts + 1
316 checkHEAD(host, URL, r, retry + 1, sanity) 310 checkHEAD(host, URL, r, retry + 1, '' ~= sanity)
317 return 311 return
318 end 312 end
319 end 313 end
@@ -323,10 +317,10 @@ checkHEAD = function (host, URL, r, retry, sanity)
323 if "#" == line:sub(1, 1) then 317 if "#" == line:sub(1, 1) then
324 code = line:sub(2, 4) 318 code = line:sub(2, 4)
325 if ("https" == PU.scheme) and ("0" ~= line:sub(6, 6)) then 319 if ("https" == PU.scheme) and ("0" ~= line:sub(6, 6)) then
326 if sanity then 320 if '' ~= sanity then
327 E(" The certificate is invalid.", PU.scheme, "URLSanity", host) 321 E(spcd .. spcd .. "The certificate is invalid.", PU.scheme, sanity, host)
328 else 322 else
329 E(" The certificate is invalid.", PU.scheme, "https", host) 323 E(spcd .. spcd .. "The certificate is invalid.", PU.scheme, "https", host)
330 end 324 end
331 end 325 end
332 elseif "http" == line:sub(1, 4):lower() then 326 elseif "http" == line:sub(1, 4):lower() then
@@ -339,39 +333,35 @@ checkHEAD = function (host, URL, r, retry, sanity)
339 os.execute('rm -f results/STATUS_' .. fname .. ' 2>/dev/null') 333 os.execute('rm -f results/STATUS_' .. fname .. ' 2>/dev/null')
340 end 334 end
341 if ("4" == tostring(code):sub(1, 1)) or ("5" == tostring(code):sub(1, 1)) then 335 if ("4" == tostring(code):sub(1, 1)) or ("5" == tostring(code):sub(1, 1)) then
342 if sanity then 336 E(spcd .. spcd .. code .. " " .. cstr .. ". " .. check .. " " .. APT.lnk(URL), PU.scheme, sanity, host)
343 E(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "URLSanity", host)
344 else
345 E(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
346 end
347 else 337 else
348 if not APT.testing(PU.scheme, host) then 338 if not APT.testing(PU.scheme, host) then
349 W("Not supported, but works " .. PU.scheme .. " " .. host .. " -> " .. URL, PU.scheme, "", host) 339 W(spcd .. spcd .. "Not supported, but works " .. PU.scheme .. " " .. APT.lnk(URL), PU.scheme, "", host)
350 end 340 end
351 I(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL) 341 I(spcd .. spcd .. code .. " " .. cstr .. ". " .. check .. " " .. APT.lnk(URL), host)
352-- timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through. 342-- timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through.
353 if (1 <= APT.options.bandwidth.value) and (nil ~= location) then 343 if (1 <= APT.options.bandwidth.value) and (nil ~= location) then
354 pu = url.parse(location, defaultURL) 344 pu = url.parse(location, defaultURL)
355 if ('http' == location:sub(1, 4)) and (pu.scheme ~= PU.scheme) then -- Sometimes a location sans scheme is returned, this is not a protocol change. 345 if ('http' == location:sub(1, 4)) and (pu.scheme ~= PU.scheme) then -- Sometimes a location sans scheme is returned, this is not a protocol change.
356 if APT.testing("Protocol") then 346 if APT.testing("Protocol") then
357 if APT.options.roundRobin.value == host then -- Coz HTTPS shouldn't happen via the round robin. 347 if APT.options.roundRobin.value == host then -- Coz HTTPS shouldn't happen via the round robin.
358 E(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host) 348 E(spcd .. spcd .. "Protocol changed during redirect! " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "Protocol", host)
359 end 349 end
360 W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host) 350 W(spcd .. spcd .. "Protocol changed during redirect! " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "Protocol", host)
361 else 351 else
362 end 352 end
363 end 353 end
364 354
365 if location == URL then 355 if location == URL then
366 E(" redirect loop! " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location, PU.scheme, "", host) 356 E(spcd .. spcd .. "Redirect loop! " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "", host)
367 elseif nil == pu.host then 357 elseif nil == pu.host then
368 I(" relative redirect. " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location) 358 I(spcd .. spcd .. "Relative redirect. " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
369 checkHEAD(host, PU.scheme .. "://" .. PU.host .. location, r + 1, retry, sanity) 359 checkHEAD(host, PU.scheme .. "://" .. PU.host .. location, r + 1, retry, '' ~= sanity)
370 elseif (PU.host == pu.host) or (host == pu.host) then 360 elseif (PU.host == pu.host) or (host == pu.host) then
371 I(" redirect to same host. " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location) 361 I(spcd .. spcd .. "Redirect to same host. " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
372 checkHEAD(pu.host, location, r + 1, retry, sanity) 362 checkHEAD(pu.host, location, r + 1, retry, '' ~= sanity)
373 else 363 else
374 I(" redirect to different host. " .. check .. " " .. host .. " -> \n" .. URL .. " -> " .. location) 364 I(spcd .. spcd .. "Redirect to different host. " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
375 --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location. 365 --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location.
376 Not good for deb.debian.org, which we should only be throwing .debs at. 366 Not good for deb.debian.org, which we should only be throwing .debs at.
377 What we do is loop through the DNS entries, and only test the specific protocol & file being tested here. 367 What we do is loop through the DNS entries, and only test the specific protocol & file being tested here.
@@ -384,10 +374,10 @@ checkHEAD = function (host, URL, r, retry, sanity)
384 if 'https' == pu.scheme then extraArgs = extraArgs .. ' --tests=-http' end 374 if 'https' == pu.scheme then extraArgs = extraArgs .. ' --tests=-http' end
385 if 'http' == pu.scheme then extraArgs = extraArgs .. ' --tests=-https' end 375 if 'http' == pu.scheme then extraArgs = extraArgs .. ' --tests=-https' end
386 local pth = path:match('^(.*/pool/).*$') 376 local pth = path:match('^(.*/pool/).*$')
387 if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E('Odd redirect path ' .. path) end 377 if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E(spcd .. spcd .. 'Odd redirect path ' .. path) end
388 I(" Now checking redirected host " .. u) 378 I(spcd .. spcd .. "Now checking redirected host " .. u .. ' &nbsp; for &nbsp; ' .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
389 APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork() 379 APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork()
390 D('logging to ' .. APT.logName(pu.host, nil, file)[2]) 380 D(spcd .. 'logging to ' .. APT.logName(pu.host, nil, file)[2])
391 end 381 end
392 end 382 end
393 end 383 end
@@ -418,11 +408,10 @@ local checkFiles = function (host, ip, path, file)
418 if nil == path then path = "" end 408 if nil == path then path = "" end
419 if nil ~= file then 409 if nil ~= file then
420 if "redir" == ip then ip = host end 410 if "redir" == ip then ip = host end
421 I(" Checking IP for file " .. host .. " -> " .. ip .. " " .. path .. " " .. file)
422 if checkTimeouts(host, "http", ip .. path .. "/" .. file) then return end 411 if checkTimeouts(host, "http", ip .. path .. "/" .. file) then return end
423 if checkTimeouts(host, "https", ip .. path .. "/" .. file) then return end 412 if checkTimeouts(host, "https", ip .. path .. "/" .. file) then return end
424 else 413 else
425 I(" Checking IP " .. host .. " -> " .. ip .. " " .. path) 414 I(" HEAD testing files for " .. host .. arw .. ip .. " " .. path, host)
426 if 1 <= APT.options.bandwidth.value then 415 if 1 <= APT.options.bandwidth.value then
427 -- Do these first, coz they are likely to fork off a different server. 416 -- Do these first, coz they are likely to fork off a different server.
428 for i, s in pairs(referenceDebs) do 417 for i, s in pairs(referenceDebs) do
@@ -467,10 +456,10 @@ checkHost = function (orig, host, path, ip, file)
467 end 456 end
468 else 457 else
469 if orig == host then 458 if orig == host then
470 D("checkHost " .. orig .. "" .. file) 459 I("Testing mirror " .. orig .. "" .. file)
471 APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork() 460 APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork()
472 D('logging to ' .. APT.logName(ph.host, nil, file)[2]) 461 D('logging to ' .. APT.logName(ph.host, nil, file)[2])
473 else D("checkHost " .. orig .. " -> " .. host) end 462 else D("checkHost " .. orig .. arw .. host) end
474 end 463 end
475end 464end
476 465
@@ -534,7 +523,7 @@ local downloads = function(host, URL, meta, release, list)
534 addDownload(host, URL, f, release, "/" .. l) 523 addDownload(host, URL, f, release, "/" .. l)
535 end 524 end
536 else 525 else
537 D('Downloading http://' .. host .. URL .. '/merged/' .. list) 526 I('Downloading ' .. APT.lnk('http://' .. host .. URL .. '/merged/' .. list))
538 f:write('url "' .. 'http://' .. host .. URL .. '/merged/' .. list .. '"\n') 527 f:write('url "' .. 'http://' .. host .. URL .. '/merged/' .. list .. '"\n')
539 f:write('output "results/' .. host .. '/merged/' .. list .. '"\n') 528 f:write('output "results/' .. host .. '/merged/' .. list .. '"\n')
540 end 529 end
@@ -582,8 +571,8 @@ local getMirrors = function ()
582 local host = "" 571 local host = ""
583 local m = {} 572 local m = {}
584 local active = true 573 local active = true
585 local URL = "http://" .. APT.options.referenceSite.value .. "/mirror_list.txt" 574 local URL = 'http://' .. APT.options.referenceSite.value .. '/mirror_list.txt'
586 I("getting mirrors.") 575 I('Downloadin gand parsing http://' .. APT.options.referenceSite.value .. '/mirror_list.txt')
587 local p, c, h = http.request(URL) 576 local p, c, h = http.request(URL)
588 if nil == p then E(c .. " fetching " .. URL) else 577 if nil == p then E(c .. " fetching " .. URL) else
589 578
@@ -683,12 +672,12 @@ local parseDebs = function(host)
683 end 672 end
684 if APT.testing("Updated") then 673 if APT.testing("Updated") then
685 if sz ~= fsz:sub(2, -2) then 674 if sz ~= fsz:sub(2, -2) then
686 E('Package size mismatch - results/' .. host .. "/merged/" .. p, 'http', 'Updated', host) 675 E('Package size mismatch for ' .. host .. "/merged/" .. p, 'http', 'Updated', host)
687 end 676 end
688 end 677 end
689 os.execute('rm -f results/' .. host .. "/merged/" .. p) 678 os.execute('rm -f results/' .. host .. "/merged/" .. p)
690 else 679 else
691 E('Failed to download - results/' .. host .. "/merged/" .. p, 'http', 'Updated', host) 680 E('Failed to download ' .. host .. "/merged/" .. p, 'http', 'Updated', host)
692 end 681 end
693 end 682 end
694 end 683 end
@@ -803,7 +792,7 @@ local parseRelease = function(host)
803 if APT.testing("Integrity") then 792 if APT.testing("Integrity") then
804 local status = APT.exe( "gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. 793 local status = APT.exe( "gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o ..
805 " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status 794 " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status
806 if 0 ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end 795 if 0 ~= status then E("GPG check failed for " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end
807-- TODO - should check the PGP sig of InRelease as well. 796-- TODO - should check the PGP sig of InRelease as well.
808 end 797 end
809 os.execute('rm results/' .. host .. '/merged/dists/' .. n .. '/' .. o) 798 os.execute('rm results/' .. host .. '/merged/dists/' .. n .. '/' .. o)
@@ -992,12 +981,11 @@ if 0 < #arg then
992 end 981 end
993 end 982 end
994 983
995 if APT.origin or APT.redir then APT.results["IPs"] = gatherIPs(pu.host) end
996
997 if not APT.logOpen(pu.host, arg[2], arg[3]) then return end 984 if not APT.logOpen(pu.host, arg[2], arg[3]) then return end
998 I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", ")) 985 I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", "))
999 if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end 986 if APT.origin or APT.redir then APT.results["IPs"] = gatherIPs(pu.host) end
1000 if nil ~= arg[3] then I(" Using file " .. arg[3]); end 987 if nil ~= arg[2] then I(" &nbsp; Using IP " .. arg[2]); ip = arg[2] end
988 if nil ~= arg[3] then I(" &nbsp; Using file " .. arg[3]); end
1001 APT.mirrors = loadfile("results/mirrors.lua")() 989 APT.mirrors = loadfile("results/mirrors.lua")()
1002 APT.results = APT.padResults(APT.results) 990 APT.results = APT.padResults(APT.results)
1003 991