diff options
Diffstat (limited to '')
-rwxr-xr-x | apt-panopticon.lua | 680 |
1 files changed, 398 insertions, 282 deletions
diff --git a/apt-panopticon.lua b/apt-panopticon.lua index 6bec6a1..4e4be94 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua | |||
@@ -5,6 +5,7 @@ local now = os.time() | |||
5 | local APT = require 'apt-panopticommon' | 5 | local APT = require 'apt-panopticommon' |
6 | local D = APT.D | 6 | local D = APT.D |
7 | local I = APT.I | 7 | local I = APT.I |
8 | local T = APT.T | ||
8 | local W = APT.W | 9 | local W = APT.W |
9 | local E = APT.E | 10 | local E = APT.E |
10 | local C = APT.C | 11 | local C = APT.C |
@@ -23,14 +24,14 @@ local releases = {"jessie", "ascii", "beowulf", "ceres"} | |||
23 | local releaseFiles = | 24 | local releaseFiles = |
24 | { | 25 | { |
25 | -- Release file. | 26 | -- Release file. |
26 | "/Release", -- 3.7 MB | 27 | "Release", -- 3.7 MB |
27 | "/Release.gpg", -- | 28 | "Release.gpg", -- |
28 | -- "/InRelease", -- 3.7 MB | 29 | -- "InRelease", -- 3.7 MB |
29 | -- "/main/binary-all/Packages.xz", -- 2.6 GB for all that changed recently. | 30 | -- "main/binary-all/Packages.xz", -- 2.6 GB for all that changed recently. |
30 | -- Contents files. -- 3.3 GB | 31 | -- Contents files. -- 3.3 GB |
31 | -- "/main/Contents-all.xz", | 32 | -- "main/Contents-all.xz", |
32 | -- "/main/Contents-amd64.xz", | 33 | -- "main/Contents-amd64.xz", |
33 | -- "/main/Contents-arm64.xz", | 34 | -- "main/Contents-arm64.xz", |
34 | -- "-security/main/Contents-all.xz", | 35 | -- "-security/main/Contents-all.xz", |
35 | -- "-security/main/Contents-amd64.xz", | 36 | -- "-security/main/Contents-amd64.xz", |
36 | -- "-security/main/Contents-arm64.xz", | 37 | -- "-security/main/Contents-arm64.xz", |
@@ -160,6 +161,7 @@ local url = require 'socket.url' | |||
160 | 161 | ||
161 | 162 | ||
162 | local ip = "" | 163 | local ip = "" |
164 | local cor = nil | ||
163 | 165 | ||
164 | 166 | ||
165 | local repoExists = function (r) | 167 | local repoExists = function (r) |
@@ -197,6 +199,7 @@ gatherIPs = function (host) | |||
197 | end | 199 | end |
198 | until nil == IPs | 200 | until nil == IPs |
199 | end | 201 | end |
202 | return IP[host] | ||
200 | end | 203 | end |
201 | 204 | ||
202 | -- Returns FTP directory listing | 205 | -- Returns FTP directory listing |
@@ -229,7 +232,7 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
229 | E("too many timeouts! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) | 232 | E("too many timeouts! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) |
230 | return | 233 | return |
231 | end | 234 | end |
232 | if 4 <= (totalTimeouts) then | 235 | if APT.options.timeouts.value <= (totalTimeouts) then |
233 | E("Way too many timeouts!", PU.scheme, "", host) | 236 | E("Way too many timeouts!", PU.scheme, "", host) |
234 | return | 237 | return |
235 | end | 238 | end |
@@ -237,7 +240,7 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
237 | E("too many redirects! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) | 240 | E("too many redirects! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) |
238 | return | 241 | return |
239 | end | 242 | end |
240 | if 4 <= retry then | 243 | if APT.options.retries.value <= retry then |
241 | E("too many retries! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) | 244 | E("too many retries! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) |
242 | return | 245 | return |
243 | end | 246 | end |
@@ -343,6 +346,8 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
343 | Still duplicates a tiny bit, but much less than the previous find based method. | 346 | Still duplicates a tiny bit, but much less than the previous find based method. |
344 | TODO - maybe we can switch to using flock like we do with the other curl calls? | 347 | TODO - maybe we can switch to using flock like we do with the other curl calls? |
345 | ]] | 348 | ]] |
349 | C('Commented out code being called in checkHEAD() for redirects that should no longer happen!') | ||
350 | --[[ | ||
346 | local file = pu.host .. "://" .. pu.path | 351 | local file = pu.host .. "://" .. pu.path |
347 | local f = io.popen(string.format('if [ ! -f results/%s.check ] ; then touch results/%s.check; echo -n "check"; fi', file:gsub("/", "_"), file:gsub("/", "_") )):read("*a") | 352 | local f = io.popen(string.format('if [ ! -f results/%s.check ] ; then touch results/%s.check; echo -n "check"; fi', file:gsub("/", "_"), file:gsub("/", "_") )):read("*a") |
348 | if (nil == f) or ("check" == f) then | 353 | if (nil == f) or ("check" == f) then |
@@ -351,28 +356,27 @@ TODO - maybe we can switch to using flock like we do with the other curl calls? | |||
351 | else | 356 | else |
352 | D(" Already checking " .. file) | 357 | D(" Already checking " .. file) |
353 | end | 358 | end |
359 | ]] | ||
354 | end | 360 | end |
355 | end | 361 | end |
356 | end | 362 | end |
357 | end | 363 | end |
358 | 364 | ||
359 | local checkTimeouts = function(host, scheme, URL) | 365 | local checkTimeouts = function(host, scheme, URL) |
360 | if APT.testing(scheme) then | 366 | totalTimeouts = totalTimeouts + timeouts; timeouts = 0 |
361 | totalTimeouts = totalTimeouts + timeouts; timeouts = 0 | 367 | checkHEAD(host, scheme .. "://" .. URL) |
362 | checkHEAD(host, scheme .. "://" .. URL) | ||
363 | if 4 <= (totalTimeouts) then | ||
364 | E("Way too many timeouts!", scheme, "", host) | ||
365 | return true | ||
366 | end | ||
367 | end | ||
368 | if APT.testing("URLSanity") then | 368 | if APT.testing("URLSanity") then |
369 | URL = URL:gsub("merged/", "merged///") | 369 | URL = URL:gsub("merged/", "merged///") |
370 | totalTimeouts = totalTimeouts + timeouts; timeouts = 0 | ||
371 | checkHEAD(host, scheme .. "://" .. URL, 0, 0, true) | 370 | checkHEAD(host, scheme .. "://" .. URL, 0, 0, true) |
372 | if 4 <= (totalTimeouts) then | 371 | end |
373 | E("Way too many timeouts!", scheme, "URLSanity", host) | 372 | if nil ~= cor then |
374 | return true | 373 | D('*>* About to resume coroutine after checkHEAD(' .. host .. ' , ' .. scheme .. ' :// ' .. URL .. ')') |
375 | end | 374 | local ok, message = coroutine.resume(cor) |
375 | if not ok then cor = nil; print(message) end | ||
376 | end | ||
377 | if APT.options.timeouts.value <= (totalTimeouts) then | ||
378 | E("Way too many timeouts!", scheme, "URLSanity", host) | ||
379 | return true | ||
376 | end | 380 | end |
377 | return false | 381 | return false |
378 | end | 382 | end |
@@ -387,19 +391,18 @@ local checkFiles = function (host, ip, path, file) | |||
387 | if checkTimeouts(host, "https", ip .. path .. "/" .. file) then return end | 391 | if checkTimeouts(host, "https", ip .. path .. "/" .. file) then return end |
388 | else | 392 | else |
389 | I(" Checking IP " .. host .. " -> " .. ip .. " " .. path) | 393 | I(" Checking IP " .. host .. " -> " .. ip .. " " .. path) |
390 | for i, s in pairs(referenceDevs) do | ||
391 | if checkTimeouts(host, "http", ip .. path .. "/" .. s) then return end | ||
392 | if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end | ||
393 | end | ||
394 | |||
395 | for i, s in pairs(releases) do | 394 | for i, s in pairs(releases) do |
396 | for j, k in pairs(releaseFiles) do | 395 | for j, k in pairs(releaseFiles) do |
397 | if repoExists(s .. k) then | 396 | if repoExists(s .. k) then |
398 | if checkTimeouts(host, "http", ip .. path .. "/merged/dists/" .. s .. k) then return end | 397 | if checkTimeouts(host, "http", ip .. path .. "/merged/dists/" .. s .. '/' .. k) then return end |
399 | if checkTimeouts(host, "https", ip .. path .. "/merged/dists/" .. s .. k) then return end | 398 | if checkTimeouts(host, "https", ip .. path .. "/merged/dists/" .. s .. '/' .. k) then return end |
400 | end | 399 | end |
401 | end | 400 | end |
402 | end | 401 | end |
402 | for i, s in pairs(referenceDevs) do | ||
403 | if checkTimeouts(host, "http", ip .. path .. "/" .. s) then return end | ||
404 | if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end | ||
405 | end | ||
403 | end | 406 | end |
404 | end | 407 | end |
405 | 408 | ||
@@ -421,127 +424,51 @@ checkHost = function (orig, host, path, ip, file) | |||
421 | D("checkHost " .. orig .. "" .. file) | 424 | D("checkHost " .. orig .. "" .. file) |
422 | if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file) end | 425 | if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file) end |
423 | else D("checkHost " .. orig .. " -> " .. host) end | 426 | else D("checkHost " .. orig .. " -> " .. host) end |
424 | local h = APT.mirrors[ph.host] | ||
425 | if nil == h then return end | ||
426 | for k, v in pairs(h.IPs) do | ||
427 | if "table" == type(v) then | ||
428 | for k1, v1 in pairs(v) do | ||
429 | if v1 == "A" then | ||
430 | if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end | ||
431 | elseif v1 == "AAAA" then | ||
432 | if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end | ||
433 | end | ||
434 | end | ||
435 | else | ||
436 | if v == "A" then | ||
437 | if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end | ||
438 | elseif v == "AAAA" then | ||
439 | if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end | ||
440 | end | ||
441 | end | ||
442 | end | ||
443 | end | 427 | end |
444 | end | 428 | end |
445 | 429 | ||
446 | 430 | ||
447 | local addDownload = function(host, URL, f, r, k) | 431 | local addDownload = function(host, URL, f, r, k) |
448 | local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename. | 432 | local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename. |
449 | if APT.checkFile("results/" .. host .. "/merged/dists/" .. r .. k) then | 433 | if APT.checkFile("results/" .. host .. "/merged/dists/" .. r .. '/' .. k) then |
450 | -- Curls "check timestamp and overwrite file" stuff sucks. | 434 | -- Curls "check timestamp and overwrite file" stuff sucks. |
451 | -- -R means the destination file gets the timestamp of the remote file. | 435 | -- -R means the destination file gets the timestamp of the remote file. |
452 | -- Can only do ONE timestamp check per command. | 436 | -- Can only do ONE timestamp check per command. |
453 | -- This doesn't work either. All downloads get all these headers. Pffft | 437 | -- This doesn't work either. All downloads get all these headers. Pffft |
454 | -- local status, ts = APT.execute('TZ="GMT" ls -l --time-style="+%a, %d %b %Y %T %Z" results/' .. host .. "/merged/dists/" .. r .. k .. ' | cut -d " " -f 6-11') | 438 | -- local status, ts = APT.execute('TZ="GMT" ls -l --time-style="+%a, %d %b %Y %T %Z" results/' .. host .. "/merged/dists/" .. r .. '/' .. k .. ' | cut -d " " -f 6-11') |
455 | -- f:write('header "If-Modified-Since: ' .. ts:sub(2, -2) .. '"\n') | 439 | -- f:write('header "If-Modified-Since: ' .. ts:sub(2, -2) .. '"\n') |
456 | -- Curl will DELETE the existing file if the timestamp fails to download a new one, unless we change directory first, | 440 | -- Curl will DELETE the existing file if the timestamp fails to download a new one, unless we change directory first, |
457 | -- which wont work with multiple files in multiple directories. WTF? | 441 | -- which wont work with multiple files in multiple directories. WTF? |
458 | os.execute(" mv results/" .. host .. "/merged/dists/" .. r .. k .. | 442 | os.execute(" mv results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. |
459 | " results/" .. host .. "/merged/dists/" .. r .. k .. ".old") | 443 | " results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. ".old") |
460 | end | 444 | end |
461 | 445 | ||
462 | D('Downloading http://' .. host .. URL .. '/merged/dists/' .. r .. k) | 446 | D('Downloading http://' .. host .. URL .. '/merged/dists/' .. r .. '/' .. k) |
463 | f:write('url "' .. 'http://' .. host .. URL .. '/merged/dists/' .. r .. k .. '"\n') | 447 | f:write('url "' .. 'http://' .. host .. URL .. '/merged/dists/' .. r .. '/' .. k .. '"\n') |
464 | f:write('output "results/' .. host .. '/merged/dists/' .. r .. k .. '"\n') | 448 | f:write('output "results/' .. host .. '/merged/dists/' .. r .. '/' .. k .. '"\n') |
465 | end | 449 | end |
466 | 450 | ||
467 | local postDownload = function(host, r, k) | 451 | local postDownload = function(host, r, k) |
468 | local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename. | 452 | local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename. |
469 | local dir = k:sub(1, 0 - (#file + 1)) | 453 | if nil == file then file = k end |
470 | os.execute("if [ -f results/" .. host .. "/merged/dists/" .. r .. k .. ".old ]" .. | 454 | os.execute("if [ -f results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. ".old ]" .. |
471 | " && [ ! -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then cp -a" .. | 455 | " && [ ! -f results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. " ]; then cp -a" .. |
472 | " results/" .. host .. "/merged/dists/" .. r .. k .. ".old" .. | 456 | " results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. ".old" .. |
473 | " results/" .. host .. "/merged/dists/" .. r .. k .. "; fi") | 457 | " results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. "; fi") |
474 | if ".gz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 gzip -dfk results/" .. host .. "/merged/dists/" .. r .. k) end | 458 | if APT.checkFile('results/' .. host .. '/merged/dists/' .. r .. '/' .. k) then |
475 | if ".xz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 xz -dfk results/" .. host .. "/merged/dists/" .. r .. k .. " 2>/dev/null") end | 459 | if ".gz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 gzip -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k) end |
476 | if APT.testing("Integrity") then | 460 | if ".xz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 xz -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. " 2>/dev/null") end |
477 | if ".gpg" == k:sub(-4, -1) then | ||
478 | local status, out = APT.execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. r .. k .. | ||
479 | " results/" .. host .. "/merged/dists/" .. r .. k:sub(1, -5) .. " 2>/dev/null") | ||
480 | if "0" ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. r .. k, "http", "Integrity", host) end | ||
481 | os.execute('rm results/' .. host .. '/merged/dists/' .. r .. k) | ||
482 | end | ||
483 | -- TODO - should check the PGP sig of InRelease as well. | ||
484 | end | ||
485 | if APT.testing("Integrity") or APT.testing("Updated") then | ||
486 | if "Packages." == file:sub(1, 9) then | ||
487 | -- TODO - compare the SHA256 sums in pkgmaster's Release for both the packed and unpacked versions. | ||
488 | -- Also note that this might get only a partial download due to maxtime. | ||
489 | if APT.options.referenceSite.value == host then | ||
490 | local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed', "w+") | ||
491 | if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed' .. ' file - ' .. e) else | ||
492 | local pp = {} | ||
493 | for l in io.lines('results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages') do | ||
494 | if "Package: " == l:sub(1, 9) then | ||
495 | if 0 ~= #pp then | ||
496 | for i = 1, 5 do | ||
497 | if nil == pp[i] then print(host .. " " .. r .. " " .. dir .. " " .. i) else Pp:write(pp[i] .. " | ") end | ||
498 | end | ||
499 | Pp:write("\n") | ||
500 | end | ||
501 | pp = {} | ||
502 | pp[1] = l:sub(10, -1) | ||
503 | elseif "Version: " == l:sub(1, 9) then | ||
504 | pp[2] = l:sub(10, -1) | ||
505 | elseif "Filename: " == l:sub(1, 10) then | ||
506 | pp[3] = l:sub(11, -1) | ||
507 | elseif "Size: " == l:sub(1, 6) then | ||
508 | pp[4] = l:sub(7, -1) | ||
509 | elseif "SHA256: " == l:sub(1, 8) then | ||
510 | pp[5] = l:sub(9, -1) | ||
511 | end | ||
512 | end | ||
513 | Pp:close() | ||
514 | os.execute('sort results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed >results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages_parsed-sorted') | ||
515 | if APT.checkFile('Packages/' .. r .. dir .. 'Packages_parsed-sorted') then | ||
516 | os.execute('diff -U 0 Packages/' .. r .. dir .. 'Packages_parsed-sorted ' .. | ||
517 | 'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages_parsed-sorted ' .. | ||
518 | ' | grep -E "^-" | grep -Ev "^\\+\\+\\+|^---" >>results/OLD_PACKAGES_' .. r .. '.txt') | ||
519 | os.execute('diff -U 0 Packages/' .. r .. dir .. 'Packages_parsed-sorted ' .. | ||
520 | 'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages_parsed-sorted ' .. | ||
521 | ' | grep -E "^\\+" | grep -Ev "^\\+\\+\\+|^---" >>results/NEW_Packages_' .. r .. '.txt') | ||
522 | -- Find the smallest new package for each release. | ||
523 | os.execute('sort -b -k 9,9 -n results/NEW_Packages_' .. r .. '.txt >results/NEW_Packages_' .. r .. '.sorted.txt') | ||
524 | os.execute('grep -s " | pool/DEBIAN/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >results/NEW_Packages_' .. r .. '.test.tmp') | ||
525 | os.execute('grep -s " | pool/DEBIAN-SECURITY/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >>results/NEW_Packages_' .. r .. '.test.tmp') | ||
526 | os.execute('grep -s " | pool/DEVUAN/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >>results/NEW_Packages_' .. r .. '.test.tmp') | ||
527 | os.execute('mv results/NEW_Packages_' .. r .. '.test.tmp results/NEW_Packages_' .. r .. '.test.txt') | ||
528 | else | ||
529 | C("Can't find file Packages/" .. r .. dir .. "Packages_parsed-sorted") | ||
530 | end | ||
531 | os.execute('mkdir -p Packages/' .. r .. dir) | ||
532 | os.execute('mv -f results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages_parsed-sorted Packages/' .. r .. dir .. 'Packages_parsed-sorted') | ||
533 | end | ||
534 | else | ||
535 | while not APT.checkFile('results/NEW_Packages_' .. r .. '.test.txt') do os.execute('sleep 10') end | ||
536 | end | ||
537 | os.execute('rm -f results/' .. host .. '/merged/dists/' .. r .. dir .. 'Packages 2>/dev/null') | ||
538 | os.execute('rm -f results/' .. host .. '/merged/dists/' .. r .. dir .. 'Packages.* 2>/dev/null') | ||
539 | end | ||
540 | end | 461 | end |
541 | end | 462 | end |
542 | 463 | ||
543 | local downloadLock = "flock -n results/curl-" | 464 | local downloadLock = "flock -n results/curl-" |
544 | local download = "curl --connect-timeout " .. APT.options.timeout.value .. " --create-dirs -f -L --max-time " .. APT.options.maxtime.value .. " -z 'results/stamp.old' -v -R " | 465 | local download = "curl" .. |
466 | " --connect-timeout " .. APT.options.timeout.value .. | ||
467 | " --create-dirs -f -L" .. | ||
468 | " --fail-early" .. | ||
469 | " --max-time " .. APT.options.maxtime.value .. | ||
470 | " --retry " .. APT.options.retries.value .. | ||
471 | " -R -v -z 'results/stamp.old' " | ||
545 | local downloads = function(host, URL, meta, release, list) | 472 | local downloads = function(host, URL, meta, release, list) |
546 | if nil == URL then URL = "" end | 473 | if nil == URL then URL = "" end |
547 | local lock = meta .. "-" .. host .. ".log" | 474 | local lock = meta .. "-" .. host .. ".log" |
@@ -600,6 +527,7 @@ local validateURL = function(m) | |||
600 | m.BaseURL = p.authority .. p.path | 527 | m.BaseURL = p.authority .. p.path |
601 | return m | 528 | return m |
602 | end | 529 | end |
530 | |||
603 | local getMirrors = function () | 531 | local getMirrors = function () |
604 | local mirrors = {} | 532 | local mirrors = {} |
605 | local host = "" | 533 | local host = "" |
@@ -621,8 +549,6 @@ local getMirrors = function () | |||
621 | end | 549 | end |
622 | host = d | 550 | host = d |
623 | m[t] = d | 551 | m[t] = d |
624 | gatherIPs(host) | ||
625 | m["IPs"] = IP[host] | ||
626 | elseif "Protocols" == t then | 552 | elseif "Protocols" == t then |
627 | local prot = {} | 553 | local prot = {} |
628 | for w in d:gmatch("(%w+)") do | 554 | for w in d:gmatch("(%w+)") do |
@@ -643,19 +569,305 @@ local getMirrors = function () | |||
643 | end | 569 | end |
644 | end | 570 | end |
645 | if APT.testing("DNSRR") then | 571 | if APT.testing("DNSRR") then |
646 | mirrors[APT.options.roundRobin.value] = { ["Protocols"] = { ["http"] = true; ["https"] = true; }; ["FQDN"] = 'deb.devuan.org'; ["Active"] = 'yes'; ["BaseURL"] = 'deb.devuan.org'; } | 572 | mirrors[APT.options.roundRobin.value] = { ["Protocols"] = { ["http"] = true; ["https"] = true; }; |
647 | gatherIPs(APT.options.roundRobin.value) | 573 | ["FQDN"] = APT.options.roundRobin.value; ["Active"] = 'yes'; ["BaseURL"] = APT.options.roundRobin.value; } |
648 | mirrors[APT.options.roundRobin.value].IPs = IP[APT.options.roundRobin.value] | ||
649 | end | ||
650 | local file, e = io.open("results/mirrors.lua", "w+") | ||
651 | if nil == file then C("opening mirrors file - " .. e) else | ||
652 | file:write(APT.dumpTable(mirrors, "", "mirrors") .. "\nreturn mirrors\n") | ||
653 | file:close() | ||
654 | end | 574 | end |
655 | return mirrors | 575 | return mirrors |
656 | end | 576 | end |
657 | 577 | ||
658 | 578 | ||
579 | local postParse = function(host, list) | ||
580 | if APT.options.referenceSite.value == host then | ||
581 | if nil ~= list then | ||
582 | local sem = 'results/NEW_' .. list.out .. '_%s.txt' | ||
583 | for i, n in pairs(releases) do | ||
584 | local f = sem:format(n) | ||
585 | if APT.checkFile(f .. '.tmp') then | ||
586 | os.execute('mv ' .. f .. '.tmp ' .. f) | ||
587 | else | ||
588 | os.execute('touch ' .. f) | ||
589 | end | ||
590 | end | ||
591 | end | ||
592 | end | ||
593 | end | ||
594 | |||
595 | local parseDebs = function(host) | ||
596 | for i, n in pairs(releases) do | ||
597 | local inFile = 'results/NEW_debs_' .. n .. '.txt' | ||
598 | local nfile, e = io.open(inFile, "r") | ||
599 | if nil == nfile then W("opening " .. inFile .. " file - " .. e) else | ||
600 | for l in nfile:lines() do | ||
601 | local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |') | ||
602 | if nil ~= p then | ||
603 | if APT.checkFile('results/' .. host .. "/merged/" .. p) then | ||
604 | local status, fsz = APT.execute('ls -l results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 5-5') | ||
605 | if APT.testing("Integrity") then | ||
606 | if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. | ||
607 | E('Package size mismatch - results/' .. host .. "/merged/" .. p .. ' should be ' .. sz .. ', but is ' .. fsz:sub(2, -2) .. '.', 'http', 'Integrity', host) | ||
608 | else | ||
609 | local status, fsha = APT.execute('sha256sum results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 1') | ||
610 | if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. host .. "/merged/" .. p, 'http', 'Integrity', host) end | ||
611 | -- TODO - maybe check the PGP key, though packages are mostly not signed. | ||
612 | end | ||
613 | end | ||
614 | if APT.testing("Updated") then | ||
615 | if sz ~= fsz:sub(2, -2) then | ||
616 | E('Package size mismatch - results/' .. host .. "/merged/" .. p, 'http', 'Updated', host) | ||
617 | end | ||
618 | end | ||
619 | os.execute('rm -f results/' .. host .. "/merged/" .. p) | ||
620 | else | ||
621 | E('Failed to download - results/' .. host .. "/merged/" .. p, 'http', 'Updated', host) | ||
622 | end | ||
623 | end | ||
624 | end | ||
625 | end | ||
626 | end | ||
627 | return nil | ||
628 | end | ||
629 | |||
630 | local parsePackages = function(host) | ||
631 | local list = {inf = 'Packages', parser = parseDebs, out = 'debs', files = {}, nextf = ''} | ||
632 | for i, n in pairs(releases) do | ||
633 | local inFile = 'results/NEW_' .. list.inf .. '_' .. n .. '.txt' | ||
634 | local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt' | ||
635 | if APT.options.referenceSite.value == host then | ||
636 | outFile = outFile .. '.tmp' | ||
637 | end | ||
638 | local dFile, e = io.open(inFile, "r") | ||
639 | if nil == dFile then W("opening " .. inFile .. " file - " .. e) else | ||
640 | for l in dFile:lines() do | ||
641 | postDownload(host, n, l) | ||
642 | |||
643 | l = '/' .. l | ||
644 | local file = l:match(".*/([%w%.%+%-_]*)$") -- Get the filename. | ||
645 | local dir = l:sub(1, 0 - (#file + 1)) | ||
646 | |||
647 | if "Packages." == file:sub(1, 9) then | ||
648 | -- TODO - compare the SHA256 sums in pkgmaster's Release for both the packed and unpacked versions. | ||
649 | -- Also note that this might get only a partial download due to maxtime. | ||
650 | if APT.options.referenceSite.value == host then | ||
651 | local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed', "w+") | ||
652 | if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed' .. ' file - ' .. e) else | ||
653 | local pp = {} | ||
654 | for l in io.lines('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages') do | ||
655 | if "Package: " == l:sub(1, 9) then | ||
656 | if 0 ~= #pp then | ||
657 | for i = 1, 5 do | ||
658 | if nil == pp[i] then print(host .. " " .. n .. " " .. dir .. " " .. i) else Pp:write(pp[i] .. " | ") end | ||
659 | end | ||
660 | Pp:write("\n") | ||
661 | end | ||
662 | pp = {} | ||
663 | pp[1] = l:sub(10, -1) | ||
664 | elseif "Version: " == l:sub(1, 9) then | ||
665 | pp[2] = l:sub(10, -1) | ||
666 | elseif "Filename: " == l:sub(1, 10) then | ||
667 | pp[3] = l:sub(11, -1) | ||
668 | elseif "Size: " == l:sub(1, 6) then | ||
669 | pp[4] = l:sub(7, -1) | ||
670 | elseif "SHA256: " == l:sub(1, 8) then | ||
671 | pp[5] = l:sub(9, -1) | ||
672 | end | ||
673 | end | ||
674 | Pp:close() | ||
675 | os.execute('sort results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed >results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages_parsed-sorted') | ||
676 | if APT.checkFile('Packages/' .. n .. dir .. 'Packages_parsed-sorted') then | ||
677 | os.execute('diff -U 0 Packages/' .. n .. dir .. 'Packages_parsed-sorted ' .. | ||
678 | 'results/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. dir .. 'Packages_parsed-sorted ' .. | ||
679 | ' | grep -E "^-" | grep -Ev "^\\+\\+\\+|^---" >>results/OLD_' .. list.out .. '_' .. n .. '.txt') | ||
680 | os.execute('diff -U 0 Packages/' .. n .. dir .. 'Packages_parsed-sorted ' .. | ||
681 | 'results/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. dir .. 'Packages_parsed-sorted ' .. | ||
682 | ' | grep -E "^\\+" | grep -Ev "^\\+\\+\\+|^---" >>results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt') | ||
683 | else | ||
684 | C("Can't find file Packages/" .. n .. dir .. "Packages_parsed-sorted") | ||
685 | end | ||
686 | os.execute('mkdir -p Packages/' .. n .. dir) | ||
687 | os.execute('mv -f results/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. dir .. 'Packages_parsed-sorted Packages/' .. n .. dir .. 'Packages_parsed-sorted') | ||
688 | end | ||
689 | else | ||
690 | end | ||
691 | os.execute('rm -fr results/' .. host .. '/merged/dists/' .. n .. dir .. ' 2>/dev/null') | ||
692 | end | ||
693 | end | ||
694 | |||
695 | if APT.checkFile('results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt') then | ||
696 | -- Sort by size. | ||
697 | os.execute('sort -b -k 9,9 -n results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt >results/NEW_' .. list.out .. '_' .. n .. '.sorted.txt') | ||
698 | os.execute('grep -s " | pool/DEBIAN/" results/NEW_' .. list.out .. '_' .. n .. '.sorted.txt 2>/dev/null | head -n 1 >' .. outFile) | ||
699 | os.execute('grep -s " | pool/DEBIAN-SECURITY/" results/NEW_' .. list.out .. '_' .. n .. '.sorted.txt 2>/dev/null | head -n 1 >>' .. outFile) | ||
700 | os.execute('grep -s " | pool/DEVUAN/" results/NEW_' .. list.out .. '_' .. n .. '.sorted.txt 2>/dev/null | head -n 1 >>' .. outFile) | ||
701 | os.execute('rm -f results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt') | ||
702 | end | ||
703 | |||
704 | end | ||
705 | |||
706 | local nfile, e = io.open(outFile, "r") | ||
707 | if nil ~= nfile then | ||
708 | -- for l in nfile:lines() do | ||
709 | local l = nfile:read('*l') | ||
710 | if nil ~= l then | ||
711 | local p = l:match('(pool/.*%.deb)') | ||
712 | if nil ~= p then | ||
713 | table.insert(list.files, p) | ||
714 | end | ||
715 | end | ||
716 | -- end | ||
717 | end | ||
718 | end | ||
719 | |||
720 | postParse(host, list) | ||
721 | return list | ||
722 | end | ||
723 | |||
724 | local parseRelease = function(host) | ||
725 | local list = {inf = 'Release', parser = parsePackages, out = 'Packages', files = {}, nextf = 'debs'} | ||
726 | for i, n in pairs(releases) do | ||
727 | for l, o in pairs(releaseFiles) do | ||
728 | if repoExists(i .. o) then | ||
729 | postDownload(host, n, o) | ||
730 | if (".gpg" == o:sub(-4, -1)) and (APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/' .. o)) then | ||
731 | if APT.testing("Integrity") then | ||
732 | local status, out = APT.execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. | ||
733 | " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5) .. " 2>/dev/null") | ||
734 | if "0" ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end | ||
735 | -- TODO - should check the PGP sig of InRelease as well. | ||
736 | end | ||
737 | os.execute('rm results/' .. host .. '/merged/dists/' .. n .. '/' .. o) | ||
738 | end | ||
739 | end | ||
740 | end | ||
741 | |||
742 | if APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/Release') then | ||
743 | os.execute('sort -k 3 results/' .. host .. '/merged/dists/' .. n .. '/Release >results/' .. host .. '/merged/dists/' .. n .. '/Release.SORTED') | ||
744 | local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt' | ||
745 | if APT.checkFile('results_old/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. '/Release.SORTED') then | ||
746 | if APT.options.referenceSite.value == host then | ||
747 | outFile = outFile .. '.tmp' | ||
748 | os.execute('diff -U 0 results_old/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. '/Release.SORTED ' .. | ||
749 | 'results/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. '/Release.SORTED ' .. | ||
750 | '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >' .. outFile) | ||
751 | -- TODO - Maybe check the date in Release, though since they are updated daily, is there any point? Perhaps it's for checking amprolla got run? | ||
752 | -- Also check if that date is in the future, apt recently got a check for that, though not sure why. | ||
753 | os.execute('rm -f results/' .. host .. '/merged/dists/' .. n .. '/Release 2>/dev/null; ') | ||
754 | else | ||
755 | -- TODO - compare to the pkgmaster copy. | ||
756 | end | ||
757 | |||
758 | -- TODO - if it's not Integrity and not reference, then just do a HEAD check and compare file times? | ||
759 | if APT.testing("Integrity") or (APT.options.referenceSite.value == host) then | ||
760 | local dfile, e = io.open(outFile, "r") | ||
761 | if nil == dfile then W("opening " .. outFile .. " file - " .. e) else | ||
762 | for l in dfile:lines() do | ||
763 | table.insert(list.files, 'dists/' .. n .. '/' .. l) | ||
764 | end | ||
765 | end | ||
766 | end | ||
767 | end | ||
768 | end | ||
769 | |||
770 | end | ||
771 | postParse(host, list) | ||
772 | return list | ||
773 | end | ||
774 | |||
775 | local parseStart = function(host) | ||
776 | local list = {inf = '', parser = parseRelease, out = 'Release', files = {}, nextf = 'Packages'} | ||
777 | for i, n in pairs(releases) do | ||
778 | local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt' | ||
779 | for l, o in pairs(releaseFiles) do | ||
780 | if repoExists(n .. o) then | ||
781 | if APT.options.referenceSite.value == host then | ||
782 | local dfile, e = io.open(outFile .. '.tmp', "a+") | ||
783 | if nil == dfile then W("opening " .. outFile .. ".tmp file - " .. e) else | ||
784 | dfile:write(o .. '\n') | ||
785 | end | ||
786 | end | ||
787 | table.insert(list.files, 'dists/' .. n .. '/' .. o) | ||
788 | end | ||
789 | end | ||
790 | end | ||
791 | postParse(host, list) | ||
792 | return list | ||
793 | end | ||
794 | |||
795 | local doDownloads = function(host, path, list) | ||
796 | while nil ~= list do | ||
797 | if 0 ~= #(list.files) then | ||
798 | for j, f in pairs(list.files) do | ||
799 | downloads(host, path, list.out, nil, f) | ||
800 | end | ||
801 | downloads(host, path, list.out, nil, '') | ||
802 | --[[ I've seen flock & curl act oddly. Perhaps flock didn't have time to start up? | ||
803 | /var/www/html/apt-panopticon/apt-panopticon/results_2019-12-22-15-00 | ||
804 | |||
805 | Mon Dec 23 01:02:54 2019 DEBUG : forking | ||
806 | |||
807 | ionice -c3 nice -n 19 flock -n results/curl-debs-pkgmaster.devuan.org.log curl --connect-timeout 5 --create-dirs -f -L --fail-early --max-time 300 --retry 3 -R -v -z 'results/stamp.old' --stderr results/curl-debs-pkgmaster.devuan.org.log -K results/curl-debs-pkgmaster.devuan.org.files | ||
808 | |||
809 | |||
810 | Mon Dec 23 01:02:54 2019 DEBUG : 0 flock -n results/curl-debs-pkgmaster.devuan.org.log commands still running. | ||
811 | Mon Dec 23 01:02:54 2019 DEBUG : *>* Resumed coroutine NO LONGER waiting on - 0 < APT.checkExes(flock -n results/curl-debs-pkgmaster.devuan.org.log | ||
812 | Mon Dec 23 01:02:54 2019 DEBUG : *** Doing list.parser() for debs | ||
813 | Mon Dec 23 01:02:54 2019 ERROR (http Updated pkgmaster.devuan.org): Failed to download - results/pkgmaster.devuan.org/merged/pool/DEBIAN/main/a/aptly/aptly_1.3.0+ds1-4_amd64.deb | ||
814 | |||
815 | drwxr-x--- 2 www-data www-data 4096 2019-12-23 01:02:57.000000000 +1000 aptly | ||
816 | -rw-r--r-- 1 www-data www-data 7129 2019-12-23 01:03:54.000000000 +1000 curl-debs-pkgmaster.devuan.org.log | ||
817 | |||
818 | ]] | ||
819 | os.execute('sleep 2') | ||
820 | while 0 < APT.checkExes(downloadLock .. list.out .. "-" .. host .. ".log") do | ||
821 | D('*<* About to yield coroutine while waiting on - 0 < APT.checkExes(' .. downloadLock .. list.out .. '-' .. host .. '.log') | ||
822 | coroutine.yield() | ||
823 | D('*>* Resumed coroutine while waiting on - 0 < APT.checkExes(' .. downloadLock .. list.out .. '-' .. host .. '.log') | ||
824 | end | ||
825 | D('*>* Resumed coroutine NO LONGER waiting on - 0 < APT.checkExes(' .. downloadLock .. list.out .. '-' .. host .. '.log') | ||
826 | |||
827 | local min, max, spd = 999999999999, 0 | ||
828 | if APT.checkFile("results/curl-" .. list.out .. "-" .. host .. ".log") then | ||
829 | for l in io.lines("results/curl-" .. list.out .. "-" .. host .. ".log") do | ||
830 | local speed, crrnt = l:match('^%c *%d+ +%d+k? +%d+ +%d+k? +%d+ +%d+ +(%d+k?) +%d+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +(%d+k?)') | ||
831 | if nil ~= speed then | ||
832 | if 'k' == speed:sub(-1, -1) then speed = speed:sub(1, -2) .. '000' end | ||
833 | if 'k' == crrnt:sub(-1, -1) then crrnt = crrnt:sub(1, -2) .. '000' end | ||
834 | speed = tonumber(speed) | ||
835 | crrnt = tonumber(crrnt) | ||
836 | if speed < min and speed ~= 0 then min = speed end | ||
837 | if speed > max then max = speed end | ||
838 | if crrnt < min and crrnt ~= 0 then min = crrnt end | ||
839 | if crrnt > max then max = crrnt end | ||
840 | end | ||
841 | if l:find('timed out') ~= nil then | ||
842 | E(" TIMEOUT " .. timeouts + 1 .. ', details in curl-' .. list.out .. '-' .. host .. '.log', 'http', '', host) | ||
843 | timeouts = timeouts + 1 | ||
844 | APT.results["timeout"] = true | ||
845 | end | ||
846 | end | ||
847 | end | ||
848 | APT.results["speed"] = {min = min, max = max} | ||
849 | end | ||
850 | |||
851 | if (APT.options.referenceSite.value ~= host) and ('' ~= list.nextf) then | ||
852 | local sem = 'results/NEW_' .. list.nextf .. '_%s.txt' | ||
853 | for i, n in pairs(releases) do | ||
854 | local f = sem:format(n) | ||
855 | while not APT.checkFile(f) do | ||
856 | D('*<* About to yield coroutine while waiting on - not APT.checkFile(' .. f .. ')') | ||
857 | coroutine.yield() | ||
858 | D('*>* Resumed coroutine while waiting on - not APT.checkFile(' .. f .. ')') | ||
859 | end | ||
860 | end | ||
861 | end | ||
862 | D('*** Doing list.parser() for ' .. list.out) | ||
863 | list = list.parser(host) | ||
864 | if APT.options.timeouts.value <= (totalTimeouts) then break end | ||
865 | end | ||
866 | D('*<<* About to end coroutine.') | ||
867 | cor = nil | ||
868 | end | ||
869 | |||
870 | |||
659 | if 0 < #arg then | 871 | if 0 < #arg then |
660 | if "/" == arg[1]:sub(-1, -1) then | 872 | if "/" == arg[1]:sub(-1, -1) then |
661 | W("slash at end of path! " .. arg[1]) | 873 | W("slash at end of path! " .. arg[1]) |
@@ -673,6 +885,8 @@ if 0 < #arg then | |||
673 | end | 885 | end |
674 | end | 886 | end |
675 | 887 | ||
888 | if APT.origin then APT.results["IPs"] = gatherIPs(pu.host) end | ||
889 | |||
676 | if nil ~= arg[2] then | 890 | if nil ~= arg[2] then |
677 | APT.logFile, e = io.open("results/LOG_" .. pu.host .. "_" .. arg[2] .. ".html", "a+") | 891 | APT.logFile, e = io.open("results/LOG_" .. pu.host .. "_" .. arg[2] .. ".html", "a+") |
678 | else | 892 | else |
@@ -681,165 +895,57 @@ if 0 < #arg then | |||
681 | if nil == APT.logFile then C("opening log file - " .. e); return end | 895 | if nil == APT.logFile then C("opening log file - " .. e); return end |
682 | APT.logPre() | 896 | APT.logPre() |
683 | I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", ")) | 897 | I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", ")) |
684 | APT.mirrors = loadfile("results/mirrors.lua")() | ||
685 | if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end | 898 | if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end |
686 | if nil ~= arg[3] then I(" Using file " .. arg[3]); end | 899 | if nil ~= arg[3] then I(" Using file " .. arg[3]); end |
687 | APT.results = APT.padResults(APT.results) | 900 | APT.results = APT.padResults(APT.results) |
901 | |||
688 | if APT.origin then | 902 | if APT.origin then |
903 | |||
904 | local file = arg[3] | ||
905 | if nil == file then file = '' end | ||
906 | local path = pu.path | ||
907 | if nil == path then path = '' end | ||
908 | local ips = APT.results["IPs"] | ||
909 | for k, v in pairs(ips) do | ||
910 | if "table" == type(v) then | ||
911 | for k1, v1 in pairs(v) do | ||
912 | if v1 == "A" then | ||
913 | if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file) end | ||
914 | elseif v1 == "AAAA" then | ||
915 | if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file) end | ||
916 | end | ||
917 | end | ||
918 | else | ||
919 | if v == "A" then | ||
920 | if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file) end | ||
921 | elseif v == "AAAA" then | ||
922 | if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file) end | ||
923 | end | ||
924 | end | ||
925 | end | ||
926 | |||
689 | if APT.testing("Integrity") or APT.testing("Updated") then | 927 | if APT.testing("Integrity") or APT.testing("Updated") then |
690 | if APT.origin and (APT.options.roundRobin.value ~= pu.host) then | 928 | if APT.origin and (APT.options.roundRobin.value ~= pu.host) then |
691 | I("Starting file downloads for " .. pu.host) | 929 | I("Starting file downloads for " .. pu.host) |
692 | downloads(pu.host, pu.path, 'Release') | 930 | D('*>* About to create coroutine.') |
931 | cor = coroutine.create(doDownloads) | ||
932 | local ok, message = coroutine.resume(cor, pu.host, pu.path, parseStart(pu.host)) | ||
933 | if not ok then cor = nil; print(message) end | ||
693 | end | 934 | end |
694 | end | 935 | end |
936 | |||
695 | checkFiles(pu.host, pu.host, pu.path); | 937 | checkFiles(pu.host, pu.host, pu.path); |
696 | else | 938 | else |
697 | checkHost(pu.host, pu.host, pu.path, arg[2], arg[3]) | 939 | checkHost(pu.host, pu.host, pu.path, arg[2], arg[3]) |
698 | end | 940 | end |
699 | 941 | ||
700 | if APT.testing("Integrity") or APT.testing("Updated") then | 942 | while nil ~= cor do |
701 | if 4 > (totalTimeouts) then | 943 | os.execute('sleep 10') |
702 | if APT.origin and (APT.options.roundRobin.value ~= pu.host) then | 944 | D('*>* About to resume coroutine before writing results.') |
703 | while 0 < APT.checkExes(downloadLock .. "Release-" .. pu.host .. ".log") do os.execute("sleep 10") end | ||
704 | |||
705 | local fcount = 0 | ||
706 | for i, n in pairs(releases) do | ||
707 | for l, o in pairs(releaseFiles) do | ||
708 | if repoExists(i .. o) then | ||
709 | postDownload(pu.host, n, o) | ||
710 | end | ||
711 | end | ||
712 | |||
713 | if APT.checkFile('results/' .. pu.host .. '/merged/dists/' .. n .. '/Release') then | ||
714 | os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED') | ||
715 | if APT.checkFile('results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED') then | ||
716 | if APT.options.referenceSite.value == pu.host then | ||
717 | os.execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. | ||
718 | 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. | ||
719 | '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.tmp') | ||
720 | -- TODO - Maybe check the date in Release, though since they are updated daily, is there any point? Perhaps it's for checking amprolla got run? | ||
721 | os.execute('rm -f results/' .. pu.host .. '/merged/dists/' .. n .. '/Release 2>/dev/null; ' .. | ||
722 | 'mv results/NEW_Release_' .. n .. '.tmp results/NEW_Release_' .. n .. '.txt') | ||
723 | else | ||
724 | -- TODO - compare to the pkgmaster copy. | ||
725 | while not APT.checkFile('results/NEW_Release_' .. n .. '.txt') do os.execute('sleep 10') end | ||
726 | end | ||
727 | |||
728 | local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") | ||
729 | if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else | ||
730 | local diff = dfile:read("*a") | ||
731 | if "" ~= diff then | ||
732 | downloads(pu.host, pu.path, 'Packages', n, diff) | ||
733 | fcount = fcount + 1 | ||
734 | end | ||
735 | end | ||
736 | end | ||
737 | end | ||
738 | |||
739 | end | ||
740 | |||
741 | if 0 ~= fcount then | ||
742 | downloads(pu.host, pu.path, 'Packages', "", "") | ||
743 | while 0 < APT.checkExes(downloadLock .. "Packages-" .. pu.host .. ".log") do os.execute("sleep 10") end | ||
744 | |||
745 | for i, n in pairs(releases) do | ||
746 | local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") | ||
747 | if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else | ||
748 | local diff = dfile:read("*a") | ||
749 | for l in diff:gmatch("\n*([^\n]+)\n*") do | ||
750 | postDownload(pu.host, n, "/" .. l) | ||
751 | end | ||
752 | end | ||
753 | if APT.options.referenceSite.value == pu.host then | ||
754 | -- In case it wasn't dealt with already. | ||
755 | os.execute('touch results/NEW_Packages_' .. n .. '.test.txt') | ||
756 | end | ||
757 | end | ||
758 | |||
759 | fcount = 0 | ||
760 | for i, n in pairs(releases) do | ||
761 | local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") | ||
762 | if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else | ||
763 | for l in nfile:lines() do | ||
764 | local p = l:match('(pool/.*%.deb)') | ||
765 | if nil ~= p then | ||
766 | downloads(pu.host, pu.path, 'package', nil, p) | ||
767 | fcount = fcount + 1 | ||
768 | end | ||
769 | end | ||
770 | end | ||
771 | end | ||
772 | |||
773 | if 0 ~= fcount then | ||
774 | downloads(pu.host, pu.path, 'package', nil, "") | ||
775 | while 0 < APT.checkExes(downloadLock .. "package-" .. pu.host .. ".log") do os.execute("sleep 10") end | ||
776 | for i, n in pairs(releases) do | ||
777 | local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") | ||
778 | if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else | ||
779 | for l in nfile:lines() do | ||
780 | local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |') | ||
781 | if nil ~= p then | ||
782 | if APT.checkFile('results/' .. pu.host .. "/merged/" .. p) then | ||
783 | local status, fsz = APT.execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5') | ||
784 | if APT.testing("Integrity") then | ||
785 | if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. | ||
786 | E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p .. ' should be ' .. sz .. ', but is ' .. fsz:sub(2, -2) .. '.', 'http', 'Integrity', pu.host) | ||
787 | else | ||
788 | local status, fsha = APT.execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') | ||
789 | if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end | ||
790 | -- TODO - maybe check the PGP key, though packages are mostly not signed. | ||
791 | end | ||
792 | end | ||
793 | if APT.testing("Updated") then | ||
794 | if sz ~= fsz:sub(2, -2) then | ||
795 | E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) | ||
796 | end | ||
797 | end | ||
798 | else | ||
799 | E('Failed to download - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) | ||
800 | end | ||
801 | end | ||
802 | end | ||
803 | end | ||
804 | end | ||
805 | end | ||
806 | else | ||
807 | for i, n in pairs(releases) do | ||
808 | os.execute('touch results/NEW_Packages_' .. n .. '.test.txt') | ||
809 | end | ||
810 | end | ||
811 | end | ||
812 | 945 | ||
813 | APT.results["timeout"] = false | 946 | local ok, message = coroutine.resume(cor) |
814 | else | 947 | if not ok then cor = nil; print(message); break end |
815 | APT.results["timeout"] = true | ||
816 | end | ||
817 | end | ||
818 | |||
819 | local min, max, spd = 999999999999, 0 | ||
820 | for i, mt in pairs({'Release', 'Packages', 'package'}) do | ||
821 | if APT.checkFile("results/curl-" .. mt .. "-" .. pu.host .. ".log") then | ||
822 | for l in io.lines("results/curl-" .. mt .. "-" .. pu.host .. ".log") do | ||
823 | local speed, crrnt = l:match('^%c *%d+ +%d+k? +%d+ +%d+k? +%d+ +%d+ +(%d+k?) +%d+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +(%d+k?)') | ||
824 | if nil ~= speed then | ||
825 | if 'k' == speed:sub(-1, -1) then speed = speed:sub(1, -2) .. '000' end | ||
826 | if 'k' == crrnt:sub(-1, -1) then crrnt = crrnt:sub(1, -2) .. '000' end | ||
827 | speed = tonumber(speed) | ||
828 | crrnt = tonumber(crrnt) | ||
829 | if speed < min and speed ~= 0 then min = speed end | ||
830 | if speed > max then max = speed end | ||
831 | if crrnt < min and crrnt ~= 0 then min = crrnt end | ||
832 | if crrnt > max then max = crrnt end | ||
833 | end | ||
834 | if l:find('timed out') ~= nil then | ||
835 | E(" TIMEOUT " .. timeouts + 1 .. ', details in curl-' .. mt .. '-' .. pu.host .. '.log', 'http', '', pu.host) | ||
836 | timeouts = timeouts + 1 | ||
837 | APT.results["timeout"] = true | ||
838 | end | ||
839 | end | ||
840 | end | ||
841 | end | 948 | end |
842 | APT.results["speed"] = {min = min, max = max} | ||
843 | 949 | ||
844 | local f = pu.host | 950 | local f = pu.host |
845 | if "" ~= ip then f = f .. "_" .. ip end | 951 | if "" ~= ip then f = f .. "_" .. ip end |
@@ -891,13 +997,23 @@ else | |||
891 | local pu = url.parse("http://" .. m.BaseURL) | 997 | local pu = url.parse("http://" .. m.BaseURL) |
892 | if APT.options.referenceSite.value ~= pu.host then | 998 | if APT.options.referenceSite.value ~= pu.host then |
893 | checkHost(m.BaseURL) | 999 | checkHost(m.BaseURL) |
894 | APT.checkExes("apt-panopticon.lua " .. sendArgs) | ||
895 | if APT.testing("Integrity") or APT.testing("Updated") then APT.checkExes(downloadLock) end | ||
896 | end | 1000 | end |
897 | end | 1001 | end |
898 | 1002 | ||
899 | while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end | 1003 | while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end |
900 | os.execute('rm -f results/*.check; rm STATUS_* 2>/dev/null') | 1004 | os.execute('rm -f results/*.check; rm STATUS_* 2>/dev/null') |
1005 | for k, v in pairs(APT.mirrors) do | ||
1006 | local f = 'results/' .. k .. '.lua' | ||
1007 | if APT.checkFile(f) then | ||
1008 | results = loadfile(f)() | ||
1009 | APT.mirrors[k]['IPs'] = results.IPs | ||
1010 | end | ||
1011 | end | ||
1012 | local file, e = io.open("results/mirrors.lua", "w+") | ||
1013 | if nil == file then C("opening mirrors file - " .. e) else | ||
1014 | file:write(APT.dumpTable(APT.mirrors, "", "mirrors") .. "\nreturn mirrors\n") | ||
1015 | file:close() | ||
1016 | end | ||
901 | 1017 | ||
902 | -- Create the reports. | 1018 | -- Create the reports. |
903 | for n, r in pairs(APT.options.reports.value) do | 1019 | for n, r in pairs(APT.options.reports.value) do |