diff options
author | onefang | 2019-12-28 22:45:36 +1000 |
---|---|---|
committer | onefang | 2019-12-28 22:45:36 +1000 |
commit | cafd4c5d4ae2de4266026d44f9340fa3de2f2b40 (patch) | |
tree | 85d972bac4592596b38b7d63e94e557cf954fce1 /apt-panopticon.lua | |
parent | Tweak the dumpTable() arguments a little. (diff) | |
download | apt-panopticon-cafd4c5d4ae2de4266026d44f9340fa3de2f2b40.zip apt-panopticon-cafd4c5d4ae2de4266026d44f9340fa3de2f2b40.tar.gz apt-panopticon-cafd4c5d4ae2de4266026d44f9340fa3de2f2b40.tar.bz2 apt-panopticon-cafd4c5d4ae2de4266026d44f9340fa3de2f2b40.tar.xz |
Finish the executable refactor, by using it and removing the original.
Diffstat (limited to 'apt-panopticon.lua')
-rwxr-xr-x | apt-panopticon.lua | 61 |
1 files changed, 34 insertions, 27 deletions
diff --git a/apt-panopticon.lua b/apt-panopticon.lua index c39a9b9..777fdf3 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua | |||
@@ -272,15 +272,16 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
272 | end | 272 | end |
273 | IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"' | 273 | IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"' |
274 | end | 274 | end |
275 | local cmd = 'ionice -c3 nice -n 19 curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' .. | 275 | local status = APT.exe( |
276 | IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' .. | 276 | 'curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' .. |
277 | hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"' | 277 | IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' .. |
278 | local status, result = APT.execute(cmd) | 278 | hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"' |
279 | ):Nice():log():Do().status | ||
279 | os.execute('cat results/"HEADERS_' .. fname .. '" >>results/"STATUS_' .. fname .. '" 2>/dev/null; rm -f results/"HEADERS_' .. fname .. '" 2>/dev/null') | 280 | os.execute('cat results/"HEADERS_' .. fname .. '" >>results/"STATUS_' .. fname .. '" 2>/dev/null; rm -f results/"HEADERS_' .. fname .. '" 2>/dev/null') |
280 | if "0" ~= status then | 281 | if 0 ~= status then |
281 | local msg = curlStatus[0 + status] | 282 | local msg = curlStatus[status] |
282 | if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end | 283 | if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end |
283 | if ("28" == status) or ("7" == status) then | 284 | if (28 == status) or (7 == status) then |
284 | if sanity then | 285 | if sanity then |
285 | T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "URLSanity", host) | 286 | T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "URLSanity", host) |
286 | else | 287 | else |
@@ -385,7 +386,8 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
385 | local pth = path:match('^(.*/pool/).*$') | 386 | local pth = path:match('^(.*/pool/).*$') |
386 | if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E('Odd redirect path ' .. path) end | 387 | if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E('Odd redirect path ' .. path) end |
387 | I(" Now checking redirected host " .. u) | 388 | I(" Now checking redirected host " .. u) |
388 | APT.fork("ionice -c3 nice -n 19 " .. downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file) | 389 | -- APT.fork("ionice -c3 nice -n 19 " .. downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file) |
390 | APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork() | ||
389 | D('logging to ' .. APT.logName(pu.host, nil, file)[2]) | 391 | D('logging to ' .. APT.logName(pu.host, nil, file)[2]) |
390 | end | 392 | end |
391 | end | 393 | end |
@@ -457,7 +459,7 @@ checkHost = function (orig, host, path, ip, file) | |||
457 | else | 459 | else |
458 | if orig == host then | 460 | if orig == host then |
459 | D("checkHost " .. orig .. "" .. file) | 461 | D("checkHost " .. orig .. "" .. file) |
460 | APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file) | 462 | APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork() |
461 | D('logging to ' .. APT.logName(ph.host, nil, file)[2]) | 463 | D('logging to ' .. APT.logName(ph.host, nil, file)[2]) |
462 | else D("checkHost " .. orig .. " -> " .. host) end | 464 | else D("checkHost " .. orig .. " -> " .. host) end |
463 | end | 465 | end |
@@ -492,8 +494,8 @@ local postDownload = function(host, r, k) | |||
492 | " results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. ".old" .. | 494 | " results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. ".old" .. |
493 | " results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. "; fi") | 495 | " results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. "; fi") |
494 | if APT.checkFile('results/' .. host .. '/merged/dists/' .. r .. '/' .. k) then | 496 | if APT.checkFile('results/' .. host .. '/merged/dists/' .. r .. '/' .. k) then |
495 | if ".gz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 gzip -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k, false) end | 497 | if ".gz" == k:sub(-3, -1) then APT.exe("gzip -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k):Nice():noErr():Do() end |
496 | if ".xz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 xz -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. " 2>/dev/null", false) end | 498 | if ".xz" == k:sub(-3, -1) then APT.exe("xz -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k):Nice():noErr():Do() end |
497 | end | 499 | end |
498 | end | 500 | end |
499 | 501 | ||
@@ -509,7 +511,7 @@ local downloads = function(host, URL, meta, release, list) | |||
509 | local files = 'curl-' .. meta .. '-' .. host .. '.files.txt' | 511 | local files = 'curl-' .. meta .. '-' .. host .. '.files.txt' |
510 | local lock = meta .. "-" .. host .. ".log.txt" | 512 | local lock = meta .. "-" .. host .. ".log.txt" |
511 | local log = "curl-" .. meta .. "-" .. host .. ".log.txt" | 513 | local log = "curl-" .. meta .. "-" .. host .. ".log.txt" |
512 | local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock .. " " .. download .. log .. " -K results/" .. files | 514 | local cm = downloadLock .. lock .. " " .. download .. log .. " -K results/" .. files |
513 | if APT.testing("IPv4") and (not APT.testing("IPv6")) then cm = cm .. ' -4' end | 515 | if APT.testing("IPv4") and (not APT.testing("IPv6")) then cm = cm .. ' -4' end |
514 | if (not APT.testing("IPv4")) and APT.testing("IPv6") then cm = cm .. ' -6' end | 516 | if (not APT.testing("IPv4")) and APT.testing("IPv6") then cm = cm .. ' -6' end |
515 | f, e = io.open("results/curl-" .. meta .. '-' .. host .. ".files.txt", "a+") | 517 | f, e = io.open("results/curl-" .. meta .. '-' .. host .. ".files.txt", "a+") |
@@ -539,7 +541,7 @@ local downloads = function(host, URL, meta, release, list) | |||
539 | end | 541 | end |
540 | end | 542 | end |
541 | f:close() | 543 | f:close() |
542 | APT.fork(cm) | 544 | APT.exe(cm):Nice():log():fork() |
543 | D('logging to <a href="' .. log .. '">' .. log .. '</a>, with <a href="' .. files .. '">these files</a>') | 545 | D('logging to <a href="' .. log .. '">' .. log .. '</a>, with <a href="' .. files .. '">these files</a>') |
544 | end | 546 | end |
545 | 547 | ||
@@ -656,12 +658,14 @@ local parseDebs = function(host) | |||
656 | local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |') | 658 | local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |') |
657 | if nil ~= p then | 659 | if nil ~= p then |
658 | if APT.checkFile('results/' .. host .. "/merged/" .. p) then | 660 | if APT.checkFile('results/' .. host .. "/merged/" .. p) then |
659 | local status, fsz = APT.execute('ls -l results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 5-5', false) | 661 | -- local status, fsz = APT.execute('ls -l results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 5-5', false) |
662 | local fsz = APT.exe('ls -l results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 5-5'):Do().result | ||
660 | if APT.testing("Integrity") then | 663 | if APT.testing("Integrity") then |
661 | if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. | 664 | if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. |
662 | E('Package size mismatch - results/' .. host .. "/merged/" .. p .. ' should be ' .. sz .. ', but is ' .. fsz:sub(2, -2) .. '.', 'http', 'Integrity', host) | 665 | E('Package size mismatch - results/' .. host .. "/merged/" .. p .. ' should be ' .. sz .. ', but is ' .. fsz:sub(2, -2) .. '.', 'http', 'Integrity', host) |
663 | else | 666 | else |
664 | local status, fsha = APT.execute('sha256sum results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 1') | 667 | -- local status, fsha = APT.execute('sha256sum results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 1') |
668 | local fsha = APT.exe('sha256sum results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 1'):Do().result | ||
665 | if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. host .. "/merged/" .. p, 'http', 'Integrity', host) end | 669 | if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. host .. "/merged/" .. p, 'http', 'Integrity', host) end |
666 | -- TODO - maybe check the PGP key, though packages are mostly not signed. | 670 | -- TODO - maybe check the PGP key, though packages are mostly not signed. |
667 | end | 671 | end |
@@ -779,7 +783,8 @@ end | |||
779 | local parseRelease = function(host) | 783 | local parseRelease = function(host) |
780 | local list = {inf = 'Release', parser = parsePackages, out = 'Packages', files = {}, nextf = 'debs'} | 784 | local list = {inf = 'Release', parser = parsePackages, out = 'Packages', files = {}, nextf = 'debs'} |
781 | local updated = false | 785 | local updated = false |
782 | local status, now = APT.execute('TZ="GMT" date "+%s"', false) | 786 | -- local now = APT.execute('TZ="GMT" date "+%s"', false) |
787 | local now = os.date('!%s') | ||
783 | now = tonumber(now:sub(2, -2)) | 788 | now = tonumber(now:sub(2, -2)) |
784 | for i, n in pairs(releases) do | 789 | for i, n in pairs(releases) do |
785 | for l, o in pairs(releaseFiles) do | 790 | for l, o in pairs(releaseFiles) do |
@@ -787,9 +792,9 @@ local parseRelease = function(host) | |||
787 | postDownload(host, n, o) | 792 | postDownload(host, n, o) |
788 | if (".gpg" == o:sub(-4, -1)) and (APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/' .. o)) then | 793 | if (".gpg" == o:sub(-4, -1)) and (APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/' .. o)) then |
789 | if APT.testing("Integrity") then | 794 | if APT.testing("Integrity") then |
790 | local status, out = APT.execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. | 795 | local status = APT.exe( "gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. |
791 | " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5) .. " 2>/dev/null") | 796 | " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status |
792 | if "0" ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end | 797 | if 0 ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end |
793 | -- TODO - should check the PGP sig of InRelease as well. | 798 | -- TODO - should check the PGP sig of InRelease as well. |
794 | end | 799 | end |
795 | os.execute('rm results/' .. host .. '/merged/dists/' .. n .. '/' .. o) | 800 | os.execute('rm results/' .. host .. '/merged/dists/' .. n .. '/' .. o) |
@@ -812,9 +817,11 @@ local parseRelease = function(host) | |||
812 | else | 817 | else |
813 | -- TODO - compare to the pkgmaster copy. | 818 | -- TODO - compare to the pkgmaster copy. |
814 | if APT.testing('Updated') then | 819 | if APT.testing('Updated') then |
815 | local status, pkt = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/pkgmaster.devuan.org/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false) | 820 | -- local status, pkt = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/pkgmaster.devuan.org/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false) |
821 | local pkt = APT.exe([[TZ="GMT" date -d "$(grep '^Date:' results/pkgmaster.devuan.org/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']]):Do().result | ||
816 | pkt = tonumber(pkt:sub(2, -2)) | 822 | pkt = tonumber(pkt:sub(2, -2)) |
817 | local status, new = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/]] .. host .. [[/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false) | 823 | -- local status, new = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/]] .. host .. [[/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false) |
824 | local new = APT.exe([[TZ="GMT" date -d "$(grep '^Date:' results/]] .. host .. [[/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']]):Do().result | ||
818 | new = tonumber(new:sub(2, -2)) | 825 | new = tonumber(new:sub(2, -2)) |
819 | local upd = pkt + APT.mirrors[host].Updated | 826 | local upd = pkt + APT.mirrors[host].Updated |
820 | local updd = pkt + (APT.mirrors[host].Updated * 1.5) -- Give the mirror time to actually do the update. | 827 | local updd = pkt + (APT.mirrors[host].Updated * 1.5) -- Give the mirror time to actually do the update. |
@@ -883,7 +890,7 @@ local doDownloads = function(host, path, list) | |||
883 | 890 | ||
884 | Mon Dec 23 01:02:54 2019 DEBUG : forking | 891 | Mon Dec 23 01:02:54 2019 DEBUG : forking |
885 | 892 | ||
886 | ionice -c3 nice -n 19 flock -n results/curl-debs-pkgmaster.devuan.org.log curl --connect-timeout 5 --create-dirs -f -L --fail-early --max-time 300 --retry 3 -R -v -z 'results/stamp.old' --stderr results/curl-debs-pkgmaster.devuan.org.log -K results/curl-debs-pkgmaster.devuan.org.files | 893 | flock -n results/curl-debs-pkgmaster.devuan.org.log curl --connect-timeout 5 --create-dirs -f -L --fail-early --max-time 300 --retry 3 -R -v -z 'results/stamp.old' --stderr results/curl-debs-pkgmaster.devuan.org.log -K results/curl-debs-pkgmaster.devuan.org.files |
887 | 894 | ||
888 | 895 | ||
889 | Mon Dec 23 01:02:54 2019 DEBUG : 0 flock -n results/curl-debs-pkgmaster.devuan.org.log commands still running. | 896 | Mon Dec 23 01:02:54 2019 DEBUG : 0 flock -n results/curl-debs-pkgmaster.devuan.org.log commands still running. |
@@ -998,17 +1005,17 @@ if 0 < #arg then | |||
998 | if "table" == type(v) then | 1005 | if "table" == type(v) then |
999 | for k1, v1 in pairs(v) do | 1006 | for k1, v1 in pairs(v) do |
1000 | if v1 == "A" then | 1007 | if v1 == "A" then |
1001 | if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file) end | 1008 | if APT.testing("IPv4") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file):Nice():log():fork() end |
1002 | elseif v1 == "AAAA" then | 1009 | elseif v1 == "AAAA" then |
1003 | if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file) end | 1010 | if APT.testing("IPv6") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file):Nice():log():fork() end |
1004 | end | 1011 | end |
1005 | D('logging to ' .. APT.logName(pu.host, k1, file)[2]) | 1012 | D('logging to ' .. APT.logName(pu.host, k1, file)[2]) |
1006 | end | 1013 | end |
1007 | else | 1014 | else |
1008 | if v == "A" then | 1015 | if v == "A" then |
1009 | if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file) end | 1016 | if APT.testing("IPv4") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file):Nice():log():fork() end |
1010 | elseif v == "AAAA" then | 1017 | elseif v == "AAAA" then |
1011 | if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file) end | 1018 | if APT.testing("IPv6") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file):Nice():log():fork() end |
1012 | end | 1019 | end |
1013 | D('logging to ' .. APT.logName(pu.host, k, file)[2]) | 1020 | D('logging to ' .. APT.logName(pu.host, k, file)[2]) |
1014 | end | 1021 | end |
@@ -1212,7 +1219,7 @@ os.execute('sleep 1') -- Wait for things to start up before checking for them. | |||
1212 | for n, r in pairs(APT.options.reports.value) do | 1219 | for n, r in pairs(APT.options.reports.value) do |
1213 | if APT.checkFile("apt-panopticon-report-" .. r .. ".lua") then | 1220 | if APT.checkFile("apt-panopticon-report-" .. r .. ".lua") then |
1214 | I("Creating " .. r .. " report.") | 1221 | I("Creating " .. r .. " report.") |
1215 | APT.execute("./apt-panopticon-report-" .. r .. ".lua " .. sendArgs) | 1222 | APT.exe("./apt-panopticon-report-" .. r .. ".lua " .. sendArgs):log():Do() |
1216 | end | 1223 | end |
1217 | end | 1224 | end |
1218 | 1225 | ||