From cafd4c5d4ae2de4266026d44f9340fa3de2f2b40 Mon Sep 17 00:00:00 2001 From: onefang Date: Sat, 28 Dec 2019 22:45:36 +1000 Subject: Finish the executable refactor, by using it and removing the original. --- apt-panopticon.lua | 61 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 27 deletions(-) (limited to 'apt-panopticon.lua') diff --git a/apt-panopticon.lua b/apt-panopticon.lua index c39a9b9..777fdf3 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua @@ -272,15 +272,16 @@ checkHEAD = function (host, URL, r, retry, sanity) end IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"' end - local cmd = 'ionice -c3 nice -n 19 curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' .. - IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' .. - hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"' - local status, result = APT.execute(cmd) + local status = APT.exe( + 'curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' .. + IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' .. + hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"' + ):Nice():log():Do().status os.execute('cat results/"HEADERS_' .. fname .. '" >>results/"STATUS_' .. fname .. '" 2>/dev/null; rm -f results/"HEADERS_' .. fname .. '" 2>/dev/null') - if "0" ~= status then - local msg = curlStatus[0 + status] + if 0 ~= status then + local msg = curlStatus[status] if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end - if ("28" == status) or ("7" == status) then + if (28 == status) or (7 == status) then if sanity then T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "URLSanity", host) else @@ -385,7 +386,8 @@ checkHEAD = function (host, URL, r, retry, sanity) local pth = path:match('^(.*/pool/).*$') if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E('Odd redirect path ' .. path) end I(" Now checking redirected host " .. u) - APT.fork("ionice -c3 nice -n 19 " .. downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file) +-- APT.fork("ionice -c3 nice -n 19 " .. downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file) + APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork() D('logging to ' .. APT.logName(pu.host, nil, file)[2]) end end @@ -457,7 +459,7 @@ checkHost = function (orig, host, path, ip, file) else if orig == host then D("checkHost " .. orig .. "" .. file) - APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file) + APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork() D('logging to ' .. APT.logName(ph.host, nil, file)[2]) else D("checkHost " .. orig .. " -> " .. host) end end @@ -492,8 +494,8 @@ local postDownload = function(host, r, k) " results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. ".old" .. " results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. "; fi") if APT.checkFile('results/' .. host .. '/merged/dists/' .. r .. '/' .. k) then - if ".gz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 gzip -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k, false) end - if ".xz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 xz -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. " 2>/dev/null", false) end + if ".gz" == k:sub(-3, -1) then APT.exe("gzip -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k):Nice():noErr():Do() end + if ".xz" == k:sub(-3, -1) then APT.exe("xz -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k):Nice():noErr():Do() end end end @@ -509,7 +511,7 @@ local downloads = function(host, URL, meta, release, list) local files = 'curl-' .. meta .. '-' .. host .. '.files.txt' local lock = meta .. "-" .. host .. ".log.txt" local log = "curl-" .. meta .. "-" .. host .. ".log.txt" - local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock .. " " .. download .. log .. " -K results/" .. files + local cm = downloadLock .. lock .. " " .. download .. log .. " -K results/" .. files if APT.testing("IPv4") and (not APT.testing("IPv6")) then cm = cm .. ' -4' end if (not APT.testing("IPv4")) and APT.testing("IPv6") then cm = cm .. ' -6' end f, e = io.open("results/curl-" .. meta .. '-' .. host .. ".files.txt", "a+") @@ -539,7 +541,7 @@ local downloads = function(host, URL, meta, release, list) end end f:close() - APT.fork(cm) + APT.exe(cm):Nice():log():fork() D('logging to ' .. log .. ', with these files') end @@ -656,12 +658,14 @@ local parseDebs = function(host) local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |') if nil ~= p then if APT.checkFile('results/' .. host .. "/merged/" .. p) then - local status, fsz = APT.execute('ls -l results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 5-5', false) +-- local status, fsz = APT.execute('ls -l results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 5-5', false) + local fsz = APT.exe('ls -l results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 5-5'):Do().result if APT.testing("Integrity") then if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. E('Package size mismatch - results/' .. host .. "/merged/" .. p .. ' should be ' .. sz .. ', but is ' .. fsz:sub(2, -2) .. '.', 'http', 'Integrity', host) else - local status, fsha = APT.execute('sha256sum results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 1') +-- local status, fsha = APT.execute('sha256sum results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 1') + local fsha = APT.exe('sha256sum results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 1'):Do().result if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. host .. "/merged/" .. p, 'http', 'Integrity', host) end -- TODO - maybe check the PGP key, though packages are mostly not signed. end @@ -779,7 +783,8 @@ end local parseRelease = function(host) local list = {inf = 'Release', parser = parsePackages, out = 'Packages', files = {}, nextf = 'debs'} local updated = false - local status, now = APT.execute('TZ="GMT" date "+%s"', false) +-- local now = APT.execute('TZ="GMT" date "+%s"', false) + local now = os.date('!%s') now = tonumber(now:sub(2, -2)) for i, n in pairs(releases) do for l, o in pairs(releaseFiles) do @@ -787,9 +792,9 @@ local parseRelease = function(host) postDownload(host, n, o) if (".gpg" == o:sub(-4, -1)) and (APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/' .. o)) then if APT.testing("Integrity") then - local status, out = APT.execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. - " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5) .. " 2>/dev/null") - if "0" ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end + local status = APT.exe( "gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. + " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status + if 0 ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end -- TODO - should check the PGP sig of InRelease as well. end os.execute('rm results/' .. host .. '/merged/dists/' .. n .. '/' .. o) @@ -812,9 +817,11 @@ local parseRelease = function(host) else -- TODO - compare to the pkgmaster copy. if APT.testing('Updated') then - local status, pkt = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/pkgmaster.devuan.org/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false) +-- local status, pkt = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/pkgmaster.devuan.org/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false) + local pkt = APT.exe([[TZ="GMT" date -d "$(grep '^Date:' results/pkgmaster.devuan.org/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']]):Do().result pkt = tonumber(pkt:sub(2, -2)) - local status, new = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/]] .. host .. [[/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false) +-- local status, new = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/]] .. host .. [[/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false) + local new = APT.exe([[TZ="GMT" date -d "$(grep '^Date:' results/]] .. host .. [[/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']]):Do().result new = tonumber(new:sub(2, -2)) local upd = pkt + APT.mirrors[host].Updated local updd = pkt + (APT.mirrors[host].Updated * 1.5) -- Give the mirror time to actually do the update. @@ -883,7 +890,7 @@ local doDownloads = function(host, path, list) Mon Dec 23 01:02:54 2019 DEBUG : forking -ionice -c3 nice -n 19 flock -n results/curl-debs-pkgmaster.devuan.org.log curl --connect-timeout 5 --create-dirs -f -L --fail-early --max-time 300 --retry 3 -R -v -z 'results/stamp.old' --stderr results/curl-debs-pkgmaster.devuan.org.log -K results/curl-debs-pkgmaster.devuan.org.files +flock -n results/curl-debs-pkgmaster.devuan.org.log curl --connect-timeout 5 --create-dirs -f -L --fail-early --max-time 300 --retry 3 -R -v -z 'results/stamp.old' --stderr results/curl-debs-pkgmaster.devuan.org.log -K results/curl-debs-pkgmaster.devuan.org.files Mon Dec 23 01:02:54 2019 DEBUG : 0 flock -n results/curl-debs-pkgmaster.devuan.org.log commands still running. @@ -998,17 +1005,17 @@ if 0 < #arg then if "table" == type(v) then for k1, v1 in pairs(v) do if v1 == "A" then - if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file) end + if APT.testing("IPv4") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file):Nice():log():fork() end elseif v1 == "AAAA" then - if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file) end + if APT.testing("IPv6") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file):Nice():log():fork() end end D('logging to ' .. APT.logName(pu.host, k1, file)[2]) end else if v == "A" then - if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file) end + if APT.testing("IPv4") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file):Nice():log():fork() end elseif v == "AAAA" then - if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file) end + if APT.testing("IPv6") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file):Nice():log():fork() end end D('logging to ' .. APT.logName(pu.host, k, file)[2]) end @@ -1212,7 +1219,7 @@ os.execute('sleep 1') -- Wait for things to start up before checking for them. for n, r in pairs(APT.options.reports.value) do if APT.checkFile("apt-panopticon-report-" .. r .. ".lua") then I("Creating " .. r .. " report.") - APT.execute("./apt-panopticon-report-" .. r .. ".lua " .. sendArgs) + APT.exe("./apt-panopticon-report-" .. r .. ".lua " .. sendArgs):log():Do() end end -- cgit v1.1