From cafd4c5d4ae2de4266026d44f9340fa3de2f2b40 Mon Sep 17 00:00:00 2001
From: onefang
Date: Sat, 28 Dec 2019 22:45:36 +1000
Subject: Finish the executable refactor, by using it and removing the
original.
---
apt-panopticon.lua | 61 ++++++++++++++++++++++++++++++------------------------
1 file changed, 34 insertions(+), 27 deletions(-)
(limited to 'apt-panopticon.lua')
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index c39a9b9..777fdf3 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -272,15 +272,16 @@ checkHEAD = function (host, URL, r, retry, sanity)
end
IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"'
end
- local cmd = 'ionice -c3 nice -n 19 curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' ..
- IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' ..
- hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"'
- local status, result = APT.execute(cmd)
+ local status = APT.exe(
+ 'curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' ..
+ IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' ..
+ hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"'
+ ):Nice():log():Do().status
os.execute('cat results/"HEADERS_' .. fname .. '" >>results/"STATUS_' .. fname .. '" 2>/dev/null; rm -f results/"HEADERS_' .. fname .. '" 2>/dev/null')
- if "0" ~= status then
- local msg = curlStatus[0 + status]
+ if 0 ~= status then
+ local msg = curlStatus[status]
if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end
- if ("28" == status) or ("7" == status) then
+ if (28 == status) or (7 == status) then
if sanity then
T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "URLSanity", host)
else
@@ -385,7 +386,8 @@ checkHEAD = function (host, URL, r, retry, sanity)
local pth = path:match('^(.*/pool/).*$')
if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E('Odd redirect path ' .. path) end
I(" Now checking redirected host " .. u)
- APT.fork("ionice -c3 nice -n 19 " .. downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file)
+-- APT.fork("ionice -c3 nice -n 19 " .. downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file)
+ APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork()
D('logging to ' .. APT.logName(pu.host, nil, file)[2])
end
end
@@ -457,7 +459,7 @@ checkHost = function (orig, host, path, ip, file)
else
if orig == host then
D("checkHost " .. orig .. "" .. file)
- APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file)
+ APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork()
D('logging to ' .. APT.logName(ph.host, nil, file)[2])
else D("checkHost " .. orig .. " -> " .. host) end
end
@@ -492,8 +494,8 @@ local postDownload = function(host, r, k)
" results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. ".old" ..
" results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. "; fi")
if APT.checkFile('results/' .. host .. '/merged/dists/' .. r .. '/' .. k) then
- if ".gz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 gzip -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k, false) end
- if ".xz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 xz -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. " 2>/dev/null", false) end
+ if ".gz" == k:sub(-3, -1) then APT.exe("gzip -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k):Nice():noErr():Do() end
+ if ".xz" == k:sub(-3, -1) then APT.exe("xz -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k):Nice():noErr():Do() end
end
end
@@ -509,7 +511,7 @@ local downloads = function(host, URL, meta, release, list)
local files = 'curl-' .. meta .. '-' .. host .. '.files.txt'
local lock = meta .. "-" .. host .. ".log.txt"
local log = "curl-" .. meta .. "-" .. host .. ".log.txt"
- local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock .. " " .. download .. log .. " -K results/" .. files
+ local cm = downloadLock .. lock .. " " .. download .. log .. " -K results/" .. files
if APT.testing("IPv4") and (not APT.testing("IPv6")) then cm = cm .. ' -4' end
if (not APT.testing("IPv4")) and APT.testing("IPv6") then cm = cm .. ' -6' end
f, e = io.open("results/curl-" .. meta .. '-' .. host .. ".files.txt", "a+")
@@ -539,7 +541,7 @@ local downloads = function(host, URL, meta, release, list)
end
end
f:close()
- APT.fork(cm)
+ APT.exe(cm):Nice():log():fork()
D('logging to ' .. log .. ', with these files')
end
@@ -656,12 +658,14 @@ local parseDebs = function(host)
local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |')
if nil ~= p then
if APT.checkFile('results/' .. host .. "/merged/" .. p) then
- local status, fsz = APT.execute('ls -l results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 5-5', false)
+-- local status, fsz = APT.execute('ls -l results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 5-5', false)
+ local fsz = APT.exe('ls -l results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 5-5'):Do().result
if APT.testing("Integrity") then
if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end.
E('Package size mismatch - results/' .. host .. "/merged/" .. p .. ' should be ' .. sz .. ', but is ' .. fsz:sub(2, -2) .. '.', 'http', 'Integrity', host)
else
- local status, fsha = APT.execute('sha256sum results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 1')
+-- local status, fsha = APT.execute('sha256sum results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 1')
+ local fsha = APT.exe('sha256sum results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 1'):Do().result
if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. host .. "/merged/" .. p, 'http', 'Integrity', host) end
-- TODO - maybe check the PGP key, though packages are mostly not signed.
end
@@ -779,7 +783,8 @@ end
local parseRelease = function(host)
local list = {inf = 'Release', parser = parsePackages, out = 'Packages', files = {}, nextf = 'debs'}
local updated = false
- local status, now = APT.execute('TZ="GMT" date "+%s"', false)
+-- local now = APT.execute('TZ="GMT" date "+%s"', false)
+ local now = os.date('!%s')
now = tonumber(now:sub(2, -2))
for i, n in pairs(releases) do
for l, o in pairs(releaseFiles) do
@@ -787,9 +792,9 @@ local parseRelease = function(host)
postDownload(host, n, o)
if (".gpg" == o:sub(-4, -1)) and (APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/' .. o)) then
if APT.testing("Integrity") then
- local status, out = APT.execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o ..
- " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5) .. " 2>/dev/null")
- if "0" ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end
+ local status = APT.exe( "gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o ..
+ " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status
+ if 0 ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end
-- TODO - should check the PGP sig of InRelease as well.
end
os.execute('rm results/' .. host .. '/merged/dists/' .. n .. '/' .. o)
@@ -812,9 +817,11 @@ local parseRelease = function(host)
else
-- TODO - compare to the pkgmaster copy.
if APT.testing('Updated') then
- local status, pkt = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/pkgmaster.devuan.org/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false)
+-- local status, pkt = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/pkgmaster.devuan.org/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false)
+ local pkt = APT.exe([[TZ="GMT" date -d "$(grep '^Date:' results/pkgmaster.devuan.org/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']]):Do().result
pkt = tonumber(pkt:sub(2, -2))
- local status, new = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/]] .. host .. [[/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false)
+-- local status, new = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/]] .. host .. [[/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false)
+ local new = APT.exe([[TZ="GMT" date -d "$(grep '^Date:' results/]] .. host .. [[/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']]):Do().result
new = tonumber(new:sub(2, -2))
local upd = pkt + APT.mirrors[host].Updated
local updd = pkt + (APT.mirrors[host].Updated * 1.5) -- Give the mirror time to actually do the update.
@@ -883,7 +890,7 @@ local doDownloads = function(host, path, list)
Mon Dec 23 01:02:54 2019 DEBUG : forking
-ionice -c3 nice -n 19 flock -n results/curl-debs-pkgmaster.devuan.org.log curl --connect-timeout 5 --create-dirs -f -L --fail-early --max-time 300 --retry 3 -R -v -z 'results/stamp.old' --stderr results/curl-debs-pkgmaster.devuan.org.log -K results/curl-debs-pkgmaster.devuan.org.files
+flock -n results/curl-debs-pkgmaster.devuan.org.log curl --connect-timeout 5 --create-dirs -f -L --fail-early --max-time 300 --retry 3 -R -v -z 'results/stamp.old' --stderr results/curl-debs-pkgmaster.devuan.org.log -K results/curl-debs-pkgmaster.devuan.org.files
Mon Dec 23 01:02:54 2019 DEBUG : 0 flock -n results/curl-debs-pkgmaster.devuan.org.log commands still running.
@@ -998,17 +1005,17 @@ if 0 < #arg then
if "table" == type(v) then
for k1, v1 in pairs(v) do
if v1 == "A" then
- if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file) end
+ if APT.testing("IPv4") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file):Nice():log():fork() end
elseif v1 == "AAAA" then
- if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file) end
+ if APT.testing("IPv6") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file):Nice():log():fork() end
end
D('logging to ' .. APT.logName(pu.host, k1, file)[2])
end
else
if v == "A" then
- if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file) end
+ if APT.testing("IPv4") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file):Nice():log():fork() end
elseif v == "AAAA" then
- if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file) end
+ if APT.testing("IPv6") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file):Nice():log():fork() end
end
D('logging to ' .. APT.logName(pu.host, k, file)[2])
end
@@ -1212,7 +1219,7 @@ os.execute('sleep 1') -- Wait for things to start up before checking for them.
for n, r in pairs(APT.options.reports.value) do
if APT.checkFile("apt-panopticon-report-" .. r .. ".lua") then
I("Creating " .. r .. " report.")
- APT.execute("./apt-panopticon-report-" .. r .. ".lua " .. sendArgs)
+ APT.exe("./apt-panopticon-report-" .. r .. ".lua " .. sendArgs):log():Do()
end
end
--
cgit v1.1