From cafd4c5d4ae2de4266026d44f9340fa3de2f2b40 Mon Sep 17 00:00:00 2001 From: onefang Date: Sat, 28 Dec 2019 22:45:36 +1000 Subject: Finish the executable refactor, by using it and removing the original. --- apt-panopticommon.lua | 54 ++++++++++++-------------------- apt-panopticon-report-email-web.lua | 4 +-- apt-panopticon-update-data.lua | 6 ++-- apt-panopticon.lua | 61 +++++++++++++++++++++---------------- 4 files changed, 60 insertions(+), 65 deletions(-) diff --git a/apt-panopticommon.lua b/apt-panopticommon.lua index 53bd674..d9293f6 100644 --- a/apt-panopticommon.lua +++ b/apt-panopticommon.lua @@ -424,43 +424,24 @@ APT.testing = function(t, host) return false end -APT.execute = function (s, logit) - if nil == logit then logit = true end - if logit then D(" executing -   " .. s .. "") end - --[[ Damn os.execute() - Lua 5.1 says it returns "a status code, which is system-dependent" - Lua 5.2 says it returns true/nil, "exit"/"signal", the status code. - I'm getting 7168 or 0. No idea what the fuck that is. - local ok, rslt, status = os.execute(s) - ]] - local f = io.popen(s .. ' ; echo "$?"', 'r') - local status = "" - local result = "" - -- The last line will be the command's returned status, collect everything else in result. - for l in f:lines() do - result = result .. status .. "\n" - status = l - end - return status, result -end - -APT.fork = function(s) - D(" forking -   " .. s .. "") - os.execute(s .. " &") -end - - APT.exe = function(c) - local exe = {status = 0, result = ''} - exe.cmd = c .. ' ' - exe.log = false + local exe = {status = 0, result = '', log = true, cmd = c .. ' '} function exe:log() self.log = true return self end function exe:Nice(c) - self.cmd = self.cmd .. 'ionice -c3 nice -n 19 ' .. c .. ' ' + if nil == c then + self.cmd = 'ionice -c3 nice -n 19 ' .. self.cmd + else + self.cmd = self.cmd .. 'ionice -c3 nice -n 19 ' .. c .. ' ' + end + return self + end + function exe:also(c) + if nil == c then c = '' else c = ' ' .. c end + self.cmd = self.cmd .. ';' .. c .. ' ' return self end function exe:And(c) @@ -482,7 +463,12 @@ APT.exe = function(c) return self end function exe:Do() - if self.log then D(" executing -   " .. self.cmd .. "") end + --[[ "The condition expression of a control structure can return any + value. Both false and nil are considered false. All values different + from nil and false are considered true (in particular, the number 0 + and the empty string are also true)." + says the docs, I beg to differ.]] + if true == self.log then D(" executing -   " .. self.cmd .. "") end --[[ Damn os.execute() Lua 5.1 says it returns "a status code, which is system-dependent" Lua 5.2 says it returns true/nil, "exit"/"signal", the status code. @@ -491,6 +477,7 @@ APT.exe = function(c) ]] local f = io.popen(self.cmd .. ' ; echo "$?"', 'r') -- The last line will be the command's returned status, collect everything else in result. + self.status = '' -- Otherwise the result starts with 0. for l in f:lines() do self.result = self.result .. self.status .. "\n" self.status = l @@ -500,14 +487,13 @@ APT.exe = function(c) end function exe:fork() self.cmd = '{ ' .. self.cmd .. '; } &' - if self.log then D(" forking -   " .. self.cmd .. "") end + if true == self.log then D(" forking -   " .. self.cmd .. "") end os.execute(self.cmd) return self end return exe end - APT.checkExes = function (exe) local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | grep -v "flock -n apt-panopticon.lock " | wc -l'):read("*l") D(count .. " " .. exe .. " commands still running.") @@ -650,7 +636,7 @@ end APT.now = 0 local status -status, APT.now = APT.execute('TZ="GMT" ls -l --time-style="+%s" results/stamp | cut -d " " -f 6-6') +APT.now = APT.exe('TZ="GMT" ls -l --time-style="+%s" results/stamp | cut -d " " -f 6-6'):Do().result APT.now = tonumber(APT.now) local start = 'now-1month' local step = '10min' diff --git a/apt-panopticon-report-email-web.lua b/apt-panopticon-report-email-web.lua index 67d4d13..3304fb2 100755 --- a/apt-panopticon-report-email-web.lua +++ b/apt-panopticon-report-email-web.lua @@ -645,8 +645,8 @@ if nil == web then C("opening mirrors file - " .. e) else "You can get the cgp graphing source code here (main repo)" .. "and here (Devuan repo).

\n" ) - local status, whn = APT.execute('TZ="GMT" ls -l1 --time-style="+%s" results/stamp | cut -d " " -f 6-6') - web:write( "

This run took " .. (os.time() - tonumber("0" .. whn:sub(2, -2))) .. " seconds.

" .. + local whn = APT.exe('TZ="GMT" ls -l1 --time-style="+%s" results/stamp | cut -d " " -f 6-6'):Do().result:sub(2, -2) + web:write( "

This run took " .. (os.time() - tonumber("0" .. whn)) .. " seconds.

" .. "\n\n") web:close() end diff --git a/apt-panopticon-update-data.lua b/apt-panopticon-update-data.lua index a26b11b..8ab9254 100755 --- a/apt-panopticon-update-data.lua +++ b/apt-panopticon-update-data.lua @@ -106,9 +106,11 @@ local files = io.popen('ls -d1 results_*') for l in files:lines() do if ('results_old' ~= l) and ('.tar.xz' ~= l:sub(25, -1)) then if APT.checkFile(l .. '/stamp') then - local status, whn = APT.execute('TZ="GMT" ls -d1 --time-style="+%s" ' .. l .. ' | cut -d " " -f 6-6') +-- local status, whn = APT.execute('TZ="GMT" ls -d1 --time-style="+%s" ' .. l .. ' | cut -d " " -f 6-6') + local whn = APT.exe('TZ="GMT" ls -d1 --time-style="+%s" ' .. l .. ' | cut -d " " -f 6-6'):Do() whn = whn:sub(2, -2) - local status, new = APT.execute('TZ="GMT" date -d "' .. whn:sub(9, 18) .. ' ' .. whn:sub(20, 21) .. ':' .. whn:sub(23, 24) .. '" "+%s"') +-- local status, new = APT.execute('TZ="GMT" date -d "' .. whn:sub(9, 18) .. ' ' .. whn:sub(20, 21) .. ':' .. whn:sub(23, 24) .. '" "+%s"') + local new = APT.exe('TZ="GMT" date -d "' .. whn:sub(9, 18) .. ' ' .. whn:sub(20, 21) .. ':' .. whn:sub(23, 24) .. '" "+%s"'):Do() APT.now = tonumber("0" .. new:sub(2, -2)) if 0 ~= APT.now then if APT.checkFile(l .. '/mirrors.lua') then diff --git a/apt-panopticon.lua b/apt-panopticon.lua index c39a9b9..777fdf3 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua @@ -272,15 +272,16 @@ checkHEAD = function (host, URL, r, retry, sanity) end IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"' end - local cmd = 'ionice -c3 nice -n 19 curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' .. - IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' .. - hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"' - local status, result = APT.execute(cmd) + local status = APT.exe( + 'curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' .. + IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' .. + hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"' + ):Nice():log():Do().status os.execute('cat results/"HEADERS_' .. fname .. '" >>results/"STATUS_' .. fname .. '" 2>/dev/null; rm -f results/"HEADERS_' .. fname .. '" 2>/dev/null') - if "0" ~= status then - local msg = curlStatus[0 + status] + if 0 ~= status then + local msg = curlStatus[status] if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end - if ("28" == status) or ("7" == status) then + if (28 == status) or (7 == status) then if sanity then T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "URLSanity", host) else @@ -385,7 +386,8 @@ checkHEAD = function (host, URL, r, retry, sanity) local pth = path:match('^(.*/pool/).*$') if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E('Odd redirect path ' .. path) end I(" Now checking redirected host " .. u) - APT.fork("ionice -c3 nice -n 19 " .. downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file) +-- APT.fork("ionice -c3 nice -n 19 " .. downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file) + APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork() D('logging to ' .. APT.logName(pu.host, nil, file)[2]) end end @@ -457,7 +459,7 @@ checkHost = function (orig, host, path, ip, file) else if orig == host then D("checkHost " .. orig .. "" .. file) - APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file) + APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork() D('logging to ' .. APT.logName(ph.host, nil, file)[2]) else D("checkHost " .. orig .. " -> " .. host) end end @@ -492,8 +494,8 @@ local postDownload = function(host, r, k) " results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. ".old" .. " results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. "; fi") if APT.checkFile('results/' .. host .. '/merged/dists/' .. r .. '/' .. k) then - if ".gz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 gzip -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k, false) end - if ".xz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 xz -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. " 2>/dev/null", false) end + if ".gz" == k:sub(-3, -1) then APT.exe("gzip -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k):Nice():noErr():Do() end + if ".xz" == k:sub(-3, -1) then APT.exe("xz -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k):Nice():noErr():Do() end end end @@ -509,7 +511,7 @@ local downloads = function(host, URL, meta, release, list) local files = 'curl-' .. meta .. '-' .. host .. '.files.txt' local lock = meta .. "-" .. host .. ".log.txt" local log = "curl-" .. meta .. "-" .. host .. ".log.txt" - local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock .. " " .. download .. log .. " -K results/" .. files + local cm = downloadLock .. lock .. " " .. download .. log .. " -K results/" .. files if APT.testing("IPv4") and (not APT.testing("IPv6")) then cm = cm .. ' -4' end if (not APT.testing("IPv4")) and APT.testing("IPv6") then cm = cm .. ' -6' end f, e = io.open("results/curl-" .. meta .. '-' .. host .. ".files.txt", "a+") @@ -539,7 +541,7 @@ local downloads = function(host, URL, meta, release, list) end end f:close() - APT.fork(cm) + APT.exe(cm):Nice():log():fork() D('logging to ' .. log .. ', with these files') end @@ -656,12 +658,14 @@ local parseDebs = function(host) local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |') if nil ~= p then if APT.checkFile('results/' .. host .. "/merged/" .. p) then - local status, fsz = APT.execute('ls -l results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 5-5', false) +-- local status, fsz = APT.execute('ls -l results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 5-5', false) + local fsz = APT.exe('ls -l results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 5-5'):Do().result if APT.testing("Integrity") then if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. E('Package size mismatch - results/' .. host .. "/merged/" .. p .. ' should be ' .. sz .. ', but is ' .. fsz:sub(2, -2) .. '.', 'http', 'Integrity', host) else - local status, fsha = APT.execute('sha256sum results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 1') +-- local status, fsha = APT.execute('sha256sum results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 1') + local fsha = APT.exe('sha256sum results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 1'):Do().result if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. host .. "/merged/" .. p, 'http', 'Integrity', host) end -- TODO - maybe check the PGP key, though packages are mostly not signed. end @@ -779,7 +783,8 @@ end local parseRelease = function(host) local list = {inf = 'Release', parser = parsePackages, out = 'Packages', files = {}, nextf = 'debs'} local updated = false - local status, now = APT.execute('TZ="GMT" date "+%s"', false) +-- local now = APT.execute('TZ="GMT" date "+%s"', false) + local now = os.date('!%s') now = tonumber(now:sub(2, -2)) for i, n in pairs(releases) do for l, o in pairs(releaseFiles) do @@ -787,9 +792,9 @@ local parseRelease = function(host) postDownload(host, n, o) if (".gpg" == o:sub(-4, -1)) and (APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/' .. o)) then if APT.testing("Integrity") then - local status, out = APT.execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. - " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5) .. " 2>/dev/null") - if "0" ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end + local status = APT.exe( "gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. + " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status + if 0 ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end -- TODO - should check the PGP sig of InRelease as well. end os.execute('rm results/' .. host .. '/merged/dists/' .. n .. '/' .. o) @@ -812,9 +817,11 @@ local parseRelease = function(host) else -- TODO - compare to the pkgmaster copy. if APT.testing('Updated') then - local status, pkt = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/pkgmaster.devuan.org/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false) +-- local status, pkt = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/pkgmaster.devuan.org/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false) + local pkt = APT.exe([[TZ="GMT" date -d "$(grep '^Date:' results/pkgmaster.devuan.org/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']]):Do().result pkt = tonumber(pkt:sub(2, -2)) - local status, new = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/]] .. host .. [[/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false) +-- local status, new = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/]] .. host .. [[/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false) + local new = APT.exe([[TZ="GMT" date -d "$(grep '^Date:' results/]] .. host .. [[/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']]):Do().result new = tonumber(new:sub(2, -2)) local upd = pkt + APT.mirrors[host].Updated local updd = pkt + (APT.mirrors[host].Updated * 1.5) -- Give the mirror time to actually do the update. @@ -883,7 +890,7 @@ local doDownloads = function(host, path, list) Mon Dec 23 01:02:54 2019 DEBUG : forking -ionice -c3 nice -n 19 flock -n results/curl-debs-pkgmaster.devuan.org.log curl --connect-timeout 5 --create-dirs -f -L --fail-early --max-time 300 --retry 3 -R -v -z 'results/stamp.old' --stderr results/curl-debs-pkgmaster.devuan.org.log -K results/curl-debs-pkgmaster.devuan.org.files +flock -n results/curl-debs-pkgmaster.devuan.org.log curl --connect-timeout 5 --create-dirs -f -L --fail-early --max-time 300 --retry 3 -R -v -z 'results/stamp.old' --stderr results/curl-debs-pkgmaster.devuan.org.log -K results/curl-debs-pkgmaster.devuan.org.files Mon Dec 23 01:02:54 2019 DEBUG : 0 flock -n results/curl-debs-pkgmaster.devuan.org.log commands still running. @@ -998,17 +1005,17 @@ if 0 < #arg then if "table" == type(v) then for k1, v1 in pairs(v) do if v1 == "A" then - if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file) end + if APT.testing("IPv4") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file):Nice():log():fork() end elseif v1 == "AAAA" then - if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file) end + if APT.testing("IPv6") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file):Nice():log():fork() end end D('logging to ' .. APT.logName(pu.host, k1, file)[2]) end else if v == "A" then - if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file) end + if APT.testing("IPv4") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file):Nice():log():fork() end elseif v == "AAAA" then - if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file) end + if APT.testing("IPv6") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file):Nice():log():fork() end end D('logging to ' .. APT.logName(pu.host, k, file)[2]) end @@ -1212,7 +1219,7 @@ os.execute('sleep 1') -- Wait for things to start up before checking for them. for n, r in pairs(APT.options.reports.value) do if APT.checkFile("apt-panopticon-report-" .. r .. ".lua") then I("Creating " .. r .. " report.") - APT.execute("./apt-panopticon-report-" .. r .. ".lua " .. sendArgs) + APT.exe("./apt-panopticon-report-" .. r .. ".lua " .. sendArgs):log():Do() end end -- cgit v1.1