Finish the executable refactor, by using it and removing the original.

4 files changed, 60 insertions, 65 deletions

diff --git a/apt-panopticommon.lua b/apt-panopticommon.lua
index 53bd674..d9293f6 100644
--- a/apt-panopticommon.lua
+++ b/apt-panopticommon.lua
@@ -424,43 +424,24 @@ APT.testing = function(t, host)
     return false
 end
 
-APT.execute = function (s, logit)
-    if nil == logit then logit = true end
-    if logit then D("  executing - &nbsp; <code>" .. s .. "</code>") end
-    --[[ Damn os.execute()
-        Lua 5.1 says it returns "a status code, which is system-dependent"
-        Lua 5.2 says it returns true/nil, "exit"/"signal", the status code.
-        I'm getting 7168 or 0.  No idea what the fuck that is.
-    local ok, rslt, status = os.execute(s)
-    ]]
-    local f = io.popen(s .. ' ; echo "$?"', 'r')
-    local status = ""
-    local result = ""
-    -- The last line will be the command's returned status, collect everything else in result.
-    for l in f:lines() do
-        result = result .. status .. "\n"
-        status = l
-    end
-    return status, result
-end
-
-APT.fork = function(s)
-    D("  forking - &nbsp; <code>" .. s .. "</code>")
-    os.execute(s .. " &")
-end
-
-
 APT.exe = function(c)
-    local exe = {status = 0, result = ''}
-    exe.cmd = c .. ' '
-    exe.log = false
+    local exe = {status = 0, result = '', log = true, cmd = c .. ' '}
 
     function exe:log()
         self.log = true
         return self
     end
     function exe:Nice(c)
-        self.cmd = self.cmd .. 'ionice -c3 nice -n 19 ' .. c .. ' '
+        if nil == c then
+            self.cmd = 'ionice -c3 nice -n 19 ' .. self.cmd
+        else
+            self.cmd = self.cmd .. 'ionice -c3 nice -n 19 ' .. c .. ' '
+        end
+        return self
+    end
+    function exe:also(c)
+        if nil == c then c = '' else c = ' ' .. c end
+        self.cmd = self.cmd .. ';' .. c .. ' '
         return self
     end
     function exe:And(c)
@@ -482,7 +463,12 @@ APT.exe = function(c)
         return self
     end
     function exe:Do()
-        if self.log then D("  executing - &nbsp; <code>" .. self.cmd .. "</code>") end
+    --[[    "The condition expression of a control structure can return any
+            value. Both false and nil are considered false. All values different
+            from nil and false are considered true (in particular, the number 0
+            and the empty string are also true)."
+        says the docs, I beg to differ.]]
+        if true == self.log then D("  executing - &nbsp; <code>" .. self.cmd .. "</code>") end
         --[[ Damn os.execute()
             Lua 5.1 says it returns "a status code, which is system-dependent"
             Lua 5.2 says it returns true/nil, "exit"/"signal", the status code.
@@ -491,6 +477,7 @@ APT.exe = function(c)
         ]]
         local f = io.popen(self.cmd .. ' ; echo "$?"', 'r')
         -- The last line will be the command's returned status, collect everything else in result.
+        self.status = ''        -- Otherwise the result starts with 0.
         for l in f:lines() do
             self.result = self.result .. self.status .. "\n"
             self.status = l
@@ -500,14 +487,13 @@ APT.exe = function(c)
     end
     function exe:fork()
         self.cmd = '{ ' .. self.cmd .. '; } &'
-        if self.log then D("  forking - &nbsp; <code>" .. self.cmd .. "</code>") end
+        if true == self.log then D("  forking - &nbsp; <code>" .. self.cmd .. "</code>") end
         os.execute(self.cmd)
         return self
     end
     return exe
 end
 
-
 APT.checkExes = function (exe)
     local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | grep -v "flock -n apt-panopticon.lock " | wc -l'):read("*l")
     D(count .. " " .. exe .. " commands still running.")
@@ -650,7 +636,7 @@ end
 
 APT.now = 0
 local status
-status, APT.now = APT.execute('TZ="GMT" ls -l --time-style="+%s" results/stamp | cut -d " " -f 6-6')
+APT.now = APT.exe('TZ="GMT" ls -l --time-style="+%s" results/stamp | cut -d " " -f 6-6'):Do().result
 APT.now = tonumber(APT.now)
 local start = 'now-1month'
 local step = '10min'
diff --git a/apt-panopticon-report-email-web.lua b/apt-panopticon-report-email-web.lua
index 67d4d13..3304fb2 100755
--- a/apt-panopticon-report-email-web.lua
+++ b/apt-panopticon-report-email-web.lua
@@ -645,8 +645,8 @@ if nil == web then C("opening mirrors file - " .. e) else
                 "You can get the cgp graphing <a href='https://sledjhamr.org/cgit/apt-panopticon_cgp/about/'>source code here (main repo)</a>" .. 
                 "and <a href='https://git.devuan.org/onefang/apt-panopticon_cgp'>here (Devuan repo)</a>.</p>\n"
                 )
-    local status, whn = APT.execute('TZ="GMT" ls -l1 --time-style="+%s" results/stamp | cut -d " " -f 6-6')
-    web:write(  "<p>This run took " .. (os.time() - tonumber("0" .. whn:sub(2, -2))) .. " seconds.</p>" ..
+    local whn = APT.exe('TZ="GMT" ls -l1 --time-style="+%s" results/stamp | cut -d " " -f 6-6'):Do().result:sub(2, -2)
+    web:write(  "<p>This run took " .. (os.time() - tonumber("0" .. whn)) .. " seconds.</p>" ..
                 "\n</body></html>\n")
     web:close()
 end
diff --git a/apt-panopticon-update-data.lua b/apt-panopticon-update-data.lua
index a26b11b..8ab9254 100755
--- a/apt-panopticon-update-data.lua
+++ b/apt-panopticon-update-data.lua
@@ -106,9 +106,11 @@ local files = io.popen('ls -d1 results_*')
 for l in files:lines() do
     if ('results_old' ~= l) and ('.tar.xz' ~= l:sub(25, -1)) then
         if APT.checkFile(l .. '/stamp') then
-            local status, whn = APT.execute('TZ="GMT" ls -d1 --time-style="+%s" ' .. l .. ' | cut -d " " -f 6-6')
+--          local status, whn = APT.execute('TZ="GMT" ls -d1 --time-style="+%s" ' .. l .. ' | cut -d " " -f 6-6')
+            local whn = APT.exe('TZ="GMT" ls -d1 --time-style="+%s" ' .. l .. ' | cut -d " " -f 6-6'):Do()
             whn = whn:sub(2, -2)
-            local status, new = APT.execute('TZ="GMT" date -d "' .. whn:sub(9, 18) .. ' ' .. whn:sub(20, 21) .. ':' .. whn:sub(23, 24) .. '" "+%s"')
+--          local status, new = APT.execute('TZ="GMT" date -d "' .. whn:sub(9, 18) .. ' ' .. whn:sub(20, 21) .. ':' .. whn:sub(23, 24) .. '" "+%s"')
+            local new = APT.exe('TZ="GMT" date -d "' .. whn:sub(9, 18) .. ' ' .. whn:sub(20, 21) .. ':' .. whn:sub(23, 24) .. '" "+%s"'):Do()
             APT.now = tonumber("0" .. new:sub(2, -2))
             if 0 ~= APT.now then
                 if APT.checkFile(l .. '/mirrors.lua') then
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index c39a9b9..777fdf3 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -272,15 +272,16 @@ checkHEAD = function (host, URL, r, retry, sanity)
         end
         IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"'
     end
-    local cmd = 'ionice -c3 nice -n 19 curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' ..
-                                             IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' ..
-                hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"'
-    local status, result = APT.execute(cmd)
+    local status = APT.exe(
+        'curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' ..
+        IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' ..
+        hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"'
+    ):Nice():log():Do().status
     os.execute('cat results/"HEADERS_' .. fname .. '" >>results/"STATUS_' .. fname .. '" 2>/dev/null; rm -f results/"HEADERS_' .. fname .. '" 2>/dev/null')
-    if "0" ~= status then
-        local msg = curlStatus[0 + status]
+    if 0 ~= status then
+        local msg = curlStatus[status]
         if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end
-        if ("28" == status) or ("7" == status) then
+        if (28 == status) or (7 == status) then
             if sanity then
                 T("  TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "URLSanity", host)
             else
@@ -385,7 +386,8 @@ checkHEAD = function (host, URL, r, retry, sanity)
                 local pth = path:match('^(.*/pool/).*$')
                 if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E('Odd redirect path ' .. path) end
                 I("  Now checking redirected host " .. u)
-                APT.fork("ionice -c3 nice -n 19 " .. downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file)
+--              APT.fork("ionice -c3 nice -n 19 " .. downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file)
+                APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork()
                 D('logging to ' .. APT.logName(pu.host, nil, file)[2])
             end
         end
@@ -457,7 +459,7 @@ checkHost = function (orig, host, path, ip, file)
     else
         if orig == host then
             D("checkHost " .. orig .. "" .. file)
-            APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file)
+            APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork()
             D('logging to ' .. APT.logName(ph.host, nil, file)[2])
         else D("checkHost " .. orig .. " -> " .. host) end
     end
@@ -492,8 +494,8 @@ local postDownload = function(host, r, k)
                     "   results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. ".old" ..
                     "   results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. "; fi")
     if APT.checkFile('results/' .. host .. '/merged/dists/' .. r .. '/' .. k) then
-        if ".gz"  == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 gzip -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k, false) end
-        if ".xz"  == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 xz   -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. " 2>/dev/null", false) end
+        if ".gz"  == k:sub(-3, -1) then APT.exe("gzip -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k):Nice():noErr():Do() end
+        if ".xz"  == k:sub(-3, -1) then APT.exe("xz   -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k):Nice():noErr():Do() end
     end
 end
 
@@ -509,7 +511,7 @@ local downloads = function(host, URL, meta, release, list)
     local files = 'curl-' .. meta .. '-' .. host .. '.files.txt'
     local lock = meta .. "-" .. host .. ".log.txt"
     local log = "curl-" .. meta .. "-" .. host .. ".log.txt"
-    local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock .. " " .. download .. log .. " -K results/" .. files
+    local cm = downloadLock .. lock .. " " .. download .. log .. " -K results/" .. files
     if      APT.testing("IPv4")  and (not APT.testing("IPv6")) then cm = cm .. ' -4' end
     if (not APT.testing("IPv4")) and      APT.testing("IPv6")  then cm = cm .. ' -6' end
     f, e = io.open("results/curl-" .. meta .. '-' .. host .. ".files.txt", "a+")
@@ -539,7 +541,7 @@ local downloads = function(host, URL, meta, release, list)
         end
     end
     f:close()
-    APT.fork(cm)
+    APT.exe(cm):Nice():log():fork()
     D('logging to <a href="' .. log .. '">' .. log .. '</a>, with <a href="' .. files .. '">these files</a>')
 end
 
@@ -656,12 +658,14 @@ local parseDebs = function(host)
                 local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |')
                 if nil ~= p then
                     if APT.checkFile('results/' .. host .. "/merged/" .. p) then
-                        local status, fsz = APT.execute('ls -l results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 5-5', false)
+--                      local status, fsz = APT.execute('ls -l results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 5-5', false)
+                        local fsz = APT.exe('ls -l results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 5-5'):Do().result
                         if APT.testing("Integrity") then
                             if sz ~= fsz:sub(2, -2) then        -- The sub bit is to slice off the EOLs at each end.
                                 E('Package size mismatch - results/' .. host .. "/merged/" .. p .. ' should be ' .. sz .. ', but is ' .. fsz:sub(2, -2) .. '.', 'http', 'Integrity', host)
                             else
-                                local status, fsha = APT.execute('sha256sum results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 1')
+--                              local status, fsha = APT.execute('sha256sum results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 1')
+                                local fsha = APT.exe('sha256sum results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 1'):Do().result
                                 if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. host .. "/merged/" .. p, 'http', 'Integrity', host) end
 -- TODO - maybe check the PGP key, though packages are mostly not signed.
                             end
@@ -779,7 +783,8 @@ end
 local parseRelease = function(host)
     local list = {inf = 'Release', parser = parsePackages, out = 'Packages', files = {}, nextf = 'debs'}
     local updated = false
-    local status, now = APT.execute('TZ="GMT" date "+%s"', false)
+--    local now = APT.execute('TZ="GMT" date "+%s"', false)
+    local now = os.date('!%s')
     now = tonumber(now:sub(2, -2))
     for i, n in pairs(releases) do
         for l, o in pairs(releaseFiles) do
@@ -787,9 +792,9 @@ local parseRelease = function(host)
                 postDownload(host, n, o)
                 if (".gpg" == o:sub(-4, -1)) and (APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/' .. o)) then
                     if APT.testing("Integrity") then
-                        local status, out = APT.execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. 
-                                                        " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5) .. " 2>/dev/null")
-                        if "0" ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end
+                        local status = APT.exe( "gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. 
+                                                " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status
+                        if 0 ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end
 -- TODO - should check the PGP sig of InRelease as well.
                     end
                     os.execute('rm results/' .. host .. '/merged/dists/' .. n .. '/' .. o)
@@ -812,9 +817,11 @@ local parseRelease = function(host)
                 else
 -- TODO - compare to the pkgmaster copy.
                     if APT.testing('Updated') then
-                        local status, pkt = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/pkgmaster.devuan.org/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false)
+--                      local status, pkt = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/pkgmaster.devuan.org/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false)
+                        local pkt = APT.exe([[TZ="GMT" date -d "$(grep '^Date:' results/pkgmaster.devuan.org/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']]):Do().result
                         pkt = tonumber(pkt:sub(2, -2))
-                        local status, new = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/]]  ..  host  ..  [[/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false)
+--                      local status, new = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/]]  ..  host  ..  [[/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false)
+                        local new = APT.exe([[TZ="GMT" date -d "$(grep '^Date:' results/]]  ..  host  ..  [[/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']]):Do().result
                         new = tonumber(new:sub(2, -2))
                         local upd  = pkt +  APT.mirrors[host].Updated
                         local updd = pkt + (APT.mirrors[host].Updated * 1.5)    -- Give the mirror time to actually do the update.
@@ -883,7 +890,7 @@ local doDownloads = function(host, path, list)
 
 Mon Dec 23 01:02:54 2019 DEBUG : forking
 
-ionice -c3 nice -n 19 flock -n results/curl-debs-pkgmaster.devuan.org.log curl --connect-timeout 5 --create-dirs -f -L --fail-early --max-time 300 --retry 3 -R -v -z 'results/stamp.old'  --stderr results/curl-debs-pkgmaster.devuan.org.log -K results/curl-debs-pkgmaster.devuan.org.files
+flock -n results/curl-debs-pkgmaster.devuan.org.log curl --connect-timeout 5 --create-dirs -f -L --fail-early --max-time 300 --retry 3 -R -v -z 'results/stamp.old'  --stderr results/curl-debs-pkgmaster.devuan.org.log -K results/curl-debs-pkgmaster.devuan.org.files
 
 
 Mon Dec 23 01:02:54 2019 DEBUG : 0 flock -n results/curl-debs-pkgmaster.devuan.org.log commands still running.
@@ -998,17 +1005,17 @@ if 0 < #arg then
                 if "table" == type(v) then
                     for k1, v1 in pairs(v) do
                         if v1 == "A" then
-                            if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file) end
+                            if APT.testing("IPv4") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file):Nice():log():fork() end
                         elseif v1 == "AAAA" then
-                            if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file) end
+                            if APT.testing("IPv6") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file):Nice():log():fork() end
                         end
                         D('logging to ' .. APT.logName(pu.host, k1, file)[2])
                     end
                 else
                     if v == "A" then
-                        if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file) end
+                        if APT.testing("IPv4") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file):Nice():log():fork() end
                     elseif v == "AAAA" then
-                        if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file) end
+                        if APT.testing("IPv6") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file):Nice():log():fork() end
                     end
                     D('logging to ' .. APT.logName(pu.host, k, file)[2])
                 end
@@ -1212,7 +1219,7 @@ os.execute('sleep 1')	-- Wait for things to start up before checking for them.
     for n, r in pairs(APT.options.reports.value) do
         if APT.checkFile("apt-panopticon-report-" .. r .. ".lua") then
             I("Creating " .. r .. " report.")
-            APT.execute("./apt-panopticon-report-" .. r .. ".lua " .. sendArgs)
+            APT.exe("./apt-panopticon-report-" .. r .. ".lua " .. sendArgs):log():Do()
         end
     end