Add yet another timeout watchdog.

5 files changed, 49 insertions, 16 deletions

diff --git a/README.md b/README.md
index 2edc3ca..7e5d6d2 100644
--- a/README.md
+++ b/README.md
@@ -55,6 +55,7 @@ installed -
 * lua-rrd
 * LuaSocket, on Debian based systems it'll be in the lua-socket package.
 * md5sum and sha256, on Debian based systems they'll be in the coreutils package.
+* timeout, on Debian based systems it'll be in the coreutils package.
 * rrdtool
 * xz, on Debian based systems it'll be in the xz-utils package.
 
diff --git a/apt-panopticommon.lua b/apt-panopticommon.lua
index da1fc3d..a2def5a 100644
--- a/apt-panopticommon.lua
+++ b/apt-panopticommon.lua
@@ -134,7 +134,7 @@ APT.parseArgs = function(args)
     local arg = {}
     local sendArgs = ""
     -- A special test to disable IPv6 tests if IPv6 isn't available.
-    if 1 == APT.exe('ip -6 addr | grep inet6 | grep " global"'):Do().status then
+    if 1 == APT.exe('ip -6 addr | grep inet6 | grep " global"'):timeout():Do().status then
         table.insert(args, '--tests=-IPv6')
     end
     if 0 ~= #(args) then
@@ -529,7 +529,7 @@ APT.tested = function(prot, test, host)
 end
 
 APT.exe = function(c)
-    local exe = {status = 0, result = '', log = true, cmd = c .. ' '}
+    local exe = {status = 0, result = '', log = true, cmd = c .. ' ', command = c}
 
     function exe:log()
         self.log = true
@@ -543,6 +543,16 @@ APT.exe = function(c)
         end
         return self
     end
+    function exe:timeout(c)
+        -- timeout returns a status of - command status if --preserve-status; "128+9" (actually 137) if --kill-after ends up being done; 124 if it had to TERM; command status if all went well.
+        -- --kill-after means "send KILL after TERM fails.
+        if nil == c then
+            self.cmd = 'timeout --kill-after=10.0 --foreground -v 42.0s ' .. self.cmd
+        else
+            self.cmd = 'timeout --kill-after=10.0 --foreground -v ' .. c .. ' ' .. self.cmd
+        end
+        return self
+    end
     function exe:also(c)
         if nil == c then c = '' else c = ' ' .. c end
         self.cmd = self.cmd .. ';' .. c .. ' '
@@ -579,17 +589,28 @@ APT.exe = function(c)
             I'm getting 7168 or 0.  No idea what the fuck that is.
         local ok, rslt, status = os.execute(s)
         ]]
-        local f = APT.readCmd(self.cmd .. ' ; echo "$?"', 'r')
+        local f = APT.readCmd(self.cmd, 'r')
         -- The last line will be the command's returned status, collect everything else in result.
         self.status = ''        -- Otherwise the result starts with 0.
+        self.result = '\n'
+        for i,l in ipairs(f) do
+            self.result = self.result .. l .. "\n"
+        end
+        f = APT.readCmd('echo "$?"', 'r')
         for i,l in ipairs(f) do
-            self.result = self.result .. self.status .. "\n"
-            self.status = l
+            self.status = tonumber(l)
+            if (137 == self.status) or (124 == self.status) then
+                print("timeout killed " .. self.status .. ' ' .. self.command)
+                E("timeout killed " .. self.status .. ' ' .. self.command)
+            elseif (0 ~= self.status) then
+                print("status |" .. self.status .. '| ' .. self.command)
+                E("status |" .. self.status .. '| ' .. self.command)
+            end
         end
-        self.status = tonumber(self.status)
         return self
     end
-    function exe:fork()
+    function exe:fork(host)
+        if nil ~= host then self.cmd = self.cmd .. ';  r=$?; if [ $r -ge 124 ]; then echo "$r ' .. host .. ' failed forked command ' .. string.gsub(self.cmd, '"', "'") .. '"; fi' end
         self.cmd = '{ ' .. self.cmd .. '; } &'
         if true == self.log then D("  forking - &nbsp; <code>" .. self.cmd .. "</code>") end
         os.execute(self.cmd)
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index 8fd39e5..93243a4 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -330,7 +330,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
         'curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' .. APT.IPv46 .. ' ' ..
         IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' ..
         hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"'
-    ):Nice():log():Do().status
+    ):timeout(APT.options.maxtime.value * 2.0):Nice():log():Do().status
     if 0 < r then
         APT.tested(PU.scheme, 'Redirects', host)
     else
@@ -356,7 +356,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
     if 0 ~= status then
         local msg = curlStatus[status]
         if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end
-        if (28 == status) or (7 == status) then
+        if (128+9 == status) or (124 == status) or (28 == status) or (7 == status) then
             T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host)
             timeouts = timeouts + 1
         else
@@ -447,7 +447,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
                     local pth = path:match('^(.*/pool/).*$')
                     if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E(spcd .. spcd .. 'Odd redirect path ' .. path) end
                     I(spcd .. spcd .. "Now checking redirected host " .. u .. ' &nbsp; for &nbsp; ' .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
-                    APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork()
+                    APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host)
                     D(spcd .. 'logging to ' .. APT.logName(pu.host, nil, file)[2])
                     APT.tested(PU.scheme, 'Redirects', host)
                 end
@@ -534,7 +534,7 @@ checkHost = function (orig, host, path, ip, file)
     else
         if orig == host then
             I("Testing mirror " .. orig .. "" .. file)
-            APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork()
+            APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(orig)
             D('logging to ' .. APT.logName(ph.host, nil, file)[2])
         else D("checkHost " .. orig .. arw .. host) end
     end
@@ -618,7 +618,7 @@ local downloads = function(host, URL, meta, release, list)
         end
     end
     f:close()
-    APT.exe(cm):Nice():log():fork()
+    APT.exe(cm):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(host)
     D('logging to <a href="' .. log .. '">' .. log .. '</a>, with <a href="' .. files .. '">these files</a>')
 end
 
@@ -1129,9 +1129,9 @@ if 0 < #arg then
                 APT.allpairs(ips,
                     function(k, v)
                         if v == "A" then
-                            if APT.testing("IPv4") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -4 ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):Nice():log():fork() end
+                            if APT.testing("IPv4") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -4 ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host) end
                         elseif v == "AAAA" then
-                            if APT.testing("IPv6") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -6 ' .. APT.IPv46 .. ' ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):Nice():log():fork() end
+                            if APT.testing("IPv6") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -6 ' .. APT.IPv46 .. ' ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host) end
                         end
                         D('logging to ' .. APT.logName(pu.host, k, file)[2])
                     end
diff --git a/laggers b/laggers
new file mode 100755
index 0000000..c5f30ee
--- /dev/null
+++ b/laggers
@@ -0,0 +1,3 @@
+#!/bin/bash
+echo "apt-panopticon processes still running -"
+ps ax -o pid,args --sort args | grep -E 'apt-panopticon\.lua | curl | dig ' | grep -v -E 'flock -n |grep -E |sh -c |timeout -k '
diff --git a/update_apt-panopticon b/update_apt-panopticon
index 8edb43d..abbc154 100755
--- a/update_apt-panopticon
+++ b/update_apt-panopticon
@@ -14,18 +14,26 @@ fi
 # Check if the lock file still exists.
 if [ -f apt-panopticon.lock ] ; then
     # Check if it's still running.
-    ps ax -eo pid,args | grep "luajit ./apt-panopticon.lua" | grep -v "grep luajit ./apt-panopticon.lua" | while read line ; do touch apt-panopticon.running ; exit ; done
+    ps ax -eo pid,args | grep "apt-panopticon.lua" | grep -v "grep apt-panopticon.lua" | while read line ; do touch apt-panopticon.running ; exit ; done
     if [ -f apt-panopticon.running ] ; then
         echo "Previous apt-panopticon still running, exiting."
+        echo "Previous apt-panopticon still running, exiting."
+        ./laggers
         rm apt-panopticon.running
         exit 1
     fi
     echo "Crashed apt-panopticon detected, removing stale lock file."
+    echo "Crashed apt-panopticon detected, removing stale lock file."
+    ./laggers
     rm apt-panopticon.lock
 fi
 
 rm ../results; ln -s apt-panopticon/results_old ../results
-flock -n apt-panopticon.lock ./apt-panopticon.lua && rm apt-panopticon.lock
+flock -n apt-panopticon.lock ionice -c3 nice -n 19 timeout --kill-after=20.0 --foreground -v 8.5m ./apt-panopticon.lua && rm apt-panopticon.lock
+if [ -f apt-panopticon.lock ] ; then
+    echo "apt-panopticon timed out."
+    ./laggers
+fi
 rm ../results; ln -s apt-panopticon/results ../results
 
 chown -R www-data:www-data *