aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
-rw-r--r--README.md1
-rw-r--r--apt-panopticommon.lua35
-rwxr-xr-xapt-panopticon.lua14
-rwxr-xr-xlaggers3
-rwxr-xr-xupdate_apt-panopticon12
5 files changed, 49 insertions, 16 deletions
diff --git a/README.md b/README.md
index 2edc3ca..7e5d6d2 100644
--- a/README.md
+++ b/README.md
@@ -55,6 +55,7 @@ installed -
55* lua-rrd 55* lua-rrd
56* LuaSocket, on Debian based systems it'll be in the lua-socket package. 56* LuaSocket, on Debian based systems it'll be in the lua-socket package.
57* md5sum and sha256, on Debian based systems they'll be in the coreutils package. 57* md5sum and sha256, on Debian based systems they'll be in the coreutils package.
58* timeout, on Debian based systems it'll be in the coreutils package.
58* rrdtool 59* rrdtool
59* xz, on Debian based systems it'll be in the xz-utils package. 60* xz, on Debian based systems it'll be in the xz-utils package.
60 61
diff --git a/apt-panopticommon.lua b/apt-panopticommon.lua
index da1fc3d..a2def5a 100644
--- a/apt-panopticommon.lua
+++ b/apt-panopticommon.lua
@@ -134,7 +134,7 @@ APT.parseArgs = function(args)
134 local arg = {} 134 local arg = {}
135 local sendArgs = "" 135 local sendArgs = ""
136 -- A special test to disable IPv6 tests if IPv6 isn't available. 136 -- A special test to disable IPv6 tests if IPv6 isn't available.
137 if 1 == APT.exe('ip -6 addr | grep inet6 | grep " global"'):Do().status then 137 if 1 == APT.exe('ip -6 addr | grep inet6 | grep " global"'):timeout():Do().status then
138 table.insert(args, '--tests=-IPv6') 138 table.insert(args, '--tests=-IPv6')
139 end 139 end
140 if 0 ~= #(args) then 140 if 0 ~= #(args) then
@@ -529,7 +529,7 @@ APT.tested = function(prot, test, host)
529end 529end
530 530
531APT.exe = function(c) 531APT.exe = function(c)
532 local exe = {status = 0, result = '', log = true, cmd = c .. ' '} 532 local exe = {status = 0, result = '', log = true, cmd = c .. ' ', command = c}
533 533
534 function exe:log() 534 function exe:log()
535 self.log = true 535 self.log = true
@@ -543,6 +543,16 @@ APT.exe = function(c)
543 end 543 end
544 return self 544 return self
545 end 545 end
546 function exe:timeout(c)
547 -- timeout returns a status of - command status if --preserve-status; "128+9" (actually 137) if --kill-after ends up being done; 124 if it had to TERM; command status if all went well.
548 -- --kill-after means "send KILL after TERM fails.
549 if nil == c then
550 self.cmd = 'timeout --kill-after=10.0 --foreground -v 42.0s ' .. self.cmd
551 else
552 self.cmd = 'timeout --kill-after=10.0 --foreground -v ' .. c .. ' ' .. self.cmd
553 end
554 return self
555 end
546 function exe:also(c) 556 function exe:also(c)
547 if nil == c then c = '' else c = ' ' .. c end 557 if nil == c then c = '' else c = ' ' .. c end
548 self.cmd = self.cmd .. ';' .. c .. ' ' 558 self.cmd = self.cmd .. ';' .. c .. ' '
@@ -579,17 +589,28 @@ APT.exe = function(c)
579 I'm getting 7168 or 0. No idea what the fuck that is. 589 I'm getting 7168 or 0. No idea what the fuck that is.
580 local ok, rslt, status = os.execute(s) 590 local ok, rslt, status = os.execute(s)
581 ]] 591 ]]
582 local f = APT.readCmd(self.cmd .. ' ; echo "$?"', 'r') 592 local f = APT.readCmd(self.cmd, 'r')
583 -- The last line will be the command's returned status, collect everything else in result. 593 -- The last line will be the command's returned status, collect everything else in result.
584 self.status = '' -- Otherwise the result starts with 0. 594 self.status = '' -- Otherwise the result starts with 0.
595 self.result = '\n'
596 for i,l in ipairs(f) do
597 self.result = self.result .. l .. "\n"
598 end
599 f = APT.readCmd('echo "$?"', 'r')
585 for i,l in ipairs(f) do 600 for i,l in ipairs(f) do
586 self.result = self.result .. self.status .. "\n" 601 self.status = tonumber(l)
587 self.status = l 602 if (137 == self.status) or (124 == self.status) then
603 print("timeout killed " .. self.status .. ' ' .. self.command)
604 E("timeout killed " .. self.status .. ' ' .. self.command)
605 elseif (0 ~= self.status) then
606 print("status |" .. self.status .. '| ' .. self.command)
607 E("status |" .. self.status .. '| ' .. self.command)
608 end
588 end 609 end
589 self.status = tonumber(self.status)
590 return self 610 return self
591 end 611 end
592 function exe:fork() 612 function exe:fork(host)
613 if nil ~= host then self.cmd = self.cmd .. '; r=$?; if [ $r -ge 124 ]; then echo "$r ' .. host .. ' failed forked command ' .. string.gsub(self.cmd, '"', "'") .. '"; fi' end
593 self.cmd = '{ ' .. self.cmd .. '; } &' 614 self.cmd = '{ ' .. self.cmd .. '; } &'
594 if true == self.log then D(" forking - &nbsp; <code>" .. self.cmd .. "</code>") end 615 if true == self.log then D(" forking - &nbsp; <code>" .. self.cmd .. "</code>") end
595 os.execute(self.cmd) 616 os.execute(self.cmd)
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index 8fd39e5..93243a4 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -330,7 +330,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
330 'curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' .. APT.IPv46 .. ' ' .. 330 'curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' .. APT.IPv46 .. ' ' ..
331 IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' .. 331 IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' ..
332 hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"' 332 hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"'
333 ):Nice():log():Do().status 333 ):timeout(APT.options.maxtime.value * 2.0):Nice():log():Do().status
334 if 0 < r then 334 if 0 < r then
335 APT.tested(PU.scheme, 'Redirects', host) 335 APT.tested(PU.scheme, 'Redirects', host)
336 else 336 else
@@ -356,7 +356,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
356 if 0 ~= status then 356 if 0 ~= status then
357 local msg = curlStatus[status] 357 local msg = curlStatus[status]
358 if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end 358 if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end
359 if (28 == status) or (7 == status) then 359 if (128+9 == status) or (124 == status) or (28 == status) or (7 == status) then
360 T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host) 360 T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host)
361 timeouts = timeouts + 1 361 timeouts = timeouts + 1
362 else 362 else
@@ -447,7 +447,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
447 local pth = path:match('^(.*/pool/).*$') 447 local pth = path:match('^(.*/pool/).*$')
448 if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E(spcd .. spcd .. 'Odd redirect path ' .. path) end 448 if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E(spcd .. spcd .. 'Odd redirect path ' .. path) end
449 I(spcd .. spcd .. "Now checking redirected host " .. u .. ' &nbsp; for &nbsp; ' .. APT.lnk(URL) .. arw .. APT.lnk(location), host) 449 I(spcd .. spcd .. "Now checking redirected host " .. u .. ' &nbsp; for &nbsp; ' .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
450 APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork() 450 APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host)
451 D(spcd .. 'logging to ' .. APT.logName(pu.host, nil, file)[2]) 451 D(spcd .. 'logging to ' .. APT.logName(pu.host, nil, file)[2])
452 APT.tested(PU.scheme, 'Redirects', host) 452 APT.tested(PU.scheme, 'Redirects', host)
453 end 453 end
@@ -534,7 +534,7 @@ checkHost = function (orig, host, path, ip, file)
534 else 534 else
535 if orig == host then 535 if orig == host then
536 I("Testing mirror " .. orig .. "" .. file) 536 I("Testing mirror " .. orig .. "" .. file)
537 APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork() 537 APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(orig)
538 D('logging to ' .. APT.logName(ph.host, nil, file)[2]) 538 D('logging to ' .. APT.logName(ph.host, nil, file)[2])
539 else D("checkHost " .. orig .. arw .. host) end 539 else D("checkHost " .. orig .. arw .. host) end
540 end 540 end
@@ -618,7 +618,7 @@ local downloads = function(host, URL, meta, release, list)
618 end 618 end
619 end 619 end
620 f:close() 620 f:close()
621 APT.exe(cm):Nice():log():fork() 621 APT.exe(cm):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(host)
622 D('logging to <a href="' .. log .. '">' .. log .. '</a>, with <a href="' .. files .. '">these files</a>') 622 D('logging to <a href="' .. log .. '">' .. log .. '</a>, with <a href="' .. files .. '">these files</a>')
623end 623end
624 624
@@ -1129,9 +1129,9 @@ if 0 < #arg then
1129 APT.allpairs(ips, 1129 APT.allpairs(ips,
1130 function(k, v) 1130 function(k, v)
1131 if v == "A" then 1131 if v == "A" then
1132 if APT.testing("IPv4") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -4 ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):Nice():log():fork() end 1132 if APT.testing("IPv4") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -4 ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host) end
1133 elseif v == "AAAA" then 1133 elseif v == "AAAA" then
1134 if APT.testing("IPv6") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -6 ' .. APT.IPv46 .. ' ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):Nice():log():fork() end 1134 if APT.testing("IPv6") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -6 ' .. APT.IPv46 .. ' ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host) end
1135 end 1135 end
1136 D('logging to ' .. APT.logName(pu.host, k, file)[2]) 1136 D('logging to ' .. APT.logName(pu.host, k, file)[2])
1137 end 1137 end
diff --git a/laggers b/laggers
new file mode 100755
index 0000000..c5f30ee
--- /dev/null
+++ b/laggers
@@ -0,0 +1,3 @@
1#!/bin/bash
2echo "apt-panopticon processes still running -"
3ps ax -o pid,args --sort args | grep -E 'apt-panopticon\.lua | curl | dig ' | grep -v -E 'flock -n |grep -E |sh -c |timeout -k '
diff --git a/update_apt-panopticon b/update_apt-panopticon
index 8edb43d..abbc154 100755
--- a/update_apt-panopticon
+++ b/update_apt-panopticon
@@ -14,18 +14,26 @@ fi
14# Check if the lock file still exists. 14# Check if the lock file still exists.
15if [ -f apt-panopticon.lock ] ; then 15if [ -f apt-panopticon.lock ] ; then
16 # Check if it's still running. 16 # Check if it's still running.
17 ps ax -eo pid,args | grep "luajit ./apt-panopticon.lua" | grep -v "grep luajit ./apt-panopticon.lua" | while read line ; do touch apt-panopticon.running ; exit ; done 17 ps ax -eo pid,args | grep "apt-panopticon.lua" | grep -v "grep apt-panopticon.lua" | while read line ; do touch apt-panopticon.running ; exit ; done
18 if [ -f apt-panopticon.running ] ; then 18 if [ -f apt-panopticon.running ] ; then
19 echo "Previous apt-panopticon still running, exiting." 19 echo "Previous apt-panopticon still running, exiting."
20 echo "Previous apt-panopticon still running, exiting."
21 ./laggers
20 rm apt-panopticon.running 22 rm apt-panopticon.running
21 exit 1 23 exit 1
22 fi 24 fi
23 echo "Crashed apt-panopticon detected, removing stale lock file." 25 echo "Crashed apt-panopticon detected, removing stale lock file."
26 echo "Crashed apt-panopticon detected, removing stale lock file."
27 ./laggers
24 rm apt-panopticon.lock 28 rm apt-panopticon.lock
25fi 29fi
26 30
27rm ../results; ln -s apt-panopticon/results_old ../results 31rm ../results; ln -s apt-panopticon/results_old ../results
28flock -n apt-panopticon.lock ./apt-panopticon.lua && rm apt-panopticon.lock 32flock -n apt-panopticon.lock ionice -c3 nice -n 19 timeout --kill-after=20.0 --foreground -v 8.5m ./apt-panopticon.lua && rm apt-panopticon.lock
33if [ -f apt-panopticon.lock ] ; then
34 echo "apt-panopticon timed out."
35 ./laggers
36fi
29rm ../results; ln -s apt-panopticon/results ../results 37rm ../results; ln -s apt-panopticon/results ../results
30 38
31chown -R www-data:www-data * 39chown -R www-data:www-data *