8 files changed, 272 insertions, 110 deletions

diff --git a/README.md b/README.md
index 2edc3ca..7e5d6d2 100644
--- a/README.md
+++ b/README.md
@@ -55,6 +55,7 @@ installed -
 * lua-rrd
 * LuaSocket, on Debian based systems it'll be in the lua-socket package.
 * md5sum and sha256, on Debian based systems they'll be in the coreutils package.
+* timeout, on Debian based systems it'll be in the coreutils package.
 * rrdtool
 * xz, on Debian based systems it'll be in the xz-utils package.
 
diff --git a/Report-web_3.html b/Report-web_3.html
index 632e6f4..a1ca70d 100644
--- a/Report-web_3.html
+++ b/Report-web_3.html
@@ -7,7 +7,13 @@
     <p><font color='aqua'><b>TIMEOUT</b></font> or <font color='blue'><b>TIMEOUT</b></font> means the mirror had too many timeouts, and tests where aborted, so there is no result for this test.</p>
     <p>NOTE: timeouts may be due to a problem on the testing computer, it might be busy with other things, or be having it's own network problems.</p>
     <p>NOTE: the speed of the server this checking script is running on might be wildly exaggerated, it doesn't have to go through the Internet to download from itself.</p>
-    <p>The DNS round robin (DNS-RR) column shows the IPs for that mirror, or <font color='grey'><b>no</b></font> if it isn't part of the DNS-RR. &nbsp; The IPs link to the testing log for that IP accessed via the DNS-RR. &nbsp; deb.devuan.org is the DNS-RR itself, so it doesn't get tested directly.</p>
+    <p>The DNS round robin (DNS-RR) column shows the IPs for that mirror, or <font color='grey'><b>no</b></font> if it isn't part of the DNS-RR.</p>
+    <p> &nbsp; The IPs link to the testing log for that IP accessed via the DNS-RR.</p>
+    <p> &nbsp; If a mirror <a href='DNS-RR_good.txt'>should be</a> it is marked with '<font color='green'><b>&#x2705;</b></font>', 
+                if it <a href='DNS-RR_bad.txt'>should not be</a> it is marked with '<font color='red'>&#x274c;</font>',
+                if it might be but still pending full testing, it is marked with '<font color='red'>&#x2753;</font>'.</p>
+    <p> &nbsp; deb.devuan.org is the DNS-RR itself, so it doesn't get tested directly.
+    </p>
     <p>The time in the Updated column is how often the mirror updates itself.</p>
     <p>Mirrors with a <font style='background-color:dimgrey'>grey background</font> are not active (though may be usable as part of the DNS-RR).</p>
     <p><font color='grey'><b>skip</b></font> means that the test hasn't been written yet.</p>
diff --git a/Report-web_TABLE.html b/Report-web_TABLE.html
index 6ee14f4..b267094 100644
--- a/Report-web_TABLE.html
+++ b/Report-web_TABLE.html
@@ -1,4 +1,4 @@
-    <table>
+    <table id=results>
     <tr>
       <th></th>
       <th class="TTitem">
diff --git a/apt-panopticommon.lua b/apt-panopticommon.lua
index 0862b77..e31c22f 100644
--- a/apt-panopticommon.lua
+++ b/apt-panopticommon.lua
@@ -3,14 +3,18 @@ local APT = {}
 -- https://oss.oetiker.ch/rrdtool/prog/rrdlua.en.html
 APT.rrd = require 'rrd'
 
+APT.version = '0.3 alpha'
+
 APT.protocols = {"ftp", "http", "https", "rsync"}
 APT.tests = {'raw', 'Integrity', 'Protocol', 'Redirects', 'Updated', 'URLSanity', 'Speed'}
-APT.releases = {"ascii", "beowulf", "chimaera", "ceres"}
+--APT.releases = {"jessie", "ascii", "beowulf", "chimaera", "daedalus", "ceres"}
+APT.releases = {"beowulf","chimaera", "daedalus", "excalibur", "ceres"}
 APT.subRels = {'backports', 'proposed-updates', 'security', 'updates'}
 APT.notExist =
 {
-    'chimaera-backports',
-    'chimaera-security',
+    'excalibur-backports',
+    'excalibur-security',
+    'excalibur-updates',
     'ceres-backports',          -- These will never exist, it's our code name for the testing suite.
     'ceres-proposed-updates',
     'ceres-updates',
@@ -50,6 +54,12 @@ APT.options =
             help = "The round robin DNS name.",
             value = "deb.devuan.org",
         },
+    roundRobinCname =
+        {
+            typ = "string",
+            help = "The round robin DNS name.",
+            value = "deb.rr.devuan.org",
+        },
     tests =
         {
             typ = "table",
@@ -125,7 +135,7 @@ APT.parseArgs = function(args)
     local arg = {}
     local sendArgs = ""
     -- A special test to disable IPv6 tests if IPv6 isn't available.
-    if 1 == APT.exe('ip -6 addr | grep inet6 | grep " global"'):Do().status then
+    if 1 == APT.exe('ip -6 addr | grep inet6 | grep " global"'):timeout():Do().status then
         table.insert(args, '--tests=-IPv6')
     end
     if 0 ~= #(args) then
@@ -139,7 +149,7 @@ APT.parseArgs = function(args)
                 end
                 os.exit()
             elseif "--version" == a then
-                print("apt-panopticon version 0.2 alpha")
+                print("apt-panopticon version " .. APT.version)
                 os.exit()
             elseif "-v" == a then
                 APT.verbosity = APT.verbosity + 1
@@ -398,7 +408,7 @@ APT.logOpen = function(host, a2, a3)
     local name = APT.logName(host, a2, a3)[1]
     if APT.checkFile(name) then return false end
     APT.logFile, e = io.open(name, "a+")
-    if nil == APT.logFile then C('opening log file (' .. name .. ') - ' .. e); return false end
+    if nil == APT.logFile then print('CRITICAL - opening log file (' .. name .. ') - ' .. e); return false end
     if nil ~= APT.logFile then
         APT.logFile:write("<html><head>\n")
         APT.logFile:write("</head><body bgcolor='black' text='white' alink='red' link='aqua' vlink='fuchsia'>\n")
@@ -415,6 +425,7 @@ APT.logPost = function()
     if nil ~= APT.logFile then
         APT.logFile:write("</body></html> \n")
         APT.logFile:close()
+        APT.logFile = nil
     end
 end
 
@@ -455,7 +466,7 @@ local log = function(v, t, s, prot, test, host)
     if nil ~= APT.logFile then
         if APT.html then
             local colour = "white"
-            if -1 == v then colour = "fuchsia" end      -- CRITICAL
+            if -1 == v then colour = "fuchsia"; print(t .. " " .. s) end        -- CRITICAL
             if  0 == v then colour = "red    " end      -- ERROR
             if  1 == v then colour = "yellow " end      -- WARNING
             if  2 == v then colour = "blue   " end      -- TIMEOUT
@@ -482,6 +493,17 @@ local E = APT.E
 local C = APT.C
 
 
+APT.readCmd = function(cmd)
+    local result = {}
+    local output = io.popen(cmd)
+    if nil ~= output then
+        for l in output:lines() do
+            table.insert(result, l)
+        end
+    end
+    return result
+end
+
 APT.debians = {}
 APT.mirrors = {}
 
@@ -508,7 +530,7 @@ APT.tested = function(prot, test, host)
 end
 
 APT.exe = function(c)
-    local exe = {status = 0, result = '', log = true, cmd = c .. ' '}
+    local exe = {status = 0, result = '', log = true, cmd = c .. ' ', command = c}
 
     function exe:log()
         self.log = true
@@ -522,6 +544,16 @@ APT.exe = function(c)
         end
         return self
     end
+    function exe:timeout(c)
+        -- timeout returns a status of - command status if --preserve-status; "128+9" (actually 137) if --kill-after ends up being done; 124 if it had to TERM; command status if all went well.
+        -- --kill-after means "send KILL after TERM fails.
+        if nil == c then
+            self.cmd = 'timeout --kill-after=10.0 --foreground 42.0s ' .. self.cmd
+        else
+            self.cmd = 'timeout --kill-after=10.0 --foreground ' .. c .. ' ' .. self.cmd
+        end
+        return self
+    end
     function exe:also(c)
         if nil == c then c = '' else c = ' ' .. c end
         self.cmd = self.cmd .. ';' .. c .. ' '
@@ -558,17 +590,28 @@ APT.exe = function(c)
             I'm getting 7168 or 0.  No idea what the fuck that is.
         local ok, rslt, status = os.execute(s)
         ]]
-        local f = io.popen(self.cmd .. ' ; echo "$?"', 'r')
+        local f = APT.readCmd(self.cmd, 'r')
         -- The last line will be the command's returned status, collect everything else in result.
         self.status = ''        -- Otherwise the result starts with 0.
-        for l in f:lines() do
-            self.result = self.result .. self.status .. "\n"
-            self.status = l
+        self.result = '\n'
+        for i,l in ipairs(f) do
+            self.result = self.result .. l .. "\n"
+        end
+        f = APT.readCmd('echo "$?"', 'r')
+        for i,l in ipairs(f) do
+            self.status = tonumber(l)
+            if (137 == self.status) or (124 == self.status) then
+                print("timeout killed " .. self.status .. ' ' .. self.command)
+                E("timeout killed " .. self.status .. ' ' .. self.command)
+            elseif (0 ~= self.status) then
+                print("status |" .. self.status .. '| ' .. self.command)
+                E("status |" .. self.status .. '| ' .. self.command)
+            end
         end
-        self.status = tonumber(self.status)
         return self
     end
-    function exe:fork()
+    function exe:fork(host)
+        if nil ~= host then self.cmd = self.cmd .. ';  r=$?; if [ $r -ge 124 ]; then echo "$r ' .. host .. ' failed forked command ' .. string.gsub(self.cmd, '"', "'") .. '"; fi' end
         self.cmd = '{ ' .. self.cmd .. '; } &'
         if true == self.log then D("  forking - &nbsp; <code>" .. self.cmd .. "</code>") end
         os.execute(self.cmd)
@@ -578,7 +621,9 @@ APT.exe = function(c)
 end
 
 APT.checkExes = function (exe)
-    local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | grep -v "flock -n apt-panopticon.lock " | wc -l'):read("*l")
+    local count = 0
+    local ps = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | grep -v "flock -n apt-panopticon.lock " | wc -l')
+    if nil ~= ps then count = ps:read("*l") end
     D(count .. " " .. exe .. " commands still running.")
     return tonumber(count)
 end
diff --git a/apt-panopticon-report-email-web.lua b/apt-panopticon-report-email-web.lua
index b21ba8a..2a13975 100755
--- a/apt-panopticon-report-email-web.lua
+++ b/apt-panopticon-report-email-web.lua
@@ -18,11 +18,13 @@ APT.debians = loadfile("results/debians.lua")()
 local revDNS = function(hosts, dom, IP)
     if APT.options.roundRobin.value ~= dom then
         if nil ~= hosts[APT.options.roundRobin.value] then
-            if nil ~= hosts[APT.options.roundRobin.value].IPs["deb.roundr.devuan.org"][IP] then
-                if APT.html then
-                    return "<font color='purple'><b>DNS-RR</b></font>"
-                else
-                    return "DNS-RR"
+            if nil ~= hosts[APT.options.roundRobin.value].IPs[APT.options.roundRobinCname.value] then
+                if nil ~= hosts[APT.options.roundRobin.value].IPs[APT.options.roundRobinCname.value][IP] then
+                    if APT.html then
+                        return "<font color='purple'><b>DNS-RR</b></font>"
+                    else
+                        return "DNS-RR"
+                    end
                 end
             end
         end
@@ -87,6 +89,7 @@ local status = function(hosts, host, results, typ)
 
     if to then
         result = "TIMEOUT"
+        hosts[host].passed = false;
         if not s then result = result .. "*" end
         if APT.html then
             if s then
@@ -102,6 +105,7 @@ local status = function(hosts, host, results, typ)
         end
     elseif 0 < e then
         result = "FAILED"
+        hosts[host].passed = false;
         if not s then result = result .. "*" end
         if APT.html then
             if s then
@@ -198,24 +202,28 @@ local DNSrrTest = function(hosts, k)
         space = ' &nbsp; '
         no = "<font color='grey'><b>no</b></font>"
     end
-    if (APT.options.roundRobin.value ~= k) and (nil ~= hosts[APT.options.roundRobin.value]) and (nil ~= hosts[k].IPs) then
+    if (APT.options.roundRobin.value ~= k) and (nil ~= hosts[APT.options.roundRobin.value]) and (nil ~= hosts[k].IPs) and ("no" ~= hosts[k].DNSRR) then
         APT.allpairs(hosts[k].IPs,
             function(i, w, k, v)
-                if nil ~= hosts[APT.options.roundRobin.value].IPs["deb.roundr.devuan.org"][i] then
-                    local log = logCount(APT.options.roundRobin.value, i)
-                    if "" ~= log then
-                        if "" == dns then dns = " " else dns = dns .. space end
-                        dns = dns .. logCount(APT.options.roundRobin.value, i)
-                    else
-                        if "" == dns then dns = " " else dns = dns .. space end
-                        if APT.html then i = "<font color='maroon'><b>" .. i .. "</b></font>" end
-                        dns = dns .. i
-                    end
-                end
+--              if nil ~= hosts[APT.options.roundRobin.value].IPs[APT.options.roundRobinCname.value] then
+--                  if nil ~= hosts[APT.options.roundRobin.value].IPs[APT.options.roundRobinCname.value][i] then
+                        local log = logCount(APT.options.roundRobin.value, i)
+                        local inRR = "<font color='green'><b>&#x2705;</b></font>"
+                        if nil ~= log:find("<font color='") then inRR = "<font color='red'>&#x274c;</font>" end
+                        if "" ~= log then
+                            if "" == dns then dns = " " else dns = dns .. space end
+                            dns = dns .. inRR .. logCount(APT.options.roundRobin.value, i)
+                        else
+                            if "" == dns then dns = " " else dns = dns .. space end
+                            if APT.html then i = "<font color='maroon'><b>" .. i .. "</b></font>" end
+                            dns = dns .. inRR .. i
+                        end
+--                  end
+--              end
             end
         )
-        if "" == dns then dns = no end
     end
+    if "" == dns then dns = no end
     return dns
 end
 
@@ -242,6 +250,7 @@ local makeTable = function(web, hosts)
             if nil == v.Active then active = 'nil' else active = v.Active end
             web:write("     <tr style='background-color:dimgrey'><th>" .. k .. "</th> ")
         end
+        hosts[k].passed = true;
         local inRR      = "<font color='green'><b>&#x2705;</b></font>"
         local ftp       = "<font color='grey'><b>skip</b></font>"
         local http      = status(hosts, k, results, "http")
@@ -260,14 +269,7 @@ local makeTable = function(web, hosts)
         local week      = '<td>&nbsp;</td><td>&nbsp;</td>'
         if nil == rate then rate = '' end
 
-        -- DNS-RR test.
---      if nil == http:         find('OK') then inRR = "<font color='red'>&#x274c;</font>" end
---      if nil == https:        find('OK') then inRR = "<font color='red'>&#x274c;</font>" end
---      if nil == protocol:     find('OK') then inRR = "<font color='red'>&#x274c;</font>" end
---      if nil == redirects:    find('OK') then inRR = "<font color='red'>&#x274c;</font>" end
---      if nil == sanity:       find('OK') then inRR = "<font color='red'>&#x274c;</font>" end
---      if nil == integrity:    find('OK') then inRR = "<font color='red'>&#x274c;</font>" end
---      if nil == updated:      find('OK') then inRR = "<font color='red'>&#x274c;</font>" end
+        if not hosts[k].passed then inRR = "<font color='red'>&#x274c;</font>" end
 
         if (APT.options.roundRobin.value ~= k) and (nil ~= hosts[APT.options.roundRobin.value]) then
             if 0 == max then
@@ -319,8 +321,10 @@ local makeTable = function(web, hosts)
             week = '<td>&nbsp;' .. percentUp .. '% up</td><td>&nbsp;' .. percentUpdated .. '% updated</td>'
 --          if ('100.00' ~= percentUp) or ('100.00' ~= percentUpdated) then inRR = "<font color='red'>&#x274c;</font>" end
         end
+        if "yes"   ~= hosts[k].DNSRR then inRR = " &nbsp; &nbsp; " end
+        if "maybe" == hosts[k].DNSRR then inRR = "<font color='red'>&#x2753;</font>" end
 
-        web:write("<td>" .. ftp .. "&nbsp;</td><td>" .. http .. "&nbsp;</td><td>" .. https .. "&nbsp;</td><td>" .. rsync .. "&nbsp;</td><td>" .. dns ..
+        web:write("<td>" .. ftp .. "&nbsp;</td><td>" .. http .. "&nbsp;</td><td>" .. https .. "&nbsp;</td><td>" .. rsync .. "&nbsp;</td><td>" .. inRR .. " " .. dns ..
                     "&nbsp;</td><td>" .. protocol .. "&nbsp;</td><td>" .. redirects .. "&nbsp;</td><td>" .. sanity .. "&nbsp;</td><td>" .. integrity .. "&nbsp;</td>" .. '<td>' .. rate ..
                     '</td><td>' ..  updated .. '</td>&nbsp;' .. spd .. "&nbsp;" .. week .."&nbsp;</tr>\n")
         if "" ~= active then
@@ -360,15 +364,14 @@ local makeIPlist = function(hosts)
         if nil ~= hosts[k].IPs then
             for l, w in pairs(hosts[k].IPs) do
                 if type(w) == "table" then
--- TODO - don't hard code deb.roundr.devuan.org.
                     -- Don't output the extra DNS-RR entries that are for admin reasons.
-                    if ((APT.options.roundRobin.value == k) and ("deb.roundr.devuan.org" == l)) or (APT.options.roundRobin.value ~= k) then 
+                    if ((APT.options.roundRobin.value == k) and (APT.options.roundRobinCname.value == l)) or (APT.options.roundRobin.value ~= k) then 
                         n[l] = {}
                         for i, u in pairs(w) do
                             if (APT.testing("IPv6") and ("AAAA" == u)) or ("A" == u) then
                                 local inRR = ""
                                 local lc = logCount(k, i)
-                                if checkRR then
+                                if checkRR and ('no' ~= hosts[k].DNSRR) then
                                     -- If there where errors, warnings, or timeouts, then it'll have that wrapped in font tags.
                                     inRR        = "<font color='green'><b>&#x2705;</b></font>"
                                     if nil ~= lc:find("<font color='") then
@@ -382,7 +385,8 @@ local makeIPlist = function(hosts)
                                         if f == nil then C("writing DNS-RR_good.txt file - " .. e) end
                                     end
                                 end
-                                if "yes" ~= hosts[k].DNSRR then inRR = "" end
+                                if "maybe" == hosts[k].DNSRR then inRR = "<font color='red'>&#x2753;</font>" end
+                                if "no"    == hosts[k].DNSRR then inRR = "" end
                                 local log = '[<a href="' .. adr .. k .. '_' .. i .. '">graphs</a>] &nbsp; '
                                 if "" == log then n[l][i] = u else n[l][log .. inRR .. ' ' .. revDNS(hosts, k, i) .. ' ' .. lc] = u end
                             end
@@ -392,7 +396,7 @@ local makeIPlist = function(hosts)
                     if (APT.testing("IPv6") and ("AAAA" == w)) or ("A" == w) then
                         local inRR      = ""
                         local lc = logCount(k, l)
-                        if checkRR then
+                        if checkRR and ('no' ~= hosts[k].DNSRR) then
                             -- If there where errors, warnings, or timeouts, then it'll have that wrapped in font tags.
                             inRR        = "<font color='green'><b>&#x2705;</b></font>"
                             if nil ~= lc:find("<font color='") then
@@ -406,7 +410,8 @@ local makeIPlist = function(hosts)
                                 if f == nil then C("writing DNS-RR_good.txt file - " .. e) end
                             end
                         end
-                        if "yes" ~= hosts[k].DNSRR then inRR = "" end
+                        if "maybe" == hosts[k].DNSRR then inRR = "<font color='red'>&#x2753;</font>" end
+                        if "no"    == hosts[k].DNSRR then inRR = "" end
                         local log = '[<a href="' .. adr .. k .. '_' .. l .. '">graphs</a>] &nbsp; '
                         if "" == log then n[l] = w else n[log .. inRR .. ' ' .. revDNS(hosts, k, l) .. ' ' .. lc] = w end
                     end
@@ -512,6 +517,17 @@ local colours =
     '00880080',
     '00088080',
     '00008880',
+    '80000080',
+    '08000080',
+    '00800080',
+    '00080080',
+    '00008080',
+    '00000880',
+    'fff00080',
+    '0fff0080',
+    '00fff080',
+    '000fff80',
+    '0000fff0',
 }
 local g = {}
 local count = 0
@@ -526,6 +542,7 @@ for k, v in APT.orderedPairs(mirrors) do
     if APT.options.roundRobin.value ~= k then
         local c = colours[count]
         local name = string.format('%32s', k)
+        if c == nil then c = 'ffffff' end
         if APT.options.referenceSite.value == k then c = 'ffffff' end
         table.insert(g, 'DEF:speedn' .. count .. '=rrd/' .. k .. '/Speed/Speed.rrd:max:MIN')
         table.insert(g, 'DEF:speedx' .. count .. '=rrd/' .. k .. '/Speed/Speed.rrd:max:MAX')
@@ -576,7 +593,8 @@ if nil == web then C("opening mirrors file - " .. e) else
                 APT.options.roundRobin.value .. " is the DNS round robin, which points to the mirrors that are part of the DNS-RR. &nbsp; " ..
                 "If an IP is part of the DNS-RR, it is marked with '<font color='purple'><b>DNS-RR</b></font>'," ..
                 " if it <a href='DNS-RR_good.txt'>should be</a> it is marked with '<font color='green'><b>&#x2705;</b></font>'," ..
-                " if it <a href='DNS-RR_bad.txt'>should not be</a> it is marked with '<font color='red'>&#x274c;</font>'. &nbsp; " ..
+                " if it <a href='DNS-RR_bad.txt'>should not be</a> it is marked with '<font color='red'>&#x274c;</font>'," ..
+                " if it might be but still pending full testing, it is marked with '<font color='red'>&#x2753;</font>'. &nbsp; " ..
                 "<br>" ..
                 APT.options.referenceSite.value .. " is the master mirror, all the others copy files from it. &nbsp; " ..
                 "</p>\n"
@@ -608,7 +626,7 @@ if nil == web then C("opening mirrors file - " .. e) else
                 "and <a href='https://git.devuan.dev/onefang/apt-panopticon_cgp'>here (Devuan repo)</a>.</p>\n"
                 )
     local whn = APT.exe('TZ="GMT" ls -dl1 --time-style="+%s" results/stamp | cut -d " " -f 6-6'):Do().result:sub(2, -2)
-    web:write(  "<p>This run took " .. (os.time() - tonumber("0" .. whn)) .. " seconds.</p>" ..
+    web:write(  "<p>This run took " .. (os.time() - tonumber("0" .. whn)) .. " seconds. &nbsp &nbsp apt-panopticon version " .. APT.version ..  "  </p>" ..
                 "\n</body></html>\n")
     web:close()
 end
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index 4019fdf..79bdd19 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -14,7 +14,6 @@ APT.html = true
 
 
 local defaultURL = {scheme = "http"}
-local releases = {"jessie", "ascii", "beowulf", "ceres"}
 local releaseFiles =
 {
     -- Release file.
@@ -37,16 +36,20 @@ local notExist =
 local referenceDebs =
 {
     -- Debian package.
-    "merged/pool/DEBIAN/main/d/debian-keyring/debian-keyring_2019.02.25_all.deb",
+    "merged/pool/DEBIAN/main/d/debian-keyring/debian-keyring_2023.12.24_all.deb",
     -- Debian security package.  NOTE this one should always be redirected?
-    "merged/pool/DEBIAN-SECURITY/updates/main/a/apt/apt-transport-https_1.4.10_amd64.deb",
+    "merged/pool/DEBIAN-SECURITY/updates/main/a/apt/apt-transport-https_1.8.2.2_all.deb",
 }
 local referenceDevs =
 {
     -- Devuan package.  NOTE this one should not get redirected, but that's more a warning than an error.
-    "merged/pool/DEVUAN/main/d/devuan-keyring/devuan-keyring_2017.10.03_all.deb",
+    "merged/pool/DEVUAN/main/d/devuan-keyring/devuan-keyring_2023.10.07_all.deb",       -- Devuan keeps changing this since the key expiry incident.
+    "merged/pool/DEVUAN/main/b/base-files/base-files_13devuan4_all.deb",
 }
 
+local keyring = "/usr/share/keyrings/devuan-archive-keyring.gpg"
+--local keyring = "/etc/apt/trusted.gpg.d/devuan-keyring-2022-archive.gpg"
+
 local curlStatus = 
 {
     [1 ] = "Unsupported protocol. This build of curl has no support for this protocol.",
@@ -170,38 +173,76 @@ local repoExists = function (r)
 end
 
 local IP = {}
-gatherIPs = function (host)
+gatherIPs = function (host, again)
+    if nil == again then again = '' end
     if nil == IP[host] then
         local IPs
         -- Takes about 30 seconds to look up the lot.
         -- I tested using dig's -f option, it didn't seem much faster.
         -- The sort -r assumes that deb.devuan.org is the first alphabetically.
-        local dig = io.popen('dig +keepopen +noall +nottlid +answer ' .. host .. ' A ' .. host .. ' AAAA ' .. host .. ' CNAME ' .. host .. ' SRV | sort -r | uniq')
-        repeat
-            IPs = dig:read("*l")
-            if nil ~= IPs then
-                for k, t, v in IPs:gmatch("([%w_%-%.]*)%.%s*IN%s*(%a*)%s*(.*)") do
-                    if "." == v:sub(-1, -1) then v = v:sub(1, -2) end
-                    if nil == IP[k] then IP[k] = {} end
-                    IP[k][v] = t
-                    D("  DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v)
-                    if t == "CNAME" then
+        if "" == host then print("Empty host name!") end
+        local dig = APT.readCmd('dig ' .. again .. ' +keepopen +noall +nottlid +answer ' .. host .. ' A ' .. host .. ' AAAA ' .. host .. ' CNAME ' .. host .. ' SRV | sort -r | uniq')
+        for i,IPs in ipairs(dig) do
+            for k, t, v in IPs:gmatch("([%w_%-%.]*)%.%s*IN%s*(%a*)%s*(.*)") do
+                if "." == v:sub(-1, -1) then v = v:sub(1, -2) end
+                if nil == IP[k] then IP[k] = {} end
+                IP[k][v] = t
+                D("  DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v)
+                if t == "CNAME" then
+                    if "" == v then
+                        if '' ~= again then
+                            print("Empty host name!  DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v)
+                        else
+                            return gatherIPs(host, '@9.9.9.11')
+                        end
+                    else
                         gatherIPs(v)
                         IP[k][v] = IP[v]
-                    elseif v == "SRV" then
-                        print("SVR record found, now what do we do?")
+                    end
+                elseif v == "SRV" then
+                    print("SVR record found, now what do we do?")
+                elseif "" == v then
+                    if '' ~= again then
+                        print("Empty host name!  DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v)
+                    else
+                        return gatherIPs(host, '@9.9.9.11')
                     end
                 end
             end
-        until nil == IPs
+        end
     end
 
     -- If this is the DNS-RR domain name, gather the IPs for the mirrors that mirror_list.txt says should be in it.
     if host == APT.options.roundRobin.value then
         for k, m in pairs(APT.mirrors) do
-            if "yes" == m.DNSRR then
+            if ("yes" == m.DNSRR) or ("maybe" == m.DNSRR) then
+                if "" == m.FQDN then
+                    print("Empty FQDN name! " .. host)
+                end
                 gatherIPs(m.FQDN)
                 IP[host][m.FQDN] = IP[m.FQDN]
+                -- Strip them out so we don't test them twice.
+                if (nil ~= IP[m.FQDN]) and (nil ~= IP[host][APT.options.roundRobinCname.value]) then
+                    for l, n in pairs(IP[m.FQDN]) do
+                        if type(n) == 'table' then
+                            for h, p in pairs(n) do
+                                for j, o in pairs(IP[host][APT.options.roundRobinCname.value]) do
+                                    if h == j then IP[host][m.FQDN][l][h] = nil end
+                                end
+                                o = 0
+                                for j in pairs(IP[host][m.FQDN][l]) do o = o + 1 end
+                                if 0 == o then IP[host][m.FQDN][l] = nil end
+                            end
+                        else
+                            for j, o in pairs(IP[host][APT.options.roundRobinCname.value]) do
+                                if l == j then IP[host][m.FQDN][l] = nil end
+                            end
+                        end
+                        o = 0
+                        for j in pairs(IP[host][m.FQDN]) do o = o + 1 end
+                        if 0 == o then IP[host][m.FQDN] = nil end
+                    end
+                end
             end
         end
     end
@@ -289,7 +330,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
         'curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' .. APT.IPv46 .. ' ' ..
         IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' ..
         hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"'
-    ):Nice():log():Do().status
+    ):timeout(APT.options.maxtime.value * 2.0):Nice():log():Do().status
     if 0 < r then
         APT.tested(PU.scheme, 'Redirects', host)
     else
@@ -315,7 +356,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
     if 0 ~= status then
         local msg = curlStatus[status]
         if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end
-        if (28 == status) or (7 == status) then
+        if (128+9 == status) or (124 == status) or (28 == status) or (7 == status) then
             T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host)
             timeouts = timeouts + 1
         else
@@ -406,7 +447,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
                     local pth = path:match('^(.*/pool/).*$')
                     if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E(spcd .. spcd .. 'Odd redirect path ' .. path) end
                     I(spcd .. spcd .. "Now checking redirected host " .. u .. ' &nbsp; for &nbsp; ' .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
-                    APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork()
+                    APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host)
                     D(spcd .. 'logging to ' .. APT.logName(pu.host, nil, file)[2])
                     APT.tested(PU.scheme, 'Redirects', host)
                 end
@@ -454,7 +495,7 @@ local checkFiles = function (host, ip, path, file)
                 if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end
             end
         end
-        for i, s in pairs(releases) do
+        for i, s in pairs(APT.releases) do
             for j, k in pairs(releaseFiles) do
                 if repoExists(s .. k) then
                     if checkTimeouts(host, "http",  ip .. path .. "/merged/dists/" .. s .. '/' .. k) then return end
@@ -493,7 +534,7 @@ checkHost = function (orig, host, path, ip, file)
     else
         if orig == host then
             I("Testing mirror " .. orig .. "" .. file)
-            APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork()
+            APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(orig)
             D('logging to ' .. APT.logName(ph.host, nil, file)[2])
         else D("checkHost " .. orig .. arw .. host) end
     end
@@ -568,7 +609,7 @@ local downloads = function(host, URL, meta, release, list)
             return
         end
     else
-        for i, s in pairs(releases) do
+        for i, s in pairs(APT.releases) do
             for j, k in pairs(releaseFiles) do
                 if repoExists(s .. k) then
                     addDownload(host, URL, f, s, k)
@@ -577,7 +618,7 @@ local downloads = function(host, URL, meta, release, list)
         end
     end
     f:close()
-    APT.exe(cm):Nice():log():fork()
+    APT.exe(cm):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(host)
     D('logging to <a href="' .. log .. '">' .. log .. '</a>, with <a href="' .. files .. '">these files</a>')
 end
 
@@ -594,7 +635,7 @@ local validateURL = function(m)
     local p = url.parse("http://" .. m.BaseURL)
     if nil == p.path then p.path = '' end
     if nil ~= p.port then p.authority = authority .. ':' .. p.port end
-    if nil == m.FDQN then W("Something wrong in FDQN from mirror_list.txt!  nil", "", "", p.authority) else
+    if nil == m.FQDN then W("Something wrong in FQDN from mirror_list.txt!  nil", "", "", p.authority) else
         if m.FQDN ~= p.authority then W("Something wrong in FDQN from mirror_list.txt!  " .. m.FDQN, "", "", p.authority) end
     end
     if nil == m.BaseURL then W("Something wrong in BaseURL from mirror_list.txt!  nil", "", "", p.authority) else
@@ -678,7 +719,7 @@ local postParse = function(host, list)
     if APT.options.referenceSite.value == host then
         if nil ~= list then
             local sem = 'results/NEW_' .. list.out .. '_%s.txt'
-            for i, n in pairs(releases) do
+            for i, n in pairs(APT.releases) do
                 local f = sem:format(n)
                 if APT.checkFile(f .. '.tmp') then
                     os.execute('mv ' .. f .. '.tmp ' .. f)
@@ -691,7 +732,7 @@ local postParse = function(host, list)
 end
 
 local parseDebs = function(host)
-    for i, n in pairs(releases) do
+    for i, n in pairs(APT.releases) do
         local inFile =  'results/NEW_debs_' .. n .. '.txt'
         local nfile, e = io.open(inFile, "r")
             if nil == nfile then W("opening " .. inFile .. " file - " .. e) else
@@ -733,7 +774,7 @@ end
 
 local parsePackages = function(host)
     local list = {inf = 'Packages', parser = parseDebs, out = 'debs', files = {}, nextf = ''}
-    for i, n in pairs(releases) do
+    for i, n in pairs(APT.releases) do
         local inFile =  'results/NEW_' .. list.inf  .. '_' .. n .. '.txt'
         local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt'
         if APT.options.referenceSite.value == host then
@@ -755,6 +796,7 @@ local parsePackages = function(host)
                         local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed', "w+")
                         if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed' .. ' file - ' .. e) else
                             local pp = {}
+-- TODO - FIX - check if this file exists first.
                             for l in io.lines('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages') do
                                 if "Package: "  == l:sub(1, 9) then
                                     if 0 ~= #pp then
@@ -786,12 +828,12 @@ local parsePackages = function(host)
                                                         ' | grep -E "^\\+" | grep -Ev "^\\+\\+\\+|^---" >>results/NEW_' .. list.out  .. '_TMP_' .. n .. '.txt')
                                 for i, s in pairs(referenceDebs) do
                                     if 0 == APT.exe('grep -q "' .. s:sub(8, -1) .. '" results/OLD_' .. list.out  .. '_' .. n .. '.txt'):log():Do().status then
-                                        print('Reference package is out of date (' .. n .. ') - ' .. s)
+                                        print('Reference package is out of date from ' .. host .. ' (' .. n .. ') - ' .. s)
                                     end
                                 end
                                 for i, s in pairs(referenceDevs) do
                                     if 0 == APT.exe('grep -q "' .. s:sub(8, -1) .. '" results/OLD_' .. list.out  .. '_' .. n .. '.txt'):log():Do().status then
-                                        print('Reference package is out of date (' .. n .. ') - ' .. s)
+                                        print('Reference package is out of date from ' .. host .. ' (' .. n .. ') - ' .. s)
                                     end
                                 end
                             else
@@ -839,13 +881,13 @@ local parseRelease = function(host)
     local list = {inf = 'Release', parser = parsePackages, out = 'Packages', files = {}, nextf = 'debs'}
     local updated = false
     local now = tonumber(os.date('%s'))
-    for i, n in pairs(releases) do
+    for i, n in pairs(APT.releases) do
         for l, o in pairs(releaseFiles) do
             if repoExists(i .. o) then
                 postDownload(host, n, o)
                 if (".gpg" == o:sub(-4, -1)) and (APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/' .. o)) then
                     if APT.testing("Integrity") then
-                        local status = APT.exe( "gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. 
+                        local status = APT.exe( "gpgv --keyring " .. keyring .. " results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. 
                                                 " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status
                         if 0 ~= status then E("GPG check failed for " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end
 -- TODO - should check the PGP sig of InRelease as well.
@@ -922,7 +964,7 @@ end
 
 local parseStart = function(host)
     local list = {inf = '', parser = parseRelease, out = 'Release', files = {}, nextf = 'Packages'}
-    for i, n in pairs(releases) do
+    for i, n in pairs(APT.releases) do
         local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt'
         for l, o in pairs(releaseFiles) do
             if repoExists(n .. o) then
@@ -991,6 +1033,7 @@ os.execute('sleep 1')	-- Wait for things to start up before checking for them.
               0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0* Connected to devuan.bio.lmu.de (141.84.43.19) port 80 (#0)
             curl: (22) The requested URL returned error: 404 Not Found
 ]]
+            local trace = {}
             local min, max, spd = 999999999999, 0
             local num = '[%d%.]+[kM]?'
             if APT.checkFile(f) then
@@ -1022,7 +1065,7 @@ os.execute('sleep 1')	-- Wait for things to start up before checking for them.
 
         if (APT.options.referenceSite.value ~= host) and ('' ~= list.nextf) then
             local sem = 'results/NEW_' .. list.nextf .. '_%s.txt'
-            for i, n in pairs(releases) do
+            for i, n in pairs(APT.releases) do
                 local f = sem:format(n)
                 while not APT.checkFile(f) do
                     D('*&lt;* About to yield coroutine while waiting on - not APT.checkFile(' .. f .. ')')
@@ -1066,7 +1109,12 @@ if 0 < #arg then
     I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", "))
     APT.mirrors = loadfile("results/mirrors.lua")()
     APT.results = APT.padResults(APT.results)
-    if APT.origin or APT.redir then APT.results["IPs"] = gatherIPs(pu.host) end
+    if APT.origin or APT.redir then
+        if "" == pu.host then
+            print("Empty pu.host name! " .. pu.host)
+        end
+        APT.results["IPs"] = gatherIPs(pu.host)
+    end
     if nil ~= arg[2] then I(" &nbsp; Using IP " .. arg[2]); ip = arg[2] end
     if nil ~= arg[3] then I(" &nbsp; Using file " .. arg[3]); end
 
@@ -1081,15 +1129,17 @@ if 0 < #arg then
                 APT.allpairs(ips,
                     function(k, v)
                         if v == "A" then
-                            if APT.testing("IPv4") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -4 ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):Nice():log():fork() end
+                            if APT.testing("IPv4") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -4 ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host) end
                         elseif v == "AAAA" then
-                            if APT.testing("IPv6") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -6 ' .. APT.IPv46 .. ' ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):Nice():log():fork() end
+                            if APT.testing("IPv6") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -6 ' .. APT.IPv46 .. ' ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host) end
                         end
                         D('logging to ' .. APT.logName(pu.host, k, file)[2])
                     end
                 )
             else
                 E("no IPs for " .. pu.host)
+                APT.logPost()
+                return
             end
         end
 
@@ -1178,16 +1228,15 @@ else
     while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 5") end
 
     local APT_args    = APT.args
-    local APT_logFile = APT.logFile
     local debians = {}
-    local srvs = io.popen('ls -1 results/*.lua')
-    for l in srvs:lines() do
+    local srvs = APT.readCmd('ls -1 results/*.lua')
+    for ii,l in ipairs(srvs) do
         local hst = l:sub(9, -5)
         if nil ~= l:find('_R%.lua') then hst = hst:sub(1, -3) end
         if (hst:find('_') == nil) and (nil == APT.mirrors[hst]) then
             local ips = loadfile(l)().IPs
             if nil ~= ips then
-                debians[hst] = {Country = '', FDQN = hst, Active = 'yes', Rate = '', BaseURL = hst, Protocols = {http = true, https = true}, Bandwidth = '', IPs = ips}
+                debians[hst] = {Country = '', FQDN = hst, Active = 'yes', Rate = '', BaseURL = hst, Protocols = {http = true, https = true}, Bandwidth = '', IPs = ips}
                 local baseFiles = {}
                 local IPfiles = {}
                 for i, a in pairs(ips) do
@@ -1198,8 +1247,8 @@ else
                         end
                     end
                 end
-                local files = io.popen('ls -1 results/LOG_' .. hst .. '_*.html')
-                for ll in files:lines() do
+                local files = APT.readCmd('ls -1 results/LOG_' .. hst .. '_*.html')
+                for iii,ll in ipairs(files) do
                     local dn = false
                     for i, a in pairs(ips) do
                         if type(a) == 'table' then
@@ -1220,6 +1269,7 @@ else
                 end
 
                 local combine = function(ip, a)
+                    local APT_logFile = APT.logFile
                     if not APT.logOpen(hst, ip) then
                         print('PROBLEM OPENING LOG FILE ' .. hst .. ' ' .. ip) 
                     else
@@ -1231,8 +1281,8 @@ else
                                 if ln:match('^' .. os.date('!%Y%-%m%-%d ') .. '.*$') then APT.logFile:write(ln .. '\n') end     -- %F isn't good enough, coz we have to escape the '-'.
                             end
                         end
+                        APT.logPost()
                     end
-                    APT.logPost()
                     APT.args = APT_args
                     APT.logFile = APT_logFile
                 end
diff --git a/laggers b/laggers
new file mode 100755
index 0000000..c5f30ee
--- /dev/null
+++ b/laggers
@@ -0,0 +1,3 @@
+#!/bin/bash
+echo "apt-panopticon processes still running -"
+ps ax -o pid,args --sort args | grep -E 'apt-panopticon\.lua | curl | dig ' | grep -v -E 'flock -n |grep -E |sh -c |timeout -k '
diff --git a/update_apt-panopticon b/update_apt-panopticon
index b70f4c3..dde18a5 100755
--- a/update_apt-panopticon
+++ b/update_apt-panopticon
@@ -1,16 +1,55 @@
 #!/bin/bash
 
-cd /var/www/html/apt-panopticon/apt-panopticon_cgp
-#git pull > /dev/null
-#chown -R mirrors:www-data *
-cd /var/www/html/apt-panopticon/apt-panopticon
-#git pull > /dev/null
-#chown -R mirrors:www-data *
-
-if [ ! -f apt-panopticon.lock ] ; then
-    rm ../results; ln -s apt-panopticon/results_old ../results
-    flock -n apt-panopticon.lock ./apt-panopticon.lua && rm apt-panopticon.lock
-    rm ../results; ln -s apt-panopticon/results ../results
+PANOPATH="/var/www/html/apt-panopticon"
+#KEEPDAYS=$((2 * 30))
+
+cd ${PANOPATH}/apt-panopticon_cgp
+if [ -d .git ] ; then
+    git pull > /dev/null
+    chown -R www-data:www-data *
+fi
+cd ${PANOPATH}/apt-panopticon
+if [ -d .git ] ; then
+    git pull > /dev/null
+    chown -R www-data:www-data *
+fi
+
+# Check if the lock file still exists.
+if [ -f apt-panopticon.lock ] ; then
+    # Check if it's still running.
+    ps ax -eo pid,args | grep "apt-panopticon.lua" | grep -v "grep apt-panopticon.lua" | while read line ; do touch apt-panopticon.running ; exit ; done
+    if [ -f apt-panopticon.running ] ; then
+        echo "Previous apt-panopticon still running, exiting."
+        echo "Previous apt-panopticon still running, exiting."
+        ./laggers
+        rm apt-panopticon.running
+        exit 1
+    fi
+    echo "Crashed apt-panopticon detected, removing stale lock file."
+    echo "Crashed apt-panopticon detected, removing stale lock file."
+    ./laggers
+    rm apt-panopticon.lock
+fi
+
+# Clean up any mess left over from failed runs.
+find results_* -maxdepth 2 ! -name "pkgmaster.devuan.org" -name "*.*" -type d -print0 | xargs -0rt /bin/rm -fr
+
+if [ "z" != "z${KEEPDAYS}" ] ; then
+    # Remove old results.
+    find . -daystart -maxdepth 2 -name "stamp"            -type f ! -mtime $((0 - ${KEEPDAYS})) -printf "%h\0" | xargs -0rt /bin/rm -fr
+    find . -daystart -maxdepth 2 -name "results_*.tar.xz" -type f ! -mtime $((0 - ${KEEPDAYS})) -print0        | xargs -0rt /bin/rm -f
+    find .           -maxdepth 1 -name "results_*"        -type d -empty -print0                               | xargs -0rt /bin/rm -fr
 fi
 
-#chown -R mirrors:www-data /var/www/html/SledjHamr/apt-panopticon/results
+rm ../results; ln -s apt-panopticon/results_old ../results
+flock -n apt-panopticon.lock ionice -c3 nice -n 19 timeout --kill-after=20.0 --foreground 8.5m ./apt-panopticon.lua && rm apt-panopticon.lock
+if [ -f apt-panopticon.lock ] ; then
+    echo "apt-panopticon timed out."
+    ./laggers
+fi
+rm ../results; ln -s apt-panopticon/results ../results
+
+chown -R www-data:www-data *
+
+# Clean up any mess left over from THIS failed run.
+find results_* -maxdepth 2 ! -name "pkgmaster.devuan.org" -name "*.*" -type d -print0 | xargs -0rt /bin/rm -fr