9 files changed, 329 insertions, 123 deletions

diff --git a/README.md b/README.md
index 2edc3ca..4d80062 100644
--- a/README.md
+++ b/README.md
@@ -29,9 +29,9 @@ The details of that are yet to be determined.
 
 
 You can get the source code from [https://sledjhamr.org/cgit/apt-panopticon/](https://sledjhamr.org/cgit/apt-panopticon/) (main repo)
-and [https://git.devuan.dev/onefang/apt-panopticon](https://git.devuan.dev/onefang/apt-panopticon) (Devuan repo).
+and [https://git.devuan.org/onefang/apt-panopticon](https://git.devuan.org/onefang/apt-panopticon) (Devuan repo).
 You can get the cgp graphing source code from [https://sledjhamr.org/cgit/apt-panopticon_cgp/]() (main repo)
-and [https://git.devuan.dev/onefang/apt-panopticon_cgp](https://git.devuan.dev/onefang/apt-panopticon_cgp) (Devuan repo).
+and [https://git.devuan.org/onefang/apt-panopticon_cgp](https://git.devuan.org/onefang/apt-panopticon_cgp) (Devuan repo).
 
 The issue tracker is at [https://sledjhamr.org/mantisbt/project_page.php?project_id=13](https://sledjhamr.org/mantisbt/project_page.php?project_id=13)
 
@@ -55,6 +55,7 @@ installed -
 * lua-rrd
 * LuaSocket, on Debian based systems it'll be in the lua-socket package.
 * md5sum and sha256, on Debian based systems they'll be in the coreutils package.
+* timeout, on Debian based systems it'll be in the coreutils package.
 * rrdtool
 * xz, on Debian based systems it'll be in the xz-utils package.
 
diff --git a/Report-web_2.html b/Report-web_2.html
index 592e8c6..2882179 100644
--- a/Report-web_2.html
+++ b/Report-web_2.html
@@ -1,5 +1,8 @@
     <p><font style='background-color:red; color:black'>EXPERIMENTAL CODE - double check all results you see here, and read the logs if it's important.</font></p>
     <p>The full list of Devuan package mirrors is available at the URL: <a href='https://pkgmaster.devuan.org/mirror_list.txt'>https://pkgmaster.devuan.org/mirror_list.txt</a></p>
-    <p>Due to the nature of the tests, some errors or warnings will be counted several times. &nbsp; The links in the table and DNS list go to the detailed testing logs.</p>
+    <p>Due to the nature of the tests, some errors or warnings will be counted several times.  There will be some duplication.</p>
+    <p>Due to the nature of the tests, some errors or warnings will be counted several times.  There will be some duplication.</p>
+    <p>Due to the nature of the tests, some errors or warnings will be counted several times.  There will be some duplication.</p>
+    <p>The links in the table and DNS list go to the detailed testing logs.</p>
 
     <hr>
diff --git a/Report-web_3.html b/Report-web_3.html
index 632e6f4..a1ca70d 100644
--- a/Report-web_3.html
+++ b/Report-web_3.html
@@ -7,7 +7,13 @@
     <p><font color='aqua'><b>TIMEOUT</b></font> or <font color='blue'><b>TIMEOUT</b></font> means the mirror had too many timeouts, and tests where aborted, so there is no result for this test.</p>
     <p>NOTE: timeouts may be due to a problem on the testing computer, it might be busy with other things, or be having it's own network problems.</p>
     <p>NOTE: the speed of the server this checking script is running on might be wildly exaggerated, it doesn't have to go through the Internet to download from itself.</p>
-    <p>The DNS round robin (DNS-RR) column shows the IPs for that mirror, or <font color='grey'><b>no</b></font> if it isn't part of the DNS-RR. &nbsp; The IPs link to the testing log for that IP accessed via the DNS-RR. &nbsp; deb.devuan.org is the DNS-RR itself, so it doesn't get tested directly.</p>
+    <p>The DNS round robin (DNS-RR) column shows the IPs for that mirror, or <font color='grey'><b>no</b></font> if it isn't part of the DNS-RR.</p>
+    <p> &nbsp; The IPs link to the testing log for that IP accessed via the DNS-RR.</p>
+    <p> &nbsp; If a mirror <a href='DNS-RR_good.txt'>should be</a> it is marked with '<font color='green'><b>&#x2705;</b></font>', 
+                if it <a href='DNS-RR_bad.txt'>should not be</a> it is marked with '<font color='red'>&#x274c;</font>',
+                if it might be but still pending full testing, it is marked with '<font color='red'>&#x2753;</font>'.</p>
+    <p> &nbsp; deb.devuan.org is the DNS-RR itself, so it doesn't get tested directly.
+    </p>
     <p>The time in the Updated column is how often the mirror updates itself.</p>
     <p>Mirrors with a <font style='background-color:dimgrey'>grey background</font> are not active (though may be usable as part of the DNS-RR).</p>
     <p><font color='grey'><b>skip</b></font> means that the test hasn't been written yet.</p>
diff --git a/Report-web_TABLE.html b/Report-web_TABLE.html
index 6ee14f4..b267094 100644
--- a/Report-web_TABLE.html
+++ b/Report-web_TABLE.html
@@ -1,4 +1,4 @@
-    <table>
+    <table id=results>
     <tr>
       <th></th>
       <th class="TTitem">
diff --git a/apt-panopticommon.lua b/apt-panopticommon.lua
index 0862b77..982ff21 100644
--- a/apt-panopticommon.lua
+++ b/apt-panopticommon.lua
@@ -3,14 +3,18 @@ local APT = {}
 -- https://oss.oetiker.ch/rrdtool/prog/rrdlua.en.html
 APT.rrd = require 'rrd'
 
+APT.version = '0.3 alpha'
+
 APT.protocols = {"ftp", "http", "https", "rsync"}
 APT.tests = {'raw', 'Integrity', 'Protocol', 'Redirects', 'Updated', 'URLSanity', 'Speed'}
-APT.releases = {"ascii", "beowulf", "chimaera", "ceres"}
+--APT.releases = {"jessie", "ascii", "beowulf", "chimaera", "daedalus", "ceres"}
+APT.releases = {"beowulf","chimaera", "daedalus", "excalibur", "ceres"}
 APT.subRels = {'backports', 'proposed-updates', 'security', 'updates'}
 APT.notExist =
 {
-    'chimaera-backports',
-    'chimaera-security',
+    'excalibur-backports',
+    'excalibur-security',
+    'excalibur-updates',
     'ceres-backports',          -- These will never exist, it's our code name for the testing suite.
     'ceres-proposed-updates',
     'ceres-updates',
@@ -50,6 +54,12 @@ APT.options =
             help = "The round robin DNS name.",
             value = "deb.devuan.org",
         },
+    roundRobinCname =
+        {
+            typ = "string",
+            help = "The round robin DNS name.",
+            value = "deb.rr.devuan.org",
+        },
     tests =
         {
             typ = "table",
@@ -80,7 +90,7 @@ APT.options =
         {
             typ = "number",
             help = "",
-            value = 5,
+            value = 24,
         },
     timeouts =
         {
@@ -125,7 +135,7 @@ APT.parseArgs = function(args)
     local arg = {}
     local sendArgs = ""
     -- A special test to disable IPv6 tests if IPv6 isn't available.
-    if 1 == APT.exe('ip -6 addr | grep inet6 | grep " global"'):Do().status then
+    if 1 == APT.exe('ip -6 addr | grep inet6 | grep " global"'):timeout():Do().status then
         table.insert(args, '--tests=-IPv6')
     end
     if 0 ~= #(args) then
@@ -139,7 +149,7 @@ APT.parseArgs = function(args)
                 end
                 os.exit()
             elseif "--version" == a then
-                print("apt-panopticon version 0.2 alpha")
+                print("apt-panopticon version " .. APT.version)
                 os.exit()
             elseif "-v" == a then
                 APT.verbosity = APT.verbosity + 1
@@ -398,7 +408,7 @@ APT.logOpen = function(host, a2, a3)
     local name = APT.logName(host, a2, a3)[1]
     if APT.checkFile(name) then return false end
     APT.logFile, e = io.open(name, "a+")
-    if nil == APT.logFile then C('opening log file (' .. name .. ') - ' .. e); return false end
+    if nil == APT.logFile then print('CRITICAL - opening log file (' .. name .. ') - ' .. e); return false end
     if nil ~= APT.logFile then
         APT.logFile:write("<html><head>\n")
         APT.logFile:write("</head><body bgcolor='black' text='white' alink='red' link='aqua' vlink='fuchsia'>\n")
@@ -415,6 +425,7 @@ APT.logPost = function()
     if nil ~= APT.logFile then
         APT.logFile:write("</body></html> \n")
         APT.logFile:close()
+        APT.logFile = nil
     end
 end
 
@@ -455,7 +466,7 @@ local log = function(v, t, s, prot, test, host)
     if nil ~= APT.logFile then
         if APT.html then
             local colour = "white"
-            if -1 == v then colour = "fuchsia" end      -- CRITICAL
+            if -1 == v then colour = "fuchsia"; print(t .. " " .. s) end        -- CRITICAL
             if  0 == v then colour = "red    " end      -- ERROR
             if  1 == v then colour = "yellow " end      -- WARNING
             if  2 == v then colour = "blue   " end      -- TIMEOUT
@@ -482,6 +493,19 @@ local E = APT.E
 local C = APT.C
 
 
+APT.readCmd = function(cmd)
+    local result = {}
+    local output = io.popen(cmd)
+    if nil ~= output then
+        for l in output:lines() do
+            table.insert(result, l)
+        end
+    end
+    -- While this does return the same things as os.execute(), it's just as useless.
+    output:close()
+    return result
+end
+
 APT.debians = {}
 APT.mirrors = {}
 
@@ -508,7 +532,7 @@ APT.tested = function(prot, test, host)
 end
 
 APT.exe = function(c)
-    local exe = {status = 0, result = '', log = true, cmd = c .. ' '}
+    local exe = {status = 0, result = '', log = true, cmd = c .. ' ', command = c}
 
     function exe:log()
         self.log = true
@@ -522,6 +546,16 @@ APT.exe = function(c)
         end
         return self
     end
+    function exe:timeout(c)
+        -- timeout returns a status of - command status if --preserve-status; "128+9" (actually 137) if --kill-after ends up being done; 124 if it had to TERM; command status if all went well.
+        -- --kill-after means "send KILL after TERM fails.
+        if nil == c then
+            self.cmd = 'timeout --kill-after=10.0 --foreground 42.0s ' .. self.cmd
+        else
+            self.cmd = 'timeout --kill-after=10.0 --foreground ' .. c .. ' ' .. self.cmd
+        end
+        return self
+    end
     function exe:also(c)
         if nil == c then c = '' else c = ' ' .. c end
         self.cmd = self.cmd .. ';' .. c .. ' '
@@ -558,17 +592,51 @@ APT.exe = function(c)
             I'm getting 7168 or 0.  No idea what the fuck that is.
         local ok, rslt, status = os.execute(s)
         ]]
-        local f = io.popen(self.cmd .. ' ; echo "$?"', 'r')
-        -- The last line will be the command's returned status, collect everything else in result.
+
+        local f = APT.readCmd(self.cmd, 'r')
+    -- The last line will be the command's returned status, collect everything else in result.
         self.status = ''        -- Otherwise the result starts with 0.
-        for l in f:lines() do
-            self.result = self.result .. self.status .. "\n"
-            self.status = l
+        self.result = '\n'
+        for i,l in ipairs(f) do
+            self.result = self.result .. l .. "\n"
+        end
+        f = APT.readCmd('echo "$?"', 'r')
+        for i,l in ipairs(f) do
+            self.status = tonumber(l)
+            if (137 == self.status) or (124 == self.status) then
+                print("timeout killed " .. self.status .. ' ' .. self.command)
+                E("timeout killed " .. self.status .. ' ' .. self.command)
+            elseif (0 ~= self.status) then
+                print("status |" .. self.status .. '| ' .. self.command)
+                E("status |" .. self.status .. '| ' .. self.command)
+            end
+        end
+
+--[[  While this is more reliable, it breaks stuff that's likely making assumptions that match the old way.
+TODO - fix it later.
+        local f = APT.readCmd(self.cmd .. '; echo "$?"', 'r')
+        -- The last line will be the command's returned status, collect everything else in result.
+        self.status = tonumber(f[#f])
+        f[#f] = nil
+        self.result = '\n'
+        for i,l in ipairs(f) do
+            self.result = self.result .. l .. "\n"
+        end
+        if (137 == self.status) or (124 == self.status) then
+            print("timeout killed " .. self.status .. ' ' .. self.command)
+            print('ERROR ' .. "timeout killed " .. self.status .. ' ' .. self.command)
+        elseif (nil == self.status) then
+            print("status |" .. "NIL" .. '| ' .. self.command)
+            E("status |" .. "NIL" .. '| ' .. self.command)
+        elseif (0 ~= self.status) then
+--          print("status |" .. self.status .. '| ' .. self.command)
+            E("status |" .. self.status .. '| ' .. self.command)
         end
-        self.status = tonumber(self.status)
+]]
         return self
     end
-    function exe:fork()
+    function exe:fork(host)
+        if nil ~= host then self.cmd = self.cmd .. ';  r=$?; if [ $r -ge 124 ]; then echo "$r ' .. host .. ' failed forked command ' .. string.gsub(self.cmd, '"', "'") .. '"; fi' end
         self.cmd = '{ ' .. self.cmd .. '; } &'
         if true == self.log then D("  forking - &nbsp; <code>" .. self.cmd .. "</code>") end
         os.execute(self.cmd)
@@ -578,7 +646,9 @@ APT.exe = function(c)
 end
 
 APT.checkExes = function (exe)
-    local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | grep -v "flock -n apt-panopticon.lock " | wc -l'):read("*l")
+    local count = 0
+    local ps = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | grep -v "flock -n apt-panopticon.lock " | wc -l')
+    if nil ~= ps then count = ps:read("*l") end
     D(count .. " " .. exe .. " commands still running.")
     return tonumber(count)
 end
diff --git a/apt-panopticon-report-email-web.lua b/apt-panopticon-report-email-web.lua
index b21ba8a..de63274 100755
--- a/apt-panopticon-report-email-web.lua
+++ b/apt-panopticon-report-email-web.lua
@@ -18,11 +18,13 @@ APT.debians = loadfile("results/debians.lua")()
 local revDNS = function(hosts, dom, IP)
     if APT.options.roundRobin.value ~= dom then
         if nil ~= hosts[APT.options.roundRobin.value] then
-            if nil ~= hosts[APT.options.roundRobin.value].IPs["deb.roundr.devuan.org"][IP] then
-                if APT.html then
-                    return "<font color='purple'><b>DNS-RR</b></font>"
-                else
-                    return "DNS-RR"
+            if nil ~= hosts[APT.options.roundRobin.value].IPs[APT.options.roundRobinCname.value] then
+                if nil ~= hosts[APT.options.roundRobin.value].IPs[APT.options.roundRobinCname.value][IP] then
+                    if APT.html then
+                        return "<font color='purple'><b>DNS-RR</b></font>"
+                    else
+                        return "DNS-RR"
+                    end
                 end
             end
         end
@@ -87,6 +89,7 @@ local status = function(hosts, host, results, typ)
 
     if to then
         result = "TIMEOUT"
+        hosts[host].passed = false;
         if not s then result = result .. "*" end
         if APT.html then
             if s then
@@ -102,6 +105,7 @@ local status = function(hosts, host, results, typ)
         end
     elseif 0 < e then
         result = "FAILED"
+        hosts[host].passed = false;
         if not s then result = result .. "*" end
         if APT.html then
             if s then
@@ -198,24 +202,28 @@ local DNSrrTest = function(hosts, k)
         space = ' &nbsp; '
         no = "<font color='grey'><b>no</b></font>"
     end
-    if (APT.options.roundRobin.value ~= k) and (nil ~= hosts[APT.options.roundRobin.value]) and (nil ~= hosts[k].IPs) then
+    if (APT.options.roundRobin.value ~= k) and (nil ~= hosts[APT.options.roundRobin.value]) and (nil ~= hosts[k].IPs) and ("no" ~= hosts[k].DNSRR) then
         APT.allpairs(hosts[k].IPs,
             function(i, w, k, v)
-                if nil ~= hosts[APT.options.roundRobin.value].IPs["deb.roundr.devuan.org"][i] then
-                    local log = logCount(APT.options.roundRobin.value, i)
-                    if "" ~= log then
-                        if "" == dns then dns = " " else dns = dns .. space end
-                        dns = dns .. logCount(APT.options.roundRobin.value, i)
-                    else
-                        if "" == dns then dns = " " else dns = dns .. space end
-                        if APT.html then i = "<font color='maroon'><b>" .. i .. "</b></font>" end
-                        dns = dns .. i
-                    end
-                end
+--              if nil ~= hosts[APT.options.roundRobin.value].IPs[APT.options.roundRobinCname.value] then
+--                  if nil ~= hosts[APT.options.roundRobin.value].IPs[APT.options.roundRobinCname.value][i] then
+                        local log = logCount(APT.options.roundRobin.value, i)
+                        local inRR = "<font color='green'><b>&#x2705;</b></font>"
+                        if nil ~= log:find("<font color='") then inRR = "<font color='red'>&#x274c;</font>" end
+                        if "" ~= log then
+                            if "" == dns then dns = " " else dns = dns .. space end
+                            dns = dns .. inRR .. logCount(APT.options.roundRobin.value, i)
+                        else
+                            if "" == dns then dns = " " else dns = dns .. space end
+                            if APT.html then i = "<font color='maroon'><b>" .. i .. "</b></font>" end
+                            dns = dns .. inRR .. i
+                        end
+--                  end
+--              end
             end
         )
-        if "" == dns then dns = no end
     end
+    if "" == dns then dns = no end
     return dns
 end
 
@@ -242,6 +250,7 @@ local makeTable = function(web, hosts)
             if nil == v.Active then active = 'nil' else active = v.Active end
             web:write("     <tr style='background-color:dimgrey'><th>" .. k .. "</th> ")
         end
+        hosts[k].passed = true;
         local inRR      = "<font color='green'><b>&#x2705;</b></font>"
         local ftp       = "<font color='grey'><b>skip</b></font>"
         local http      = status(hosts, k, results, "http")
@@ -260,14 +269,7 @@ local makeTable = function(web, hosts)
         local week      = '<td>&nbsp;</td><td>&nbsp;</td>'
         if nil == rate then rate = '' end
 
-        -- DNS-RR test.
---      if nil == http:         find('OK') then inRR = "<font color='red'>&#x274c;</font>" end
---      if nil == https:        find('OK') then inRR = "<font color='red'>&#x274c;</font>" end
---      if nil == protocol:     find('OK') then inRR = "<font color='red'>&#x274c;</font>" end
---      if nil == redirects:    find('OK') then inRR = "<font color='red'>&#x274c;</font>" end
---      if nil == sanity:       find('OK') then inRR = "<font color='red'>&#x274c;</font>" end
---      if nil == integrity:    find('OK') then inRR = "<font color='red'>&#x274c;</font>" end
---      if nil == updated:      find('OK') then inRR = "<font color='red'>&#x274c;</font>" end
+        if not hosts[k].passed then inRR = "<font color='red'>&#x274c;</font>" end
 
         if (APT.options.roundRobin.value ~= k) and (nil ~= hosts[APT.options.roundRobin.value]) then
             if 0 == max then
@@ -319,8 +321,10 @@ local makeTable = function(web, hosts)
             week = '<td>&nbsp;' .. percentUp .. '% up</td><td>&nbsp;' .. percentUpdated .. '% updated</td>'
 --          if ('100.00' ~= percentUp) or ('100.00' ~= percentUpdated) then inRR = "<font color='red'>&#x274c;</font>" end
         end
+        if "yes"   ~= hosts[k].DNSRR then inRR = " &nbsp; &nbsp; " end
+        if "maybe" == hosts[k].DNSRR then inRR = "<font color='red'>&#x2753;</font>" end
 
-        web:write("<td>" .. ftp .. "&nbsp;</td><td>" .. http .. "&nbsp;</td><td>" .. https .. "&nbsp;</td><td>" .. rsync .. "&nbsp;</td><td>" .. dns ..
+        web:write("<td>" .. ftp .. "&nbsp;</td><td>" .. http .. "&nbsp;</td><td>" .. https .. "&nbsp;</td><td>" .. rsync .. "&nbsp;</td><td>" .. inRR .. " " .. dns ..
                     "&nbsp;</td><td>" .. protocol .. "&nbsp;</td><td>" .. redirects .. "&nbsp;</td><td>" .. sanity .. "&nbsp;</td><td>" .. integrity .. "&nbsp;</td>" .. '<td>' .. rate ..
                     '</td><td>' ..  updated .. '</td>&nbsp;' .. spd .. "&nbsp;" .. week .."&nbsp;</tr>\n")
         if "" ~= active then
@@ -360,15 +364,14 @@ local makeIPlist = function(hosts)
         if nil ~= hosts[k].IPs then
             for l, w in pairs(hosts[k].IPs) do
                 if type(w) == "table" then
--- TODO - don't hard code deb.roundr.devuan.org.
                     -- Don't output the extra DNS-RR entries that are for admin reasons.
-                    if ((APT.options.roundRobin.value == k) and ("deb.roundr.devuan.org" == l)) or (APT.options.roundRobin.value ~= k) then 
+                    if ((APT.options.roundRobin.value == k) and (APT.options.roundRobinCname.value == l)) or (APT.options.roundRobin.value ~= k) then 
                         n[l] = {}
                         for i, u in pairs(w) do
                             if (APT.testing("IPv6") and ("AAAA" == u)) or ("A" == u) then
                                 local inRR = ""
                                 local lc = logCount(k, i)
-                                if checkRR then
+                                if checkRR and ('no' ~= hosts[k].DNSRR) then
                                     -- If there where errors, warnings, or timeouts, then it'll have that wrapped in font tags.
                                     inRR        = "<font color='green'><b>&#x2705;</b></font>"
                                     if nil ~= lc:find("<font color='") then
@@ -382,7 +385,8 @@ local makeIPlist = function(hosts)
                                         if f == nil then C("writing DNS-RR_good.txt file - " .. e) end
                                     end
                                 end
-                                if "yes" ~= hosts[k].DNSRR then inRR = "" end
+                                if "maybe" == hosts[k].DNSRR then inRR = "<font color='red'>&#x2753;</font>" end
+                                if "no"    == hosts[k].DNSRR then inRR = "" end
                                 local log = '[<a href="' .. adr .. k .. '_' .. i .. '">graphs</a>] &nbsp; '
                                 if "" == log then n[l][i] = u else n[l][log .. inRR .. ' ' .. revDNS(hosts, k, i) .. ' ' .. lc] = u end
                             end
@@ -392,7 +396,7 @@ local makeIPlist = function(hosts)
                     if (APT.testing("IPv6") and ("AAAA" == w)) or ("A" == w) then
                         local inRR      = ""
                         local lc = logCount(k, l)
-                        if checkRR then
+                        if checkRR and ('no' ~= hosts[k].DNSRR) then
                             -- If there where errors, warnings, or timeouts, then it'll have that wrapped in font tags.
                             inRR        = "<font color='green'><b>&#x2705;</b></font>"
                             if nil ~= lc:find("<font color='") then
@@ -406,7 +410,8 @@ local makeIPlist = function(hosts)
                                 if f == nil then C("writing DNS-RR_good.txt file - " .. e) end
                             end
                         end
-                        if "yes" ~= hosts[k].DNSRR then inRR = "" end
+                        if "maybe" == hosts[k].DNSRR then inRR = "<font color='red'>&#x2753;</font>" end
+                        if "no"    == hosts[k].DNSRR then inRR = "" end
                         local log = '[<a href="' .. adr .. k .. '_' .. l .. '">graphs</a>] &nbsp; '
                         if "" == log then n[l] = w else n[log .. inRR .. ' ' .. revDNS(hosts, k, l) .. ' ' .. lc] = w end
                     end
@@ -439,7 +444,9 @@ if nil == email then C("opening mirrors file - " .. e) else
                 "The full results of the mirror checking is available at the URLs:\n\n" ..
                 " https://borta.devuan.dev/apt-panopticon/results/Report-web.html\n    (updated once every hour)\n" ..
                 " https://sledjhamr.org/apt-panopticon/results/Report-web.html\n    (updated once every ten minutes)\n\n" ..
-                "Due to the nature of the tests, some errors or warnings will be \ncounted several times.  " ..
+                "Due to the nature of the tests, some errors or warnings will be \ncounted several times.  There will be some duplication.\n\n" ..
+                "Due to the nature of the tests, some errors or warnings will be \ncounted several times.  There will be some duplication.\n\n" ..
+                "Due to the nature of the tests, some errors or warnings will be \ncounted several times.  There will be some duplication.\n\n" ..
                 "Refer to the logs on the web page for details.\n\n" ..
                 "Please see below the current status of the Devuan Package Mirror \nnetwork:\n\n" ..
                 "==== package mirror status " .. os.date("!%F %H:%M") .. " GMT ====\n" ..
@@ -479,9 +486,9 @@ if nil == email then C("opening mirrors file - " .. e) else
                 "* This means that this protocol isn't actually supported, but the test was run ayway.\n\n" ..
                 "Thanks for your precious help in ensuring that Devuan GNU+Linux \nremains a universal, stable, dependable, free operating system.\n\n" ..
                 "You can get the source code from https://sledjhamr.org/cgit/apt-panopticon/about/ (main repo)\n" ..
-                "and from https://git.devuan.dev/onefang/apt-panopticon' (Devuan repo).\n" ..
+                "and from https://git.devuan.org/onefang/apt-panopticon' (Devuan repo).\n" ..
                 "You can get the cgp graphing source code from https://sledjhamr.org/cgit/apt-panopticon_cgp/about/ (main repo)\n" .. 
-                "and https://git.devuan.dev/onefang/apt-panopticon_cgp (Devuan repo)\n\n" ..
+                "and https://git.devuan.org/onefang/apt-panopticon_cgp (Devuan repo)\n\n" ..
                 "Love\n\n" ..
                 "The Dev1Devs\n\n")
     email:close()
@@ -512,6 +519,17 @@ local colours =
     '00880080',
     '00088080',
     '00008880',
+    '80000080',
+    '08000080',
+    '00800080',
+    '00080080',
+    '00008080',
+    '00000880',
+    'fff00080',
+    '0fff0080',
+    '00fff080',
+    '000fff80',
+    '0000fff0',
 }
 local g = {}
 local count = 0
@@ -526,6 +544,7 @@ for k, v in APT.orderedPairs(mirrors) do
     if APT.options.roundRobin.value ~= k then
         local c = colours[count]
         local name = string.format('%32s', k)
+        if c == nil then c = 'ffffff' end
         if APT.options.referenceSite.value == k then c = 'ffffff' end
         table.insert(g, 'DEF:speedn' .. count .. '=rrd/' .. k .. '/Speed/Speed.rrd:max:MIN')
         table.insert(g, 'DEF:speedx' .. count .. '=rrd/' .. k .. '/Speed/Speed.rrd:max:MAX')
@@ -576,7 +595,8 @@ if nil == web then C("opening mirrors file - " .. e) else
                 APT.options.roundRobin.value .. " is the DNS round robin, which points to the mirrors that are part of the DNS-RR. &nbsp; " ..
                 "If an IP is part of the DNS-RR, it is marked with '<font color='purple'><b>DNS-RR</b></font>'," ..
                 " if it <a href='DNS-RR_good.txt'>should be</a> it is marked with '<font color='green'><b>&#x2705;</b></font>'," ..
-                " if it <a href='DNS-RR_bad.txt'>should not be</a> it is marked with '<font color='red'>&#x274c;</font>'. &nbsp; " ..
+                " if it <a href='DNS-RR_bad.txt'>should not be</a> it is marked with '<font color='red'>&#x274c;</font>'," ..
+                " if it might be but still pending full testing, it is marked with '<font color='red'>&#x2753;</font>'. &nbsp; " ..
                 "<br>" ..
                 APT.options.referenceSite.value .. " is the master mirror, all the others copy files from it. &nbsp; " ..
                 "</p>\n"
@@ -603,12 +623,12 @@ if nil == web then C("opening mirrors file - " .. e) else
     web:write( "<hr>\n<hr>\n<p>The <a href='Report-email.txt'>email report</a>. &nbsp; " ..
                 "All <a href='../results'>the logs and other output</a>. &nbsp; " ..
                 "You can get the <a href='https://sledjhamr.org/cgit/apt-panopticon/about/'>source code here (main repo)</a>" ..
-                "and <a href='https://git.devuan.dev/onefang/apt-panopticon'>here (Devuan repo)</a>. &nbsp;" ..
+                "and <a href='https://git.devuan.org/onefang/apt-panopticon'>here (Devuan repo)</a>. &nbsp;" ..
                 "You can get the cgp graphing <a href='https://sledjhamr.org/cgit/apt-panopticon_cgp/about/'>source code here (main repo)</a>" .. 
-                "and <a href='https://git.devuan.dev/onefang/apt-panopticon_cgp'>here (Devuan repo)</a>.</p>\n"
+                "and <a href='https://git.devuan.org/onefang/apt-panopticon_cgp'>here (Devuan repo)</a>.</p>\n"
                 )
     local whn = APT.exe('TZ="GMT" ls -dl1 --time-style="+%s" results/stamp | cut -d " " -f 6-6'):Do().result:sub(2, -2)
-    web:write(  "<p>This run took " .. (os.time() - tonumber("0" .. whn)) .. " seconds.</p>" ..
+    web:write(  "<p>This run took " .. (os.time() - tonumber("0" .. whn)) .. " seconds. &nbsp &nbsp apt-panopticon version " .. APT.version ..  "  </p>" ..
                 "\n</body></html>\n")
     web:close()
 end
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index 4019fdf..50e6177 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -14,7 +14,6 @@ APT.html = true
 
 
 local defaultURL = {scheme = "http"}
-local releases = {"jessie", "ascii", "beowulf", "ceres"}
 local releaseFiles =
 {
     -- Release file.
@@ -22,7 +21,7 @@ local releaseFiles =
     "Release.gpg",                      --
 --    "InRelease",                      -- 3.7 MB
 --    "main/binary-all/Packages.xz",    -- 2.6 GB for all that changed recently.
-    -- Contents files.                  -- 3.3 GB
+--    Contents files.                   -- 3.3 GB
 --    "main/Contents-all.xz",
 --    "main/Contents-amd64.xz",
 --    "main/Contents-arm64.xz",
@@ -37,16 +36,21 @@ local notExist =
 local referenceDebs =
 {
     -- Debian package.
-    "merged/pool/DEBIAN/main/d/debian-keyring/debian-keyring_2019.02.25_all.deb",
+    "merged/pool/DEBIAN/main/d/debian-keyring/debian-keyring_2025.03.23_all.deb",
     -- Debian security package.  NOTE this one should always be redirected?
-    "merged/pool/DEBIAN-SECURITY/updates/main/a/apt/apt-transport-https_1.4.10_amd64.deb",
+    "merged/pool/DEBIAN-SECURITY/updates/main/e/exim4/exim4_4.96-15+deb12u7_all.deb",
 }
 local referenceDevs =
 {
     -- Devuan package.  NOTE this one should not get redirected, but that's more a warning than an error.
-    "merged/pool/DEVUAN/main/d/devuan-keyring/devuan-keyring_2017.10.03_all.deb",
+    "merged/pool/DEVUAN/main/d/devuan-keyring/devuan-keyring_2025.06.02_all.deb",       -- Devuan keeps changing this since the key expiry incident.
+    -- Hmmm used to be _all, but it was split to the individual archs.  Seems pointless, each one is the same size as the previous all.
+    "merged/pool/DEVUAN/main/b/base-files/base-files_13.8devuan1_amd64.deb",
 }
 
+local keyring = "/usr/share/keyrings/devuan-archive-keyring.gpg"
+--local keyring = "/etc/apt/trusted.gpg.d/devuan-keyring-2022-archive.gpg"
+
 local curlStatus = 
 {
     [1 ] = "Unsupported protocol. This build of curl has no support for this protocol.",
@@ -170,38 +174,76 @@ local repoExists = function (r)
 end
 
 local IP = {}
-gatherIPs = function (host)
+gatherIPs = function (host, again)
+    if nil == again then again = '' end
     if nil == IP[host] then
         local IPs
         -- Takes about 30 seconds to look up the lot.
         -- I tested using dig's -f option, it didn't seem much faster.
         -- The sort -r assumes that deb.devuan.org is the first alphabetically.
-        local dig = io.popen('dig +keepopen +noall +nottlid +answer ' .. host .. ' A ' .. host .. ' AAAA ' .. host .. ' CNAME ' .. host .. ' SRV | sort -r | uniq')
-        repeat
-            IPs = dig:read("*l")
-            if nil ~= IPs then
-                for k, t, v in IPs:gmatch("([%w_%-%.]*)%.%s*IN%s*(%a*)%s*(.*)") do
-                    if "." == v:sub(-1, -1) then v = v:sub(1, -2) end
-                    if nil == IP[k] then IP[k] = {} end
-                    IP[k][v] = t
-                    D("  DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v)
-                    if t == "CNAME" then
+        if "" == host then print("Empty host name!") end
+        local dig = APT.readCmd('dig ' .. again .. ' +keepopen +noall +nottlid +answer ' .. host .. ' A ' .. host .. ' AAAA ' .. host .. ' CNAME ' .. host .. ' SRV | sort -r | uniq')
+        for i,IPs in ipairs(dig) do
+            for k, t, v in IPs:gmatch("([%w_%-%.]*)%.%s*IN%s*(%a*)%s*(.*)") do
+                if "." == v:sub(-1, -1) then v = v:sub(1, -2) end
+                if nil == IP[k] then IP[k] = {} end
+                IP[k][v] = t
+                D("  DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v)
+                if t == "CNAME" then
+                    if "" == v then
+                        if '' ~= again then
+                            print("Empty host name!  DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v)
+                        else
+                            return gatherIPs(host, '@9.9.9.11')
+                        end
+                    else
                         gatherIPs(v)
                         IP[k][v] = IP[v]
-                    elseif v == "SRV" then
-                        print("SVR record found, now what do we do?")
+                    end
+                elseif v == "SRV" then
+                    print("SVR record found, now what do we do?")
+                elseif "" == v then
+                    if '' ~= again then
+                        print("Empty host name!  DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v)
+                    else
+                        return gatherIPs(host, '@9.9.9.11')
                     end
                 end
             end
-        until nil == IPs
+        end
     end
 
     -- If this is the DNS-RR domain name, gather the IPs for the mirrors that mirror_list.txt says should be in it.
     if host == APT.options.roundRobin.value then
         for k, m in pairs(APT.mirrors) do
-            if "yes" == m.DNSRR then
+            if ("yes" == m.DNSRR) or ("maybe" == m.DNSRR) then
+                if "" == m.FQDN then
+                    print("Empty FQDN name! " .. host)
+                end
                 gatherIPs(m.FQDN)
                 IP[host][m.FQDN] = IP[m.FQDN]
+                -- Strip them out so we don't test them twice.
+                if (nil ~= IP[m.FQDN]) and (nil ~= IP[host][APT.options.roundRobinCname.value]) then
+                    for l, n in pairs(IP[m.FQDN]) do
+                        if type(n) == 'table' then
+                            for h, p in pairs(n) do
+                                for j, o in pairs(IP[host][APT.options.roundRobinCname.value]) do
+                                    if h == j then IP[host][m.FQDN][l][h] = nil end
+                                end
+                                o = 0
+                                for j in pairs(IP[host][m.FQDN][l]) do o = o + 1 end
+                                if 0 == o then IP[host][m.FQDN][l] = nil end
+                            end
+                        else
+                            for j, o in pairs(IP[host][APT.options.roundRobinCname.value]) do
+                                if l == j then IP[host][m.FQDN][l] = nil end
+                            end
+                        end
+                        o = 0
+                        for j in pairs(IP[host][m.FQDN]) do o = o + 1 end
+                        if 0 == o then IP[host][m.FQDN] = nil end
+                    end
+                end
             end
         end
     end
@@ -235,7 +277,11 @@ checkHEAD = function (host, URL, r, retry, sanity)
         if "http" == PU.scheme then
             hdr = '-H "Host: ' .. host .. '"'
         end
-        IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"'
+        if '-6' == APT.IPv46 then
+            IP = '--connect-to "' .. pu.host .. '::[' .. PU.host .. ']:"'
+        else
+            IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"'
+        end
         fname = host .. "_" .. pu.host .. '_' .. PU.host .. "_" .. PU.path:gsub("/", "_") .. ".txt"
     end
     os.execute('rm -f results/HEADERS_' .. fname .. ' 2>/dev/null; rm -f results/STATUS_' .. fname .. ' 2>/dev/null; touch results/STATUS_' .. fname)
@@ -266,6 +312,10 @@ checkHEAD = function (host, URL, r, retry, sanity)
     if "https" == PU.scheme and APT.options.roundRobin.value == host then
         I(spcd .. "Not testing " .. APT.lnk(URL) .. " mirrors wont have the correct HTTPS certificate for the round robin.", host)
         return
+-- TODO - For some odd reason, sometimes one of these is nil.  Dig deeper to figure out why.
+    elseif "https" == PU.scheme and (APT.mirrors[host] ~= nil) and (APT.mirrors[host].Protocols ~= nil) and (not APT.mirrors[host].Protocols.https) then
+        I(spcd .. "Not testing " .. APT.lnk(URL) .. " host doesn't support HTTPS.", host)
+        return
     else
         I(spcd .. check .. " " .. APT.lnk(URL), host)
     end
@@ -289,7 +339,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
         'curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' .. APT.IPv46 .. ' ' ..
         IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' ..
         hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"'
-    ):Nice():log():Do().status
+    ):timeout(APT.options.maxtime.value * 2.0):Nice():log():Do().status
     if 0 < r then
         APT.tested(PU.scheme, 'Redirects', host)
     else
@@ -315,7 +365,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
     if 0 ~= status then
         local msg = curlStatus[status]
         if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end
-        if (28 == status) or (7 == status) then
+        if (128+9 == status) or (124 == status) or (28 == status) or (7 == status) then
             T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host)
             timeouts = timeouts + 1
         else
@@ -406,7 +456,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
                     local pth = path:match('^(.*/pool/).*$')
                     if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E(spcd .. spcd .. 'Odd redirect path ' .. path) end
                     I(spcd .. spcd .. "Now checking redirected host " .. u .. ' &nbsp; for &nbsp; ' .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
-                    APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork()
+                    APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host)
                     D(spcd .. 'logging to ' .. APT.logName(pu.host, nil, file)[2])
                     APT.tested(PU.scheme, 'Redirects', host)
                 end
@@ -454,7 +504,7 @@ local checkFiles = function (host, ip, path, file)
                 if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end
             end
         end
-        for i, s in pairs(releases) do
+        for i, s in pairs(APT.releases) do
             for j, k in pairs(releaseFiles) do
                 if repoExists(s .. k) then
                     if checkTimeouts(host, "http",  ip .. path .. "/merged/dists/" .. s .. '/' .. k) then return end
@@ -493,7 +543,7 @@ checkHost = function (orig, host, path, ip, file)
     else
         if orig == host then
             I("Testing mirror " .. orig .. "" .. file)
-            APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork()
+            APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(orig)
             D('logging to ' .. APT.logName(ph.host, nil, file)[2])
         else D("checkHost " .. orig .. arw .. host) end
     end
@@ -568,7 +618,7 @@ local downloads = function(host, URL, meta, release, list)
             return
         end
     else
-        for i, s in pairs(releases) do
+        for i, s in pairs(APT.releases) do
             for j, k in pairs(releaseFiles) do
                 if repoExists(s .. k) then
                     addDownload(host, URL, f, s, k)
@@ -577,12 +627,16 @@ local downloads = function(host, URL, meta, release, list)
         end
     end
     f:close()
-    APT.exe(cm):Nice():log():fork()
+    APT.exe(cm):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(host)
     D('logging to <a href="' .. log .. '">' .. log .. '</a>, with <a href="' .. files .. '">these files</a>')
 end
 
 
 local validateURL = function(m)
+    if "http://" == m.BaseURL:sub(1, 7) then
+        W("HTTP at beginning of BaseURL in mirror_list.txt!  " .. m.BaseURL, "", "", m.FQDN)
+        m.BaseURL = m.BaseURL:sub(1, -2)
+    end
     if " " == m.BaseURL:sub(-1, -1) then
         W("space at end of BaseURL in mirror_list.txt!  " .. m.BaseURL, "", "", m.FQDN)
         m.BaseURL = m.BaseURL:sub(1, -2)
@@ -594,8 +648,8 @@ local validateURL = function(m)
     local p = url.parse("http://" .. m.BaseURL)
     if nil == p.path then p.path = '' end
     if nil ~= p.port then p.authority = authority .. ':' .. p.port end
-    if nil == m.FDQN then W("Something wrong in FDQN from mirror_list.txt!  nil", "", "", p.authority) else
-        if m.FQDN ~= p.authority then W("Something wrong in FDQN from mirror_list.txt!  " .. m.FDQN, "", "", p.authority) end
+    if nil == m.FQDN then W("Something wrong in FQDN from mirror_list.txt!  nil", "", "", p.authority) else
+        if m.FQDN ~= p.authority then W("Something wrong in FQDN from mirror_list.txt!  " .. m.FQDN, "", "", p.authority) end
     end
     if nil == m.BaseURL then W("Something wrong in BaseURL from mirror_list.txt!  nil", "", "", p.authority) else
         if m.BaseURL ~= (p.authority .. p.path) then W("Something wrong in BaseURL from mirror_list.txt!  " .. m.BaseURL, "", "", p.authority) end
@@ -678,7 +732,7 @@ local postParse = function(host, list)
     if APT.options.referenceSite.value == host then
         if nil ~= list then
             local sem = 'results/NEW_' .. list.out .. '_%s.txt'
-            for i, n in pairs(releases) do
+            for i, n in pairs(APT.releases) do
                 local f = sem:format(n)
                 if APT.checkFile(f .. '.tmp') then
                     os.execute('mv ' .. f .. '.tmp ' .. f)
@@ -691,7 +745,7 @@ local postParse = function(host, list)
 end
 
 local parseDebs = function(host)
-    for i, n in pairs(releases) do
+    for i, n in pairs(APT.releases) do
         local inFile =  'results/NEW_debs_' .. n .. '.txt'
         local nfile, e = io.open(inFile, "r")
             if nil == nfile then W("opening " .. inFile .. " file - " .. e) else
@@ -733,7 +787,7 @@ end
 
 local parsePackages = function(host)
     local list = {inf = 'Packages', parser = parseDebs, out = 'debs', files = {}, nextf = ''}
-    for i, n in pairs(releases) do
+    for i, n in pairs(APT.releases) do
         local inFile =  'results/NEW_' .. list.inf  .. '_' .. n .. '.txt'
         local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt'
         if APT.options.referenceSite.value == host then
@@ -755,6 +809,7 @@ local parsePackages = function(host)
                         local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed', "w+")
                         if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed' .. ' file - ' .. e) else
                             local pp = {}
+-- TODO - FIX - check if this file exists first.
                             for l in io.lines('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages') do
                                 if "Package: "  == l:sub(1, 9) then
                                     if 0 ~= #pp then
@@ -786,12 +841,12 @@ local parsePackages = function(host)
                                                         ' | grep -E "^\\+" | grep -Ev "^\\+\\+\\+|^---" >>results/NEW_' .. list.out  .. '_TMP_' .. n .. '.txt')
                                 for i, s in pairs(referenceDebs) do
                                     if 0 == APT.exe('grep -q "' .. s:sub(8, -1) .. '" results/OLD_' .. list.out  .. '_' .. n .. '.txt'):log():Do().status then
-                                        print('Reference package is out of date (' .. n .. ') - ' .. s)
+                                        print('Reference package is out of date from ' .. host .. ' (' .. n .. ') - ' .. s)
                                     end
                                 end
                                 for i, s in pairs(referenceDevs) do
                                     if 0 == APT.exe('grep -q "' .. s:sub(8, -1) .. '" results/OLD_' .. list.out  .. '_' .. n .. '.txt'):log():Do().status then
-                                        print('Reference package is out of date (' .. n .. ') - ' .. s)
+                                        print('Reference package is out of date from ' .. host .. ' (' .. n .. ') - ' .. s)
                                     end
                                 end
                             else
@@ -839,13 +894,13 @@ local parseRelease = function(host)
     local list = {inf = 'Release', parser = parsePackages, out = 'Packages', files = {}, nextf = 'debs'}
     local updated = false
     local now = tonumber(os.date('%s'))
-    for i, n in pairs(releases) do
+    for i, n in pairs(APT.releases) do
         for l, o in pairs(releaseFiles) do
             if repoExists(i .. o) then
                 postDownload(host, n, o)
                 if (".gpg" == o:sub(-4, -1)) and (APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/' .. o)) then
                     if APT.testing("Integrity") then
-                        local status = APT.exe( "gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. 
+                        local status = APT.exe( "gpgv --keyring " .. keyring .. " results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. 
                                                 " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status
                         if 0 ~= status then E("GPG check failed for " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end
 -- TODO - should check the PGP sig of InRelease as well.
@@ -922,7 +977,7 @@ end
 
 local parseStart = function(host)
     local list = {inf = '', parser = parseRelease, out = 'Release', files = {}, nextf = 'Packages'}
-    for i, n in pairs(releases) do
+    for i, n in pairs(APT.releases) do
         local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt'
         for l, o in pairs(releaseFiles) do
             if repoExists(n .. o) then
@@ -991,6 +1046,7 @@ os.execute('sleep 1')	-- Wait for things to start up before checking for them.
               0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0* Connected to devuan.bio.lmu.de (141.84.43.19) port 80 (#0)
             curl: (22) The requested URL returned error: 404 Not Found
 ]]
+            local trace = {}
             local min, max, spd = 999999999999, 0
             local num = '[%d%.]+[kM]?'
             if APT.checkFile(f) then
@@ -1022,7 +1078,7 @@ os.execute('sleep 1')	-- Wait for things to start up before checking for them.
 
         if (APT.options.referenceSite.value ~= host) and ('' ~= list.nextf) then
             local sem = 'results/NEW_' .. list.nextf .. '_%s.txt'
-            for i, n in pairs(releases) do
+            for i, n in pairs(APT.releases) do
                 local f = sem:format(n)
                 while not APT.checkFile(f) do
                     D('*&lt;* About to yield coroutine while waiting on - not APT.checkFile(' .. f .. ')')
@@ -1066,7 +1122,12 @@ if 0 < #arg then
     I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", "))
     APT.mirrors = loadfile("results/mirrors.lua")()
     APT.results = APT.padResults(APT.results)
-    if APT.origin or APT.redir then APT.results["IPs"] = gatherIPs(pu.host) end
+    if APT.origin or APT.redir then
+        if "" == pu.host then
+            print("Empty pu.host name! " .. pu.host)
+        end
+        APT.results["IPs"] = gatherIPs(pu.host)
+    end
     if nil ~= arg[2] then I(" &nbsp; Using IP " .. arg[2]); ip = arg[2] end
     if nil ~= arg[3] then I(" &nbsp; Using file " .. arg[3]); end
 
@@ -1081,15 +1142,17 @@ if 0 < #arg then
                 APT.allpairs(ips,
                     function(k, v)
                         if v == "A" then
-                            if APT.testing("IPv4") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -4 ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):Nice():log():fork() end
+                            if APT.testing("IPv4") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -4 ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host) end
                         elseif v == "AAAA" then
-                            if APT.testing("IPv6") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -6 ' .. APT.IPv46 .. ' ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):Nice():log():fork() end
+                            if APT.testing("IPv6") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -6 ' .. APT.IPv46 .. ' ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host) end
                         end
                         D('logging to ' .. APT.logName(pu.host, k, file)[2])
                     end
                 )
             else
                 E("no IPs for " .. pu.host)
+                APT.logPost()
+                return
             end
         end
 
@@ -1108,6 +1171,7 @@ if 0 < #arg then
             checkFiles(pu.host, pu.host, pu.path:sub(1, -1), file);
         end
     else
+        if "-6" == APT.IPv46 then arg[2] = '[' .. arg[2] .. ']' end
         checkHost(pu.host, pu.host, pu.path, arg[2], arg[3])
     end
 
@@ -1178,16 +1242,15 @@ else
     while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 5") end
 
     local APT_args    = APT.args
-    local APT_logFile = APT.logFile
     local debians = {}
-    local srvs = io.popen('ls -1 results/*.lua')
-    for l in srvs:lines() do
+    local srvs = APT.readCmd('ls -1 results/*.lua')
+    for ii,l in ipairs(srvs) do
         local hst = l:sub(9, -5)
         if nil ~= l:find('_R%.lua') then hst = hst:sub(1, -3) end
         if (hst:find('_') == nil) and (nil == APT.mirrors[hst]) then
             local ips = loadfile(l)().IPs
             if nil ~= ips then
-                debians[hst] = {Country = '', FDQN = hst, Active = 'yes', Rate = '', BaseURL = hst, Protocols = {http = true, https = true}, Bandwidth = '', IPs = ips}
+                debians[hst] = {Country = '', FQDN = hst, Active = 'yes', Rate = '', BaseURL = hst, Protocols = {http = true, https = true}, Bandwidth = '', IPs = ips}
                 local baseFiles = {}
                 local IPfiles = {}
                 for i, a in pairs(ips) do
@@ -1198,8 +1261,8 @@ else
                         end
                     end
                 end
-                local files = io.popen('ls -1 results/LOG_' .. hst .. '_*.html')
-                for ll in files:lines() do
+                local files = APT.readCmd('ls -1 results/LOG_' .. hst .. '_*.html')
+                for iii,ll in ipairs(files) do
                     local dn = false
                     for i, a in pairs(ips) do
                         if type(a) == 'table' then
@@ -1220,6 +1283,7 @@ else
                 end
 
                 local combine = function(ip, a)
+                    local APT_logFile = APT.logFile
                     if not APT.logOpen(hst, ip) then
                         print('PROBLEM OPENING LOG FILE ' .. hst .. ' ' .. ip) 
                     else
@@ -1231,8 +1295,8 @@ else
                                 if ln:match('^' .. os.date('!%Y%-%m%-%d ') .. '.*$') then APT.logFile:write(ln .. '\n') end     -- %F isn't good enough, coz we have to escape the '-'.
                             end
                         end
+                        APT.logPost()
                     end
-                    APT.logPost()
                     APT.args = APT_args
                     APT.logFile = APT_logFile
                 end
diff --git a/laggers b/laggers
new file mode 100755
index 0000000..c5f30ee
--- /dev/null
+++ b/laggers
@@ -0,0 +1,3 @@
+#!/bin/bash
+echo "apt-panopticon processes still running -"
+ps ax -o pid,args --sort args | grep -E 'apt-panopticon\.lua | curl | dig ' | grep -v -E 'flock -n |grep -E |sh -c |timeout -k '
diff --git a/update_apt-panopticon b/update_apt-panopticon
index b70f4c3..969cc8f 100755
--- a/update_apt-panopticon
+++ b/update_apt-panopticon
@@ -1,16 +1,55 @@
 #!/bin/bash
 
-cd /var/www/html/apt-panopticon/apt-panopticon_cgp
-#git pull > /dev/null
-#chown -R mirrors:www-data *
-cd /var/www/html/apt-panopticon/apt-panopticon
-#git pull > /dev/null
-#chown -R mirrors:www-data *
-
-if [ ! -f apt-panopticon.lock ] ; then
-    rm ../results; ln -s apt-panopticon/results_old ../results
-    flock -n apt-panopticon.lock ./apt-panopticon.lua && rm apt-panopticon.lock
-    rm ../results; ln -s apt-panopticon/results ../results
+PANOPATH="/var/www/html/apt-panopticon"
+#KEEPDAYS=$((2 * 30))
+
+cd ${PANOPATH}/apt-panopticon_cgp
+if [ -d .git ] ; then
+    git pull > /dev/null
+    chown -hR www-data:www-data *
+fi
+cd ${PANOPATH}/apt-panopticon
+if [ -d .git ] ; then
+    git pull > /dev/null
+    chown -hR www-data:www-data *
+fi
+
+# Check if the lock file still exists.
+if [ -f apt-panopticon.lock ] ; then
+    # Check if it's still running.
+    ps ax -eo pid,args | grep "apt-panopticon.lua" | grep -v "grep apt-panopticon.lua" | while read line ; do touch apt-panopticon.running ; exit ; done
+    if [ -f apt-panopticon.running ] ; then
+        echo "Previous apt-panopticon still running, exiting."
+        echo "Previous apt-panopticon still running, exiting."
+        ./laggers
+        rm apt-panopticon.running
+        exit 1
+    fi
+    echo "Crashed apt-panopticon detected, removing stale lock file."
+    echo "Crashed apt-panopticon detected, removing stale lock file."
+    ./laggers
+    rm apt-panopticon.lock
+fi
+
+# Clean up any mess left over from failed runs.
+find results_* -maxdepth 2 ! -name "pkgmaster.devuan.org" -name "*.*" -type d -print0 | xargs -0rt /bin/rm -fr
+
+if [ "z" != "z${KEEPDAYS}" ] ; then
+    # Remove old results.
+    find . -daystart -maxdepth 2 -name "stamp"            -type f ! -mtime $((0 - ${KEEPDAYS})) -printf "%h\0" | xargs -0rt /bin/rm -fr
+    find . -daystart -maxdepth 2 -name "results_*.tar.xz" -type f ! -mtime $((0 - ${KEEPDAYS})) -print0        | xargs -0rt /bin/rm -f
+    find .           -maxdepth 1 -name "results_*"        -type d -empty -print0                               | xargs -0rt /bin/rm -fr
 fi
 
-#chown -R mirrors:www-data /var/www/html/SledjHamr/apt-panopticon/results
+rm ../results; ln -s apt-panopticon/results_old ../results
+flock -n apt-panopticon.lock ionice -c3 nice -n 19 timeout --kill-after=20.0 --foreground 8.5m ./apt-panopticon.lua && rm apt-panopticon.lock
+if [ -f apt-panopticon.lock ] ; then
+    echo "apt-panopticon timed out."
+    ./laggers
+fi
+rm ../results; ln -s apt-panopticon/results ../results
+
+chown -hR www-data:www-data *
+
+# Clean up any mess left over from THIS failed run.
+find results_* -maxdepth 2 ! -name "pkgmaster.devuan.org" -name "*.*" -type d -print0 | xargs -0rt /bin/rm -fr