1 files changed, 96 insertions, 46 deletions

diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index 4019fdf..bee4eac 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -14,7 +14,6 @@ APT.html = true
 
 
 local defaultURL = {scheme = "http"}
-local releases = {"jessie", "ascii", "beowulf", "ceres"}
 local releaseFiles =
 {
     -- Release file.
@@ -37,16 +36,20 @@ local notExist =
 local referenceDebs =
 {
     -- Debian package.
-    "merged/pool/DEBIAN/main/d/debian-keyring/debian-keyring_2019.02.25_all.deb",
+    "merged/pool/DEBIAN/main/d/debian-keyring/debian-keyring_2024.03.24_all.deb",
     -- Debian security package.  NOTE this one should always be redirected?
-    "merged/pool/DEBIAN-SECURITY/updates/main/a/apt/apt-transport-https_1.4.10_amd64.deb",
+    "merged/pool/DEBIAN-SECURITY/updates/main/a/apt/apt-transport-https_1.8.2.2_all.deb",
 }
 local referenceDevs =
 {
     -- Devuan package.  NOTE this one should not get redirected, but that's more a warning than an error.
-    "merged/pool/DEVUAN/main/d/devuan-keyring/devuan-keyring_2017.10.03_all.deb",
+    "merged/pool/DEVUAN/main/d/devuan-keyring/devuan-keyring_2023.10.07_all.deb",       -- Devuan keeps changing this since the key expiry incident.
+    "merged/pool/DEVUAN/main/b/base-files/base-files_13.1devuan1_all.deb",
 }
 
+local keyring = "/usr/share/keyrings/devuan-archive-keyring.gpg"
+--local keyring = "/etc/apt/trusted.gpg.d/devuan-keyring-2022-archive.gpg"
+
 local curlStatus = 
 {
     [1 ] = "Unsupported protocol. This build of curl has no support for this protocol.",
@@ -170,38 +173,76 @@ local repoExists = function (r)
 end
 
 local IP = {}
-gatherIPs = function (host)
+gatherIPs = function (host, again)
+    if nil == again then again = '' end
     if nil == IP[host] then
         local IPs
         -- Takes about 30 seconds to look up the lot.
         -- I tested using dig's -f option, it didn't seem much faster.
         -- The sort -r assumes that deb.devuan.org is the first alphabetically.
-        local dig = io.popen('dig +keepopen +noall +nottlid +answer ' .. host .. ' A ' .. host .. ' AAAA ' .. host .. ' CNAME ' .. host .. ' SRV | sort -r | uniq')
-        repeat
-            IPs = dig:read("*l")
-            if nil ~= IPs then
-                for k, t, v in IPs:gmatch("([%w_%-%.]*)%.%s*IN%s*(%a*)%s*(.*)") do
-                    if "." == v:sub(-1, -1) then v = v:sub(1, -2) end
-                    if nil == IP[k] then IP[k] = {} end
-                    IP[k][v] = t
-                    D("  DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v)
-                    if t == "CNAME" then
+        if "" == host then print("Empty host name!") end
+        local dig = APT.readCmd('dig ' .. again .. ' +keepopen +noall +nottlid +answer ' .. host .. ' A ' .. host .. ' AAAA ' .. host .. ' CNAME ' .. host .. ' SRV | sort -r | uniq')
+        for i,IPs in ipairs(dig) do
+            for k, t, v in IPs:gmatch("([%w_%-%.]*)%.%s*IN%s*(%a*)%s*(.*)") do
+                if "." == v:sub(-1, -1) then v = v:sub(1, -2) end
+                if nil == IP[k] then IP[k] = {} end
+                IP[k][v] = t
+                D("  DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v)
+                if t == "CNAME" then
+                    if "" == v then
+                        if '' ~= again then
+                            print("Empty host name!  DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v)
+                        else
+                            return gatherIPs(host, '@9.9.9.11')
+                        end
+                    else
                         gatherIPs(v)
                         IP[k][v] = IP[v]
-                    elseif v == "SRV" then
-                        print("SVR record found, now what do we do?")
+                    end
+                elseif v == "SRV" then
+                    print("SVR record found, now what do we do?")
+                elseif "" == v then
+                    if '' ~= again then
+                        print("Empty host name!  DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v)
+                    else
+                        return gatherIPs(host, '@9.9.9.11')
                     end
                 end
             end
-        until nil == IPs
+        end
     end
 
     -- If this is the DNS-RR domain name, gather the IPs for the mirrors that mirror_list.txt says should be in it.
     if host == APT.options.roundRobin.value then
         for k, m in pairs(APT.mirrors) do
-            if "yes" == m.DNSRR then
+            if ("yes" == m.DNSRR) or ("maybe" == m.DNSRR) then
+                if "" == m.FQDN then
+                    print("Empty FQDN name! " .. host)
+                end
                 gatherIPs(m.FQDN)
                 IP[host][m.FQDN] = IP[m.FQDN]
+                -- Strip them out so we don't test them twice.
+                if (nil ~= IP[m.FQDN]) and (nil ~= IP[host][APT.options.roundRobinCname.value]) then
+                    for l, n in pairs(IP[m.FQDN]) do
+                        if type(n) == 'table' then
+                            for h, p in pairs(n) do
+                                for j, o in pairs(IP[host][APT.options.roundRobinCname.value]) do
+                                    if h == j then IP[host][m.FQDN][l][h] = nil end
+                                end
+                                o = 0
+                                for j in pairs(IP[host][m.FQDN][l]) do o = o + 1 end
+                                if 0 == o then IP[host][m.FQDN][l] = nil end
+                            end
+                        else
+                            for j, o in pairs(IP[host][APT.options.roundRobinCname.value]) do
+                                if l == j then IP[host][m.FQDN][l] = nil end
+                            end
+                        end
+                        o = 0
+                        for j in pairs(IP[host][m.FQDN]) do o = o + 1 end
+                        if 0 == o then IP[host][m.FQDN] = nil end
+                    end
+                end
             end
         end
     end
@@ -289,7 +330,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
         'curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' .. APT.IPv46 .. ' ' ..
         IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' ..
         hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"'
-    ):Nice():log():Do().status
+    ):timeout(APT.options.maxtime.value * 2.0):Nice():log():Do().status
     if 0 < r then
         APT.tested(PU.scheme, 'Redirects', host)
     else
@@ -315,7 +356,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
     if 0 ~= status then
         local msg = curlStatus[status]
         if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end
-        if (28 == status) or (7 == status) then
+        if (128+9 == status) or (124 == status) or (28 == status) or (7 == status) then
             T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host)
             timeouts = timeouts + 1
         else
@@ -406,7 +447,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
                     local pth = path:match('^(.*/pool/).*$')
                     if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E(spcd .. spcd .. 'Odd redirect path ' .. path) end
                     I(spcd .. spcd .. "Now checking redirected host " .. u .. ' &nbsp; for &nbsp; ' .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
-                    APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork()
+                    APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host)
                     D(spcd .. 'logging to ' .. APT.logName(pu.host, nil, file)[2])
                     APT.tested(PU.scheme, 'Redirects', host)
                 end
@@ -454,7 +495,7 @@ local checkFiles = function (host, ip, path, file)
                 if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end
             end
         end
-        for i, s in pairs(releases) do
+        for i, s in pairs(APT.releases) do
             for j, k in pairs(releaseFiles) do
                 if repoExists(s .. k) then
                     if checkTimeouts(host, "http",  ip .. path .. "/merged/dists/" .. s .. '/' .. k) then return end
@@ -493,7 +534,7 @@ checkHost = function (orig, host, path, ip, file)
     else
         if orig == host then
             I("Testing mirror " .. orig .. "" .. file)
-            APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork()
+            APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(orig)
             D('logging to ' .. APT.logName(ph.host, nil, file)[2])
         else D("checkHost " .. orig .. arw .. host) end
     end
@@ -568,7 +609,7 @@ local downloads = function(host, URL, meta, release, list)
             return
         end
     else
-        for i, s in pairs(releases) do
+        for i, s in pairs(APT.releases) do
             for j, k in pairs(releaseFiles) do
                 if repoExists(s .. k) then
                     addDownload(host, URL, f, s, k)
@@ -577,7 +618,7 @@ local downloads = function(host, URL, meta, release, list)
         end
     end
     f:close()
-    APT.exe(cm):Nice():log():fork()
+    APT.exe(cm):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(host)
     D('logging to <a href="' .. log .. '">' .. log .. '</a>, with <a href="' .. files .. '">these files</a>')
 end
 
@@ -594,7 +635,7 @@ local validateURL = function(m)
     local p = url.parse("http://" .. m.BaseURL)
     if nil == p.path then p.path = '' end
     if nil ~= p.port then p.authority = authority .. ':' .. p.port end
-    if nil == m.FDQN then W("Something wrong in FDQN from mirror_list.txt!  nil", "", "", p.authority) else
+    if nil == m.FQDN then W("Something wrong in FQDN from mirror_list.txt!  nil", "", "", p.authority) else
         if m.FQDN ~= p.authority then W("Something wrong in FDQN from mirror_list.txt!  " .. m.FDQN, "", "", p.authority) end
     end
     if nil == m.BaseURL then W("Something wrong in BaseURL from mirror_list.txt!  nil", "", "", p.authority) else
@@ -678,7 +719,7 @@ local postParse = function(host, list)
     if APT.options.referenceSite.value == host then
         if nil ~= list then
             local sem = 'results/NEW_' .. list.out .. '_%s.txt'
-            for i, n in pairs(releases) do
+            for i, n in pairs(APT.releases) do
                 local f = sem:format(n)
                 if APT.checkFile(f .. '.tmp') then
                     os.execute('mv ' .. f .. '.tmp ' .. f)
@@ -691,7 +732,7 @@ local postParse = function(host, list)
 end
 
 local parseDebs = function(host)
-    for i, n in pairs(releases) do
+    for i, n in pairs(APT.releases) do
         local inFile =  'results/NEW_debs_' .. n .. '.txt'
         local nfile, e = io.open(inFile, "r")
             if nil == nfile then W("opening " .. inFile .. " file - " .. e) else
@@ -733,7 +774,7 @@ end
 
 local parsePackages = function(host)
     local list = {inf = 'Packages', parser = parseDebs, out = 'debs', files = {}, nextf = ''}
-    for i, n in pairs(releases) do
+    for i, n in pairs(APT.releases) do
         local inFile =  'results/NEW_' .. list.inf  .. '_' .. n .. '.txt'
         local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt'
         if APT.options.referenceSite.value == host then
@@ -755,6 +796,7 @@ local parsePackages = function(host)
                         local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed', "w+")
                         if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed' .. ' file - ' .. e) else
                             local pp = {}
+-- TODO - FIX - check if this file exists first.
                             for l in io.lines('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages') do
                                 if "Package: "  == l:sub(1, 9) then
                                     if 0 ~= #pp then
@@ -786,12 +828,12 @@ local parsePackages = function(host)
                                                         ' | grep -E "^\\+" | grep -Ev "^\\+\\+\\+|^---" >>results/NEW_' .. list.out  .. '_TMP_' .. n .. '.txt')
                                 for i, s in pairs(referenceDebs) do
                                     if 0 == APT.exe('grep -q "' .. s:sub(8, -1) .. '" results/OLD_' .. list.out  .. '_' .. n .. '.txt'):log():Do().status then
-                                        print('Reference package is out of date (' .. n .. ') - ' .. s)
+                                        print('Reference package is out of date from ' .. host .. ' (' .. n .. ') - ' .. s)
                                     end
                                 end
                                 for i, s in pairs(referenceDevs) do
                                     if 0 == APT.exe('grep -q "' .. s:sub(8, -1) .. '" results/OLD_' .. list.out  .. '_' .. n .. '.txt'):log():Do().status then
-                                        print('Reference package is out of date (' .. n .. ') - ' .. s)
+                                        print('Reference package is out of date from ' .. host .. ' (' .. n .. ') - ' .. s)
                                     end
                                 end
                             else
@@ -839,13 +881,13 @@ local parseRelease = function(host)
     local list = {inf = 'Release', parser = parsePackages, out = 'Packages', files = {}, nextf = 'debs'}
     local updated = false
     local now = tonumber(os.date('%s'))
-    for i, n in pairs(releases) do
+    for i, n in pairs(APT.releases) do
         for l, o in pairs(releaseFiles) do
             if repoExists(i .. o) then
                 postDownload(host, n, o)
                 if (".gpg" == o:sub(-4, -1)) and (APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/' .. o)) then
                     if APT.testing("Integrity") then
-                        local status = APT.exe( "gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. 
+                        local status = APT.exe( "gpgv --keyring " .. keyring .. " results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. 
                                                 " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status
                         if 0 ~= status then E("GPG check failed for " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end
 -- TODO - should check the PGP sig of InRelease as well.
@@ -922,7 +964,7 @@ end
 
 local parseStart = function(host)
     local list = {inf = '', parser = parseRelease, out = 'Release', files = {}, nextf = 'Packages'}
-    for i, n in pairs(releases) do
+    for i, n in pairs(APT.releases) do
         local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt'
         for l, o in pairs(releaseFiles) do
             if repoExists(n .. o) then
@@ -991,6 +1033,7 @@ os.execute('sleep 1')	-- Wait for things to start up before checking for them.
               0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0* Connected to devuan.bio.lmu.de (141.84.43.19) port 80 (#0)
             curl: (22) The requested URL returned error: 404 Not Found
 ]]
+            local trace = {}
             local min, max, spd = 999999999999, 0
             local num = '[%d%.]+[kM]?'
             if APT.checkFile(f) then
@@ -1022,7 +1065,7 @@ os.execute('sleep 1')	-- Wait for things to start up before checking for them.
 
         if (APT.options.referenceSite.value ~= host) and ('' ~= list.nextf) then
             local sem = 'results/NEW_' .. list.nextf .. '_%s.txt'
-            for i, n in pairs(releases) do
+            for i, n in pairs(APT.releases) do
                 local f = sem:format(n)
                 while not APT.checkFile(f) do
                     D('*&lt;* About to yield coroutine while waiting on - not APT.checkFile(' .. f .. ')')
@@ -1066,7 +1109,12 @@ if 0 < #arg then
     I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", "))
     APT.mirrors = loadfile("results/mirrors.lua")()
     APT.results = APT.padResults(APT.results)
-    if APT.origin or APT.redir then APT.results["IPs"] = gatherIPs(pu.host) end
+    if APT.origin or APT.redir then
+        if "" == pu.host then
+            print("Empty pu.host name! " .. pu.host)
+        end
+        APT.results["IPs"] = gatherIPs(pu.host)
+    end
     if nil ~= arg[2] then I(" &nbsp; Using IP " .. arg[2]); ip = arg[2] end
     if nil ~= arg[3] then I(" &nbsp; Using file " .. arg[3]); end
 
@@ -1081,15 +1129,17 @@ if 0 < #arg then
                 APT.allpairs(ips,
                     function(k, v)
                         if v == "A" then
-                            if APT.testing("IPv4") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -4 ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):Nice():log():fork() end
+                            if APT.testing("IPv4") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -4 ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host) end
                         elseif v == "AAAA" then
-                            if APT.testing("IPv6") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -6 ' .. APT.IPv46 .. ' ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):Nice():log():fork() end
+                            if APT.testing("IPv6") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -6 ' .. APT.IPv46 .. ' ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host) end
                         end
                         D('logging to ' .. APT.logName(pu.host, k, file)[2])
                     end
                 )
             else
                 E("no IPs for " .. pu.host)
+                APT.logPost()
+                return
             end
         end
 
@@ -1178,16 +1228,15 @@ else
     while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 5") end
 
     local APT_args    = APT.args
-    local APT_logFile = APT.logFile
     local debians = {}
-    local srvs = io.popen('ls -1 results/*.lua')
-    for l in srvs:lines() do
+    local srvs = APT.readCmd('ls -1 results/*.lua')
+    for ii,l in ipairs(srvs) do
         local hst = l:sub(9, -5)
         if nil ~= l:find('_R%.lua') then hst = hst:sub(1, -3) end
         if (hst:find('_') == nil) and (nil == APT.mirrors[hst]) then
             local ips = loadfile(l)().IPs
             if nil ~= ips then
-                debians[hst] = {Country = '', FDQN = hst, Active = 'yes', Rate = '', BaseURL = hst, Protocols = {http = true, https = true}, Bandwidth = '', IPs = ips}
+                debians[hst] = {Country = '', FQDN = hst, Active = 'yes', Rate = '', BaseURL = hst, Protocols = {http = true, https = true}, Bandwidth = '', IPs = ips}
                 local baseFiles = {}
                 local IPfiles = {}
                 for i, a in pairs(ips) do
@@ -1198,8 +1247,8 @@ else
                         end
                     end
                 end
-                local files = io.popen('ls -1 results/LOG_' .. hst .. '_*.html')
-                for ll in files:lines() do
+                local files = APT.readCmd('ls -1 results/LOG_' .. hst .. '_*.html')
+                for iii,ll in ipairs(files) do
                     local dn = false
                     for i, a in pairs(ips) do
                         if type(a) == 'table' then
@@ -1220,6 +1269,7 @@ else
                 end
 
                 local combine = function(ip, a)
+                    local APT_logFile = APT.logFile
                     if not APT.logOpen(hst, ip) then
                         print('PROBLEM OPENING LOG FILE ' .. hst .. ' ' .. ip) 
                     else
@@ -1231,8 +1281,8 @@ else
                                 if ln:match('^' .. os.date('!%Y%-%m%-%d ') .. '.*$') then APT.logFile:write(ln .. '\n') end     -- %F isn't good enough, coz we have to escape the '-'.
                             end
                         end
+                        APT.logPost()
                     end
-                    APT.logPost()
                     APT.args = APT_args
                     APT.logFile = APT_logFile
                 end