aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/apt-panopticon.lua
diff options
context:
space:
mode:
Diffstat (limited to 'apt-panopticon.lua')
-rwxr-xr-xapt-panopticon.lua162
1 files changed, 113 insertions, 49 deletions
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index 4019fdf..50e6177 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -14,7 +14,6 @@ APT.html = true
14 14
15 15
16local defaultURL = {scheme = "http"} 16local defaultURL = {scheme = "http"}
17local releases = {"jessie", "ascii", "beowulf", "ceres"}
18local releaseFiles = 17local releaseFiles =
19{ 18{
20 -- Release file. 19 -- Release file.
@@ -22,7 +21,7 @@ local releaseFiles =
22 "Release.gpg", -- 21 "Release.gpg", --
23-- "InRelease", -- 3.7 MB 22-- "InRelease", -- 3.7 MB
24-- "main/binary-all/Packages.xz", -- 2.6 GB for all that changed recently. 23-- "main/binary-all/Packages.xz", -- 2.6 GB for all that changed recently.
25 -- Contents files. -- 3.3 GB 24-- Contents files. -- 3.3 GB
26-- "main/Contents-all.xz", 25-- "main/Contents-all.xz",
27-- "main/Contents-amd64.xz", 26-- "main/Contents-amd64.xz",
28-- "main/Contents-arm64.xz", 27-- "main/Contents-arm64.xz",
@@ -37,16 +36,21 @@ local notExist =
37local referenceDebs = 36local referenceDebs =
38{ 37{
39 -- Debian package. 38 -- Debian package.
40 "merged/pool/DEBIAN/main/d/debian-keyring/debian-keyring_2019.02.25_all.deb", 39 "merged/pool/DEBIAN/main/d/debian-keyring/debian-keyring_2025.03.23_all.deb",
41 -- Debian security package. NOTE this one should always be redirected? 40 -- Debian security package. NOTE this one should always be redirected?
42 "merged/pool/DEBIAN-SECURITY/updates/main/a/apt/apt-transport-https_1.4.10_amd64.deb", 41 "merged/pool/DEBIAN-SECURITY/updates/main/e/exim4/exim4_4.96-15+deb12u7_all.deb",
43} 42}
44local referenceDevs = 43local referenceDevs =
45{ 44{
46 -- Devuan package. NOTE this one should not get redirected, but that's more a warning than an error. 45 -- Devuan package. NOTE this one should not get redirected, but that's more a warning than an error.
47 "merged/pool/DEVUAN/main/d/devuan-keyring/devuan-keyring_2017.10.03_all.deb", 46 "merged/pool/DEVUAN/main/d/devuan-keyring/devuan-keyring_2025.06.02_all.deb", -- Devuan keeps changing this since the key expiry incident.
47 -- Hmmm used to be _all, but it was split to the individual archs. Seems pointless, each one is the same size as the previous all.
48 "merged/pool/DEVUAN/main/b/base-files/base-files_13.8devuan1_amd64.deb",
48} 49}
49 50
51local keyring = "/usr/share/keyrings/devuan-archive-keyring.gpg"
52--local keyring = "/etc/apt/trusted.gpg.d/devuan-keyring-2022-archive.gpg"
53
50local curlStatus = 54local curlStatus =
51{ 55{
52 [1 ] = "Unsupported protocol. This build of curl has no support for this protocol.", 56 [1 ] = "Unsupported protocol. This build of curl has no support for this protocol.",
@@ -170,38 +174,76 @@ local repoExists = function (r)
170end 174end
171 175
172local IP = {} 176local IP = {}
173gatherIPs = function (host) 177gatherIPs = function (host, again)
178 if nil == again then again = '' end
174 if nil == IP[host] then 179 if nil == IP[host] then
175 local IPs 180 local IPs
176 -- Takes about 30 seconds to look up the lot. 181 -- Takes about 30 seconds to look up the lot.
177 -- I tested using dig's -f option, it didn't seem much faster. 182 -- I tested using dig's -f option, it didn't seem much faster.
178 -- The sort -r assumes that deb.devuan.org is the first alphabetically. 183 -- The sort -r assumes that deb.devuan.org is the first alphabetically.
179 local dig = io.popen('dig +keepopen +noall +nottlid +answer ' .. host .. ' A ' .. host .. ' AAAA ' .. host .. ' CNAME ' .. host .. ' SRV | sort -r | uniq') 184 if "" == host then print("Empty host name!") end
180 repeat 185 local dig = APT.readCmd('dig ' .. again .. ' +keepopen +noall +nottlid +answer ' .. host .. ' A ' .. host .. ' AAAA ' .. host .. ' CNAME ' .. host .. ' SRV | sort -r | uniq')
181 IPs = dig:read("*l") 186 for i,IPs in ipairs(dig) do
182 if nil ~= IPs then 187 for k, t, v in IPs:gmatch("([%w_%-%.]*)%.%s*IN%s*(%a*)%s*(.*)") do
183 for k, t, v in IPs:gmatch("([%w_%-%.]*)%.%s*IN%s*(%a*)%s*(.*)") do 188 if "." == v:sub(-1, -1) then v = v:sub(1, -2) end
184 if "." == v:sub(-1, -1) then v = v:sub(1, -2) end 189 if nil == IP[k] then IP[k] = {} end
185 if nil == IP[k] then IP[k] = {} end 190 IP[k][v] = t
186 IP[k][v] = t 191 D(" DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v)
187 D(" DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v) 192 if t == "CNAME" then
188 if t == "CNAME" then 193 if "" == v then
194 if '' ~= again then
195 print("Empty host name! DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v)
196 else
197 return gatherIPs(host, '@9.9.9.11')
198 end
199 else
189 gatherIPs(v) 200 gatherIPs(v)
190 IP[k][v] = IP[v] 201 IP[k][v] = IP[v]
191 elseif v == "SRV" then 202 end
192 print("SVR record found, now what do we do?") 203 elseif v == "SRV" then
204 print("SVR record found, now what do we do?")
205 elseif "" == v then
206 if '' ~= again then
207 print("Empty host name! DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v)
208 else
209 return gatherIPs(host, '@9.9.9.11')
193 end 210 end
194 end 211 end
195 end 212 end
196 until nil == IPs 213 end
197 end 214 end
198 215
199 -- If this is the DNS-RR domain name, gather the IPs for the mirrors that mirror_list.txt says should be in it. 216 -- If this is the DNS-RR domain name, gather the IPs for the mirrors that mirror_list.txt says should be in it.
200 if host == APT.options.roundRobin.value then 217 if host == APT.options.roundRobin.value then
201 for k, m in pairs(APT.mirrors) do 218 for k, m in pairs(APT.mirrors) do
202 if "yes" == m.DNSRR then 219 if ("yes" == m.DNSRR) or ("maybe" == m.DNSRR) then
220 if "" == m.FQDN then
221 print("Empty FQDN name! " .. host)
222 end
203 gatherIPs(m.FQDN) 223 gatherIPs(m.FQDN)
204 IP[host][m.FQDN] = IP[m.FQDN] 224 IP[host][m.FQDN] = IP[m.FQDN]
225 -- Strip them out so we don't test them twice.
226 if (nil ~= IP[m.FQDN]) and (nil ~= IP[host][APT.options.roundRobinCname.value]) then
227 for l, n in pairs(IP[m.FQDN]) do
228 if type(n) == 'table' then
229 for h, p in pairs(n) do
230 for j, o in pairs(IP[host][APT.options.roundRobinCname.value]) do
231 if h == j then IP[host][m.FQDN][l][h] = nil end
232 end
233 o = 0
234 for j in pairs(IP[host][m.FQDN][l]) do o = o + 1 end
235 if 0 == o then IP[host][m.FQDN][l] = nil end
236 end
237 else
238 for j, o in pairs(IP[host][APT.options.roundRobinCname.value]) do
239 if l == j then IP[host][m.FQDN][l] = nil end
240 end
241 end
242 o = 0
243 for j in pairs(IP[host][m.FQDN]) do o = o + 1 end
244 if 0 == o then IP[host][m.FQDN] = nil end
245 end
246 end
205 end 247 end
206 end 248 end
207 end 249 end
@@ -235,7 +277,11 @@ checkHEAD = function (host, URL, r, retry, sanity)
235 if "http" == PU.scheme then 277 if "http" == PU.scheme then
236 hdr = '-H "Host: ' .. host .. '"' 278 hdr = '-H "Host: ' .. host .. '"'
237 end 279 end
238 IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"' 280 if '-6' == APT.IPv46 then
281 IP = '--connect-to "' .. pu.host .. '::[' .. PU.host .. ']:"'
282 else
283 IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"'
284 end
239 fname = host .. "_" .. pu.host .. '_' .. PU.host .. "_" .. PU.path:gsub("/", "_") .. ".txt" 285 fname = host .. "_" .. pu.host .. '_' .. PU.host .. "_" .. PU.path:gsub("/", "_") .. ".txt"
240 end 286 end
241 os.execute('rm -f results/HEADERS_' .. fname .. ' 2>/dev/null; rm -f results/STATUS_' .. fname .. ' 2>/dev/null; touch results/STATUS_' .. fname) 287 os.execute('rm -f results/HEADERS_' .. fname .. ' 2>/dev/null; rm -f results/STATUS_' .. fname .. ' 2>/dev/null; touch results/STATUS_' .. fname)
@@ -266,6 +312,10 @@ checkHEAD = function (host, URL, r, retry, sanity)
266 if "https" == PU.scheme and APT.options.roundRobin.value == host then 312 if "https" == PU.scheme and APT.options.roundRobin.value == host then
267 I(spcd .. "Not testing " .. APT.lnk(URL) .. " mirrors wont have the correct HTTPS certificate for the round robin.", host) 313 I(spcd .. "Not testing " .. APT.lnk(URL) .. " mirrors wont have the correct HTTPS certificate for the round robin.", host)
268 return 314 return
315-- TODO - For some odd reason, sometimes one of these is nil. Dig deeper to figure out why.
316 elseif "https" == PU.scheme and (APT.mirrors[host] ~= nil) and (APT.mirrors[host].Protocols ~= nil) and (not APT.mirrors[host].Protocols.https) then
317 I(spcd .. "Not testing " .. APT.lnk(URL) .. " host doesn't support HTTPS.", host)
318 return
269 else 319 else
270 I(spcd .. check .. " " .. APT.lnk(URL), host) 320 I(spcd .. check .. " " .. APT.lnk(URL), host)
271 end 321 end
@@ -289,7 +339,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
289 'curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' .. APT.IPv46 .. ' ' .. 339 'curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' .. APT.IPv46 .. ' ' ..
290 IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' .. 340 IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' ..
291 hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"' 341 hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"'
292 ):Nice():log():Do().status 342 ):timeout(APT.options.maxtime.value * 2.0):Nice():log():Do().status
293 if 0 < r then 343 if 0 < r then
294 APT.tested(PU.scheme, 'Redirects', host) 344 APT.tested(PU.scheme, 'Redirects', host)
295 else 345 else
@@ -315,7 +365,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
315 if 0 ~= status then 365 if 0 ~= status then
316 local msg = curlStatus[status] 366 local msg = curlStatus[status]
317 if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end 367 if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end
318 if (28 == status) or (7 == status) then 368 if (128+9 == status) or (124 == status) or (28 == status) or (7 == status) then
319 T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host) 369 T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host)
320 timeouts = timeouts + 1 370 timeouts = timeouts + 1
321 else 371 else
@@ -406,7 +456,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
406 local pth = path:match('^(.*/pool/).*$') 456 local pth = path:match('^(.*/pool/).*$')
407 if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E(spcd .. spcd .. 'Odd redirect path ' .. path) end 457 if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E(spcd .. spcd .. 'Odd redirect path ' .. path) end
408 I(spcd .. spcd .. "Now checking redirected host " .. u .. ' &nbsp; for &nbsp; ' .. APT.lnk(URL) .. arw .. APT.lnk(location), host) 458 I(spcd .. spcd .. "Now checking redirected host " .. u .. ' &nbsp; for &nbsp; ' .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
409 APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork() 459 APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host)
410 D(spcd .. 'logging to ' .. APT.logName(pu.host, nil, file)[2]) 460 D(spcd .. 'logging to ' .. APT.logName(pu.host, nil, file)[2])
411 APT.tested(PU.scheme, 'Redirects', host) 461 APT.tested(PU.scheme, 'Redirects', host)
412 end 462 end
@@ -454,7 +504,7 @@ local checkFiles = function (host, ip, path, file)
454 if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end 504 if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end
455 end 505 end
456 end 506 end
457 for i, s in pairs(releases) do 507 for i, s in pairs(APT.releases) do
458 for j, k in pairs(releaseFiles) do 508 for j, k in pairs(releaseFiles) do
459 if repoExists(s .. k) then 509 if repoExists(s .. k) then
460 if checkTimeouts(host, "http", ip .. path .. "/merged/dists/" .. s .. '/' .. k) then return end 510 if checkTimeouts(host, "http", ip .. path .. "/merged/dists/" .. s .. '/' .. k) then return end
@@ -493,7 +543,7 @@ checkHost = function (orig, host, path, ip, file)
493 else 543 else
494 if orig == host then 544 if orig == host then
495 I("Testing mirror " .. orig .. "" .. file) 545 I("Testing mirror " .. orig .. "" .. file)
496 APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork() 546 APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(orig)
497 D('logging to ' .. APT.logName(ph.host, nil, file)[2]) 547 D('logging to ' .. APT.logName(ph.host, nil, file)[2])
498 else D("checkHost " .. orig .. arw .. host) end 548 else D("checkHost " .. orig .. arw .. host) end
499 end 549 end
@@ -568,7 +618,7 @@ local downloads = function(host, URL, meta, release, list)
568 return 618 return
569 end 619 end
570 else 620 else
571 for i, s in pairs(releases) do 621 for i, s in pairs(APT.releases) do
572 for j, k in pairs(releaseFiles) do 622 for j, k in pairs(releaseFiles) do
573 if repoExists(s .. k) then 623 if repoExists(s .. k) then
574 addDownload(host, URL, f, s, k) 624 addDownload(host, URL, f, s, k)
@@ -577,12 +627,16 @@ local downloads = function(host, URL, meta, release, list)
577 end 627 end
578 end 628 end
579 f:close() 629 f:close()
580 APT.exe(cm):Nice():log():fork() 630 APT.exe(cm):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(host)
581 D('logging to <a href="' .. log .. '">' .. log .. '</a>, with <a href="' .. files .. '">these files</a>') 631 D('logging to <a href="' .. log .. '">' .. log .. '</a>, with <a href="' .. files .. '">these files</a>')
582end 632end
583 633
584 634
585local validateURL = function(m) 635local validateURL = function(m)
636 if "http://" == m.BaseURL:sub(1, 7) then
637 W("HTTP at beginning of BaseURL in mirror_list.txt! " .. m.BaseURL, "", "", m.FQDN)
638 m.BaseURL = m.BaseURL:sub(1, -2)
639 end
586 if " " == m.BaseURL:sub(-1, -1) then 640 if " " == m.BaseURL:sub(-1, -1) then
587 W("space at end of BaseURL in mirror_list.txt! " .. m.BaseURL, "", "", m.FQDN) 641 W("space at end of BaseURL in mirror_list.txt! " .. m.BaseURL, "", "", m.FQDN)
588 m.BaseURL = m.BaseURL:sub(1, -2) 642 m.BaseURL = m.BaseURL:sub(1, -2)
@@ -594,8 +648,8 @@ local validateURL = function(m)
594 local p = url.parse("http://" .. m.BaseURL) 648 local p = url.parse("http://" .. m.BaseURL)
595 if nil == p.path then p.path = '' end 649 if nil == p.path then p.path = '' end
596 if nil ~= p.port then p.authority = authority .. ':' .. p.port end 650 if nil ~= p.port then p.authority = authority .. ':' .. p.port end
597 if nil == m.FDQN then W("Something wrong in FDQN from mirror_list.txt! nil", "", "", p.authority) else 651 if nil == m.FQDN then W("Something wrong in FQDN from mirror_list.txt! nil", "", "", p.authority) else
598 if m.FQDN ~= p.authority then W("Something wrong in FDQN from mirror_list.txt! " .. m.FDQN, "", "", p.authority) end 652 if m.FQDN ~= p.authority then W("Something wrong in FQDN from mirror_list.txt! " .. m.FQDN, "", "", p.authority) end
599 end 653 end
600 if nil == m.BaseURL then W("Something wrong in BaseURL from mirror_list.txt! nil", "", "", p.authority) else 654 if nil == m.BaseURL then W("Something wrong in BaseURL from mirror_list.txt! nil", "", "", p.authority) else
601 if m.BaseURL ~= (p.authority .. p.path) then W("Something wrong in BaseURL from mirror_list.txt! " .. m.BaseURL, "", "", p.authority) end 655 if m.BaseURL ~= (p.authority .. p.path) then W("Something wrong in BaseURL from mirror_list.txt! " .. m.BaseURL, "", "", p.authority) end
@@ -678,7 +732,7 @@ local postParse = function(host, list)
678 if APT.options.referenceSite.value == host then 732 if APT.options.referenceSite.value == host then
679 if nil ~= list then 733 if nil ~= list then
680 local sem = 'results/NEW_' .. list.out .. '_%s.txt' 734 local sem = 'results/NEW_' .. list.out .. '_%s.txt'
681 for i, n in pairs(releases) do 735 for i, n in pairs(APT.releases) do
682 local f = sem:format(n) 736 local f = sem:format(n)
683 if APT.checkFile(f .. '.tmp') then 737 if APT.checkFile(f .. '.tmp') then
684 os.execute('mv ' .. f .. '.tmp ' .. f) 738 os.execute('mv ' .. f .. '.tmp ' .. f)
@@ -691,7 +745,7 @@ local postParse = function(host, list)
691end 745end
692 746
693local parseDebs = function(host) 747local parseDebs = function(host)
694 for i, n in pairs(releases) do 748 for i, n in pairs(APT.releases) do
695 local inFile = 'results/NEW_debs_' .. n .. '.txt' 749 local inFile = 'results/NEW_debs_' .. n .. '.txt'
696 local nfile, e = io.open(inFile, "r") 750 local nfile, e = io.open(inFile, "r")
697 if nil == nfile then W("opening " .. inFile .. " file - " .. e) else 751 if nil == nfile then W("opening " .. inFile .. " file - " .. e) else
@@ -733,7 +787,7 @@ end
733 787
734local parsePackages = function(host) 788local parsePackages = function(host)
735 local list = {inf = 'Packages', parser = parseDebs, out = 'debs', files = {}, nextf = ''} 789 local list = {inf = 'Packages', parser = parseDebs, out = 'debs', files = {}, nextf = ''}
736 for i, n in pairs(releases) do 790 for i, n in pairs(APT.releases) do
737 local inFile = 'results/NEW_' .. list.inf .. '_' .. n .. '.txt' 791 local inFile = 'results/NEW_' .. list.inf .. '_' .. n .. '.txt'
738 local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt' 792 local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt'
739 if APT.options.referenceSite.value == host then 793 if APT.options.referenceSite.value == host then
@@ -755,6 +809,7 @@ local parsePackages = function(host)
755 local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed', "w+") 809 local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed', "w+")
756 if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed' .. ' file - ' .. e) else 810 if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed' .. ' file - ' .. e) else
757 local pp = {} 811 local pp = {}
812-- TODO - FIX - check if this file exists first.
758 for l in io.lines('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages') do 813 for l in io.lines('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages') do
759 if "Package: " == l:sub(1, 9) then 814 if "Package: " == l:sub(1, 9) then
760 if 0 ~= #pp then 815 if 0 ~= #pp then
@@ -786,12 +841,12 @@ local parsePackages = function(host)
786 ' | grep -E "^\\+" | grep -Ev "^\\+\\+\\+|^---" >>results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt') 841 ' | grep -E "^\\+" | grep -Ev "^\\+\\+\\+|^---" >>results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt')
787 for i, s in pairs(referenceDebs) do 842 for i, s in pairs(referenceDebs) do
788 if 0 == APT.exe('grep -q "' .. s:sub(8, -1) .. '" results/OLD_' .. list.out .. '_' .. n .. '.txt'):log():Do().status then 843 if 0 == APT.exe('grep -q "' .. s:sub(8, -1) .. '" results/OLD_' .. list.out .. '_' .. n .. '.txt'):log():Do().status then
789 print('Reference package is out of date (' .. n .. ') - ' .. s) 844 print('Reference package is out of date from ' .. host .. ' (' .. n .. ') - ' .. s)
790 end 845 end
791 end 846 end
792 for i, s in pairs(referenceDevs) do 847 for i, s in pairs(referenceDevs) do
793 if 0 == APT.exe('grep -q "' .. s:sub(8, -1) .. '" results/OLD_' .. list.out .. '_' .. n .. '.txt'):log():Do().status then 848 if 0 == APT.exe('grep -q "' .. s:sub(8, -1) .. '" results/OLD_' .. list.out .. '_' .. n .. '.txt'):log():Do().status then
794 print('Reference package is out of date (' .. n .. ') - ' .. s) 849 print('Reference package is out of date from ' .. host .. ' (' .. n .. ') - ' .. s)
795 end 850 end
796 end 851 end
797 else 852 else
@@ -839,13 +894,13 @@ local parseRelease = function(host)
839 local list = {inf = 'Release', parser = parsePackages, out = 'Packages', files = {}, nextf = 'debs'} 894 local list = {inf = 'Release', parser = parsePackages, out = 'Packages', files = {}, nextf = 'debs'}
840 local updated = false 895 local updated = false
841 local now = tonumber(os.date('%s')) 896 local now = tonumber(os.date('%s'))
842 for i, n in pairs(releases) do 897 for i, n in pairs(APT.releases) do
843 for l, o in pairs(releaseFiles) do 898 for l, o in pairs(releaseFiles) do
844 if repoExists(i .. o) then 899 if repoExists(i .. o) then
845 postDownload(host, n, o) 900 postDownload(host, n, o)
846 if (".gpg" == o:sub(-4, -1)) and (APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/' .. o)) then 901 if (".gpg" == o:sub(-4, -1)) and (APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/' .. o)) then
847 if APT.testing("Integrity") then 902 if APT.testing("Integrity") then
848 local status = APT.exe( "gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. 903 local status = APT.exe( "gpgv --keyring " .. keyring .. " results/" .. host .. "/merged/dists/" .. n .. '/' .. o ..
849 " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status 904 " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status
850 if 0 ~= status then E("GPG check failed for " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end 905 if 0 ~= status then E("GPG check failed for " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end
851-- TODO - should check the PGP sig of InRelease as well. 906-- TODO - should check the PGP sig of InRelease as well.
@@ -922,7 +977,7 @@ end
922 977
923local parseStart = function(host) 978local parseStart = function(host)
924 local list = {inf = '', parser = parseRelease, out = 'Release', files = {}, nextf = 'Packages'} 979 local list = {inf = '', parser = parseRelease, out = 'Release', files = {}, nextf = 'Packages'}
925 for i, n in pairs(releases) do 980 for i, n in pairs(APT.releases) do
926 local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt' 981 local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt'
927 for l, o in pairs(releaseFiles) do 982 for l, o in pairs(releaseFiles) do
928 if repoExists(n .. o) then 983 if repoExists(n .. o) then
@@ -991,6 +1046,7 @@ os.execute('sleep 1') -- Wait for things to start up before checking for them.
991 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Connected to devuan.bio.lmu.de (141.84.43.19) port 80 (#0) 1046 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Connected to devuan.bio.lmu.de (141.84.43.19) port 80 (#0)
992 curl: (22) The requested URL returned error: 404 Not Found 1047 curl: (22) The requested URL returned error: 404 Not Found
993]] 1048]]
1049 local trace = {}
994 local min, max, spd = 999999999999, 0 1050 local min, max, spd = 999999999999, 0
995 local num = '[%d%.]+[kM]?' 1051 local num = '[%d%.]+[kM]?'
996 if APT.checkFile(f) then 1052 if APT.checkFile(f) then
@@ -1022,7 +1078,7 @@ os.execute('sleep 1') -- Wait for things to start up before checking for them.
1022 1078
1023 if (APT.options.referenceSite.value ~= host) and ('' ~= list.nextf) then 1079 if (APT.options.referenceSite.value ~= host) and ('' ~= list.nextf) then
1024 local sem = 'results/NEW_' .. list.nextf .. '_%s.txt' 1080 local sem = 'results/NEW_' .. list.nextf .. '_%s.txt'
1025 for i, n in pairs(releases) do 1081 for i, n in pairs(APT.releases) do
1026 local f = sem:format(n) 1082 local f = sem:format(n)
1027 while not APT.checkFile(f) do 1083 while not APT.checkFile(f) do
1028 D('*&lt;* About to yield coroutine while waiting on - not APT.checkFile(' .. f .. ')') 1084 D('*&lt;* About to yield coroutine while waiting on - not APT.checkFile(' .. f .. ')')
@@ -1066,7 +1122,12 @@ if 0 < #arg then
1066 I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", ")) 1122 I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", "))
1067 APT.mirrors = loadfile("results/mirrors.lua")() 1123 APT.mirrors = loadfile("results/mirrors.lua")()
1068 APT.results = APT.padResults(APT.results) 1124 APT.results = APT.padResults(APT.results)
1069 if APT.origin or APT.redir then APT.results["IPs"] = gatherIPs(pu.host) end 1125 if APT.origin or APT.redir then
1126 if "" == pu.host then
1127 print("Empty pu.host name! " .. pu.host)
1128 end
1129 APT.results["IPs"] = gatherIPs(pu.host)
1130 end
1070 if nil ~= arg[2] then I(" &nbsp; Using IP " .. arg[2]); ip = arg[2] end 1131 if nil ~= arg[2] then I(" &nbsp; Using IP " .. arg[2]); ip = arg[2] end
1071 if nil ~= arg[3] then I(" &nbsp; Using file " .. arg[3]); end 1132 if nil ~= arg[3] then I(" &nbsp; Using file " .. arg[3]); end
1072 1133
@@ -1081,15 +1142,17 @@ if 0 < #arg then
1081 APT.allpairs(ips, 1142 APT.allpairs(ips,
1082 function(k, v) 1143 function(k, v)
1083 if v == "A" then 1144 if v == "A" then
1084 if APT.testing("IPv4") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -4 ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):Nice():log():fork() end 1145 if APT.testing("IPv4") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -4 ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host) end
1085 elseif v == "AAAA" then 1146 elseif v == "AAAA" then
1086 if APT.testing("IPv6") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -6 ' .. APT.IPv46 .. ' ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):Nice():log():fork() end 1147 if APT.testing("IPv6") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -6 ' .. APT.IPv46 .. ' ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host) end
1087 end 1148 end
1088 D('logging to ' .. APT.logName(pu.host, k, file)[2]) 1149 D('logging to ' .. APT.logName(pu.host, k, file)[2])
1089 end 1150 end
1090 ) 1151 )
1091 else 1152 else
1092 E("no IPs for " .. pu.host) 1153 E("no IPs for " .. pu.host)
1154 APT.logPost()
1155 return
1093 end 1156 end
1094 end 1157 end
1095 1158
@@ -1108,6 +1171,7 @@ if 0 < #arg then
1108 checkFiles(pu.host, pu.host, pu.path:sub(1, -1), file); 1171 checkFiles(pu.host, pu.host, pu.path:sub(1, -1), file);
1109 end 1172 end
1110 else 1173 else
1174 if "-6" == APT.IPv46 then arg[2] = '[' .. arg[2] .. ']' end
1111 checkHost(pu.host, pu.host, pu.path, arg[2], arg[3]) 1175 checkHost(pu.host, pu.host, pu.path, arg[2], arg[3])
1112 end 1176 end
1113 1177
@@ -1178,16 +1242,15 @@ else
1178 while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 5") end 1242 while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 5") end
1179 1243
1180 local APT_args = APT.args 1244 local APT_args = APT.args
1181 local APT_logFile = APT.logFile
1182 local debians = {} 1245 local debians = {}
1183 local srvs = io.popen('ls -1 results/*.lua') 1246 local srvs = APT.readCmd('ls -1 results/*.lua')
1184 for l in srvs:lines() do 1247 for ii,l in ipairs(srvs) do
1185 local hst = l:sub(9, -5) 1248 local hst = l:sub(9, -5)
1186 if nil ~= l:find('_R%.lua') then hst = hst:sub(1, -3) end 1249 if nil ~= l:find('_R%.lua') then hst = hst:sub(1, -3) end
1187 if (hst:find('_') == nil) and (nil == APT.mirrors[hst]) then 1250 if (hst:find('_') == nil) and (nil == APT.mirrors[hst]) then
1188 local ips = loadfile(l)().IPs 1251 local ips = loadfile(l)().IPs
1189 if nil ~= ips then 1252 if nil ~= ips then
1190 debians[hst] = {Country = '', FDQN = hst, Active = 'yes', Rate = '', BaseURL = hst, Protocols = {http = true, https = true}, Bandwidth = '', IPs = ips} 1253 debians[hst] = {Country = '', FQDN = hst, Active = 'yes', Rate = '', BaseURL = hst, Protocols = {http = true, https = true}, Bandwidth = '', IPs = ips}
1191 local baseFiles = {} 1254 local baseFiles = {}
1192 local IPfiles = {} 1255 local IPfiles = {}
1193 for i, a in pairs(ips) do 1256 for i, a in pairs(ips) do
@@ -1198,8 +1261,8 @@ else
1198 end 1261 end
1199 end 1262 end
1200 end 1263 end
1201 local files = io.popen('ls -1 results/LOG_' .. hst .. '_*.html') 1264 local files = APT.readCmd('ls -1 results/LOG_' .. hst .. '_*.html')
1202 for ll in files:lines() do 1265 for iii,ll in ipairs(files) do
1203 local dn = false 1266 local dn = false
1204 for i, a in pairs(ips) do 1267 for i, a in pairs(ips) do
1205 if type(a) == 'table' then 1268 if type(a) == 'table' then
@@ -1220,6 +1283,7 @@ else
1220 end 1283 end
1221 1284
1222 local combine = function(ip, a) 1285 local combine = function(ip, a)
1286 local APT_logFile = APT.logFile
1223 if not APT.logOpen(hst, ip) then 1287 if not APT.logOpen(hst, ip) then
1224 print('PROBLEM OPENING LOG FILE ' .. hst .. ' ' .. ip) 1288 print('PROBLEM OPENING LOG FILE ' .. hst .. ' ' .. ip)
1225 else 1289 else
@@ -1231,8 +1295,8 @@ else
1231 if ln:match('^' .. os.date('!%Y%-%m%-%d ') .. '.*$') then APT.logFile:write(ln .. '\n') end -- %F isn't good enough, coz we have to escape the '-'. 1295 if ln:match('^' .. os.date('!%Y%-%m%-%d ') .. '.*$') then APT.logFile:write(ln .. '\n') end -- %F isn't good enough, coz we have to escape the '-'.
1232 end 1296 end
1233 end 1297 end
1298 APT.logPost()
1234 end 1299 end
1235 APT.logPost()
1236 APT.args = APT_args 1300 APT.args = APT_args
1237 APT.logFile = APT_logFile 1301 APT.logFile = APT_logFile
1238 end 1302 end