diff options
Diffstat (limited to 'apt-panopticon.lua')
-rwxr-xr-x | apt-panopticon.lua | 162 |
1 files changed, 113 insertions, 49 deletions
diff --git a/apt-panopticon.lua b/apt-panopticon.lua index 4019fdf..50e6177 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua | |||
@@ -14,7 +14,6 @@ APT.html = true | |||
14 | 14 | ||
15 | 15 | ||
16 | local defaultURL = {scheme = "http"} | 16 | local defaultURL = {scheme = "http"} |
17 | local releases = {"jessie", "ascii", "beowulf", "ceres"} | ||
18 | local releaseFiles = | 17 | local releaseFiles = |
19 | { | 18 | { |
20 | -- Release file. | 19 | -- Release file. |
@@ -22,7 +21,7 @@ local releaseFiles = | |||
22 | "Release.gpg", -- | 21 | "Release.gpg", -- |
23 | -- "InRelease", -- 3.7 MB | 22 | -- "InRelease", -- 3.7 MB |
24 | -- "main/binary-all/Packages.xz", -- 2.6 GB for all that changed recently. | 23 | -- "main/binary-all/Packages.xz", -- 2.6 GB for all that changed recently. |
25 | -- Contents files. -- 3.3 GB | 24 | -- Contents files. -- 3.3 GB |
26 | -- "main/Contents-all.xz", | 25 | -- "main/Contents-all.xz", |
27 | -- "main/Contents-amd64.xz", | 26 | -- "main/Contents-amd64.xz", |
28 | -- "main/Contents-arm64.xz", | 27 | -- "main/Contents-arm64.xz", |
@@ -37,16 +36,21 @@ local notExist = | |||
37 | local referenceDebs = | 36 | local referenceDebs = |
38 | { | 37 | { |
39 | -- Debian package. | 38 | -- Debian package. |
40 | "merged/pool/DEBIAN/main/d/debian-keyring/debian-keyring_2019.02.25_all.deb", | 39 | "merged/pool/DEBIAN/main/d/debian-keyring/debian-keyring_2025.03.23_all.deb", |
41 | -- Debian security package. NOTE this one should always be redirected? | 40 | -- Debian security package. NOTE this one should always be redirected? |
42 | "merged/pool/DEBIAN-SECURITY/updates/main/a/apt/apt-transport-https_1.4.10_amd64.deb", | 41 | "merged/pool/DEBIAN-SECURITY/updates/main/e/exim4/exim4_4.96-15+deb12u7_all.deb", |
43 | } | 42 | } |
44 | local referenceDevs = | 43 | local referenceDevs = |
45 | { | 44 | { |
46 | -- Devuan package. NOTE this one should not get redirected, but that's more a warning than an error. | 45 | -- Devuan package. NOTE this one should not get redirected, but that's more a warning than an error. |
47 | "merged/pool/DEVUAN/main/d/devuan-keyring/devuan-keyring_2017.10.03_all.deb", | 46 | "merged/pool/DEVUAN/main/d/devuan-keyring/devuan-keyring_2025.06.02_all.deb", -- Devuan keeps changing this since the key expiry incident. |
47 | -- Hmmm used to be _all, but it was split to the individual archs. Seems pointless, each one is the same size as the previous all. | ||
48 | "merged/pool/DEVUAN/main/b/base-files/base-files_13.8devuan1_amd64.deb", | ||
48 | } | 49 | } |
49 | 50 | ||
51 | local keyring = "/usr/share/keyrings/devuan-archive-keyring.gpg" | ||
52 | --local keyring = "/etc/apt/trusted.gpg.d/devuan-keyring-2022-archive.gpg" | ||
53 | |||
50 | local curlStatus = | 54 | local curlStatus = |
51 | { | 55 | { |
52 | [1 ] = "Unsupported protocol. This build of curl has no support for this protocol.", | 56 | [1 ] = "Unsupported protocol. This build of curl has no support for this protocol.", |
@@ -170,38 +174,76 @@ local repoExists = function (r) | |||
170 | end | 174 | end |
171 | 175 | ||
172 | local IP = {} | 176 | local IP = {} |
173 | gatherIPs = function (host) | 177 | gatherIPs = function (host, again) |
178 | if nil == again then again = '' end | ||
174 | if nil == IP[host] then | 179 | if nil == IP[host] then |
175 | local IPs | 180 | local IPs |
176 | -- Takes about 30 seconds to look up the lot. | 181 | -- Takes about 30 seconds to look up the lot. |
177 | -- I tested using dig's -f option, it didn't seem much faster. | 182 | -- I tested using dig's -f option, it didn't seem much faster. |
178 | -- The sort -r assumes that deb.devuan.org is the first alphabetically. | 183 | -- The sort -r assumes that deb.devuan.org is the first alphabetically. |
179 | local dig = io.popen('dig +keepopen +noall +nottlid +answer ' .. host .. ' A ' .. host .. ' AAAA ' .. host .. ' CNAME ' .. host .. ' SRV | sort -r | uniq') | 184 | if "" == host then print("Empty host name!") end |
180 | repeat | 185 | local dig = APT.readCmd('dig ' .. again .. ' +keepopen +noall +nottlid +answer ' .. host .. ' A ' .. host .. ' AAAA ' .. host .. ' CNAME ' .. host .. ' SRV | sort -r | uniq') |
181 | IPs = dig:read("*l") | 186 | for i,IPs in ipairs(dig) do |
182 | if nil ~= IPs then | 187 | for k, t, v in IPs:gmatch("([%w_%-%.]*)%.%s*IN%s*(%a*)%s*(.*)") do |
183 | for k, t, v in IPs:gmatch("([%w_%-%.]*)%.%s*IN%s*(%a*)%s*(.*)") do | 188 | if "." == v:sub(-1, -1) then v = v:sub(1, -2) end |
184 | if "." == v:sub(-1, -1) then v = v:sub(1, -2) end | 189 | if nil == IP[k] then IP[k] = {} end |
185 | if nil == IP[k] then IP[k] = {} end | 190 | IP[k][v] = t |
186 | IP[k][v] = t | 191 | D(" DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v) |
187 | D(" DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v) | 192 | if t == "CNAME" then |
188 | if t == "CNAME" then | 193 | if "" == v then |
194 | if '' ~= again then | ||
195 | print("Empty host name! DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v) | ||
196 | else | ||
197 | return gatherIPs(host, '@9.9.9.11') | ||
198 | end | ||
199 | else | ||
189 | gatherIPs(v) | 200 | gatherIPs(v) |
190 | IP[k][v] = IP[v] | 201 | IP[k][v] = IP[v] |
191 | elseif v == "SRV" then | 202 | end |
192 | print("SVR record found, now what do we do?") | 203 | elseif v == "SRV" then |
204 | print("SVR record found, now what do we do?") | ||
205 | elseif "" == v then | ||
206 | if '' ~= again then | ||
207 | print("Empty host name! DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v) | ||
208 | else | ||
209 | return gatherIPs(host, '@9.9.9.11') | ||
193 | end | 210 | end |
194 | end | 211 | end |
195 | end | 212 | end |
196 | until nil == IPs | 213 | end |
197 | end | 214 | end |
198 | 215 | ||
199 | -- If this is the DNS-RR domain name, gather the IPs for the mirrors that mirror_list.txt says should be in it. | 216 | -- If this is the DNS-RR domain name, gather the IPs for the mirrors that mirror_list.txt says should be in it. |
200 | if host == APT.options.roundRobin.value then | 217 | if host == APT.options.roundRobin.value then |
201 | for k, m in pairs(APT.mirrors) do | 218 | for k, m in pairs(APT.mirrors) do |
202 | if "yes" == m.DNSRR then | 219 | if ("yes" == m.DNSRR) or ("maybe" == m.DNSRR) then |
220 | if "" == m.FQDN then | ||
221 | print("Empty FQDN name! " .. host) | ||
222 | end | ||
203 | gatherIPs(m.FQDN) | 223 | gatherIPs(m.FQDN) |
204 | IP[host][m.FQDN] = IP[m.FQDN] | 224 | IP[host][m.FQDN] = IP[m.FQDN] |
225 | -- Strip them out so we don't test them twice. | ||
226 | if (nil ~= IP[m.FQDN]) and (nil ~= IP[host][APT.options.roundRobinCname.value]) then | ||
227 | for l, n in pairs(IP[m.FQDN]) do | ||
228 | if type(n) == 'table' then | ||
229 | for h, p in pairs(n) do | ||
230 | for j, o in pairs(IP[host][APT.options.roundRobinCname.value]) do | ||
231 | if h == j then IP[host][m.FQDN][l][h] = nil end | ||
232 | end | ||
233 | o = 0 | ||
234 | for j in pairs(IP[host][m.FQDN][l]) do o = o + 1 end | ||
235 | if 0 == o then IP[host][m.FQDN][l] = nil end | ||
236 | end | ||
237 | else | ||
238 | for j, o in pairs(IP[host][APT.options.roundRobinCname.value]) do | ||
239 | if l == j then IP[host][m.FQDN][l] = nil end | ||
240 | end | ||
241 | end | ||
242 | o = 0 | ||
243 | for j in pairs(IP[host][m.FQDN]) do o = o + 1 end | ||
244 | if 0 == o then IP[host][m.FQDN] = nil end | ||
245 | end | ||
246 | end | ||
205 | end | 247 | end |
206 | end | 248 | end |
207 | end | 249 | end |
@@ -235,7 +277,11 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
235 | if "http" == PU.scheme then | 277 | if "http" == PU.scheme then |
236 | hdr = '-H "Host: ' .. host .. '"' | 278 | hdr = '-H "Host: ' .. host .. '"' |
237 | end | 279 | end |
238 | IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"' | 280 | if '-6' == APT.IPv46 then |
281 | IP = '--connect-to "' .. pu.host .. '::[' .. PU.host .. ']:"' | ||
282 | else | ||
283 | IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"' | ||
284 | end | ||
239 | fname = host .. "_" .. pu.host .. '_' .. PU.host .. "_" .. PU.path:gsub("/", "_") .. ".txt" | 285 | fname = host .. "_" .. pu.host .. '_' .. PU.host .. "_" .. PU.path:gsub("/", "_") .. ".txt" |
240 | end | 286 | end |
241 | os.execute('rm -f results/HEADERS_' .. fname .. ' 2>/dev/null; rm -f results/STATUS_' .. fname .. ' 2>/dev/null; touch results/STATUS_' .. fname) | 287 | os.execute('rm -f results/HEADERS_' .. fname .. ' 2>/dev/null; rm -f results/STATUS_' .. fname .. ' 2>/dev/null; touch results/STATUS_' .. fname) |
@@ -266,6 +312,10 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
266 | if "https" == PU.scheme and APT.options.roundRobin.value == host then | 312 | if "https" == PU.scheme and APT.options.roundRobin.value == host then |
267 | I(spcd .. "Not testing " .. APT.lnk(URL) .. " mirrors wont have the correct HTTPS certificate for the round robin.", host) | 313 | I(spcd .. "Not testing " .. APT.lnk(URL) .. " mirrors wont have the correct HTTPS certificate for the round robin.", host) |
268 | return | 314 | return |
315 | -- TODO - For some odd reason, sometimes one of these is nil. Dig deeper to figure out why. | ||
316 | elseif "https" == PU.scheme and (APT.mirrors[host] ~= nil) and (APT.mirrors[host].Protocols ~= nil) and (not APT.mirrors[host].Protocols.https) then | ||
317 | I(spcd .. "Not testing " .. APT.lnk(URL) .. " host doesn't support HTTPS.", host) | ||
318 | return | ||
269 | else | 319 | else |
270 | I(spcd .. check .. " " .. APT.lnk(URL), host) | 320 | I(spcd .. check .. " " .. APT.lnk(URL), host) |
271 | end | 321 | end |
@@ -289,7 +339,7 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
289 | 'curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' .. APT.IPv46 .. ' ' .. | 339 | 'curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' .. APT.IPv46 .. ' ' .. |
290 | IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' .. | 340 | IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' .. |
291 | hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"' | 341 | hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"' |
292 | ):Nice():log():Do().status | 342 | ):timeout(APT.options.maxtime.value * 2.0):Nice():log():Do().status |
293 | if 0 < r then | 343 | if 0 < r then |
294 | APT.tested(PU.scheme, 'Redirects', host) | 344 | APT.tested(PU.scheme, 'Redirects', host) |
295 | else | 345 | else |
@@ -315,7 +365,7 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
315 | if 0 ~= status then | 365 | if 0 ~= status then |
316 | local msg = curlStatus[status] | 366 | local msg = curlStatus[status] |
317 | if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end | 367 | if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end |
318 | if (28 == status) or (7 == status) then | 368 | if (128+9 == status) or (124 == status) or (28 == status) or (7 == status) then |
319 | T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host) | 369 | T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host) |
320 | timeouts = timeouts + 1 | 370 | timeouts = timeouts + 1 |
321 | else | 371 | else |
@@ -406,7 +456,7 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
406 | local pth = path:match('^(.*/pool/).*$') | 456 | local pth = path:match('^(.*/pool/).*$') |
407 | if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E(spcd .. spcd .. 'Odd redirect path ' .. path) end | 457 | if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E(spcd .. spcd .. 'Odd redirect path ' .. path) end |
408 | I(spcd .. spcd .. "Now checking redirected host " .. u .. ' for ' .. APT.lnk(URL) .. arw .. APT.lnk(location), host) | 458 | I(spcd .. spcd .. "Now checking redirected host " .. u .. ' for ' .. APT.lnk(URL) .. arw .. APT.lnk(location), host) |
409 | APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork() | 459 | APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host) |
410 | D(spcd .. 'logging to ' .. APT.logName(pu.host, nil, file)[2]) | 460 | D(spcd .. 'logging to ' .. APT.logName(pu.host, nil, file)[2]) |
411 | APT.tested(PU.scheme, 'Redirects', host) | 461 | APT.tested(PU.scheme, 'Redirects', host) |
412 | end | 462 | end |
@@ -454,7 +504,7 @@ local checkFiles = function (host, ip, path, file) | |||
454 | if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end | 504 | if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end |
455 | end | 505 | end |
456 | end | 506 | end |
457 | for i, s in pairs(releases) do | 507 | for i, s in pairs(APT.releases) do |
458 | for j, k in pairs(releaseFiles) do | 508 | for j, k in pairs(releaseFiles) do |
459 | if repoExists(s .. k) then | 509 | if repoExists(s .. k) then |
460 | if checkTimeouts(host, "http", ip .. path .. "/merged/dists/" .. s .. '/' .. k) then return end | 510 | if checkTimeouts(host, "http", ip .. path .. "/merged/dists/" .. s .. '/' .. k) then return end |
@@ -493,7 +543,7 @@ checkHost = function (orig, host, path, ip, file) | |||
493 | else | 543 | else |
494 | if orig == host then | 544 | if orig == host then |
495 | I("Testing mirror " .. orig .. "" .. file) | 545 | I("Testing mirror " .. orig .. "" .. file) |
496 | APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork() | 546 | APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(orig) |
497 | D('logging to ' .. APT.logName(ph.host, nil, file)[2]) | 547 | D('logging to ' .. APT.logName(ph.host, nil, file)[2]) |
498 | else D("checkHost " .. orig .. arw .. host) end | 548 | else D("checkHost " .. orig .. arw .. host) end |
499 | end | 549 | end |
@@ -568,7 +618,7 @@ local downloads = function(host, URL, meta, release, list) | |||
568 | return | 618 | return |
569 | end | 619 | end |
570 | else | 620 | else |
571 | for i, s in pairs(releases) do | 621 | for i, s in pairs(APT.releases) do |
572 | for j, k in pairs(releaseFiles) do | 622 | for j, k in pairs(releaseFiles) do |
573 | if repoExists(s .. k) then | 623 | if repoExists(s .. k) then |
574 | addDownload(host, URL, f, s, k) | 624 | addDownload(host, URL, f, s, k) |
@@ -577,12 +627,16 @@ local downloads = function(host, URL, meta, release, list) | |||
577 | end | 627 | end |
578 | end | 628 | end |
579 | f:close() | 629 | f:close() |
580 | APT.exe(cm):Nice():log():fork() | 630 | APT.exe(cm):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(host) |
581 | D('logging to <a href="' .. log .. '">' .. log .. '</a>, with <a href="' .. files .. '">these files</a>') | 631 | D('logging to <a href="' .. log .. '">' .. log .. '</a>, with <a href="' .. files .. '">these files</a>') |
582 | end | 632 | end |
583 | 633 | ||
584 | 634 | ||
585 | local validateURL = function(m) | 635 | local validateURL = function(m) |
636 | if "http://" == m.BaseURL:sub(1, 7) then | ||
637 | W("HTTP at beginning of BaseURL in mirror_list.txt! " .. m.BaseURL, "", "", m.FQDN) | ||
638 | m.BaseURL = m.BaseURL:sub(1, -2) | ||
639 | end | ||
586 | if " " == m.BaseURL:sub(-1, -1) then | 640 | if " " == m.BaseURL:sub(-1, -1) then |
587 | W("space at end of BaseURL in mirror_list.txt! " .. m.BaseURL, "", "", m.FQDN) | 641 | W("space at end of BaseURL in mirror_list.txt! " .. m.BaseURL, "", "", m.FQDN) |
588 | m.BaseURL = m.BaseURL:sub(1, -2) | 642 | m.BaseURL = m.BaseURL:sub(1, -2) |
@@ -594,8 +648,8 @@ local validateURL = function(m) | |||
594 | local p = url.parse("http://" .. m.BaseURL) | 648 | local p = url.parse("http://" .. m.BaseURL) |
595 | if nil == p.path then p.path = '' end | 649 | if nil == p.path then p.path = '' end |
596 | if nil ~= p.port then p.authority = authority .. ':' .. p.port end | 650 | if nil ~= p.port then p.authority = authority .. ':' .. p.port end |
597 | if nil == m.FDQN then W("Something wrong in FDQN from mirror_list.txt! nil", "", "", p.authority) else | 651 | if nil == m.FQDN then W("Something wrong in FQDN from mirror_list.txt! nil", "", "", p.authority) else |
598 | if m.FQDN ~= p.authority then W("Something wrong in FDQN from mirror_list.txt! " .. m.FDQN, "", "", p.authority) end | 652 | if m.FQDN ~= p.authority then W("Something wrong in FQDN from mirror_list.txt! " .. m.FQDN, "", "", p.authority) end |
599 | end | 653 | end |
600 | if nil == m.BaseURL then W("Something wrong in BaseURL from mirror_list.txt! nil", "", "", p.authority) else | 654 | if nil == m.BaseURL then W("Something wrong in BaseURL from mirror_list.txt! nil", "", "", p.authority) else |
601 | if m.BaseURL ~= (p.authority .. p.path) then W("Something wrong in BaseURL from mirror_list.txt! " .. m.BaseURL, "", "", p.authority) end | 655 | if m.BaseURL ~= (p.authority .. p.path) then W("Something wrong in BaseURL from mirror_list.txt! " .. m.BaseURL, "", "", p.authority) end |
@@ -678,7 +732,7 @@ local postParse = function(host, list) | |||
678 | if APT.options.referenceSite.value == host then | 732 | if APT.options.referenceSite.value == host then |
679 | if nil ~= list then | 733 | if nil ~= list then |
680 | local sem = 'results/NEW_' .. list.out .. '_%s.txt' | 734 | local sem = 'results/NEW_' .. list.out .. '_%s.txt' |
681 | for i, n in pairs(releases) do | 735 | for i, n in pairs(APT.releases) do |
682 | local f = sem:format(n) | 736 | local f = sem:format(n) |
683 | if APT.checkFile(f .. '.tmp') then | 737 | if APT.checkFile(f .. '.tmp') then |
684 | os.execute('mv ' .. f .. '.tmp ' .. f) | 738 | os.execute('mv ' .. f .. '.tmp ' .. f) |
@@ -691,7 +745,7 @@ local postParse = function(host, list) | |||
691 | end | 745 | end |
692 | 746 | ||
693 | local parseDebs = function(host) | 747 | local parseDebs = function(host) |
694 | for i, n in pairs(releases) do | 748 | for i, n in pairs(APT.releases) do |
695 | local inFile = 'results/NEW_debs_' .. n .. '.txt' | 749 | local inFile = 'results/NEW_debs_' .. n .. '.txt' |
696 | local nfile, e = io.open(inFile, "r") | 750 | local nfile, e = io.open(inFile, "r") |
697 | if nil == nfile then W("opening " .. inFile .. " file - " .. e) else | 751 | if nil == nfile then W("opening " .. inFile .. " file - " .. e) else |
@@ -733,7 +787,7 @@ end | |||
733 | 787 | ||
734 | local parsePackages = function(host) | 788 | local parsePackages = function(host) |
735 | local list = {inf = 'Packages', parser = parseDebs, out = 'debs', files = {}, nextf = ''} | 789 | local list = {inf = 'Packages', parser = parseDebs, out = 'debs', files = {}, nextf = ''} |
736 | for i, n in pairs(releases) do | 790 | for i, n in pairs(APT.releases) do |
737 | local inFile = 'results/NEW_' .. list.inf .. '_' .. n .. '.txt' | 791 | local inFile = 'results/NEW_' .. list.inf .. '_' .. n .. '.txt' |
738 | local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt' | 792 | local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt' |
739 | if APT.options.referenceSite.value == host then | 793 | if APT.options.referenceSite.value == host then |
@@ -755,6 +809,7 @@ local parsePackages = function(host) | |||
755 | local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed', "w+") | 809 | local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed', "w+") |
756 | if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed' .. ' file - ' .. e) else | 810 | if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed' .. ' file - ' .. e) else |
757 | local pp = {} | 811 | local pp = {} |
812 | -- TODO - FIX - check if this file exists first. | ||
758 | for l in io.lines('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages') do | 813 | for l in io.lines('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages') do |
759 | if "Package: " == l:sub(1, 9) then | 814 | if "Package: " == l:sub(1, 9) then |
760 | if 0 ~= #pp then | 815 | if 0 ~= #pp then |
@@ -786,12 +841,12 @@ local parsePackages = function(host) | |||
786 | ' | grep -E "^\\+" | grep -Ev "^\\+\\+\\+|^---" >>results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt') | 841 | ' | grep -E "^\\+" | grep -Ev "^\\+\\+\\+|^---" >>results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt') |
787 | for i, s in pairs(referenceDebs) do | 842 | for i, s in pairs(referenceDebs) do |
788 | if 0 == APT.exe('grep -q "' .. s:sub(8, -1) .. '" results/OLD_' .. list.out .. '_' .. n .. '.txt'):log():Do().status then | 843 | if 0 == APT.exe('grep -q "' .. s:sub(8, -1) .. '" results/OLD_' .. list.out .. '_' .. n .. '.txt'):log():Do().status then |
789 | print('Reference package is out of date (' .. n .. ') - ' .. s) | 844 | print('Reference package is out of date from ' .. host .. ' (' .. n .. ') - ' .. s) |
790 | end | 845 | end |
791 | end | 846 | end |
792 | for i, s in pairs(referenceDevs) do | 847 | for i, s in pairs(referenceDevs) do |
793 | if 0 == APT.exe('grep -q "' .. s:sub(8, -1) .. '" results/OLD_' .. list.out .. '_' .. n .. '.txt'):log():Do().status then | 848 | if 0 == APT.exe('grep -q "' .. s:sub(8, -1) .. '" results/OLD_' .. list.out .. '_' .. n .. '.txt'):log():Do().status then |
794 | print('Reference package is out of date (' .. n .. ') - ' .. s) | 849 | print('Reference package is out of date from ' .. host .. ' (' .. n .. ') - ' .. s) |
795 | end | 850 | end |
796 | end | 851 | end |
797 | else | 852 | else |
@@ -839,13 +894,13 @@ local parseRelease = function(host) | |||
839 | local list = {inf = 'Release', parser = parsePackages, out = 'Packages', files = {}, nextf = 'debs'} | 894 | local list = {inf = 'Release', parser = parsePackages, out = 'Packages', files = {}, nextf = 'debs'} |
840 | local updated = false | 895 | local updated = false |
841 | local now = tonumber(os.date('%s')) | 896 | local now = tonumber(os.date('%s')) |
842 | for i, n in pairs(releases) do | 897 | for i, n in pairs(APT.releases) do |
843 | for l, o in pairs(releaseFiles) do | 898 | for l, o in pairs(releaseFiles) do |
844 | if repoExists(i .. o) then | 899 | if repoExists(i .. o) then |
845 | postDownload(host, n, o) | 900 | postDownload(host, n, o) |
846 | if (".gpg" == o:sub(-4, -1)) and (APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/' .. o)) then | 901 | if (".gpg" == o:sub(-4, -1)) and (APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/' .. o)) then |
847 | if APT.testing("Integrity") then | 902 | if APT.testing("Integrity") then |
848 | local status = APT.exe( "gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. | 903 | local status = APT.exe( "gpgv --keyring " .. keyring .. " results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. |
849 | " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status | 904 | " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status |
850 | if 0 ~= status then E("GPG check failed for " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end | 905 | if 0 ~= status then E("GPG check failed for " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end |
851 | -- TODO - should check the PGP sig of InRelease as well. | 906 | -- TODO - should check the PGP sig of InRelease as well. |
@@ -922,7 +977,7 @@ end | |||
922 | 977 | ||
923 | local parseStart = function(host) | 978 | local parseStart = function(host) |
924 | local list = {inf = '', parser = parseRelease, out = 'Release', files = {}, nextf = 'Packages'} | 979 | local list = {inf = '', parser = parseRelease, out = 'Release', files = {}, nextf = 'Packages'} |
925 | for i, n in pairs(releases) do | 980 | for i, n in pairs(APT.releases) do |
926 | local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt' | 981 | local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt' |
927 | for l, o in pairs(releaseFiles) do | 982 | for l, o in pairs(releaseFiles) do |
928 | if repoExists(n .. o) then | 983 | if repoExists(n .. o) then |
@@ -991,6 +1046,7 @@ os.execute('sleep 1') -- Wait for things to start up before checking for them. | |||
991 | 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Connected to devuan.bio.lmu.de (141.84.43.19) port 80 (#0) | 1046 | 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Connected to devuan.bio.lmu.de (141.84.43.19) port 80 (#0) |
992 | curl: (22) The requested URL returned error: 404 Not Found | 1047 | curl: (22) The requested URL returned error: 404 Not Found |
993 | ]] | 1048 | ]] |
1049 | local trace = {} | ||
994 | local min, max, spd = 999999999999, 0 | 1050 | local min, max, spd = 999999999999, 0 |
995 | local num = '[%d%.]+[kM]?' | 1051 | local num = '[%d%.]+[kM]?' |
996 | if APT.checkFile(f) then | 1052 | if APT.checkFile(f) then |
@@ -1022,7 +1078,7 @@ os.execute('sleep 1') -- Wait for things to start up before checking for them. | |||
1022 | 1078 | ||
1023 | if (APT.options.referenceSite.value ~= host) and ('' ~= list.nextf) then | 1079 | if (APT.options.referenceSite.value ~= host) and ('' ~= list.nextf) then |
1024 | local sem = 'results/NEW_' .. list.nextf .. '_%s.txt' | 1080 | local sem = 'results/NEW_' .. list.nextf .. '_%s.txt' |
1025 | for i, n in pairs(releases) do | 1081 | for i, n in pairs(APT.releases) do |
1026 | local f = sem:format(n) | 1082 | local f = sem:format(n) |
1027 | while not APT.checkFile(f) do | 1083 | while not APT.checkFile(f) do |
1028 | D('*<* About to yield coroutine while waiting on - not APT.checkFile(' .. f .. ')') | 1084 | D('*<* About to yield coroutine while waiting on - not APT.checkFile(' .. f .. ')') |
@@ -1066,7 +1122,12 @@ if 0 < #arg then | |||
1066 | I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", ")) | 1122 | I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", ")) |
1067 | APT.mirrors = loadfile("results/mirrors.lua")() | 1123 | APT.mirrors = loadfile("results/mirrors.lua")() |
1068 | APT.results = APT.padResults(APT.results) | 1124 | APT.results = APT.padResults(APT.results) |
1069 | if APT.origin or APT.redir then APT.results["IPs"] = gatherIPs(pu.host) end | 1125 | if APT.origin or APT.redir then |
1126 | if "" == pu.host then | ||
1127 | print("Empty pu.host name! " .. pu.host) | ||
1128 | end | ||
1129 | APT.results["IPs"] = gatherIPs(pu.host) | ||
1130 | end | ||
1070 | if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end | 1131 | if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end |
1071 | if nil ~= arg[3] then I(" Using file " .. arg[3]); end | 1132 | if nil ~= arg[3] then I(" Using file " .. arg[3]); end |
1072 | 1133 | ||
@@ -1081,15 +1142,17 @@ if 0 < #arg then | |||
1081 | APT.allpairs(ips, | 1142 | APT.allpairs(ips, |
1082 | function(k, v) | 1143 | function(k, v) |
1083 | if v == "A" then | 1144 | if v == "A" then |
1084 | if APT.testing("IPv4") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -4 ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):Nice():log():fork() end | 1145 | if APT.testing("IPv4") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -4 ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host) end |
1085 | elseif v == "AAAA" then | 1146 | elseif v == "AAAA" then |
1086 | if APT.testing("IPv6") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -6 ' .. APT.IPv46 .. ' ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):Nice():log():fork() end | 1147 | if APT.testing("IPv6") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -6 ' .. APT.IPv46 .. ' ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host) end |
1087 | end | 1148 | end |
1088 | D('logging to ' .. APT.logName(pu.host, k, file)[2]) | 1149 | D('logging to ' .. APT.logName(pu.host, k, file)[2]) |
1089 | end | 1150 | end |
1090 | ) | 1151 | ) |
1091 | else | 1152 | else |
1092 | E("no IPs for " .. pu.host) | 1153 | E("no IPs for " .. pu.host) |
1154 | APT.logPost() | ||
1155 | return | ||
1093 | end | 1156 | end |
1094 | end | 1157 | end |
1095 | 1158 | ||
@@ -1108,6 +1171,7 @@ if 0 < #arg then | |||
1108 | checkFiles(pu.host, pu.host, pu.path:sub(1, -1), file); | 1171 | checkFiles(pu.host, pu.host, pu.path:sub(1, -1), file); |
1109 | end | 1172 | end |
1110 | else | 1173 | else |
1174 | if "-6" == APT.IPv46 then arg[2] = '[' .. arg[2] .. ']' end | ||
1111 | checkHost(pu.host, pu.host, pu.path, arg[2], arg[3]) | 1175 | checkHost(pu.host, pu.host, pu.path, arg[2], arg[3]) |
1112 | end | 1176 | end |
1113 | 1177 | ||
@@ -1178,16 +1242,15 @@ else | |||
1178 | while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 5") end | 1242 | while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 5") end |
1179 | 1243 | ||
1180 | local APT_args = APT.args | 1244 | local APT_args = APT.args |
1181 | local APT_logFile = APT.logFile | ||
1182 | local debians = {} | 1245 | local debians = {} |
1183 | local srvs = io.popen('ls -1 results/*.lua') | 1246 | local srvs = APT.readCmd('ls -1 results/*.lua') |
1184 | for l in srvs:lines() do | 1247 | for ii,l in ipairs(srvs) do |
1185 | local hst = l:sub(9, -5) | 1248 | local hst = l:sub(9, -5) |
1186 | if nil ~= l:find('_R%.lua') then hst = hst:sub(1, -3) end | 1249 | if nil ~= l:find('_R%.lua') then hst = hst:sub(1, -3) end |
1187 | if (hst:find('_') == nil) and (nil == APT.mirrors[hst]) then | 1250 | if (hst:find('_') == nil) and (nil == APT.mirrors[hst]) then |
1188 | local ips = loadfile(l)().IPs | 1251 | local ips = loadfile(l)().IPs |
1189 | if nil ~= ips then | 1252 | if nil ~= ips then |
1190 | debians[hst] = {Country = '', FDQN = hst, Active = 'yes', Rate = '', BaseURL = hst, Protocols = {http = true, https = true}, Bandwidth = '', IPs = ips} | 1253 | debians[hst] = {Country = '', FQDN = hst, Active = 'yes', Rate = '', BaseURL = hst, Protocols = {http = true, https = true}, Bandwidth = '', IPs = ips} |
1191 | local baseFiles = {} | 1254 | local baseFiles = {} |
1192 | local IPfiles = {} | 1255 | local IPfiles = {} |
1193 | for i, a in pairs(ips) do | 1256 | for i, a in pairs(ips) do |
@@ -1198,8 +1261,8 @@ else | |||
1198 | end | 1261 | end |
1199 | end | 1262 | end |
1200 | end | 1263 | end |
1201 | local files = io.popen('ls -1 results/LOG_' .. hst .. '_*.html') | 1264 | local files = APT.readCmd('ls -1 results/LOG_' .. hst .. '_*.html') |
1202 | for ll in files:lines() do | 1265 | for iii,ll in ipairs(files) do |
1203 | local dn = false | 1266 | local dn = false |
1204 | for i, a in pairs(ips) do | 1267 | for i, a in pairs(ips) do |
1205 | if type(a) == 'table' then | 1268 | if type(a) == 'table' then |
@@ -1220,6 +1283,7 @@ else | |||
1220 | end | 1283 | end |
1221 | 1284 | ||
1222 | local combine = function(ip, a) | 1285 | local combine = function(ip, a) |
1286 | local APT_logFile = APT.logFile | ||
1223 | if not APT.logOpen(hst, ip) then | 1287 | if not APT.logOpen(hst, ip) then |
1224 | print('PROBLEM OPENING LOG FILE ' .. hst .. ' ' .. ip) | 1288 | print('PROBLEM OPENING LOG FILE ' .. hst .. ' ' .. ip) |
1225 | else | 1289 | else |
@@ -1231,8 +1295,8 @@ else | |||
1231 | if ln:match('^' .. os.date('!%Y%-%m%-%d ') .. '.*$') then APT.logFile:write(ln .. '\n') end -- %F isn't good enough, coz we have to escape the '-'. | 1295 | if ln:match('^' .. os.date('!%Y%-%m%-%d ') .. '.*$') then APT.logFile:write(ln .. '\n') end -- %F isn't good enough, coz we have to escape the '-'. |
1232 | end | 1296 | end |
1233 | end | 1297 | end |
1298 | APT.logPost() | ||
1234 | end | 1299 | end |
1235 | APT.logPost() | ||
1236 | APT.args = APT_args | 1300 | APT.args = APT_args |
1237 | APT.logFile = APT_logFile | 1301 | APT.logFile = APT_logFile |
1238 | end | 1302 | end |