diff options
Diffstat (limited to 'apt-panopticon.lua')
| -rwxr-xr-x | apt-panopticon.lua | 162 |
1 files changed, 113 insertions, 49 deletions
diff --git a/apt-panopticon.lua b/apt-panopticon.lua index 4019fdf..3c990c6 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua | |||
| @@ -14,7 +14,6 @@ APT.html = true | |||
| 14 | 14 | ||
| 15 | 15 | ||
| 16 | local defaultURL = {scheme = "http"} | 16 | local defaultURL = {scheme = "http"} |
| 17 | local releases = {"jessie", "ascii", "beowulf", "ceres"} | ||
| 18 | local releaseFiles = | 17 | local releaseFiles = |
| 19 | { | 18 | { |
| 20 | -- Release file. | 19 | -- Release file. |
| @@ -22,7 +21,7 @@ local releaseFiles = | |||
| 22 | "Release.gpg", -- | 21 | "Release.gpg", -- |
| 23 | -- "InRelease", -- 3.7 MB | 22 | -- "InRelease", -- 3.7 MB |
| 24 | -- "main/binary-all/Packages.xz", -- 2.6 GB for all that changed recently. | 23 | -- "main/binary-all/Packages.xz", -- 2.6 GB for all that changed recently. |
| 25 | -- Contents files. -- 3.3 GB | 24 | -- Contents files. -- 3.3 GB |
| 26 | -- "main/Contents-all.xz", | 25 | -- "main/Contents-all.xz", |
| 27 | -- "main/Contents-amd64.xz", | 26 | -- "main/Contents-amd64.xz", |
| 28 | -- "main/Contents-arm64.xz", | 27 | -- "main/Contents-arm64.xz", |
| @@ -37,16 +36,21 @@ local notExist = | |||
| 37 | local referenceDebs = | 36 | local referenceDebs = |
| 38 | { | 37 | { |
| 39 | -- Debian package. | 38 | -- Debian package. |
| 40 | "merged/pool/DEBIAN/main/d/debian-keyring/debian-keyring_2019.02.25_all.deb", | 39 | "merged/pool/DEBIAN/main/d/debian-keyring/debian-keyring_2025.07.26_all.deb", |
| 41 | -- Debian security package. NOTE this one should always be redirected? | 40 | -- Debian security package. NOTE this one should always be redirected? |
| 42 | "merged/pool/DEBIAN-SECURITY/updates/main/a/apt/apt-transport-https_1.4.10_amd64.deb", | 41 | "merged/pool/DEBIAN-SECURITY/updates/main/e/exim4/exim4_4.96-15+deb12u7_all.deb", |
| 43 | } | 42 | } |
| 44 | local referenceDevs = | 43 | local referenceDevs = |
| 45 | { | 44 | { |
| 46 | -- Devuan package. NOTE this one should not get redirected, but that's more a warning than an error. | 45 | -- Devuan package. NOTE this one should not get redirected, but that's more a warning than an error. |
| 47 | "merged/pool/DEVUAN/main/d/devuan-keyring/devuan-keyring_2017.10.03_all.deb", | 46 | "merged/pool/DEVUAN/main/d/devuan-keyring/devuan-keyring_2025.08.09_all.deb", -- Devuan keeps changing this since the key expiry incident. |
| 47 | -- Hmmm used to be _all, but it was split to the individual archs. Seems pointless, each one is the same size as the previous all. | ||
| 48 | "merged/pool/DEVUAN/main/b/base-files/base-files_14devuan1_amd64.deb", | ||
| 48 | } | 49 | } |
| 49 | 50 | ||
| 51 | local keyring = "/usr/share/keyrings/devuan-archive-keyring.gpg" | ||
| 52 | --local keyring = "/etc/apt/trusted.gpg.d/devuan-keyring-2022-archive.gpg" | ||
| 53 | |||
| 50 | local curlStatus = | 54 | local curlStatus = |
| 51 | { | 55 | { |
| 52 | [1 ] = "Unsupported protocol. This build of curl has no support for this protocol.", | 56 | [1 ] = "Unsupported protocol. This build of curl has no support for this protocol.", |
| @@ -170,38 +174,76 @@ local repoExists = function (r) | |||
| 170 | end | 174 | end |
| 171 | 175 | ||
| 172 | local IP = {} | 176 | local IP = {} |
| 173 | gatherIPs = function (host) | 177 | gatherIPs = function (host, again) |
| 178 | if nil == again then again = '' end | ||
| 174 | if nil == IP[host] then | 179 | if nil == IP[host] then |
| 175 | local IPs | 180 | local IPs |
| 176 | -- Takes about 30 seconds to look up the lot. | 181 | -- Takes about 30 seconds to look up the lot. |
| 177 | -- I tested using dig's -f option, it didn't seem much faster. | 182 | -- I tested using dig's -f option, it didn't seem much faster. |
| 178 | -- The sort -r assumes that deb.devuan.org is the first alphabetically. | 183 | -- The sort -r assumes that deb.devuan.org is the first alphabetically. |
| 179 | local dig = io.popen('dig +keepopen +noall +nottlid +answer ' .. host .. ' A ' .. host .. ' AAAA ' .. host .. ' CNAME ' .. host .. ' SRV | sort -r | uniq') | 184 | if "" == host then print("Empty host name!") end |
| 180 | repeat | 185 | local dig = APT.readCmd('dig ' .. again .. ' +keepopen +noall +nottlid +answer ' .. host .. ' A ' .. host .. ' AAAA ' .. host .. ' CNAME ' .. host .. ' SRV | sort -r | uniq') |
| 181 | IPs = dig:read("*l") | 186 | for i,IPs in ipairs(dig) do |
| 182 | if nil ~= IPs then | 187 | for k, t, v in IPs:gmatch("([%w_%-%.]*)%.%s*IN%s*(%a*)%s*(.*)") do |
| 183 | for k, t, v in IPs:gmatch("([%w_%-%.]*)%.%s*IN%s*(%a*)%s*(.*)") do | 188 | if "." == v:sub(-1, -1) then v = v:sub(1, -2) end |
| 184 | if "." == v:sub(-1, -1) then v = v:sub(1, -2) end | 189 | if nil == IP[k] then IP[k] = {} end |
| 185 | if nil == IP[k] then IP[k] = {} end | 190 | IP[k][v] = t |
| 186 | IP[k][v] = t | 191 | D(" DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v) |
| 187 | D(" DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v) | 192 | if t == "CNAME" then |
| 188 | if t == "CNAME" then | 193 | if "" == v then |
| 194 | if '' ~= again then | ||
| 195 | print("Empty host name! DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v) | ||
| 196 | else | ||
| 197 | return gatherIPs(host, '@9.9.9.11') | ||
| 198 | end | ||
| 199 | else | ||
| 189 | gatherIPs(v) | 200 | gatherIPs(v) |
| 190 | IP[k][v] = IP[v] | 201 | IP[k][v] = IP[v] |
| 191 | elseif v == "SRV" then | 202 | end |
| 192 | print("SVR record found, now what do we do?") | 203 | elseif v == "SRV" then |
| 204 | print("SVR record found, now what do we do?") | ||
| 205 | elseif "" == v then | ||
| 206 | if '' ~= again then | ||
| 207 | print("Empty host name! DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v) | ||
| 208 | else | ||
| 209 | return gatherIPs(host, '@9.9.9.11') | ||
| 193 | end | 210 | end |
| 194 | end | 211 | end |
| 195 | end | 212 | end |
| 196 | until nil == IPs | 213 | end |
| 197 | end | 214 | end |
| 198 | 215 | ||
| 199 | -- If this is the DNS-RR domain name, gather the IPs for the mirrors that mirror_list.txt says should be in it. | 216 | -- If this is the DNS-RR domain name, gather the IPs for the mirrors that mirror_list.txt says should be in it. |
| 200 | if host == APT.options.roundRobin.value then | 217 | if host == APT.options.roundRobin.value then |
| 201 | for k, m in pairs(APT.mirrors) do | 218 | for k, m in pairs(APT.mirrors) do |
| 202 | if "yes" == m.DNSRR then | 219 | if ("yes" == m.DNSRR) or ("maybe" == m.DNSRR) then |
| 220 | if "" == m.FQDN then | ||
| 221 | print("Empty FQDN name! " .. host) | ||
| 222 | end | ||
| 203 | gatherIPs(m.FQDN) | 223 | gatherIPs(m.FQDN) |
| 204 | IP[host][m.FQDN] = IP[m.FQDN] | 224 | IP[host][m.FQDN] = IP[m.FQDN] |
| 225 | -- Strip them out so we don't test them twice. | ||
| 226 | if (nil ~= IP[m.FQDN]) and (nil ~= IP[host][APT.options.roundRobinCname.value]) then | ||
| 227 | for l, n in pairs(IP[m.FQDN]) do | ||
| 228 | if type(n) == 'table' then | ||
| 229 | for h, p in pairs(n) do | ||
| 230 | for j, o in pairs(IP[host][APT.options.roundRobinCname.value]) do | ||
| 231 | if h == j then IP[host][m.FQDN][l][h] = nil end | ||
| 232 | end | ||
| 233 | o = 0 | ||
| 234 | for j in pairs(IP[host][m.FQDN][l]) do o = o + 1 end | ||
| 235 | if 0 == o then IP[host][m.FQDN][l] = nil end | ||
| 236 | end | ||
| 237 | else | ||
| 238 | for j, o in pairs(IP[host][APT.options.roundRobinCname.value]) do | ||
| 239 | if l == j then IP[host][m.FQDN][l] = nil end | ||
| 240 | end | ||
| 241 | end | ||
| 242 | o = 0 | ||
| 243 | for j in pairs(IP[host][m.FQDN]) do o = o + 1 end | ||
| 244 | if 0 == o then IP[host][m.FQDN] = nil end | ||
| 245 | end | ||
| 246 | end | ||
| 205 | end | 247 | end |
| 206 | end | 248 | end |
| 207 | end | 249 | end |
| @@ -235,7 +277,11 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
| 235 | if "http" == PU.scheme then | 277 | if "http" == PU.scheme then |
| 236 | hdr = '-H "Host: ' .. host .. '"' | 278 | hdr = '-H "Host: ' .. host .. '"' |
| 237 | end | 279 | end |
| 238 | IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"' | 280 | if '-6' == APT.IPv46 then |
| 281 | IP = '--connect-to "' .. pu.host .. '::[' .. PU.host .. ']:"' | ||
| 282 | else | ||
| 283 | IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"' | ||
| 284 | end | ||
| 239 | fname = host .. "_" .. pu.host .. '_' .. PU.host .. "_" .. PU.path:gsub("/", "_") .. ".txt" | 285 | fname = host .. "_" .. pu.host .. '_' .. PU.host .. "_" .. PU.path:gsub("/", "_") .. ".txt" |
| 240 | end | 286 | end |
| 241 | os.execute('rm -f results/HEADERS_' .. fname .. ' 2>/dev/null; rm -f results/STATUS_' .. fname .. ' 2>/dev/null; touch results/STATUS_' .. fname) | 287 | os.execute('rm -f results/HEADERS_' .. fname .. ' 2>/dev/null; rm -f results/STATUS_' .. fname .. ' 2>/dev/null; touch results/STATUS_' .. fname) |
| @@ -266,6 +312,10 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
| 266 | if "https" == PU.scheme and APT.options.roundRobin.value == host then | 312 | if "https" == PU.scheme and APT.options.roundRobin.value == host then |
| 267 | I(spcd .. "Not testing " .. APT.lnk(URL) .. " mirrors wont have the correct HTTPS certificate for the round robin.", host) | 313 | I(spcd .. "Not testing " .. APT.lnk(URL) .. " mirrors wont have the correct HTTPS certificate for the round robin.", host) |
| 268 | return | 314 | return |
| 315 | -- TODO - For some odd reason, sometimes one of these is nil. Dig deeper to figure out why. | ||
| 316 | elseif "https" == PU.scheme and (APT.mirrors[host] ~= nil) and (APT.mirrors[host].Protocols ~= nil) and (not APT.mirrors[host].Protocols.https) then | ||
| 317 | I(spcd .. "Not testing " .. APT.lnk(URL) .. " host doesn't support HTTPS.", host) | ||
| 318 | return | ||
| 269 | else | 319 | else |
| 270 | I(spcd .. check .. " " .. APT.lnk(URL), host) | 320 | I(spcd .. check .. " " .. APT.lnk(URL), host) |
| 271 | end | 321 | end |
| @@ -289,7 +339,7 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
| 289 | 'curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' .. APT.IPv46 .. ' ' .. | 339 | 'curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' .. APT.IPv46 .. ' ' .. |
| 290 | IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' .. | 340 | IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' .. |
| 291 | hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"' | 341 | hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"' |
| 292 | ):Nice():log():Do().status | 342 | ):timeout(APT.options.maxtime.value * 2.0):Nice():log():Do().status |
| 293 | if 0 < r then | 343 | if 0 < r then |
| 294 | APT.tested(PU.scheme, 'Redirects', host) | 344 | APT.tested(PU.scheme, 'Redirects', host) |
| 295 | else | 345 | else |
| @@ -315,7 +365,7 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
| 315 | if 0 ~= status then | 365 | if 0 ~= status then |
| 316 | local msg = curlStatus[status] | 366 | local msg = curlStatus[status] |
| 317 | if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end | 367 | if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end |
| 318 | if (28 == status) or (7 == status) then | 368 | if (128+9 == status) or (124 == status) or (28 == status) or (7 == status) then |
| 319 | T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host) | 369 | T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host) |
| 320 | timeouts = timeouts + 1 | 370 | timeouts = timeouts + 1 |
| 321 | else | 371 | else |
| @@ -406,7 +456,7 @@ checkHEAD = function (host, URL, r, retry, sanity) | |||
| 406 | local pth = path:match('^(.*/pool/).*$') | 456 | local pth = path:match('^(.*/pool/).*$') |
| 407 | if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E(spcd .. spcd .. 'Odd redirect path ' .. path) end | 457 | if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E(spcd .. spcd .. 'Odd redirect path ' .. path) end |
| 408 | I(spcd .. spcd .. "Now checking redirected host " .. u .. ' for ' .. APT.lnk(URL) .. arw .. APT.lnk(location), host) | 458 | I(spcd .. spcd .. "Now checking redirected host " .. u .. ' for ' .. APT.lnk(URL) .. arw .. APT.lnk(location), host) |
| 409 | APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork() | 459 | APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host) |
| 410 | D(spcd .. 'logging to ' .. APT.logName(pu.host, nil, file)[2]) | 460 | D(spcd .. 'logging to ' .. APT.logName(pu.host, nil, file)[2]) |
| 411 | APT.tested(PU.scheme, 'Redirects', host) | 461 | APT.tested(PU.scheme, 'Redirects', host) |
| 412 | end | 462 | end |
| @@ -454,7 +504,7 @@ local checkFiles = function (host, ip, path, file) | |||
| 454 | if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end | 504 | if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end |
| 455 | end | 505 | end |
| 456 | end | 506 | end |
| 457 | for i, s in pairs(releases) do | 507 | for i, s in pairs(APT.releases) do |
| 458 | for j, k in pairs(releaseFiles) do | 508 | for j, k in pairs(releaseFiles) do |
| 459 | if repoExists(s .. k) then | 509 | if repoExists(s .. k) then |
| 460 | if checkTimeouts(host, "http", ip .. path .. "/merged/dists/" .. s .. '/' .. k) then return end | 510 | if checkTimeouts(host, "http", ip .. path .. "/merged/dists/" .. s .. '/' .. k) then return end |
| @@ -493,7 +543,7 @@ checkHost = function (orig, host, path, ip, file) | |||
| 493 | else | 543 | else |
| 494 | if orig == host then | 544 | if orig == host then |
| 495 | I("Testing mirror " .. orig .. "" .. file) | 545 | I("Testing mirror " .. orig .. "" .. file) |
| 496 | APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork() | 546 | APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(orig) |
| 497 | D('logging to ' .. APT.logName(ph.host, nil, file)[2]) | 547 | D('logging to ' .. APT.logName(ph.host, nil, file)[2]) |
| 498 | else D("checkHost " .. orig .. arw .. host) end | 548 | else D("checkHost " .. orig .. arw .. host) end |
| 499 | end | 549 | end |
| @@ -568,7 +618,7 @@ local downloads = function(host, URL, meta, release, list) | |||
| 568 | return | 618 | return |
| 569 | end | 619 | end |
| 570 | else | 620 | else |
| 571 | for i, s in pairs(releases) do | 621 | for i, s in pairs(APT.releases) do |
| 572 | for j, k in pairs(releaseFiles) do | 622 | for j, k in pairs(releaseFiles) do |
| 573 | if repoExists(s .. k) then | 623 | if repoExists(s .. k) then |
| 574 | addDownload(host, URL, f, s, k) | 624 | addDownload(host, URL, f, s, k) |
| @@ -577,12 +627,16 @@ local downloads = function(host, URL, meta, release, list) | |||
| 577 | end | 627 | end |
| 578 | end | 628 | end |
| 579 | f:close() | 629 | f:close() |
| 580 | APT.exe(cm):Nice():log():fork() | 630 | APT.exe(cm):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(host) |
| 581 | D('logging to <a href="' .. log .. '">' .. log .. '</a>, with <a href="' .. files .. '">these files</a>') | 631 | D('logging to <a href="' .. log .. '">' .. log .. '</a>, with <a href="' .. files .. '">these files</a>') |
| 582 | end | 632 | end |
| 583 | 633 | ||
| 584 | 634 | ||
| 585 | local validateURL = function(m) | 635 | local validateURL = function(m) |
| 636 | if "http://" == m.BaseURL:sub(1, 7) then | ||
| 637 | W("HTTP at beginning of BaseURL in mirror_list.txt! " .. m.BaseURL, "", "", m.FQDN) | ||
| 638 | m.BaseURL = m.BaseURL:sub(1, -2) | ||
| 639 | end | ||
| 586 | if " " == m.BaseURL:sub(-1, -1) then | 640 | if " " == m.BaseURL:sub(-1, -1) then |
| 587 | W("space at end of BaseURL in mirror_list.txt! " .. m.BaseURL, "", "", m.FQDN) | 641 | W("space at end of BaseURL in mirror_list.txt! " .. m.BaseURL, "", "", m.FQDN) |
| 588 | m.BaseURL = m.BaseURL:sub(1, -2) | 642 | m.BaseURL = m.BaseURL:sub(1, -2) |
| @@ -594,8 +648,8 @@ local validateURL = function(m) | |||
| 594 | local p = url.parse("http://" .. m.BaseURL) | 648 | local p = url.parse("http://" .. m.BaseURL) |
| 595 | if nil == p.path then p.path = '' end | 649 | if nil == p.path then p.path = '' end |
| 596 | if nil ~= p.port then p.authority = authority .. ':' .. p.port end | 650 | if nil ~= p.port then p.authority = authority .. ':' .. p.port end |
| 597 | if nil == m.FDQN then W("Something wrong in FDQN from mirror_list.txt! nil", "", "", p.authority) else | 651 | if nil == m.FQDN then W("Something wrong in FQDN from mirror_list.txt! nil", "", "", p.authority) else |
| 598 | if m.FQDN ~= p.authority then W("Something wrong in FDQN from mirror_list.txt! " .. m.FDQN, "", "", p.authority) end | 652 | if m.FQDN ~= p.authority then W("Something wrong in FQDN from mirror_list.txt! " .. m.FQDN, "", "", p.authority) end |
| 599 | end | 653 | end |
| 600 | if nil == m.BaseURL then W("Something wrong in BaseURL from mirror_list.txt! nil", "", "", p.authority) else | 654 | if nil == m.BaseURL then W("Something wrong in BaseURL from mirror_list.txt! nil", "", "", p.authority) else |
| 601 | if m.BaseURL ~= (p.authority .. p.path) then W("Something wrong in BaseURL from mirror_list.txt! " .. m.BaseURL, "", "", p.authority) end | 655 | if m.BaseURL ~= (p.authority .. p.path) then W("Something wrong in BaseURL from mirror_list.txt! " .. m.BaseURL, "", "", p.authority) end |
| @@ -678,7 +732,7 @@ local postParse = function(host, list) | |||
| 678 | if APT.options.referenceSite.value == host then | 732 | if APT.options.referenceSite.value == host then |
| 679 | if nil ~= list then | 733 | if nil ~= list then |
| 680 | local sem = 'results/NEW_' .. list.out .. '_%s.txt' | 734 | local sem = 'results/NEW_' .. list.out .. '_%s.txt' |
| 681 | for i, n in pairs(releases) do | 735 | for i, n in pairs(APT.releases) do |
| 682 | local f = sem:format(n) | 736 | local f = sem:format(n) |
| 683 | if APT.checkFile(f .. '.tmp') then | 737 | if APT.checkFile(f .. '.tmp') then |
| 684 | os.execute('mv ' .. f .. '.tmp ' .. f) | 738 | os.execute('mv ' .. f .. '.tmp ' .. f) |
| @@ -691,7 +745,7 @@ local postParse = function(host, list) | |||
| 691 | end | 745 | end |
| 692 | 746 | ||
| 693 | local parseDebs = function(host) | 747 | local parseDebs = function(host) |
| 694 | for i, n in pairs(releases) do | 748 | for i, n in pairs(APT.releases) do |
| 695 | local inFile = 'results/NEW_debs_' .. n .. '.txt' | 749 | local inFile = 'results/NEW_debs_' .. n .. '.txt' |
| 696 | local nfile, e = io.open(inFile, "r") | 750 | local nfile, e = io.open(inFile, "r") |
| 697 | if nil == nfile then W("opening " .. inFile .. " file - " .. e) else | 751 | if nil == nfile then W("opening " .. inFile .. " file - " .. e) else |
| @@ -733,7 +787,7 @@ end | |||
| 733 | 787 | ||
| 734 | local parsePackages = function(host) | 788 | local parsePackages = function(host) |
| 735 | local list = {inf = 'Packages', parser = parseDebs, out = 'debs', files = {}, nextf = ''} | 789 | local list = {inf = 'Packages', parser = parseDebs, out = 'debs', files = {}, nextf = ''} |
| 736 | for i, n in pairs(releases) do | 790 | for i, n in pairs(APT.releases) do |
| 737 | local inFile = 'results/NEW_' .. list.inf .. '_' .. n .. '.txt' | 791 | local inFile = 'results/NEW_' .. list.inf .. '_' .. n .. '.txt' |
| 738 | local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt' | 792 | local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt' |
| 739 | if APT.options.referenceSite.value == host then | 793 | if APT.options.referenceSite.value == host then |
| @@ -755,6 +809,7 @@ local parsePackages = function(host) | |||
| 755 | local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed', "w+") | 809 | local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed', "w+") |
| 756 | if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed' .. ' file - ' .. e) else | 810 | if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed' .. ' file - ' .. e) else |
| 757 | local pp = {} | 811 | local pp = {} |
| 812 | -- TODO - FIX - check if this file exists first. | ||
| 758 | for l in io.lines('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages') do | 813 | for l in io.lines('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages') do |
| 759 | if "Package: " == l:sub(1, 9) then | 814 | if "Package: " == l:sub(1, 9) then |
| 760 | if 0 ~= #pp then | 815 | if 0 ~= #pp then |
| @@ -786,12 +841,12 @@ local parsePackages = function(host) | |||
| 786 | ' | grep -E "^\\+" | grep -Ev "^\\+\\+\\+|^---" >>results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt') | 841 | ' | grep -E "^\\+" | grep -Ev "^\\+\\+\\+|^---" >>results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt') |
| 787 | for i, s in pairs(referenceDebs) do | 842 | for i, s in pairs(referenceDebs) do |
| 788 | if 0 == APT.exe('grep -q "' .. s:sub(8, -1) .. '" results/OLD_' .. list.out .. '_' .. n .. '.txt'):log():Do().status then | 843 | if 0 == APT.exe('grep -q "' .. s:sub(8, -1) .. '" results/OLD_' .. list.out .. '_' .. n .. '.txt'):log():Do().status then |
| 789 | print('Reference package is out of date (' .. n .. ') - ' .. s) | 844 | print('Reference package is out of date from ' .. host .. ' (' .. n .. ') - ' .. s) |
| 790 | end | 845 | end |
| 791 | end | 846 | end |
| 792 | for i, s in pairs(referenceDevs) do | 847 | for i, s in pairs(referenceDevs) do |
| 793 | if 0 == APT.exe('grep -q "' .. s:sub(8, -1) .. '" results/OLD_' .. list.out .. '_' .. n .. '.txt'):log():Do().status then | 848 | if 0 == APT.exe('grep -q "' .. s:sub(8, -1) .. '" results/OLD_' .. list.out .. '_' .. n .. '.txt'):log():Do().status then |
| 794 | print('Reference package is out of date (' .. n .. ') - ' .. s) | 849 | print('Reference package is out of date from ' .. host .. ' (' .. n .. ') - ' .. s) |
| 795 | end | 850 | end |
| 796 | end | 851 | end |
| 797 | else | 852 | else |
| @@ -839,13 +894,13 @@ local parseRelease = function(host) | |||
| 839 | local list = {inf = 'Release', parser = parsePackages, out = 'Packages', files = {}, nextf = 'debs'} | 894 | local list = {inf = 'Release', parser = parsePackages, out = 'Packages', files = {}, nextf = 'debs'} |
| 840 | local updated = false | 895 | local updated = false |
| 841 | local now = tonumber(os.date('%s')) | 896 | local now = tonumber(os.date('%s')) |
| 842 | for i, n in pairs(releases) do | 897 | for i, n in pairs(APT.releases) do |
| 843 | for l, o in pairs(releaseFiles) do | 898 | for l, o in pairs(releaseFiles) do |
| 844 | if repoExists(i .. o) then | 899 | if repoExists(i .. o) then |
| 845 | postDownload(host, n, o) | 900 | postDownload(host, n, o) |
| 846 | if (".gpg" == o:sub(-4, -1)) and (APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/' .. o)) then | 901 | if (".gpg" == o:sub(-4, -1)) and (APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/' .. o)) then |
| 847 | if APT.testing("Integrity") then | 902 | if APT.testing("Integrity") then |
| 848 | local status = APT.exe( "gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. | 903 | local status = APT.exe( "gpgv --keyring " .. keyring .. " results/" .. host .. "/merged/dists/" .. n .. '/' .. o .. |
| 849 | " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status | 904 | " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status |
| 850 | if 0 ~= status then E("GPG check failed for " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end | 905 | if 0 ~= status then E("GPG check failed for " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end |
| 851 | -- TODO - should check the PGP sig of InRelease as well. | 906 | -- TODO - should check the PGP sig of InRelease as well. |
| @@ -922,7 +977,7 @@ end | |||
| 922 | 977 | ||
| 923 | local parseStart = function(host) | 978 | local parseStart = function(host) |
| 924 | local list = {inf = '', parser = parseRelease, out = 'Release', files = {}, nextf = 'Packages'} | 979 | local list = {inf = '', parser = parseRelease, out = 'Release', files = {}, nextf = 'Packages'} |
| 925 | for i, n in pairs(releases) do | 980 | for i, n in pairs(APT.releases) do |
| 926 | local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt' | 981 | local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt' |
| 927 | for l, o in pairs(releaseFiles) do | 982 | for l, o in pairs(releaseFiles) do |
| 928 | if repoExists(n .. o) then | 983 | if repoExists(n .. o) then |
| @@ -991,6 +1046,7 @@ os.execute('sleep 1') -- Wait for things to start up before checking for them. | |||
| 991 | 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Connected to devuan.bio.lmu.de (141.84.43.19) port 80 (#0) | 1046 | 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Connected to devuan.bio.lmu.de (141.84.43.19) port 80 (#0) |
| 992 | curl: (22) The requested URL returned error: 404 Not Found | 1047 | curl: (22) The requested URL returned error: 404 Not Found |
| 993 | ]] | 1048 | ]] |
| 1049 | local trace = {} | ||
| 994 | local min, max, spd = 999999999999, 0 | 1050 | local min, max, spd = 999999999999, 0 |
| 995 | local num = '[%d%.]+[kM]?' | 1051 | local num = '[%d%.]+[kM]?' |
| 996 | if APT.checkFile(f) then | 1052 | if APT.checkFile(f) then |
| @@ -1022,7 +1078,7 @@ os.execute('sleep 1') -- Wait for things to start up before checking for them. | |||
| 1022 | 1078 | ||
| 1023 | if (APT.options.referenceSite.value ~= host) and ('' ~= list.nextf) then | 1079 | if (APT.options.referenceSite.value ~= host) and ('' ~= list.nextf) then |
| 1024 | local sem = 'results/NEW_' .. list.nextf .. '_%s.txt' | 1080 | local sem = 'results/NEW_' .. list.nextf .. '_%s.txt' |
| 1025 | for i, n in pairs(releases) do | 1081 | for i, n in pairs(APT.releases) do |
| 1026 | local f = sem:format(n) | 1082 | local f = sem:format(n) |
| 1027 | while not APT.checkFile(f) do | 1083 | while not APT.checkFile(f) do |
| 1028 | D('*<* About to yield coroutine while waiting on - not APT.checkFile(' .. f .. ')') | 1084 | D('*<* About to yield coroutine while waiting on - not APT.checkFile(' .. f .. ')') |
| @@ -1066,7 +1122,12 @@ if 0 < #arg then | |||
| 1066 | I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", ")) | 1122 | I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", ")) |
| 1067 | APT.mirrors = loadfile("results/mirrors.lua")() | 1123 | APT.mirrors = loadfile("results/mirrors.lua")() |
| 1068 | APT.results = APT.padResults(APT.results) | 1124 | APT.results = APT.padResults(APT.results) |
| 1069 | if APT.origin or APT.redir then APT.results["IPs"] = gatherIPs(pu.host) end | 1125 | if APT.origin or APT.redir then |
| 1126 | if "" == pu.host then | ||
| 1127 | print("Empty pu.host name! " .. pu.host) | ||
| 1128 | end | ||
| 1129 | APT.results["IPs"] = gatherIPs(pu.host) | ||
| 1130 | end | ||
| 1070 | if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end | 1131 | if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end |
| 1071 | if nil ~= arg[3] then I(" Using file " .. arg[3]); end | 1132 | if nil ~= arg[3] then I(" Using file " .. arg[3]); end |
| 1072 | 1133 | ||
| @@ -1081,15 +1142,17 @@ if 0 < #arg then | |||
| 1081 | APT.allpairs(ips, | 1142 | APT.allpairs(ips, |
| 1082 | function(k, v) | 1143 | function(k, v) |
| 1083 | if v == "A" then | 1144 | if v == "A" then |
| 1084 | if APT.testing("IPv4") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -4 ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):Nice():log():fork() end | 1145 | if APT.testing("IPv4") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -4 ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host) end |
| 1085 | elseif v == "AAAA" then | 1146 | elseif v == "AAAA" then |
| 1086 | if APT.testing("IPv6") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -6 ' .. APT.IPv46 .. ' ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):Nice():log():fork() end | 1147 | if APT.testing("IPv6") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -6 ' .. APT.IPv46 .. ' ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):timeout(APT.options.maxtime.value * 2.0):Nice():log():fork(pu.host) end |
| 1087 | end | 1148 | end |
| 1088 | D('logging to ' .. APT.logName(pu.host, k, file)[2]) | 1149 | D('logging to ' .. APT.logName(pu.host, k, file)[2]) |
| 1089 | end | 1150 | end |
| 1090 | ) | 1151 | ) |
| 1091 | else | 1152 | else |
| 1092 | E("no IPs for " .. pu.host) | 1153 | E("no IPs for " .. pu.host) |
| 1154 | APT.logPost() | ||
| 1155 | return | ||
| 1093 | end | 1156 | end |
| 1094 | end | 1157 | end |
| 1095 | 1158 | ||
| @@ -1108,6 +1171,7 @@ if 0 < #arg then | |||
| 1108 | checkFiles(pu.host, pu.host, pu.path:sub(1, -1), file); | 1171 | checkFiles(pu.host, pu.host, pu.path:sub(1, -1), file); |
| 1109 | end | 1172 | end |
| 1110 | else | 1173 | else |
| 1174 | if "-6" == APT.IPv46 then arg[2] = '[' .. arg[2] .. ']' end | ||
| 1111 | checkHost(pu.host, pu.host, pu.path, arg[2], arg[3]) | 1175 | checkHost(pu.host, pu.host, pu.path, arg[2], arg[3]) |
| 1112 | end | 1176 | end |
| 1113 | 1177 | ||
| @@ -1178,16 +1242,15 @@ else | |||
| 1178 | while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 5") end | 1242 | while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 5") end |
| 1179 | 1243 | ||
| 1180 | local APT_args = APT.args | 1244 | local APT_args = APT.args |
| 1181 | local APT_logFile = APT.logFile | ||
| 1182 | local debians = {} | 1245 | local debians = {} |
| 1183 | local srvs = io.popen('ls -1 results/*.lua') | 1246 | local srvs = APT.readCmd('ls -1 results/*.lua') |
| 1184 | for l in srvs:lines() do | 1247 | for ii,l in ipairs(srvs) do |
| 1185 | local hst = l:sub(9, -5) | 1248 | local hst = l:sub(9, -5) |
| 1186 | if nil ~= l:find('_R%.lua') then hst = hst:sub(1, -3) end | 1249 | if nil ~= l:find('_R%.lua') then hst = hst:sub(1, -3) end |
| 1187 | if (hst:find('_') == nil) and (nil == APT.mirrors[hst]) then | 1250 | if (hst:find('_') == nil) and (nil == APT.mirrors[hst]) then |
| 1188 | local ips = loadfile(l)().IPs | 1251 | local ips = loadfile(l)().IPs |
| 1189 | if nil ~= ips then | 1252 | if nil ~= ips then |
| 1190 | debians[hst] = {Country = '', FDQN = hst, Active = 'yes', Rate = '', BaseURL = hst, Protocols = {http = true, https = true}, Bandwidth = '', IPs = ips} | 1253 | debians[hst] = {Country = '', FQDN = hst, Active = 'yes', Rate = '', BaseURL = hst, Protocols = {http = true, https = true}, Bandwidth = '', IPs = ips} |
| 1191 | local baseFiles = {} | 1254 | local baseFiles = {} |
| 1192 | local IPfiles = {} | 1255 | local IPfiles = {} |
| 1193 | for i, a in pairs(ips) do | 1256 | for i, a in pairs(ips) do |
| @@ -1198,8 +1261,8 @@ else | |||
| 1198 | end | 1261 | end |
| 1199 | end | 1262 | end |
| 1200 | end | 1263 | end |
| 1201 | local files = io.popen('ls -1 results/LOG_' .. hst .. '_*.html') | 1264 | local files = APT.readCmd('ls -1 results/LOG_' .. hst .. '_*.html') |
| 1202 | for ll in files:lines() do | 1265 | for iii,ll in ipairs(files) do |
| 1203 | local dn = false | 1266 | local dn = false |
| 1204 | for i, a in pairs(ips) do | 1267 | for i, a in pairs(ips) do |
| 1205 | if type(a) == 'table' then | 1268 | if type(a) == 'table' then |
| @@ -1220,6 +1283,7 @@ else | |||
| 1220 | end | 1283 | end |
| 1221 | 1284 | ||
| 1222 | local combine = function(ip, a) | 1285 | local combine = function(ip, a) |
| 1286 | local APT_logFile = APT.logFile | ||
| 1223 | if not APT.logOpen(hst, ip) then | 1287 | if not APT.logOpen(hst, ip) then |
| 1224 | print('PROBLEM OPENING LOG FILE ' .. hst .. ' ' .. ip) | 1288 | print('PROBLEM OPENING LOG FILE ' .. hst .. ' ' .. ip) |
| 1225 | else | 1289 | else |
| @@ -1231,8 +1295,8 @@ else | |||
| 1231 | if ln:match('^' .. os.date('!%Y%-%m%-%d ') .. '.*$') then APT.logFile:write(ln .. '\n') end -- %F isn't good enough, coz we have to escape the '-'. | 1295 | if ln:match('^' .. os.date('!%Y%-%m%-%d ') .. '.*$') then APT.logFile:write(ln .. '\n') end -- %F isn't good enough, coz we have to escape the '-'. |
| 1232 | end | 1296 | end |
| 1233 | end | 1297 | end |
| 1298 | APT.logPost() | ||
| 1234 | end | 1299 | end |
| 1235 | APT.logPost() | ||
| 1236 | APT.args = APT_args | 1300 | APT.args = APT_args |
| 1237 | APT.logFile = APT_logFile | 1301 | APT.logFile = APT_logFile |
| 1238 | end | 1302 | end |
