From cc872d3eba68f08ee365b8771d8504c28da9020d Mon Sep 17 00:00:00 2001 From: onefang Date: Tue, 5 Nov 2019 15:38:24 +1000 Subject: Rename project to apt-panopticon. It used to be mirror-checker-lua. --- mirror-checker.lua | 634 ----------------------------------------------------- 1 file changed, 634 deletions(-) delete mode 100755 mirror-checker.lua (limited to 'mirror-checker.lua') diff --git a/mirror-checker.lua b/mirror-checker.lua deleted file mode 100755 index 01082d9..0000000 --- a/mirror-checker.lua +++ /dev/null @@ -1,634 +0,0 @@ -#!/usr/bin/env luajit - - -local args = {...} - ---[[ TODO - What to do about HTTPS://deb.devuan.org/ redirects. - Some mirrors give a 404. - Sledjhamr gives a 404, coz it's not listening on 443 for deb.devuan.org. - Some mirrors give a 200. - They shouldn't have the proper certificate, but are giving a result anyway. -]] - -origin = false -verbosity = -1 -keep = false --- TODO - Should actually implement this. -fork = true -options = -{ - referenceSite = - { - typ = "string", - help = "", - value = "pkgmaster.devuan.org", - }, - roundRobin = - { - typ = "string", - help = "", - value = "deb.devuan.org", - }, - tests = - { - typ = "table", - help = "", - value = - { - "IPv4", - "IPv6", --- "ftp", - "http", - "https", --- "rsync", --- "DNS-RR", - "Protocol", --- "URL-Sanity", --- "Integrity", --- "Updated", - }, - }, -} - -local defaultURL = {scheme = "http"} -local downloadLock = "flock -n results/wget-" -local download = "wget --timeout=300 -np -N -r -P results " -- Note wget has a default read timeout of 900 seconds (15 minutes). -local releases = {"jessie", "ascii", "beowulf", "ceres"} -local releaseFiles = -{ - -- Release file. - "/Release", - "/InRelease", - "/main/binary-all/Packages.gz", - -- Contents files. - "/main/Contents-all.gz", - "/main/Contents-amd64.gz", - "/main/Contents-arm64.gz", - "-security/main/Contents-all.gz", - "-security/main/Contents-amd64.gz", - "-security/main/Contents-arm64.gz", -} -local notExist = -{ - "ceres-security" -- This will never exist, it's our code name for the testing suite. -} -local referenceDebs = -{ - -- Debian package. - "merged/pool/DEBIAN/main/d/dash/dash_0.5.8-2.4_amd64.deb", - -- Debian security package. NOTE this one should always be redirected? - "merged/pool/DEBIAN-SECURITY/updates/main/a/apt/apt-transport-https_1.4.9_amd64.deb", -} -local referenceDevs = -{ - -- Devuan package. NOTE this one should not get redirected, but that's more a warning than an error. - "merged/pool/DEVUAN/main/d/desktop-base/desktop-base_2.0.3_all.deb", --- "merged/pool/DEVUAN/main/u/util-linux/util-linux_2.32.1-0.1+devuan2.1_amd64.deb", -} -local arg = {} -local sendArgs = "" -local logFile - - -local socket = require 'socket' -local ftp = require 'socket.ftp' -local http = require 'socket.http' -local https = require 'ssl.https' -- See https://github.com/brunoos/luasec/wiki/LuaSec-0.6 for docs. -local url = require 'socket.url' - - --- Use this to dump a table to a string. -dumpTable = function (table, space, name) - local r = "" - if "" == space then r = r .. space .. name .. " =\n" else r = r .. space .. "[" .. name .. "] =\n" end - r = r .. space .. "{\n" - r = r .. dumpTableSub(table, space .. " ") - if "" == space then r = r .. space .. "}\n" else r = r .. space .. "},\n" end - return r -end -dumpTableSub = function (table, space) - local r = "" - for k, v in pairs(table) do - if type(k) == "string" then k = '"' .. k .. '"' end - if type(v) == "table" then - r = r .. dumpTable(v, space, k) - elseif type(v) == "string" then - r = r .. space .. "[" .. k .. "] = '" .. v .. "';\n" - elseif type(v) == "function" then - r = r .. space .. "[" .. k .. "] = function ();\n" - elseif type(v) == "userdata" then - r = r .. space .. "userdata " .. "[" .. k .. "];\n" - elseif type(v) == "boolean" then - if (v) then - r = r .. space .. "[" .. k .. "] = true;\n" - else - r = r .. space .. "[" .. k .. "] = false;\n" - end - else - r = r .. space .. "[" .. k .. "] = " .. v .. ";\n" - end - end - return r -end - -local ip = "" -local results = {} - -local log = function(v, t, s, prot, test, host) - local x = "" - if nil == prot then prot = "" end - if nil ~= test then x = x .. test else test = "" end - if nil ~= host then - if #x > 0 then x = x .. " " end - x = x .. host - end - if #x > 0 then - t = t .. "(" .. x .. ")" - if "" == test then - if v == 0 then results[prot].errors = results[prot].errors + 1 end - if v == 1 then results[prot].warnings = results[prot].warnings + 1 end - else - if v == 0 then results[prot][test].errors = results[prot][test].errors + 1 end - if v == 1 then results[prot][test].warnings = results[prot][test].warnings + 1 end - end - end - if v <= verbosity then - if 3 <= verbosity then t = os.date() .. " " .. t end - print(t .. ": " .. s) - end - if nil ~= logFile then - logFile:write(os.date() .. " " .. t .. ": " .. s .. "\n") - logFile:flush() - end -end -local D = function(s) log(3, "DEBUG ", s) end -local I = function(s) log(2, "INFO ", s) end -local W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end -local E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end -local C = function(s) log(-1, "CRITICAL", s) end - -local mirrors = {} - -local testing = function(t, host) - for i, v in pairs(options.tests.value) do - if t == v then - local h = mirrors[host] - if nil == h then return true end - if true == h["Protocols"][t] then return true else D("Skipping " .. t .. " checks for " .. host) end - end - end - return false -end - -local checkExes = function (exe) - local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | wc -l'):read("*l") - D(count .. " " .. exe .. " commands still running.") - return tonumber(count) -end - -local repoExists = function (r) - r = r:match("([%a-]*)") - if nil == r then return false end - for k, v in pairs(notExist) do - if v == r then return false end - end - return true -end - -local IP = {} -gatherIPs = function (host) - if nil == IP[host] then - local IPs - local dig = io.popen('dig +keepopen +noall +nottlid +answer ' .. host .. ' A ' .. host .. ' AAAA ' .. host .. ' CNAME ' .. host .. ' SRV | sort -r | uniq') - repeat - IPs = dig:read("*l") - if nil ~= IPs then - for k, t, v in IPs:gmatch("([%w_%-%.]*)%.%s*IN%s*(%a*)%s*(.*)") do - if "." == v:sub(-1, -1) then v = v:sub(1, -2) end - if nil == IP[k] then IP[k] = {} end - IP[k][v] = t - D(" DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v) - if t == "CNAME" then - gatherIPs(v) - IP[k][v] = IP[v] - elseif v == "SRV" then - print("SVR record found, now what do we do?") - end - end - end - until nil == IPs - end -end - --- Returns FTP directory listing -local nlst = function (u) - local t = {} - local p = url.parse(u) - p.command = "nlst" - p.sink = ltn12.sink.table(t) - local r, e = ftp.get(p) - return r and table.concat(t), e -end - -local timeouts = 0; -checkHEAD = function (host, URL, r, retry) - if nil == r then r = 0 end - if nil == retry then retry = 0 end - local check = "Checking file" - local PU = url.parse(URL, defaultURL) - local pu = url.parse(PU.scheme .. "://" .. host, defaultURL) - if 0 < r then - check = "Redirecting to" - end - if 0 < retry then - os.execute("sleep " .. math.random(1, 4)) - check = "Retry " .. retry .. " " .. check - end - if 3 < timeouts then - E("too many timeouts! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) - return - end - if 20 < r then - E("too many redirects! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) - return - end - if 4 < retry then - E("too many retries! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) - return - end - D(PU.scheme .. " :// " .. check .. " " .. host .. " -> " .. URL) - if not testing(PU.scheme, host) then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL); return end - -- TODO - Perhaps we should try it anyway, and mark it as a warning if it DOES work? - if "https" == PU.scheme and options.roundRobin.value == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert."); return end - local hd = {} - if pu.host ~= PU.host then hd = {Host = host} end - local htp = http; - if PU.scheme == "https" then htp = https end - -- NOTE - the docs for lua-sec say that redirect isn't supported is version 0.6, no idea if that means it ignores redirections like we want. - -- TODO - find out! - -- The protocol and options are lua-sec specific arguments. - local p, c, h, s = htp.request{method = "HEAD", redirect = false, url = URL, headers = hd, protocol = "any", options = "all"} - if nil == s then s = "" end - if nil == p then - E(" " .. c .. " " .. s .. "! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) - -- So far the only errors are "timeout", "Network is unreachable", and "closed", and I suspect "closed" is due to saturating my bandwidth. - if "timeout" == c then timeouts = timeouts + 1 end - if ("closed" == c) or ("Network is unreachable" == c) or ("timeout" == c) then checkHEAD(host, URL, r, retry + 1, timeouts) end - else - if ("4" == tostring(c):sub(1, 1)) or ("5" == tostring(c):sub(1, 1)) then - E(" " .. c .. " " .. s .. ". " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) - else - I(" " .. c .. " " .. s .. ". " .. check .. " " .. host .. " -> " .. URL) - timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through. - end - l = h.location - if nil ~= l then - pu = url.parse(l, defaultURL) - if (pu.scheme ~= PU.scheme) then - if testing("Protocol") then W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l, PU.scheme, "Protocol", host) end - if (pu.host == host) and pu.path == PU.path then D("Not testing protocol change " .. URL .. " -> " .. l); return end - end - - if l == URL then - E(" redirect loop! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, PU.scheme, host) - elseif nil == pu.host then - I(" relative redirect. " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l) - checkHEAD(host, PU.scheme .. "://" .. PU.host .. l, r + 1) - elseif (PU.host == pu.host) or (host == pu.host) then - checkHEAD(pu.host, l, r + 1) - else - --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location. - Not good for deb.debian.org, which we should only be throwing .debs at. - What we do is loop through the DNS entries, and only test the specific protocol & file being tested here. - - This is what I came up with for checking if we are already testing a specific URL. - Still duplicates a tiny bit, but much less than the previous find based method. - ]] - local file = pu.host .. "://" .. pu.path - local f = io.popen(string.format('if [ ! -f results/%s.check ] ; then touch results/%s.check; echo -n "check"; fi', file:gsub("/", "_"), file:gsub("/", "_") )):read("*a") - if (nil == f) or ("check" == f) then - I(" Now checking redirected host " .. file) - checkHost(pu.host, pu.host, nil, "redir", pu.path) - else - D(" Already checking " .. file) - end - end - end - end -end - -local checkFiles = function (host, ip, path, file) - if nil == path then path = "" end - if nil ~= file then - if "redir" == ip then ip = host end - I(" Checking IP for file " .. host .. " -> " .. ip .. " " .. path .. " " .. file) - if testing("http", host) then checkHEAD(host, "http://" .. ip .. path .. "/" .. file) end - if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/" .. file) end - else - I(" Checking IP " .. host .. " -> " .. ip .. " " .. path) - for i, s in pairs(referenceDevs) do - if testing("http", host) then checkHEAD(host, "http://" .. ip .. path .. "/" .. s) end - if 3 < timeouts then return end - if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/" .. s) end - if 3 < timeouts then return end - end - - for i, s in pairs(releases) do - for j, k in pairs(releaseFiles) do - if repoExists(s .. k) then - if testing("http", host) then checkHEAD(host, "http://" .. ip .. path .. "/merged/dists/" .. s .. k) end - if 3 < timeouts then return end - if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/merged/dists/" .. s .. k) end - if 3 < timeouts then return end - end - end - end - end -end - -local execute = function (s) - D(" executing " .. s) - os.execute(s) -end - -checkHost = function (orig, host, path, ip, file) - if nil == host then host = orig end - if nil == path then path = "" end - if nil == file then file = "" end - local ph = url.parse("http://" .. host) - if (nil ~= ip) and ("redir" ~= ip) then - local po = url.parse("http://" .. orig) - if "" ~= file then - D("checking redirected file " .. po.host .. " " .. file) - checkFiles(po.host, ip, path, file) - else - checkFiles(po.host, ip, path) - end - else - if orig == host then - D("checkHost " .. orig .. "" .. file) - if testing("IPv4") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file .." &") end - else D("checkHost " .. orig .. " -> " .. host) end - local h = mirrors[ph.host] - if nil == h then return end - for k, v in pairs(h.IPs) do - if "table" == type(v) then - for k1, v1 in pairs(v) do - if v1 == "A" then - if testing("IPv4") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file .." &") end - elseif v1 == "AAAA" then - if testing("IPv6") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k1 .. "] " .. file .. " &") end - end - end - else - if v == "A" then - if testing("IPv4") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file .." &") end - elseif v == "AAAA" then - if testing("IPv6") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k .. "] " .. file .. " &") end - end - end - end - end -end - -local downloads = function (cut, host, URL) - if 0 ~= cut then cd = " --cut-dirs=" .. cut .. " " else cd = "" end - if nil == URL then URL = "/" end - local lock = "%s-" .. host .. ".log " - local log = " --rejected-log=results/wget-%s_REJECTS-" .. host .. ".log -a results/wget-%s-" .. host .. ".log " - I("starting file download commands for " .. host .. " " .. URL) - local cm = "ionice -c3 " .. downloadLock .. lock:format("debs") .. download .. log:format("debs", "debs") .. cd - for i, s in pairs(referenceDevs) do - cm = cm .. " https://" .. host .. URL .. "/" .. s - end - for i, s in pairs(referenceDebs) do - cm = cm .. " https://" .. host .. URL .. "/" .. s - end - for i, s in pairs(releases) do - execute(cm .. " &") - cm = "ionice -c3 " .. downloadLock .. lock:format(s) .. download .. log:format(s, s) .. cd - if repoExists(s .. k) then - for j, k in pairs(releaseFiles) do - cm = cm .. " https://" .. host .. URL .. "/merged/dists/" .. s .. k - end - end - end - execute(cm .. " &") -end - -local getMirrors = function () - local mirrors = {} - local host = "" - local m = {} - local active = true - local URL = "https://" .. options.referenceSite.value .. "/mirror_list.txt" - I("getting mirrors.") - local p, c, h = http.request(URL) - if nil == p then E(c .. " fetching " .. URL) else - for l in p:gmatch("\n*([^\n]+)\n*") do - local t, d = l:match("(%a*):%s*(.*)") - d = string.lower(d) - if "FQDN" == t then - if "" ~= host then - if active then mirrors[host] = m end - m = {} - active = true - end - host = d - m[t] = d - gatherIPs(host) - m["IPs"] = IP[host] - elseif "Protocols" == t then - local prot = {} - for w in d:gmatch("(%w+)") do - prot[w] = true; - end - m[t] = prot - elseif "Active" == t and nil == d:find("yes", 1, true) then - W("Mirror " .. host .. " is not active - " .. d) - active = false --- TODO - Should do some input validation on BaseURL, and everything else. - else - m[t] = d - end - end - if "" ~= host and active then - mirrors[host] = m - end - end - mirrors[options.roundRobin.value] = { ["Protocols"] = { ["http"] = true; ["https"] = true; }; ["FQDN"] = 'deb.devuan.org'; ["Active"] = 'yes'; ["BaseURL"] = 'deb.devuan.org'; } - gatherIPs(options.roundRobin.value) - mirrors[options.roundRobin.value].IPs = IP[options.roundRobin.value] - local file, e = io.open("results/mirrors.lua", "w+") - if nil == file then C("opening mirrors file - " .. e) else - file:write(dumpTable(mirrors, "", "mirrors") .. "\nreturn mirrors\n") - file:close() - end - return mirrors -end - - -if 0 ~= #args then - local option = "" - for i, a in pairs(args) do - if ("--help" == a) or ("-h" == a) then - print("I should write some docs, huh? Read README.md for instructions.") - elseif "--version" == a then - print("mirror-checker-lua version 0.1 WIP development version") - elseif "-v" == a then - verbosity = verbosity + 1 - sendArgs = sendArgs .. a .. " " - elseif "-q" == a then - verbosity = -1 - sendArgs = sendArgs .. a .. " " - elseif "-k" == a then - keep = true - elseif "-n" == a then - fork = false - elseif "-o" == a then - origin = true - elseif "--" == a:sub(1, 2) then - local s, e = a:find("=") - if nil == s then e = -1 end - option = a:sub(3, e - 1) - local o = options[option] - if nil == o then - print("Unknown option --" .. option) - option = "" - else - option = a - sendArgs = sendArgs .. a .. " " - local s, e = a:find("=") - if nil == s then e = 0 end - option = a:sub(3, e - 1) - if "table" == options[option].typ then - local result = {} - for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do - local f = t:sub(1, 1) - local n = t:sub(2, -1) - if ("+" ~= f) and ("-" ~= f) then - table.insert(result, t) - end - end - if 0 ~= #result then - options[option].value = result - else - for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do - local f = t:sub(1, 1) - local n = t:sub(2, -1) - if "+" == f then - table.insert(options[option].value, n) - elseif "-" == f then - local r = {} - for i, k in pairs(options[option].value) do - if k ~= n then table.insert(r, k) end - end - options[option].value = r - end - end - end - else - options[option].value = a - end - option = "" - end - elseif "-" == a:sub(1, 1) then - print("Unknown option " .. a) - else - table.insert(arg, a) - end - end -end - ---print(dumpTable(options.tests.value, "", "tests")) - -execute("mkdir -p results") - -if 0 < #arg then - if "/" == arg[1]:sub(-1, -1) then - W("slash at end of path! " .. arg[1]) - arg[1] = arg[1]:sub(1, -2) - end - if " " == arg[1]:sub(-1, -1) then - W("space at end of path! " .. arg[1]) - arg[1] = arg[1]:sub(1, -2) - end - local pu = url.parse("http://" .. arg[1]) - if nil ~= arg[2] then - logFile, e = io.open("results/" .. pu.host .. "_" .. arg[2] .. ".log", "a+") - else - logFile, e = io.open("results/" .. pu.host .. ".log", "a+") - end - if nil == logFile then C("opening log file - " .. e); return end - I("Starting tests for " ..arg[1] .. " with these tests - " .. table.concat(options.tests.value, ", ")) - mirrors = loadfile("results/mirrors.lua")() - if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end - if nil ~= arg[3] then I(" Using file " .. arg[3]); end - - for k, v in pairs{"ftp", "http", "https", "rsync"} do - if testing(v) then - local tests = {errors = 0; warnings = 0} - if testing("Integrity") then tests.Integrity = {errors = 0; warnings = 0} end - if testing("Protocol") then tests.Protocol = {errors = 0; warnings = 0} end - if testing("Updated") then tests.Updated = {errors = 0; warnings = 0} end - if testing("URL-Sanity") then tests.URL_Sanity = {errors = 0; warnings = 0} end - results[v] = tests - end - end - - if testing("Integrity") or testing("Updated") then - if nil == arg[3] then - if not keep then execute("rm -fr results/" .. pu.host) end - cut = 0 - for t in arg[1]:gmatch("(/)") do - cut = cut + 1 - end - downloads(cut, pu.host, pu.path) - checkExes("mirror-checker.lua " .. sendArgs) - checkExes(downloadLock) - end - end - if origin then - checkFiles(pu.host, pu.host, pu.path); - else - checkHost(pu.host, pu.host, pu.path, arg[2], arg[3]) - end - logFile:close() - local rfile, e = io.open("results/" .. pu.host .. "_" .. ip .. ".lua", "w+") - if nil == rfile then C("opening results file - " .. e) else - rfile:write(dumpTable(results, "", "results") .. "\nreturn results\n") - rfile:close() - end -else - if not keep then os.execute("rm -f results/*.log") end - os.execute("rm -f results/*.check") - os.execute("mkdir -p results; touch results/stamp") - logFile, e = io.open("results/mirror-checker-lua.log", "a+") - if nil == logFile then C("opening log file - " .. e); return end - I("Starting tests " .. table.concat(options.tests.value, ", ")) - execute("mkdir -p results") - mirrors = getMirrors() - checkHost(options.referenceSite.value) - for k, m in pairs(mirrors) do - if "/" == m.BaseURL:sub(-1, -1) then - W("slash at end of BaseURL in mirror_list.txt! " .. m.BaseURL) - m.BaseURL = m.BaseURL:sub(1, -2) - end - if " " == m.BaseURL:sub(-1, -1) then - W("space at end of BaseURL in mirror_list.txt! " .. m.BaseURL) - m.BaseURL = m.BaseURL:sub(1, -2) - end - local pu = url.parse("http://" .. m.BaseURL) - if options.referenceSite.value ~= pu.host then - checkHost(m.BaseURL) - checkExes("mirror-checker.lua " .. sendArgs) - if testing("Integrity") or testing("Updated") then checkExes(downloadLock) end - end - end - while 1 <= checkExes("mirror-checker.lua " .. sendArgs) do os.execute("sleep 10") end - if testing("Integrity") or testing("Updated") then - while 0 < checkExes(downloadLock) do os.execute("sleep 10") end - end - os.execute("rm -f results/*.check") - logFile:close() -end -- cgit v1.1