From cc872d3eba68f08ee365b8771d8504c28da9020d Mon Sep 17 00:00:00 2001 From: onefang Date: Tue, 5 Nov 2019 15:38:24 +1000 Subject: Rename project to apt-panopticon. It used to be mirror-checker-lua. --- README.md | 81 +++---- apt-panopticon.lua | 634 +++++++++++++++++++++++++++++++++++++++++++++++++++++ mirror-checker.lua | 634 ----------------------------------------------------- 3 files changed, 675 insertions(+), 674 deletions(-) create mode 100755 apt-panopticon.lua delete mode 100755 mirror-checker.lua diff --git a/README.md b/README.md index 5ac7f9b..94b3c65 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,11 @@ This is currently under development, not everything has been written yet. Some of this document mentions some of the things that are not written yet. -mirror-checker-lua is a Lua script used by the Devuan mirror admins -(maybe, if they like it) to check the health of Devuan Linux package -mirrors. Originally there was bash scripts for this job, then Evilham -wrote some Python scripts, now onefang has written it in Lua. We all -have different tastes in languages. lol +apt-panopticon is a Lua script used by the Devuan mirror admins (maybe, +if they like it) to check the health of Devuan Linux package mirrors. +Originally there was bash scripts for this job, then Evilham wrote some +Python scripts, now onefang has written it in Lua. We all have different +tastes in languages. lol The main difference is that this Lua version tries to do everything, and will be maintained. Currently the shell scripts and Python scripts are @@ -17,7 +17,7 @@ badgered him about his Python scripts. It should also be much easier to use, the previous scripts needed some work before you could run them, this one you just download and run. -The source code is at [https://sledjhamr.org/cgit/mirror-checker-lua/](https://sledjhamr.org/cgit/mirror-checker-lua/) +The source code is at [https://sledjhamr.org/cgit/apt-panopticon/](https://sledjhamr.org/cgit/apt-panopticon/) The issue tracker is at [https://sledjhamr.org/mantisbt/project_page.php?project_id=13](https://sledjhamr.org/mantisbt/project_page.php?project_id=13) @@ -25,7 +25,7 @@ The issue tracker is at [https://sledjhamr.org/mantisbt/project_page.php?project Installation. ------------- -Download the source. You may want to put the mirror-checker.lua script +Download the source. You may want to put the apt-panopticon.lua script in someplace like `/usr/local/bin` and make sure it is executable. It should run on any recent Linux, you'll need to have the following @@ -52,57 +52,58 @@ and `results/web` directories, with the notification emails and web pages Note that unlike typical commands, you can't run single character options together, so this is wrong - - $ ./mirror-checker.lua -vvv + $ ./apt-panopticon.lua -vvv Instead do this - - $ ./mirror-checker.lua -v -v -v + $ ./apt-panopticon.lua -v -v -v Just run the script to do all of the tests - - $ ./mirror-checker.lua + $ ./apt-panopticon.lua Which will print any errors. If you don't want to see errors - - $ ./mirror-checker.lua -q + $ ./apt-panopticon.lua -q If you want to see warnings as well (as usual, the more `-v` options, the more details) - - $ ./mirror-checker.lua -v + $ ./apt-panopticon.lua -v Or use the usual options for the help and version number (not written yet) - - $ ./mirror-checker.lua -h - $ ./mirror-checker.lua --help - $ ./mirror-checker.lua --version + $ ./apt-panopticon.lua -h + $ ./apt-panopticon.lua --help + $ ./apt-panopticon.lua --version To run the tests on a specific mirror, for example pkgmaster.devuan.org - - $ ./mirror-checker.lua pkgmaster.devuan.org + $ ./apt-panopticon.lua pkgmaster.devuan.org You can use the `--tests` option to tune which tests are run, for example to stop IPv6 tests, coz you don't have IPv6 - - $ ./mirror-checker.lua --tests=-IPv6 + $ ./apt-panopticon.lua --tests=-IPv6 To do the same, but not run the HTTPS tests either - - $ ./mirror-checker.lua --tests=-IPv6,-https + $ ./apt-panopticon.lua --tests=-IPv6,-https To only run the HTTP integrity tests, only on IPv6 - - $ ./mirror-checker.lua --tests=http,Integrity,IPv6 + $ ./apt-panopticon.lua --tests=http,Integrity,IPv6 The tests. ---------- The basic test is to find all the IPs for a mirror, including any CNAMES, -then send HTTP HEAD requests to those IPs, with HOST headers for that mirror, -and follow any redirections, doing the same for those redirections. -Unless a specific mirror is given on the command line, the mirror_list.txt -file from pkgmaster.devuan.org is used to select mirrors to test. +then send HTTP HEAD requests to those IPs, with HOST headers for that +mirror, and follow any redirections, doing the same for those +redirections. Unless a specific mirror is given on the command line, the +mirror_list.txt file from pkgmaster.devuan.org is used to select mirrors +to test. The --tests= option can be used to adjust the list of tests performed. @@ -147,7 +148,8 @@ negative argument deselects a test. Examples are given above. --referenceSite -The mirror to use as a reference for the tests, the default is pkgmaster.devuan.org. +The mirror to use as a reference for the tests, the default is +pkgmaster.devuan.org. --roundRobin @@ -187,7 +189,7 @@ Delete results/*.check. touch results/stamp -Open results/mirror-checker-lua.log for message logging. +Open results/apt-panopticon.log for message logging. Download mirror_list.txt from the reference site. Build a table of Active mirrors keyed by the FDQN, include the listed Protocols as a sub @@ -221,12 +223,12 @@ that host. For each IPv4 and IPv6 address, fork a copy of the script something like this (including any arguments originally provided to the script) - -ionice -c3 ./mirror-checker.lua example.com/path x.x.x.x & +ionice -c3 ./apt-panopticon.lua example.com/path x.x.x.x & -ionice -c3 ./mirror-checker.lua example.com/path [x:x:x:x:x:x] & +ionice -c3 ./apt-panopticon.lua example.com/path [x:x:x:x:x:x] & -For each CNAME, it checkHost() the host, but with the CNAME as a -second argument. +For each CNAME, it checkHost() the host, but with the CNAME as a second +argument. SRV reconds don't do anything yet, coz I have yet to see one from my test environment, so can't test it. @@ -234,8 +236,7 @@ environment, so can't test it. Each forked call of the script from above does this - -Open results/mirror-checker-lua_example.com_x.x.x.x.log for message -logging. +Open results/example.com_x.x.x.x.log for message logging. Loads the mirrors table from results/mirrors.lua. @@ -245,18 +246,18 @@ actually perform the Integrity and Updated tests now, those haven't been written yet. Note that currently this downloads 4GB per mirror. Calls checkHost() with the host as first and second arguments, and -includes the IP this time. The inclusion of the IP causes -checkHost() to call checkFiles(). +includes the IP this time. The inclusion of the IP causes checkHost() to +call checkFiles(). checkFiles() will call checkHEAD() for each of the reference files. -checkHEAD() uses LuaSocket (or LuaSec for HTTPS) to send a HEAD request to -the IP, with a Host header set to the original host name. Redirects will -not be followed by that request. If the request returns a redirect, then -checkHEAD() is called recursively. If the redirect is to some host we are -not already checking, we call checkHost() on it, with an IP of -"redir". This causes checkHost() to bypass the test that would -otherwise call checkFiles(), instead gathering the IPs and fork as usual. +checkHEAD() uses LuaSocket (or LuaSec for HTTPS) to send a HEAD request +to the IP, with a Host header set to the original host name. Redirects +will not be followed by that request. If the request returns a redirect, +then checkHEAD() is called recursively. If the redirect is to some host +we are not already checking, we call checkHost() on it, with an IP of +"redir". This causes checkHost() to bypass the test that would otherwise +call checkFiles(), instead gathering the IPs and fork as usual. diff --git a/apt-panopticon.lua b/apt-panopticon.lua new file mode 100755 index 0000000..c68ec48 --- /dev/null +++ b/apt-panopticon.lua @@ -0,0 +1,634 @@ +#!/usr/bin/env luajit + + +local args = {...} + +--[[ TODO - What to do about HTTPS://deb.devuan.org/ redirects. + Some mirrors give a 404. + Sledjhamr gives a 404, coz it's not listening on 443 for deb.devuan.org. + Some mirrors give a 200. + They shouldn't have the proper certificate, but are giving a result anyway. +]] + +origin = false +verbosity = -1 +keep = false +-- TODO - Should actually implement this. +fork = true +options = +{ + referenceSite = + { + typ = "string", + help = "", + value = "pkgmaster.devuan.org", + }, + roundRobin = + { + typ = "string", + help = "", + value = "deb.devuan.org", + }, + tests = + { + typ = "table", + help = "", + value = + { + "IPv4", + "IPv6", +-- "ftp", + "http", + "https", +-- "rsync", +-- "DNS-RR", + "Protocol", +-- "URL-Sanity", +-- "Integrity", +-- "Updated", + }, + }, +} + +local defaultURL = {scheme = "http"} +local downloadLock = "flock -n results/wget-" +local download = "wget --timeout=300 -np -N -r -P results " -- Note wget has a default read timeout of 900 seconds (15 minutes). +local releases = {"jessie", "ascii", "beowulf", "ceres"} +local releaseFiles = +{ + -- Release file. + "/Release", + "/InRelease", + "/main/binary-all/Packages.gz", + -- Contents files. + "/main/Contents-all.gz", + "/main/Contents-amd64.gz", + "/main/Contents-arm64.gz", + "-security/main/Contents-all.gz", + "-security/main/Contents-amd64.gz", + "-security/main/Contents-arm64.gz", +} +local notExist = +{ + "ceres-security" -- This will never exist, it's our code name for the testing suite. +} +local referenceDebs = +{ + -- Debian package. + "merged/pool/DEBIAN/main/d/dash/dash_0.5.8-2.4_amd64.deb", + -- Debian security package. NOTE this one should always be redirected? + "merged/pool/DEBIAN-SECURITY/updates/main/a/apt/apt-transport-https_1.4.9_amd64.deb", +} +local referenceDevs = +{ + -- Devuan package. NOTE this one should not get redirected, but that's more a warning than an error. + "merged/pool/DEVUAN/main/d/desktop-base/desktop-base_2.0.3_all.deb", +-- "merged/pool/DEVUAN/main/u/util-linux/util-linux_2.32.1-0.1+devuan2.1_amd64.deb", +} +local arg = {} +local sendArgs = "" +local logFile + + +local socket = require 'socket' +local ftp = require 'socket.ftp' +local http = require 'socket.http' +local https = require 'ssl.https' -- See https://github.com/brunoos/luasec/wiki/LuaSec-0.6 for docs. +local url = require 'socket.url' + + +-- Use this to dump a table to a string. +dumpTable = function (table, space, name) + local r = "" + if "" == space then r = r .. space .. name .. " =\n" else r = r .. space .. "[" .. name .. "] =\n" end + r = r .. space .. "{\n" + r = r .. dumpTableSub(table, space .. " ") + if "" == space then r = r .. space .. "}\n" else r = r .. space .. "},\n" end + return r +end +dumpTableSub = function (table, space) + local r = "" + for k, v in pairs(table) do + if type(k) == "string" then k = '"' .. k .. '"' end + if type(v) == "table" then + r = r .. dumpTable(v, space, k) + elseif type(v) == "string" then + r = r .. space .. "[" .. k .. "] = '" .. v .. "';\n" + elseif type(v) == "function" then + r = r .. space .. "[" .. k .. "] = function ();\n" + elseif type(v) == "userdata" then + r = r .. space .. "userdata " .. "[" .. k .. "];\n" + elseif type(v) == "boolean" then + if (v) then + r = r .. space .. "[" .. k .. "] = true;\n" + else + r = r .. space .. "[" .. k .. "] = false;\n" + end + else + r = r .. space .. "[" .. k .. "] = " .. v .. ";\n" + end + end + return r +end + +local ip = "" +local results = {} + +local log = function(v, t, s, prot, test, host) + local x = "" + if nil == prot then prot = "" end + if nil ~= test then x = x .. test else test = "" end + if nil ~= host then + if #x > 0 then x = x .. " " end + x = x .. host + end + if #x > 0 then + t = t .. "(" .. x .. ")" + if "" == test then + if v == 0 then results[prot].errors = results[prot].errors + 1 end + if v == 1 then results[prot].warnings = results[prot].warnings + 1 end + else + if v == 0 then results[prot][test].errors = results[prot][test].errors + 1 end + if v == 1 then results[prot][test].warnings = results[prot][test].warnings + 1 end + end + end + if v <= verbosity then + if 3 <= verbosity then t = os.date() .. " " .. t end + print(t .. ": " .. s) + end + if nil ~= logFile then + logFile:write(os.date() .. " " .. t .. ": " .. s .. "\n") + logFile:flush() + end +end +local D = function(s) log(3, "DEBUG ", s) end +local I = function(s) log(2, "INFO ", s) end +local W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end +local E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end +local C = function(s) log(-1, "CRITICAL", s) end + +local mirrors = {} + +local testing = function(t, host) + for i, v in pairs(options.tests.value) do + if t == v then + local h = mirrors[host] + if nil == h then return true end + if true == h["Protocols"][t] then return true else D("Skipping " .. t .. " checks for " .. host) end + end + end + return false +end + +local checkExes = function (exe) + local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | wc -l'):read("*l") + D(count .. " " .. exe .. " commands still running.") + return tonumber(count) +end + +local repoExists = function (r) + r = r:match("([%a-]*)") + if nil == r then return false end + for k, v in pairs(notExist) do + if v == r then return false end + end + return true +end + +local IP = {} +gatherIPs = function (host) + if nil == IP[host] then + local IPs + local dig = io.popen('dig +keepopen +noall +nottlid +answer ' .. host .. ' A ' .. host .. ' AAAA ' .. host .. ' CNAME ' .. host .. ' SRV | sort -r | uniq') + repeat + IPs = dig:read("*l") + if nil ~= IPs then + for k, t, v in IPs:gmatch("([%w_%-%.]*)%.%s*IN%s*(%a*)%s*(.*)") do + if "." == v:sub(-1, -1) then v = v:sub(1, -2) end + if nil == IP[k] then IP[k] = {} end + IP[k][v] = t + D(" DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v) + if t == "CNAME" then + gatherIPs(v) + IP[k][v] = IP[v] + elseif v == "SRV" then + print("SVR record found, now what do we do?") + end + end + end + until nil == IPs + end +end + +-- Returns FTP directory listing +local nlst = function (u) + local t = {} + local p = url.parse(u) + p.command = "nlst" + p.sink = ltn12.sink.table(t) + local r, e = ftp.get(p) + return r and table.concat(t), e +end + +local timeouts = 0; +checkHEAD = function (host, URL, r, retry) + if nil == r then r = 0 end + if nil == retry then retry = 0 end + local check = "Checking file" + local PU = url.parse(URL, defaultURL) + local pu = url.parse(PU.scheme .. "://" .. host, defaultURL) + if 0 < r then + check = "Redirecting to" + end + if 0 < retry then + os.execute("sleep " .. math.random(1, 4)) + check = "Retry " .. retry .. " " .. check + end + if 3 < timeouts then + E("too many timeouts! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) + return + end + if 20 < r then + E("too many redirects! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) + return + end + if 4 < retry then + E("too many retries! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) + return + end + D(PU.scheme .. " :// " .. check .. " " .. host .. " -> " .. URL) + if not testing(PU.scheme, host) then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL); return end + -- TODO - Perhaps we should try it anyway, and mark it as a warning if it DOES work? + if "https" == PU.scheme and options.roundRobin.value == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert."); return end + local hd = {} + if pu.host ~= PU.host then hd = {Host = host} end + local htp = http; + if PU.scheme == "https" then htp = https end + -- NOTE - the docs for lua-sec say that redirect isn't supported is version 0.6, no idea if that means it ignores redirections like we want. + -- TODO - find out! + -- The protocol and options are lua-sec specific arguments. + local p, c, h, s = htp.request{method = "HEAD", redirect = false, url = URL, headers = hd, protocol = "any", options = "all"} + if nil == s then s = "" end + if nil == p then + E(" " .. c .. " " .. s .. "! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) + -- So far the only errors are "timeout", "Network is unreachable", and "closed", and I suspect "closed" is due to saturating my bandwidth. + if "timeout" == c then timeouts = timeouts + 1 end + if ("closed" == c) or ("Network is unreachable" == c) or ("timeout" == c) then checkHEAD(host, URL, r, retry + 1, timeouts) end + else + if ("4" == tostring(c):sub(1, 1)) or ("5" == tostring(c):sub(1, 1)) then + E(" " .. c .. " " .. s .. ". " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) + else + I(" " .. c .. " " .. s .. ". " .. check .. " " .. host .. " -> " .. URL) + timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through. + end + l = h.location + if nil ~= l then + pu = url.parse(l, defaultURL) + if (pu.scheme ~= PU.scheme) then + if testing("Protocol") then W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l, PU.scheme, "Protocol", host) end + if (pu.host == host) and pu.path == PU.path then D("Not testing protocol change " .. URL .. " -> " .. l); return end + end + + if l == URL then + E(" redirect loop! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, PU.scheme, host) + elseif nil == pu.host then + I(" relative redirect. " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l) + checkHEAD(host, PU.scheme .. "://" .. PU.host .. l, r + 1) + elseif (PU.host == pu.host) or (host == pu.host) then + checkHEAD(pu.host, l, r + 1) + else + --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location. + Not good for deb.debian.org, which we should only be throwing .debs at. + What we do is loop through the DNS entries, and only test the specific protocol & file being tested here. + + This is what I came up with for checking if we are already testing a specific URL. + Still duplicates a tiny bit, but much less than the previous find based method. + ]] + local file = pu.host .. "://" .. pu.path + local f = io.popen(string.format('if [ ! -f results/%s.check ] ; then touch results/%s.check; echo -n "check"; fi', file:gsub("/", "_"), file:gsub("/", "_") )):read("*a") + if (nil == f) or ("check" == f) then + I(" Now checking redirected host " .. file) + checkHost(pu.host, pu.host, nil, "redir", pu.path) + else + D(" Already checking " .. file) + end + end + end + end +end + +local checkFiles = function (host, ip, path, file) + if nil == path then path = "" end + if nil ~= file then + if "redir" == ip then ip = host end + I(" Checking IP for file " .. host .. " -> " .. ip .. " " .. path .. " " .. file) + if testing("http", host) then checkHEAD(host, "http://" .. ip .. path .. "/" .. file) end + if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/" .. file) end + else + I(" Checking IP " .. host .. " -> " .. ip .. " " .. path) + for i, s in pairs(referenceDevs) do + if testing("http", host) then checkHEAD(host, "http://" .. ip .. path .. "/" .. s) end + if 3 < timeouts then return end + if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/" .. s) end + if 3 < timeouts then return end + end + + for i, s in pairs(releases) do + for j, k in pairs(releaseFiles) do + if repoExists(s .. k) then + if testing("http", host) then checkHEAD(host, "http://" .. ip .. path .. "/merged/dists/" .. s .. k) end + if 3 < timeouts then return end + if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/merged/dists/" .. s .. k) end + if 3 < timeouts then return end + end + end + end + end +end + +local execute = function (s) + D(" executing " .. s) + os.execute(s) +end + +checkHost = function (orig, host, path, ip, file) + if nil == host then host = orig end + if nil == path then path = "" end + if nil == file then file = "" end + local ph = url.parse("http://" .. host) + if (nil ~= ip) and ("redir" ~= ip) then + local po = url.parse("http://" .. orig) + if "" ~= file then + D("checking redirected file " .. po.host .. " " .. file) + checkFiles(po.host, ip, path, file) + else + checkFiles(po.host, ip, path) + end + else + if orig == host then + D("checkHost " .. orig .. "" .. file) + if testing("IPv4") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file .." &") end + else D("checkHost " .. orig .. " -> " .. host) end + local h = mirrors[ph.host] + if nil == h then return end + for k, v in pairs(h.IPs) do + if "table" == type(v) then + for k1, v1 in pairs(v) do + if v1 == "A" then + if testing("IPv4") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file .." &") end + elseif v1 == "AAAA" then + if testing("IPv6") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k1 .. "] " .. file .. " &") end + end + end + else + if v == "A" then + if testing("IPv4") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file .." &") end + elseif v == "AAAA" then + if testing("IPv6") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k .. "] " .. file .. " &") end + end + end + end + end +end + +local downloads = function (cut, host, URL) + if 0 ~= cut then cd = " --cut-dirs=" .. cut .. " " else cd = "" end + if nil == URL then URL = "/" end + local lock = "%s-" .. host .. ".log " + local log = " --rejected-log=results/wget-%s_REJECTS-" .. host .. ".log -a results/wget-%s-" .. host .. ".log " + I("starting file download commands for " .. host .. " " .. URL) + local cm = "ionice -c3 " .. downloadLock .. lock:format("debs") .. download .. log:format("debs", "debs") .. cd + for i, s in pairs(referenceDevs) do + cm = cm .. " https://" .. host .. URL .. "/" .. s + end + for i, s in pairs(referenceDebs) do + cm = cm .. " https://" .. host .. URL .. "/" .. s + end + for i, s in pairs(releases) do + execute(cm .. " &") + cm = "ionice -c3 " .. downloadLock .. lock:format(s) .. download .. log:format(s, s) .. cd + if repoExists(s .. k) then + for j, k in pairs(releaseFiles) do + cm = cm .. " https://" .. host .. URL .. "/merged/dists/" .. s .. k + end + end + end + execute(cm .. " &") +end + +local getMirrors = function () + local mirrors = {} + local host = "" + local m = {} + local active = true + local URL = "https://" .. options.referenceSite.value .. "/mirror_list.txt" + I("getting mirrors.") + local p, c, h = http.request(URL) + if nil == p then E(c .. " fetching " .. URL) else + for l in p:gmatch("\n*([^\n]+)\n*") do + local t, d = l:match("(%a*):%s*(.*)") + d = string.lower(d) + if "FQDN" == t then + if "" ~= host then + if active then mirrors[host] = m end + m = {} + active = true + end + host = d + m[t] = d + gatherIPs(host) + m["IPs"] = IP[host] + elseif "Protocols" == t then + local prot = {} + for w in d:gmatch("(%w+)") do + prot[w] = true; + end + m[t] = prot + elseif "Active" == t and nil == d:find("yes", 1, true) then + W("Mirror " .. host .. " is not active - " .. d) + active = false +-- TODO - Should do some input validation on BaseURL, and everything else. + else + m[t] = d + end + end + if "" ~= host and active then + mirrors[host] = m + end + end + mirrors[options.roundRobin.value] = { ["Protocols"] = { ["http"] = true; ["https"] = true; }; ["FQDN"] = 'deb.devuan.org'; ["Active"] = 'yes'; ["BaseURL"] = 'deb.devuan.org'; } + gatherIPs(options.roundRobin.value) + mirrors[options.roundRobin.value].IPs = IP[options.roundRobin.value] + local file, e = io.open("results/mirrors.lua", "w+") + if nil == file then C("opening mirrors file - " .. e) else + file:write(dumpTable(mirrors, "", "mirrors") .. "\nreturn mirrors\n") + file:close() + end + return mirrors +end + + +if 0 ~= #args then + local option = "" + for i, a in pairs(args) do + if ("--help" == a) or ("-h" == a) then + print("I should write some docs, huh? Read README.md for instructions.") + elseif "--version" == a then + print("apt-panopticon version 0.1 WIP development version") + elseif "-v" == a then + verbosity = verbosity + 1 + sendArgs = sendArgs .. a .. " " + elseif "-q" == a then + verbosity = -1 + sendArgs = sendArgs .. a .. " " + elseif "-k" == a then + keep = true + elseif "-n" == a then + fork = false + elseif "-o" == a then + origin = true + elseif "--" == a:sub(1, 2) then + local s, e = a:find("=") + if nil == s then e = -1 end + option = a:sub(3, e - 1) + local o = options[option] + if nil == o then + print("Unknown option --" .. option) + option = "" + else + option = a + sendArgs = sendArgs .. a .. " " + local s, e = a:find("=") + if nil == s then e = 0 end + option = a:sub(3, e - 1) + if "table" == options[option].typ then + local result = {} + for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do + local f = t:sub(1, 1) + local n = t:sub(2, -1) + if ("+" ~= f) and ("-" ~= f) then + table.insert(result, t) + end + end + if 0 ~= #result then + options[option].value = result + else + for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do + local f = t:sub(1, 1) + local n = t:sub(2, -1) + if "+" == f then + table.insert(options[option].value, n) + elseif "-" == f then + local r = {} + for i, k in pairs(options[option].value) do + if k ~= n then table.insert(r, k) end + end + options[option].value = r + end + end + end + else + options[option].value = a + end + option = "" + end + elseif "-" == a:sub(1, 1) then + print("Unknown option " .. a) + else + table.insert(arg, a) + end + end +end + +--print(dumpTable(options.tests.value, "", "tests")) + +execute("mkdir -p results") + +if 0 < #arg then + if "/" == arg[1]:sub(-1, -1) then + W("slash at end of path! " .. arg[1]) + arg[1] = arg[1]:sub(1, -2) + end + if " " == arg[1]:sub(-1, -1) then + W("space at end of path! " .. arg[1]) + arg[1] = arg[1]:sub(1, -2) + end + local pu = url.parse("http://" .. arg[1]) + if nil ~= arg[2] then + logFile, e = io.open("results/" .. pu.host .. "_" .. arg[2] .. ".log", "a+") + else + logFile, e = io.open("results/" .. pu.host .. ".log", "a+") + end + if nil == logFile then C("opening log file - " .. e); return end + I("Starting tests for " ..arg[1] .. " with these tests - " .. table.concat(options.tests.value, ", ")) + mirrors = loadfile("results/mirrors.lua")() + if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end + if nil ~= arg[3] then I(" Using file " .. arg[3]); end + + for k, v in pairs{"ftp", "http", "https", "rsync"} do + if testing(v) then + local tests = {errors = 0; warnings = 0} + if testing("Integrity") then tests.Integrity = {errors = 0; warnings = 0} end + if testing("Protocol") then tests.Protocol = {errors = 0; warnings = 0} end + if testing("Updated") then tests.Updated = {errors = 0; warnings = 0} end + if testing("URL-Sanity") then tests.URL_Sanity = {errors = 0; warnings = 0} end + results[v] = tests + end + end + + if testing("Integrity") or testing("Updated") then + if nil == arg[3] then + if not keep then execute("rm -fr results/" .. pu.host) end + cut = 0 + for t in arg[1]:gmatch("(/)") do + cut = cut + 1 + end + downloads(cut, pu.host, pu.path) + checkExes("apt-panopticon.lua " .. sendArgs) + checkExes(downloadLock) + end + end + if origin then + checkFiles(pu.host, pu.host, pu.path); + else + checkHost(pu.host, pu.host, pu.path, arg[2], arg[3]) + end + logFile:close() + local rfile, e = io.open("results/" .. pu.host .. "_" .. ip .. ".lua", "w+") + if nil == rfile then C("opening results file - " .. e) else + rfile:write(dumpTable(results, "", "results") .. "\nreturn results\n") + rfile:close() + end +else + if not keep then os.execute("rm -f results/*.log") end + os.execute("rm -f results/*.check") + os.execute("mkdir -p results; touch results/stamp") + logFile, e = io.open("results/apt-panopticon.log", "a+") + if nil == logFile then C("opening log file - " .. e); return end + I("Starting tests " .. table.concat(options.tests.value, ", ")) + execute("mkdir -p results") + mirrors = getMirrors() + checkHost(options.referenceSite.value) + for k, m in pairs(mirrors) do + if "/" == m.BaseURL:sub(-1, -1) then + W("slash at end of BaseURL in mirror_list.txt! " .. m.BaseURL) + m.BaseURL = m.BaseURL:sub(1, -2) + end + if " " == m.BaseURL:sub(-1, -1) then + W("space at end of BaseURL in mirror_list.txt! " .. m.BaseURL) + m.BaseURL = m.BaseURL:sub(1, -2) + end + local pu = url.parse("http://" .. m.BaseURL) + if options.referenceSite.value ~= pu.host then + checkHost(m.BaseURL) + checkExes("apt-panopticon.lua " .. sendArgs) + if testing("Integrity") or testing("Updated") then checkExes(downloadLock) end + end + end + while 1 <= checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end + if testing("Integrity") or testing("Updated") then + while 0 < checkExes(downloadLock) do os.execute("sleep 10") end + end + os.execute("rm -f results/*.check") + logFile:close() +end diff --git a/mirror-checker.lua b/mirror-checker.lua deleted file mode 100755 index 01082d9..0000000 --- a/mirror-checker.lua +++ /dev/null @@ -1,634 +0,0 @@ -#!/usr/bin/env luajit - - -local args = {...} - ---[[ TODO - What to do about HTTPS://deb.devuan.org/ redirects. - Some mirrors give a 404. - Sledjhamr gives a 404, coz it's not listening on 443 for deb.devuan.org. - Some mirrors give a 200. - They shouldn't have the proper certificate, but are giving a result anyway. -]] - -origin = false -verbosity = -1 -keep = false --- TODO - Should actually implement this. -fork = true -options = -{ - referenceSite = - { - typ = "string", - help = "", - value = "pkgmaster.devuan.org", - }, - roundRobin = - { - typ = "string", - help = "", - value = "deb.devuan.org", - }, - tests = - { - typ = "table", - help = "", - value = - { - "IPv4", - "IPv6", --- "ftp", - "http", - "https", --- "rsync", --- "DNS-RR", - "Protocol", --- "URL-Sanity", --- "Integrity", --- "Updated", - }, - }, -} - -local defaultURL = {scheme = "http"} -local downloadLock = "flock -n results/wget-" -local download = "wget --timeout=300 -np -N -r -P results " -- Note wget has a default read timeout of 900 seconds (15 minutes). -local releases = {"jessie", "ascii", "beowulf", "ceres"} -local releaseFiles = -{ - -- Release file. - "/Release", - "/InRelease", - "/main/binary-all/Packages.gz", - -- Contents files. - "/main/Contents-all.gz", - "/main/Contents-amd64.gz", - "/main/Contents-arm64.gz", - "-security/main/Contents-all.gz", - "-security/main/Contents-amd64.gz", - "-security/main/Contents-arm64.gz", -} -local notExist = -{ - "ceres-security" -- This will never exist, it's our code name for the testing suite. -} -local referenceDebs = -{ - -- Debian package. - "merged/pool/DEBIAN/main/d/dash/dash_0.5.8-2.4_amd64.deb", - -- Debian security package. NOTE this one should always be redirected? - "merged/pool/DEBIAN-SECURITY/updates/main/a/apt/apt-transport-https_1.4.9_amd64.deb", -} -local referenceDevs = -{ - -- Devuan package. NOTE this one should not get redirected, but that's more a warning than an error. - "merged/pool/DEVUAN/main/d/desktop-base/desktop-base_2.0.3_all.deb", --- "merged/pool/DEVUAN/main/u/util-linux/util-linux_2.32.1-0.1+devuan2.1_amd64.deb", -} -local arg = {} -local sendArgs = "" -local logFile - - -local socket = require 'socket' -local ftp = require 'socket.ftp' -local http = require 'socket.http' -local https = require 'ssl.https' -- See https://github.com/brunoos/luasec/wiki/LuaSec-0.6 for docs. -local url = require 'socket.url' - - --- Use this to dump a table to a string. -dumpTable = function (table, space, name) - local r = "" - if "" == space then r = r .. space .. name .. " =\n" else r = r .. space .. "[" .. name .. "] =\n" end - r = r .. space .. "{\n" - r = r .. dumpTableSub(table, space .. " ") - if "" == space then r = r .. space .. "}\n" else r = r .. space .. "},\n" end - return r -end -dumpTableSub = function (table, space) - local r = "" - for k, v in pairs(table) do - if type(k) == "string" then k = '"' .. k .. '"' end - if type(v) == "table" then - r = r .. dumpTable(v, space, k) - elseif type(v) == "string" then - r = r .. space .. "[" .. k .. "] = '" .. v .. "';\n" - elseif type(v) == "function" then - r = r .. space .. "[" .. k .. "] = function ();\n" - elseif type(v) == "userdata" then - r = r .. space .. "userdata " .. "[" .. k .. "];\n" - elseif type(v) == "boolean" then - if (v) then - r = r .. space .. "[" .. k .. "] = true;\n" - else - r = r .. space .. "[" .. k .. "] = false;\n" - end - else - r = r .. space .. "[" .. k .. "] = " .. v .. ";\n" - end - end - return r -end - -local ip = "" -local results = {} - -local log = function(v, t, s, prot, test, host) - local x = "" - if nil == prot then prot = "" end - if nil ~= test then x = x .. test else test = "" end - if nil ~= host then - if #x > 0 then x = x .. " " end - x = x .. host - end - if #x > 0 then - t = t .. "(" .. x .. ")" - if "" == test then - if v == 0 then results[prot].errors = results[prot].errors + 1 end - if v == 1 then results[prot].warnings = results[prot].warnings + 1 end - else - if v == 0 then results[prot][test].errors = results[prot][test].errors + 1 end - if v == 1 then results[prot][test].warnings = results[prot][test].warnings + 1 end - end - end - if v <= verbosity then - if 3 <= verbosity then t = os.date() .. " " .. t end - print(t .. ": " .. s) - end - if nil ~= logFile then - logFile:write(os.date() .. " " .. t .. ": " .. s .. "\n") - logFile:flush() - end -end -local D = function(s) log(3, "DEBUG ", s) end -local I = function(s) log(2, "INFO ", s) end -local W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end -local E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end -local C = function(s) log(-1, "CRITICAL", s) end - -local mirrors = {} - -local testing = function(t, host) - for i, v in pairs(options.tests.value) do - if t == v then - local h = mirrors[host] - if nil == h then return true end - if true == h["Protocols"][t] then return true else D("Skipping " .. t .. " checks for " .. host) end - end - end - return false -end - -local checkExes = function (exe) - local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | wc -l'):read("*l") - D(count .. " " .. exe .. " commands still running.") - return tonumber(count) -end - -local repoExists = function (r) - r = r:match("([%a-]*)") - if nil == r then return false end - for k, v in pairs(notExist) do - if v == r then return false end - end - return true -end - -local IP = {} -gatherIPs = function (host) - if nil == IP[host] then - local IPs - local dig = io.popen('dig +keepopen +noall +nottlid +answer ' .. host .. ' A ' .. host .. ' AAAA ' .. host .. ' CNAME ' .. host .. ' SRV | sort -r | uniq') - repeat - IPs = dig:read("*l") - if nil ~= IPs then - for k, t, v in IPs:gmatch("([%w_%-%.]*)%.%s*IN%s*(%a*)%s*(.*)") do - if "." == v:sub(-1, -1) then v = v:sub(1, -2) end - if nil == IP[k] then IP[k] = {} end - IP[k][v] = t - D(" DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v) - if t == "CNAME" then - gatherIPs(v) - IP[k][v] = IP[v] - elseif v == "SRV" then - print("SVR record found, now what do we do?") - end - end - end - until nil == IPs - end -end - --- Returns FTP directory listing -local nlst = function (u) - local t = {} - local p = url.parse(u) - p.command = "nlst" - p.sink = ltn12.sink.table(t) - local r, e = ftp.get(p) - return r and table.concat(t), e -end - -local timeouts = 0; -checkHEAD = function (host, URL, r, retry) - if nil == r then r = 0 end - if nil == retry then retry = 0 end - local check = "Checking file" - local PU = url.parse(URL, defaultURL) - local pu = url.parse(PU.scheme .. "://" .. host, defaultURL) - if 0 < r then - check = "Redirecting to" - end - if 0 < retry then - os.execute("sleep " .. math.random(1, 4)) - check = "Retry " .. retry .. " " .. check - end - if 3 < timeouts then - E("too many timeouts! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) - return - end - if 20 < r then - E("too many redirects! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) - return - end - if 4 < retry then - E("too many retries! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) - return - end - D(PU.scheme .. " :// " .. check .. " " .. host .. " -> " .. URL) - if not testing(PU.scheme, host) then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL); return end - -- TODO - Perhaps we should try it anyway, and mark it as a warning if it DOES work? - if "https" == PU.scheme and options.roundRobin.value == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert."); return end - local hd = {} - if pu.host ~= PU.host then hd = {Host = host} end - local htp = http; - if PU.scheme == "https" then htp = https end - -- NOTE - the docs for lua-sec say that redirect isn't supported is version 0.6, no idea if that means it ignores redirections like we want. - -- TODO - find out! - -- The protocol and options are lua-sec specific arguments. - local p, c, h, s = htp.request{method = "HEAD", redirect = false, url = URL, headers = hd, protocol = "any", options = "all"} - if nil == s then s = "" end - if nil == p then - E(" " .. c .. " " .. s .. "! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) - -- So far the only errors are "timeout", "Network is unreachable", and "closed", and I suspect "closed" is due to saturating my bandwidth. - if "timeout" == c then timeouts = timeouts + 1 end - if ("closed" == c) or ("Network is unreachable" == c) or ("timeout" == c) then checkHEAD(host, URL, r, retry + 1, timeouts) end - else - if ("4" == tostring(c):sub(1, 1)) or ("5" == tostring(c):sub(1, 1)) then - E(" " .. c .. " " .. s .. ". " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) - else - I(" " .. c .. " " .. s .. ". " .. check .. " " .. host .. " -> " .. URL) - timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through. - end - l = h.location - if nil ~= l then - pu = url.parse(l, defaultURL) - if (pu.scheme ~= PU.scheme) then - if testing("Protocol") then W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l, PU.scheme, "Protocol", host) end - if (pu.host == host) and pu.path == PU.path then D("Not testing protocol change " .. URL .. " -> " .. l); return end - end - - if l == URL then - E(" redirect loop! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, PU.scheme, host) - elseif nil == pu.host then - I(" relative redirect. " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l) - checkHEAD(host, PU.scheme .. "://" .. PU.host .. l, r + 1) - elseif (PU.host == pu.host) or (host == pu.host) then - checkHEAD(pu.host, l, r + 1) - else - --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location. - Not good for deb.debian.org, which we should only be throwing .debs at. - What we do is loop through the DNS entries, and only test the specific protocol & file being tested here. - - This is what I came up with for checking if we are already testing a specific URL. - Still duplicates a tiny bit, but much less than the previous find based method. - ]] - local file = pu.host .. "://" .. pu.path - local f = io.popen(string.format('if [ ! -f results/%s.check ] ; then touch results/%s.check; echo -n "check"; fi', file:gsub("/", "_"), file:gsub("/", "_") )):read("*a") - if (nil == f) or ("check" == f) then - I(" Now checking redirected host " .. file) - checkHost(pu.host, pu.host, nil, "redir", pu.path) - else - D(" Already checking " .. file) - end - end - end - end -end - -local checkFiles = function (host, ip, path, file) - if nil == path then path = "" end - if nil ~= file then - if "redir" == ip then ip = host end - I(" Checking IP for file " .. host .. " -> " .. ip .. " " .. path .. " " .. file) - if testing("http", host) then checkHEAD(host, "http://" .. ip .. path .. "/" .. file) end - if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/" .. file) end - else - I(" Checking IP " .. host .. " -> " .. ip .. " " .. path) - for i, s in pairs(referenceDevs) do - if testing("http", host) then checkHEAD(host, "http://" .. ip .. path .. "/" .. s) end - if 3 < timeouts then return end - if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/" .. s) end - if 3 < timeouts then return end - end - - for i, s in pairs(releases) do - for j, k in pairs(releaseFiles) do - if repoExists(s .. k) then - if testing("http", host) then checkHEAD(host, "http://" .. ip .. path .. "/merged/dists/" .. s .. k) end - if 3 < timeouts then return end - if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/merged/dists/" .. s .. k) end - if 3 < timeouts then return end - end - end - end - end -end - -local execute = function (s) - D(" executing " .. s) - os.execute(s) -end - -checkHost = function (orig, host, path, ip, file) - if nil == host then host = orig end - if nil == path then path = "" end - if nil == file then file = "" end - local ph = url.parse("http://" .. host) - if (nil ~= ip) and ("redir" ~= ip) then - local po = url.parse("http://" .. orig) - if "" ~= file then - D("checking redirected file " .. po.host .. " " .. file) - checkFiles(po.host, ip, path, file) - else - checkFiles(po.host, ip, path) - end - else - if orig == host then - D("checkHost " .. orig .. "" .. file) - if testing("IPv4") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file .." &") end - else D("checkHost " .. orig .. " -> " .. host) end - local h = mirrors[ph.host] - if nil == h then return end - for k, v in pairs(h.IPs) do - if "table" == type(v) then - for k1, v1 in pairs(v) do - if v1 == "A" then - if testing("IPv4") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file .." &") end - elseif v1 == "AAAA" then - if testing("IPv6") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k1 .. "] " .. file .. " &") end - end - end - else - if v == "A" then - if testing("IPv4") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file .." &") end - elseif v == "AAAA" then - if testing("IPv6") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k .. "] " .. file .. " &") end - end - end - end - end -end - -local downloads = function (cut, host, URL) - if 0 ~= cut then cd = " --cut-dirs=" .. cut .. " " else cd = "" end - if nil == URL then URL = "/" end - local lock = "%s-" .. host .. ".log " - local log = " --rejected-log=results/wget-%s_REJECTS-" .. host .. ".log -a results/wget-%s-" .. host .. ".log " - I("starting file download commands for " .. host .. " " .. URL) - local cm = "ionice -c3 " .. downloadLock .. lock:format("debs") .. download .. log:format("debs", "debs") .. cd - for i, s in pairs(referenceDevs) do - cm = cm .. " https://" .. host .. URL .. "/" .. s - end - for i, s in pairs(referenceDebs) do - cm = cm .. " https://" .. host .. URL .. "/" .. s - end - for i, s in pairs(releases) do - execute(cm .. " &") - cm = "ionice -c3 " .. downloadLock .. lock:format(s) .. download .. log:format(s, s) .. cd - if repoExists(s .. k) then - for j, k in pairs(releaseFiles) do - cm = cm .. " https://" .. host .. URL .. "/merged/dists/" .. s .. k - end - end - end - execute(cm .. " &") -end - -local getMirrors = function () - local mirrors = {} - local host = "" - local m = {} - local active = true - local URL = "https://" .. options.referenceSite.value .. "/mirror_list.txt" - I("getting mirrors.") - local p, c, h = http.request(URL) - if nil == p then E(c .. " fetching " .. URL) else - for l in p:gmatch("\n*([^\n]+)\n*") do - local t, d = l:match("(%a*):%s*(.*)") - d = string.lower(d) - if "FQDN" == t then - if "" ~= host then - if active then mirrors[host] = m end - m = {} - active = true - end - host = d - m[t] = d - gatherIPs(host) - m["IPs"] = IP[host] - elseif "Protocols" == t then - local prot = {} - for w in d:gmatch("(%w+)") do - prot[w] = true; - end - m[t] = prot - elseif "Active" == t and nil == d:find("yes", 1, true) then - W("Mirror " .. host .. " is not active - " .. d) - active = false --- TODO - Should do some input validation on BaseURL, and everything else. - else - m[t] = d - end - end - if "" ~= host and active then - mirrors[host] = m - end - end - mirrors[options.roundRobin.value] = { ["Protocols"] = { ["http"] = true; ["https"] = true; }; ["FQDN"] = 'deb.devuan.org'; ["Active"] = 'yes'; ["BaseURL"] = 'deb.devuan.org'; } - gatherIPs(options.roundRobin.value) - mirrors[options.roundRobin.value].IPs = IP[options.roundRobin.value] - local file, e = io.open("results/mirrors.lua", "w+") - if nil == file then C("opening mirrors file - " .. e) else - file:write(dumpTable(mirrors, "", "mirrors") .. "\nreturn mirrors\n") - file:close() - end - return mirrors -end - - -if 0 ~= #args then - local option = "" - for i, a in pairs(args) do - if ("--help" == a) or ("-h" == a) then - print("I should write some docs, huh? Read README.md for instructions.") - elseif "--version" == a then - print("mirror-checker-lua version 0.1 WIP development version") - elseif "-v" == a then - verbosity = verbosity + 1 - sendArgs = sendArgs .. a .. " " - elseif "-q" == a then - verbosity = -1 - sendArgs = sendArgs .. a .. " " - elseif "-k" == a then - keep = true - elseif "-n" == a then - fork = false - elseif "-o" == a then - origin = true - elseif "--" == a:sub(1, 2) then - local s, e = a:find("=") - if nil == s then e = -1 end - option = a:sub(3, e - 1) - local o = options[option] - if nil == o then - print("Unknown option --" .. option) - option = "" - else - option = a - sendArgs = sendArgs .. a .. " " - local s, e = a:find("=") - if nil == s then e = 0 end - option = a:sub(3, e - 1) - if "table" == options[option].typ then - local result = {} - for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do - local f = t:sub(1, 1) - local n = t:sub(2, -1) - if ("+" ~= f) and ("-" ~= f) then - table.insert(result, t) - end - end - if 0 ~= #result then - options[option].value = result - else - for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do - local f = t:sub(1, 1) - local n = t:sub(2, -1) - if "+" == f then - table.insert(options[option].value, n) - elseif "-" == f then - local r = {} - for i, k in pairs(options[option].value) do - if k ~= n then table.insert(r, k) end - end - options[option].value = r - end - end - end - else - options[option].value = a - end - option = "" - end - elseif "-" == a:sub(1, 1) then - print("Unknown option " .. a) - else - table.insert(arg, a) - end - end -end - ---print(dumpTable(options.tests.value, "", "tests")) - -execute("mkdir -p results") - -if 0 < #arg then - if "/" == arg[1]:sub(-1, -1) then - W("slash at end of path! " .. arg[1]) - arg[1] = arg[1]:sub(1, -2) - end - if " " == arg[1]:sub(-1, -1) then - W("space at end of path! " .. arg[1]) - arg[1] = arg[1]:sub(1, -2) - end - local pu = url.parse("http://" .. arg[1]) - if nil ~= arg[2] then - logFile, e = io.open("results/" .. pu.host .. "_" .. arg[2] .. ".log", "a+") - else - logFile, e = io.open("results/" .. pu.host .. ".log", "a+") - end - if nil == logFile then C("opening log file - " .. e); return end - I("Starting tests for " ..arg[1] .. " with these tests - " .. table.concat(options.tests.value, ", ")) - mirrors = loadfile("results/mirrors.lua")() - if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end - if nil ~= arg[3] then I(" Using file " .. arg[3]); end - - for k, v in pairs{"ftp", "http", "https", "rsync"} do - if testing(v) then - local tests = {errors = 0; warnings = 0} - if testing("Integrity") then tests.Integrity = {errors = 0; warnings = 0} end - if testing("Protocol") then tests.Protocol = {errors = 0; warnings = 0} end - if testing("Updated") then tests.Updated = {errors = 0; warnings = 0} end - if testing("URL-Sanity") then tests.URL_Sanity = {errors = 0; warnings = 0} end - results[v] = tests - end - end - - if testing("Integrity") or testing("Updated") then - if nil == arg[3] then - if not keep then execute("rm -fr results/" .. pu.host) end - cut = 0 - for t in arg[1]:gmatch("(/)") do - cut = cut + 1 - end - downloads(cut, pu.host, pu.path) - checkExes("mirror-checker.lua " .. sendArgs) - checkExes(downloadLock) - end - end - if origin then - checkFiles(pu.host, pu.host, pu.path); - else - checkHost(pu.host, pu.host, pu.path, arg[2], arg[3]) - end - logFile:close() - local rfile, e = io.open("results/" .. pu.host .. "_" .. ip .. ".lua", "w+") - if nil == rfile then C("opening results file - " .. e) else - rfile:write(dumpTable(results, "", "results") .. "\nreturn results\n") - rfile:close() - end -else - if not keep then os.execute("rm -f results/*.log") end - os.execute("rm -f results/*.check") - os.execute("mkdir -p results; touch results/stamp") - logFile, e = io.open("results/mirror-checker-lua.log", "a+") - if nil == logFile then C("opening log file - " .. e); return end - I("Starting tests " .. table.concat(options.tests.value, ", ")) - execute("mkdir -p results") - mirrors = getMirrors() - checkHost(options.referenceSite.value) - for k, m in pairs(mirrors) do - if "/" == m.BaseURL:sub(-1, -1) then - W("slash at end of BaseURL in mirror_list.txt! " .. m.BaseURL) - m.BaseURL = m.BaseURL:sub(1, -2) - end - if " " == m.BaseURL:sub(-1, -1) then - W("space at end of BaseURL in mirror_list.txt! " .. m.BaseURL) - m.BaseURL = m.BaseURL:sub(1, -2) - end - local pu = url.parse("http://" .. m.BaseURL) - if options.referenceSite.value ~= pu.host then - checkHost(m.BaseURL) - checkExes("mirror-checker.lua " .. sendArgs) - if testing("Integrity") or testing("Updated") then checkExes(downloadLock) end - end - end - while 1 <= checkExes("mirror-checker.lua " .. sendArgs) do os.execute("sleep 10") end - if testing("Integrity") or testing("Updated") then - while 0 < checkExes(downloadLock) do os.execute("sleep 10") end - end - os.execute("rm -f results/*.check") - logFile:close() -end -- cgit v1.1