From 22fc09cfd2305850063d2bbdd3af76237caf2e73 Mon Sep 17 00:00:00 2001 From: onefang Date: Tue, 25 Jun 2019 15:08:34 +1000 Subject: Add the actual source code, and the basic documentation. Still need to write the help output. --- README.md | 95 ++++++++++++ mirror-checker.lua | 420 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 515 insertions(+) create mode 100644 README.md create mode 100755 mirror-checker.lua diff --git a/README.md b/README.md new file mode 100644 index 0000000..415025c --- /dev/null +++ b/README.md @@ -0,0 +1,95 @@ +Lua script for checking the health of Devuan Linux package mirrors. + +This is currently under development, not everything has been written yet. + +mirror-checker-lua is a Lua script used by the Devuan mirror admins +(maybe, if they like it) to check the health of Devuan Linux package +mirrors. Originally there was bash scripts for this job, then Evilham +wrote some Python scripts, now onefang has written it in Lua. We all +have different tastes in languages. lol + +The main difference is that this Lua version tries to do everything, and +will be maintained. Currently the shell scripts and Python scripts are +actually being used I think. Evilham asked me to write this, after I +badgered him about his Python scripts. + +The source code is at https://sledjhamr.org/cgit/mirror-checker-lua/ + +The issue tracker is at https://sledjhamr.org/mantisbt/project_page.php?project_id=13 + + +Installation. +------------- + +Download the source. You may want to put the actual mirror-checker.lua +in someplace liku /usr/local/bin and make sure it is executable. + +It should run on any recent Linux, you'll need to have the following +installed - + +Luajit + +wget + +dig, part of BIND. On Debian based systems it'll be in the dnsutils +package. + +LuaSocket, on Debian based systems it'll be in the lua-socket package. + +ionice, on Debian based systems it'll be in the util-linux package. + + +Using it. +--------- + +These examples assume you are running it from the source code directory. +A directory will be created called results, it'll be full of log files +and any files that get downloaded. There will also be results/email and +results/web directories, with the notification email and web pages with +the easy to read emails (once I write that bit). + +Note that unlike typical commands, you can't run single character options +together, so this is wrong - + +./mirror-checker.lua -vvv + +Instead do this - + +./mirror-checker.lua -v -v -v + +Just run the script to do all of the tests - + +./mirror-checker.lua + +Which will print any errors. If you don't want to see errors - + +./mirror-checker.lua -q + +If you want to see warnings to (as usual, the more -v options, the more +details) - + +./mirror-checker.lua -v + +Or use the usual options for the help and version number - + +./mirror-checker.lua -h +./mirror-checker.lua --help +./mirror-checker.lua --version + +To run the tests on a specific mirror, for example pkgmaster.devuan.org - + +./mirror-checker.lua pkgmaster.devuan.org + +You can use the --tests option to tune which tests are run, for example +to stop IPv6 tests, coz you don't have IPv6 - + +./mirror-checker.lua --tests=-IPv6 + +To do the same, but not run the HTTPS tests either - + +./mirror-checker.lua --tests=-IPv6,-https + +To only run the http integrity tests, only on IPv6 - + +./mirror-checker.lua --tests=http,Integrity,IPv6 + diff --git a/mirror-checker.lua b/mirror-checker.lua new file mode 100755 index 0000000..34b2ec8 --- /dev/null +++ b/mirror-checker.lua @@ -0,0 +1,420 @@ +#!/usr/bin/env luajit + + +local args = {...} + + +verbosity = 0 +keep = false +options = +{ + referenceSite = + { + typ = "string", + help = "", + value = "pkgmaster.devuan.org", + }, + tests = + { + typ = "table", + help = "", + value = + { + "IPv4", + "IPv6", +-- "ftp", + "http", + "https", +-- "rsync", + "DNS-RR", + "Protocol", +-- "URL-Sanity", +-- "Integrity", +-- "Updated", + }, + }, +} + +local defaultURL = {scheme = "http"} +local download = "wget -np -N -r -P results " +local releases = {"jessie", "ascii", "beowulf", "ceres"} +local releaseFiles = +{ + -- Release file. + "/Release", + "/InRelease", + "/main/binary-all/Packages.gz", + -- Contents files. + "/main/Contents-all.gz", + "/main/Contents-amd64.gz", + "/main/Contents-arm64.gz", + "-security/main/Contents-all.gz", + "-security/main/Contents-amd64.gz", + "-security/main/Contents-arm64.gz", +} +local referenceDebs = +{ + -- Devuan package. NOTE this one likely should not get redirected, but that's more a warning than an error. + "merged/pool/DEVUAN/main/d/desktop-base/desktop-base_2.0.3_all.deb", + -- Debian package. + "merged/pool/DEBIAN/main/d/dash/dash_0.5.8-2.4_amd64.deb", + -- Debian security package. NOTE this one should always be redirected? + "merged/pool/DEBIAN-SECURITY/updates/main/a/apt/apt-transport-https_1.4.9_amd64.deb", +} +local arg = {} +local sendArgs = "" +local log + + +local socket = require 'socket' +local ftp = require 'socket.ftp' +local http = require 'socket.http' +local url = require 'socket.url' + + +-- Use this to print a table. +printTable = function (table, space, name) + print(space .. name .. ": ") + print(space .. "{") + printTableSub(table, space .. " ") + print(space .. "}") +end +printTableSub = function (table, space) + for k, v in pairs(table) do + if type(v) == "table" then + printTable(v, space, k) + elseif type(v) == "string" then + print(space .. k .. ': "' .. v .. '";') + elseif type(v) == "function" then + print(space .. "function " .. k .. "();") + elseif type(v) == "userdata" then + print(space .. "userdata " .. k .. ";") + elseif type(v) == "boolean" then + if (v) then + print(space .. "boolean " .. k .. " TRUE ;") + else + print(space .. "boolean " .. k .. " FALSE ;") + end + else + print(space .. k .. ": " .. v .. ";") + end + end +end + +local log = function(v, t, s) + if v <= verbosity then + if 3 <= verbosity then t = os.date() .. " " .. t end + print(t .. ": " .. s) + end + if nil ~= log then + log:write(os.date() .. " " .. t .. ": " .. s .. "/n") + log:flush() + end +end +local D = function(s) log(3, "DEBUG ", s) end +local I = function(s) log(2, "INFO ", s) end +local W = function(s) log(1, "WARNING ", s) end +local E = function(s) log(0, "ERROR ", s) end +local C = function(s) log(-1, "CRITICAL", s) end + +local testing = function(t) + for i, v in pairs(options.tests.value) do + if t == v then return true end + end + return false +end + +local checkExes = function (exe) + local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | wc -l'):read("*l") + D(count .. " " .. exe .. " commands left.") + return tonumber(count) +end + + +local IP = {} +local gatherIPs = function (host) + if nil == IP[host] then + local IPs + local dig = io.popen('dig +keepopen +noall +nottlid +answer ' .. host .. ' A ' .. host .. ' AAAA ' .. host .. ' CNAME ' .. host .. ' SRV | sort | uniq') + repeat + IPs = dig:read("*l") + if nil ~= IPs then + for k, t, v in IPs:gmatch("([%w_%-%.]*)%.%s*IN%s*(%a*)%s*(.*)") do + if "." == v:sub(-1, -1) then v = v:sub(1, -2) end + if nil == IP[k] then IP[k] = {} end + IP[k][v] = t + end + end + until nil == IPs + end +end + +-- Returns FTP directory listing +local nlst = function (u) + local t = {} + local p = url.parse(u) + p.command = "nlst" + p.sink = ltn12.sink.table(t) + local r, e = ftp.get(p) + return r and table.concat(t), e +end + +checkURL = function (host, URL, r) + if nil == r then r = 0 end + local check = "Checking file" + if 0 < r then + check = "Redirecting to" +-- checkIP(host) + end + if 10 < r then + E("too many redirects! " .. check .. " " .. host .. " -> " .. URL) + return + end + local PU = url.parse(URL, defaultURL) + D(" " .. PU.scheme .. " :// " .. check .. " " .. host .. " -> " .. URL) + if not testing(PU.scheme) then D("not testing " .. PU.scheme .. " " .. host .. " -> " .. URL); return end + local hd = {Host = host} + local p, c, h, s = http.request{method = "HEAD", redirect = false, url = URL, headers = hd} + if nil == p then E(c .. "! " .. check .. " " .. host .. " -> " .. URL) else + l = h.location + if nil ~= l then + local pu = url.parse(l, defaultURL) + if l == URL then + E("redirect loop! " .. check .. " " .. host .. " -> " .. URL) + else + if nil == pu.host then + W("no location host! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l) + checkURL(host, PU.scheme .. "://" .. PU.host .. l, r + 1) + else + if testing("Protocol") and pu.scheme ~= PU.scheme then + W("protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l) + end + checkURL(pu.host, l, r + 1) + end + end + end + end +end + +local checkPaths = function (host, ip, path) + I(" Checking IP " .. host .. " -> " .. ip .. " " .. path) + for i, s in pairs(referenceDebs) do + if testing("http") then checkURL(host, "http://" .. ip .. path .. s) end + if testing("https") then checkURL(host, "https://" .. ip .. path .. s) end + end + + for i, s in pairs(releases) do + for j, k in pairs(releaseFiles) do + if testing("http") then checkURL(host, "http://" .. ip .. path .. "merged/dists/" .. s .. k) end + if testing("https") then checkURL(host, "https://" .. ip .. path .. "merged/dists/" .. s .. k) end + end + end +end + +local execute = function (s) + D("executing " .. s) + os.execute(s) +end + +forkIP = function (orig, host) + if nil == host then host = orig end + local po = url.parse("http://" .. orig, defaultURL) + local ph = url.parse("http://" .. host, defaultURL) + gatherIPs(ph.host) + for k, v in pairs(IP[ph.host]) do + if v == "A" then + if testing("IPv4") then execute("sleep 1; ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. host .. " " .. k .. " &") end + elseif v == "AAAA" then + if testing("IPv6") then execute("sleep 1; ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. host .. " [" .. k .. "] &") end + elseif v == "CNAME" then + forkIP(orig, k) + end + end +end + +checkIP = function (orig, host, path, ip) + if nil ~= ip then + checkPaths(orig, ip, path) + else + D("checkIP " .. orig .. " " .. host) + gatherIPs(host) + for k, v in pairs(IP[host]) do + if v == "A" then + if testing("IPv4") then checkPaths(orig, k, path) end + elseif v == "AAAA" then + if testing("IPv6") then checkPaths(orig, "[" .. k .. "]", path) end + elseif v == "CNAME" then + checkIP(orig, k, path) + end + end + end +end + +local checkHost = function (host, path, ip) + if nil == path then path = "/" else + if "/" == path:sub(-1, -1) then + W("slash at end of BaseURL in mirror_list.txt! " .. host .. " " .. path) + else + path = path .. "/" + end + end + checkIP(host, host, path, ip) +end + +local downloads = function (host, URL, IP) + if nil == URL then URL = "/" end + if nil == IP then IP = "" else IP = "-" .. IP end + local log = " --rejected-log=results/wget-%s_REJECTS-" .. host .. IP .. ".log -a results/wget-%s-" .. host .. IP ..".log " + I("starting file download commands for " .. host .. " " .. URL) + local cm = "ionice -c3 " .. download .. log:format("debs", "debs") + for i, s in pairs(referenceDebs) do + cm = cm .. " https://" .. host .. URL .. "/" .. s + end + for i, s in pairs(releases) do + execute(cm .. " &") + cm = "ionice -c3 " .. download .. log:format(s, s) + for j, k in pairs(releaseFiles) do + cm = cm .. " https://" .. host .. URL .. "/merged/dists/" .. s .. k + end + end + execute(cm .. " &") +end + +local getMirrors = function () + local mirrors = {} + local host = "" + local m = {} + local URL = "https://" .. options.referenceSite.value .. "/mirror_list.txt" + I("getting mirrors.") + local p, c, h = http.request(URL) + if nil == p then E(c .. " fetching " .. URL) else + for l in p:gmatch("\n*([^\n]+)\n*") do + local t, d = l:match("(%a*):%s*(.*)") + if "FQDN" == t then + if "" ~= host then + mirrors[host] = m + m = {} + end + host = d + end + m[t] = d + end + if "" ~= host then + mirrors[host] = m + end + end + return mirrors +end + + +if 0 ~= #args then + local option = "" + for i, a in pairs(args) do + if ("--help" == a) or ("-h" == a) then + print("I should write some docs, huh?") + elseif "--version" == a then + print("mirror-checker-lua version 0.1 alpha") + elseif "-v" == a then + verbosity = verbosity + 1 + sendArgs = sendArgs .. a .. " " + elseif "-q" == a then + verbosity = -1 + sendArgs = sendArgs .. a .. " " + elseif "-k" == a then + keep = true + elseif "--" == a:sub(1, 2) then + local s, e = a:find("=") + if nil == s then e = -1 end + option = a:sub(3, e - 1) + local o = options[option] + if nil == o then + print("Unknown option --" .. option) + option = "" + else + option = a + sendArgs = sendArgs .. a .. " " + local s, e = a:find("=") + if nil == s then e = 0 end + option = a:sub(3, e - 1) + if "table" == options[option].typ then + local result = {} + for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do + local f = t:sub(1, 1) + local n = t:sub(2, -1) + if ("+" ~= f) and ("-" ~= f) then + table.insert(result, t) + end + end + if 0 ~= #result then + options[option].value = result + else + for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do + local f = t:sub(1, 1) + local n = t:sub(2, -1) + if "+" == f then + table.insert(options[option].value, n) + elseif "-" == f then + local r = {} + for i, k in pairs(options[option].value) do + if k ~= n then table.insert(r, k) end + end + options[option].value = r + end + end + end + else + options[option].value = a + end + option = "" + end + elseif "-" == a:sub(1, 1) then + print("Unknown option " .. a) + else + table.insert(arg, a) + end + end +end + +--printTable(options.tests.value, "", "tests") + +execute("mkdir -p results") + +if 0 < #arg then + if nil ~= arg[2] + log = io.open ("mirror-checker-lua_" .. arg[1] .. "_" .. arg[2] .. ".log", "a+") + else + log = io.open ("mirror-checker-lua_" .. arg[1] .. ".log", "a+" [, mode]) + end + local pu = url.parse("http://" .. arg[1], defaultURL) + I("Starting tests for " ..pu.host .. " with these tests - " .. table.concat(options.tests.value, ", ")) + if nil ~= arg[2] then I(" Using IP " .. arg[2]) end + if testing("Integrity") or testing("Updated") then + if not keep then execute("rm -fr results/" .. pu.host) end + downloads(pu.host, pu.path, arg[2]) + checkExes("mirror-checker.lua " .. sendArgs) + checkExes(download) + end + checkHost(pu.host, pu.path, arg[2]) +else + if not keep then os.execute("rm -f results/*.log") end + log = io.open ("mirror-checker-lua.log", "a+") + I("Starting tests " .. table.concat(options.tests.value, ", ")) + execute("mkdir -p results") + local mirrors = getMirrors() + mirrors[options.referenceSite.value] = nil +-- checkHost(options.referenceSite.value) + forkIP(options.referenceSite.value) +-- checkHost("deb.devuan.org") + forkIP("deb.devuan.org") + for k, m in pairs(mirrors) do + local pu = url.parse("http://" .. m.BaseURL, defaultURL) +-- checkHost(pu.host) + forkIP(m.BaseURL) + checkExes("mirror-checker.lua " .. sendArgs) + if testing("Integrity") or testing("Updated") then checkExes(download) end + end + while 1 <= checkExes("mirror-checker.lua " .. sendArgs) do os.execute("sleep 30") end + if testing("Integrity") or testing("Updated") then + while 1 < checkExes(download) do os.execute("sleep 30") end + end +end -- cgit v1.1