Add the actual source code, and the basic documentation.

Still need to write the help output.

2 files changed, 515 insertions, 0 deletions

diff --git a/README.md b/README.md
new file mode 100644
index 0000000..415025c
--- /dev/null
+++ b/README.md
@@ -0,0 +1,95 @@
+Lua script for checking the health of Devuan Linux package mirrors.
+
+This is currently under development, not everything has been written yet.
+
+mirror-checker-lua is a Lua script used by the Devuan mirror admins
+(maybe, if they like it) to check the health of Devuan Linux package
+mirrors.  Originally there was bash scripts for this job, then Evilham
+wrote some Python scripts, now onefang has written it in Lua.  We all
+have different tastes in languages.  lol
+
+The main difference is that this Lua version tries to do everything, and
+will be maintained.  Currently the shell scripts and Python scripts are
+actually being used I think.  Evilham asked me to write this, after I
+badgered him about his Python scripts.
+
+The source code is at https://sledjhamr.org/cgit/mirror-checker-lua/
+
+The issue tracker is at https://sledjhamr.org/mantisbt/project_page.php?project_id=13
+
+
+Installation.
+-------------
+
+Download the source.  You may want to put the actual mirror-checker.lua
+in someplace liku /usr/local/bin and make sure it is executable.
+
+It should run on any recent Linux, you'll need to have the following
+installed -
+
+Luajit
+
+wget
+
+dig, part of BIND.  On Debian based systems it'll be in the dnsutils
+package.
+
+LuaSocket, on Debian based systems it'll be in the lua-socket package.
+
+ionice, on Debian based systems it'll be in the util-linux package.
+
+
+Using it.
+---------
+
+These examples assume you are running it from the source code directory. 
+A directory will be created called results, it'll be full of log files
+and any files that get downloaded.  There will also be results/email and
+results/web directories, with the notification email and web pages with
+the easy to read emails (once I write that bit).
+
+Note that unlike typical commands, you can't run single character options
+together, so this is wrong -
+
+./mirror-checker.lua -vvv
+
+Instead do this -
+
+./mirror-checker.lua -v -v -v
+
+Just run the script to do all of the tests -
+
+./mirror-checker.lua
+
+Which will print any errors.  If you don't want to see errors -
+
+./mirror-checker.lua -q
+
+If you want to see warnings to (as usual, the more -v options, the more
+details) -
+
+./mirror-checker.lua -v
+
+Or use the usual options for the help and version number -
+
+./mirror-checker.lua -h 
+./mirror-checker.lua --help
+./mirror-checker.lua --version
+
+To run the tests on a specific mirror, for example pkgmaster.devuan.org -
+
+./mirror-checker.lua pkgmaster.devuan.org
+
+You can use the --tests option to tune which tests are run, for example
+to stop IPv6 tests, coz you don't have IPv6 -
+
+./mirror-checker.lua --tests=-IPv6
+
+To do the same, but not run the HTTPS tests either -
+
+./mirror-checker.lua --tests=-IPv6,-https
+
+To only run the http integrity tests, only on IPv6 -
+
+./mirror-checker.lua --tests=http,Integrity,IPv6
+
diff --git a/mirror-checker.lua b/mirror-checker.lua
new file mode 100755
index 0000000..34b2ec8
--- /dev/null
+++ b/mirror-checker.lua
@@ -0,0 +1,420 @@
+#!/usr/bin/env luajit
+
+
+local args = {...}
+
+
+verbosity = 0
+keep = false
+options =
+{
+    referenceSite =
+        {
+            typ = "string",
+            help = "",
+            value = "pkgmaster.devuan.org",
+        },
+    tests =
+        {
+            typ = "table",
+            help = "",
+            value =
+            {
+                "IPv4",
+                "IPv6",
+--              "ftp",
+                "http",
+                "https",
+--              "rsync",
+                "DNS-RR",
+                "Protocol",
+--              "URL-Sanity",
+--              "Integrity",
+--              "Updated",
+            },
+        },
+}
+
+local defaultURL = {scheme = "http"}
+local download = "wget -np -N -r -P results "
+local releases = {"jessie", "ascii", "beowulf", "ceres"}
+local releaseFiles =
+{
+    -- Release file.
+    "/Release",
+    "/InRelease",
+    "/main/binary-all/Packages.gz",
+    -- Contents files.
+    "/main/Contents-all.gz",
+    "/main/Contents-amd64.gz",
+    "/main/Contents-arm64.gz",
+    "-security/main/Contents-all.gz",
+    "-security/main/Contents-amd64.gz",
+    "-security/main/Contents-arm64.gz",
+}
+local referenceDebs =
+{
+    -- Devuan package.  NOTE this one likely should not get redirected, but that's more a warning than an error.
+    "merged/pool/DEVUAN/main/d/desktop-base/desktop-base_2.0.3_all.deb",
+    -- Debian package.
+    "merged/pool/DEBIAN/main/d/dash/dash_0.5.8-2.4_amd64.deb",
+    -- Debian security package.  NOTE this one should always be redirected?
+    "merged/pool/DEBIAN-SECURITY/updates/main/a/apt/apt-transport-https_1.4.9_amd64.deb",
+}
+local arg = {}
+local sendArgs = ""
+local log
+
+
+local socket = require 'socket'
+local ftp = require 'socket.ftp'
+local http = require 'socket.http'
+local url = require 'socket.url'
+
+
+-- Use this to print a table.
+printTable = function (table, space, name)
+    print(space .. name .. ": ")
+    print(space .. "{")
+    printTableSub(table, space .. "  ")
+    print(space .. "}")
+end
+printTableSub = function (table, space)
+    for k, v in pairs(table) do 
+        if type(v) == "table" then
+            printTable(v, space, k)
+        elseif type(v) == "string" then
+            print(space .. k .. ': "' .. v .. '";')
+        elseif type(v) == "function" then
+            print(space .. "function " .. k .. "();")
+        elseif type(v) == "userdata" then
+            print(space .. "userdata " .. k .. ";")
+        elseif type(v) == "boolean" then
+            if (v) then
+                print(space .. "boolean " .. k .. " TRUE ;")
+            else
+                print(space .. "boolean " .. k .. " FALSE ;")
+            end
+        else
+            print(space .. k .. ": " .. v .. ";")
+        end
+    end
+end
+
+local log = function(v, t, s)
+    if v <= verbosity then
+        if 3 <= verbosity then t = os.date() .. " " .. t end
+        print(t .. ": " .. s)
+    end
+    if nil ~= log then
+        log:write(os.date() .. " " .. t .. ": " .. s .. "/n")
+        log:flush()
+    end
+end
+local D = function(s) log(3,  "DEBUG   ", s) end
+local I = function(s) log(2,  "INFO    ", s) end
+local W = function(s) log(1,  "WARNING ", s) end
+local E = function(s) log(0,  "ERROR   ", s) end
+local C = function(s) log(-1, "CRITICAL", s) end
+
+local testing = function(t)
+    for i, v in pairs(options.tests.value) do
+        if t == v then return true end
+    end
+    return false
+end
+
+local checkExes = function (exe)
+    local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | wc -l'):read("*l")
+    D(count .. " " .. exe .. " commands left.")
+    return tonumber(count)
+end
+
+
+local IP = {}
+local gatherIPs = function (host)
+    if nil == IP[host] then
+        local IPs
+        local dig = io.popen('dig +keepopen +noall +nottlid +answer ' .. host .. ' A ' .. host .. ' AAAA ' .. host .. ' CNAME ' .. host .. ' SRV | sort | uniq')
+        repeat
+            IPs = dig:read("*l")
+            if nil ~= IPs then
+                for k, t, v in IPs:gmatch("([%w_%-%.]*)%.%s*IN%s*(%a*)%s*(.*)") do
+                    if "." == v:sub(-1, -1) then v = v:sub(1, -2) end
+                    if nil == IP[k] then IP[k] = {} end
+                    IP[k][v] = t
+                end
+            end
+        until nil == IPs
+    end
+end
+
+-- Returns FTP directory listing
+local nlst = function (u)
+    local t = {}
+    local p = url.parse(u)
+    p.command = "nlst"
+    p.sink = ltn12.sink.table(t)
+    local r, e = ftp.get(p)
+    return r and table.concat(t), e
+end
+
+checkURL = function (host, URL, r)
+    if nil == r then r = 0 end
+    local check = "Checking file"
+    if 0 < r then
+        check = "Redirecting to"
+--        checkIP(host)
+    end
+    if 10 < r then
+        E("too many redirects!  " .. check .. " " .. host .. " -> " .. URL)
+        return
+    end
+    local PU = url.parse(URL, defaultURL)
+    D("   " .. PU.scheme .. " ://  " .. check .. " " .. host .. " -> " .. URL)
+    if not testing(PU.scheme) then D("not testing " .. PU.scheme .. " " .. host .. " -> " .. URL);  return end
+    local hd = {Host = host}
+    local p, c, h, s = http.request{method = "HEAD", redirect = false, url = URL, headers = hd}
+    if nil == p then E(c .. "!   " .. check .. " " .. host .. " -> " .. URL) else
+        l = h.location
+        if nil ~= l then
+            local pu = url.parse(l, defaultURL)
+            if l == URL then
+                E("redirect loop!  " .. check .. " " .. host .. " -> " .. URL)
+            else
+                if nil == pu.host then
+                    W("no location host!  " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l)
+                    checkURL(host, PU.scheme .. "://" .. PU.host .. l, r + 1)
+                else
+                    if testing("Protocol") and pu.scheme ~= PU.scheme then
+                        W("protocol changed during redirect!  " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l)
+                    end
+                    checkURL(pu.host, l, r + 1)
+                end
+            end
+        end
+    end
+end
+
+local checkPaths = function (host, ip, path)
+    I("  Checking IP " .. host .. " -> " .. ip .. " " .. path)
+    for i, s in pairs(referenceDebs) do
+        if testing("http")  then checkURL(host, "http://"  .. ip .. path .. s) end
+        if testing("https") then checkURL(host, "https://" .. ip .. path .. s) end
+    end
+
+    for i, s in pairs(releases) do
+        for j, k in pairs(releaseFiles) do
+            if testing("http")  then checkURL(host, "http://"  .. ip .. path .. "merged/dists/" .. s .. k) end
+            if testing("https") then checkURL(host, "https://" .. ip .. path .. "merged/dists/" .. s .. k) end
+        end
+    end
+end
+
+local execute = function (s)
+    D("executing " .. s)
+    os.execute(s)
+end
+
+forkIP = function (orig, host)
+    if nil == host then host = orig end
+    local po = url.parse("http://" .. orig, defaultURL)
+    local ph = url.parse("http://" .. host, defaultURL)
+    gatherIPs(ph.host)
+    for k, v in pairs(IP[ph.host]) do
+        if v == "A" then
+            if testing("IPv4") then execute("sleep 1; ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. host .. " " .. k .. " &") end
+        elseif v == "AAAA" then
+            if testing("IPv6") then execute("sleep 1; ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. host .. " [" .. k .. "] &") end
+        elseif v == "CNAME" then
+            forkIP(orig, k)
+        end
+    end
+end
+
+checkIP = function (orig, host, path, ip)
+    if nil ~= ip then
+        checkPaths(orig, ip, path)
+    else
+        D("checkIP " .. orig .. " " .. host)
+        gatherIPs(host)
+        for k, v in pairs(IP[host]) do
+            if v == "A" then
+                if testing("IPv4") then checkPaths(orig, k, path) end
+            elseif v == "AAAA" then
+                if testing("IPv6") then checkPaths(orig, "[" .. k .. "]", path) end
+            elseif v == "CNAME" then
+                checkIP(orig, k, path)
+            end
+        end
+    end
+end
+
+local checkHost = function (host, path, ip) 
+    if nil == path then path = "/" else
+        if "/" == path:sub(-1, -1) then
+            W("slash at end of BaseURL in mirror_list.txt!  " .. host .. " " .. path)
+        else
+            path = path .. "/"
+        end
+    end
+    checkIP(host, host, path, ip)
+end
+
+local downloads = function (host, URL, IP)
+    if nil == URL then URL = "/" end
+    if nil == IP then IP = "" else IP = "-" .. IP end
+    local log = " --rejected-log=results/wget-%s_REJECTS-" .. host .. IP .. ".log -a results/wget-%s-" .. host .. IP ..".log "
+    I("starting file download commands for " .. host .. " " .. URL)
+    local cm = "ionice -c3 " .. download .. log:format("debs", "debs")
+    for i, s in pairs(referenceDebs) do
+        cm = cm .. " https://" .. host .. URL .. "/" .. s
+    end
+    for i, s in pairs(releases) do
+        execute(cm .. " &")
+        cm = "ionice -c3 " .. download .. log:format(s, s)
+        for j, k in pairs(releaseFiles) do
+            cm = cm .. " https://" .. host .. URL .. "/merged/dists/" .. s .. k
+        end
+    end
+    execute(cm .. " &")
+end
+
+local getMirrors = function ()
+    local mirrors = {}
+    local host = ""
+    local m = {}
+    local URL = "https://" .. options.referenceSite.value .. "/mirror_list.txt"
+    I("getting mirrors.")
+    local p, c, h = http.request(URL)
+    if nil == p then E(c .. " fetching " .. URL) else
+        for l in p:gmatch("\n*([^\n]+)\n*") do
+            local t, d = l:match("(%a*):%s*(.*)")
+            if "FQDN" == t then
+                if "" ~= host then
+                    mirrors[host] = m
+                    m = {}
+                end
+                host = d
+            end
+            m[t] = d
+        end
+        if "" ~= host then
+            mirrors[host] = m
+        end
+    end
+    return mirrors
+end
+
+
+if 0 ~= #args then
+    local option = ""
+    for i, a in pairs(args) do
+        if ("--help" == a) or ("-h" == a) then
+            print("I should write some docs, huh?")
+        elseif "--version" == a then
+            print("mirror-checker-lua version 0.1 alpha")
+        elseif "-v" == a then
+            verbosity = verbosity + 1
+            sendArgs = sendArgs .. a .. " "
+        elseif "-q" == a then
+            verbosity = -1
+            sendArgs = sendArgs .. a .. " "
+        elseif "-k" == a then
+            keep = true
+        elseif "--" == a:sub(1, 2) then
+            local s, e = a:find("=")
+            if nil == s then e = -1 end
+            option = a:sub(3, e - 1)
+            local o = options[option]
+            if nil == o then
+                print("Unknown option --" .. option)
+                option = ""
+            else
+                option = a
+                sendArgs = sendArgs .. a .. " "
+                local s, e = a:find("=")
+                if nil == s then e = 0 end
+                option = a:sub(3, e - 1)
+                if "table" == options[option].typ then
+                    local result = {}
+                    for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do
+                        local f = t:sub(1, 1)
+                        local n = t:sub(2, -1)
+                        if ("+" ~= f) and ("-" ~= f) then
+                            table.insert(result, t)
+                        end
+                    end
+                    if 0 ~= #result then
+                        options[option].value = result
+                    else
+                        for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do
+                            local f = t:sub(1, 1)
+                            local n = t:sub(2, -1)
+                            if "+" == f then
+                                table.insert(options[option].value, n)
+                            elseif "-" == f then
+                                local r = {}
+                                for i, k in pairs(options[option].value) do
+                                    if k ~= n then table.insert(r, k) end
+                                end
+                                options[option].value = r
+                            end
+                        end
+                    end
+                else
+                    options[option].value = a
+                end
+                option = ""
+            end
+        elseif "-" == a:sub(1, 1) then
+            print("Unknown option " .. a)
+        else
+            table.insert(arg, a)
+        end
+    end
+end
+
+--printTable(options.tests.value, "", "tests")
+
+execute("mkdir -p results")
+
+if 0 < #arg then
+    if nil ~= arg[2]
+        log = io.open ("mirror-checker-lua_" .. arg[1] .. "_" .. arg[2] .. ".log", "a+")
+    else
+        log = io.open ("mirror-checker-lua_" .. arg[1] .. ".log", "a+" [, mode])
+    end
+    local pu = url.parse("http://" .. arg[1], defaultURL)
+    I("Starting tests for " ..pu.host .. " with these tests - " .. table.concat(options.tests.value, ", "))
+    if nil ~= arg[2] then I("  Using IP " .. arg[2]) end
+    if testing("Integrity") or testing("Updated") then
+        if not keep then execute("rm -fr results/" .. pu.host) end
+        downloads(pu.host, pu.path, arg[2])
+        checkExes("mirror-checker.lua " .. sendArgs)
+        checkExes(download)
+    end
+    checkHost(pu.host, pu.path, arg[2])
+else
+    if not keep then os.execute("rm -f results/*.log") end
+    log = io.open ("mirror-checker-lua.log", "a+")
+    I("Starting tests " .. table.concat(options.tests.value, ", "))
+    execute("mkdir -p results")
+    local mirrors = getMirrors()
+    mirrors[options.referenceSite.value] = nil
+--    checkHost(options.referenceSite.value)
+    forkIP(options.referenceSite.value)
+--    checkHost("deb.devuan.org")
+    forkIP("deb.devuan.org")
+    for k, m in pairs(mirrors) do
+        local pu = url.parse("http://" .. m.BaseURL, defaultURL)
+--      checkHost(pu.host)
+        forkIP(m.BaseURL)
+        checkExes("mirror-checker.lua " .. sendArgs)
+        if testing("Integrity") or testing("Updated") then checkExes(download) end
+    end
+    while 1 <= checkExes("mirror-checker.lua " .. sendArgs) do os.execute("sleep 30") end
+    if testing("Integrity") or testing("Updated") then
+        while 1 < checkExes(download) do os.execute("sleep 30") end
+    end
+end