path: root/mirror-checker.lua

#!/usr/bin/env luajit


local args = {...}


verbosity = 0
keep = false
options =
{
    referenceSite =
	{
	    typ = "string",
	    help = "",
	    value = "pkgmaster.devuan.org",
	},
    tests =
	{
	    typ = "table",
	    help = "",
	    value =
	    {
	        "IPv4",
	        "IPv6",
--	        "ftp",
	        "http",
	        "https",
--	        "rsync",
	        "DNS-RR",
	        "Protocol",
--	        "URL-Sanity",
--	        "Integrity",
--	        "Updated",
	    },
	},
}

local defaultURL = {scheme = "http"}
local download = "wget -np -N -r -P results "
local releases = {"jessie", "ascii", "beowulf", "ceres"}
local releaseFiles =
{
    -- Release file.
    "/Release",
    "/InRelease",
    "/main/binary-all/Packages.gz",
    -- Contents files.
    "/main/Contents-all.gz",
    "/main/Contents-amd64.gz",
    "/main/Contents-arm64.gz",
    "-security/main/Contents-all.gz",
    "-security/main/Contents-amd64.gz",
    "-security/main/Contents-arm64.gz",
}
local referenceDebs =
{
    -- Devuan package.  NOTE this one likely should not get redirected, but that's more a warning than an error.
    "merged/pool/DEVUAN/main/d/desktop-base/desktop-base_2.0.3_all.deb",
    -- Debian package.
    "merged/pool/DEBIAN/main/d/dash/dash_0.5.8-2.4_amd64.deb",
    -- Debian security package.  NOTE this one should always be redirected?
    "merged/pool/DEBIAN-SECURITY/updates/main/a/apt/apt-transport-https_1.4.9_amd64.deb",
}
local arg = {}
local sendArgs = ""
local log


local socket = require 'socket'
local ftp = require 'socket.ftp'
local http = require 'socket.http'
local url = require 'socket.url'


-- Use this to print a table.
printTable = function (table, space, name)
    print(space .. name .. ": ")
    print(space .. "{")
    printTableSub(table, space .. "  ")
    print(space .. "}")
end
printTableSub = function (table, space)
    for k, v in pairs(table) do 
	if type(v) == "table" then
	    printTable(v, space, k)
	elseif type(v) == "string" then
	    print(space .. k .. ': "' .. v .. '";')
	elseif type(v) == "function" then
	    print(space .. "function " .. k .. "();")
	elseif type(v) == "userdata" then
	    print(space .. "userdata " .. k .. ";")
	elseif type(v) == "boolean" then
	    if (v) then
		print(space .. "boolean " .. k .. " TRUE ;")
	    else
		print(space .. "boolean " .. k .. " FALSE ;")
	    end
	else
	    print(space .. k .. ": " .. v .. ";")
	end
    end
end

local log = function(v, t, s)
    if v <= verbosity then
	if 3 <= verbosity then t = os.date() .. " " .. t end
	print(t .. ": " .. s)
    end
    if nil ~= log then
	log:write(os.date() .. " " .. t .. ": " .. s .. "/n")
	log:flush()
    end
end
local D = function(s) log(3,  "DEBUG   ", s) end
local I = function(s) log(2,  "INFO    ", s) end
local W = function(s) log(1,  "WARNING ", s) end
local E = function(s) log(0,  "ERROR   ", s) end
local C = function(s) log(-1, "CRITICAL", s) end

local testing = function(t)
    for i, v in pairs(options.tests.value) do
	if t == v then return true end
    end
    return false
end

local checkExes = function (exe)
    local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | wc -l'):read("*l")
    D(count .. " " .. exe .. " commands left.")
    return tonumber(count)
end


local IP = {}
local gatherIPs = function (host)
    if nil == IP[host] then
        local IPs
	local dig = io.popen('dig +keepopen +noall +nottlid +answer ' .. host .. ' A ' .. host .. ' AAAA ' .. host .. ' CNAME ' .. host .. ' SRV | sort | uniq')
	repeat
	    IPs = dig:read("*l")
	    if nil ~= IPs then
		for k, t, v in IPs:gmatch("([%w_%-%.]*)%.%s*IN%s*(%a*)%s*(.*)") do
		    if "." == v:sub(-1, -1) then v = v:sub(1, -2) end
		    if nil == IP[k] then IP[k] = {} end
		    IP[k][v] = t
		end
	    end
	until nil == IPs
    end
end

-- Returns FTP directory listing
local nlst = function (u)
    local t = {}
    local p = url.parse(u)
    p.command = "nlst"
    p.sink = ltn12.sink.table(t)
    local r, e = ftp.get(p)
    return r and table.concat(t), e
end

checkURL = function (host, URL, r)
    if nil == r then r = 0 end
    local check = "Checking file"
    if 0 < r then
	check = "Redirecting to"
--        checkIP(host)
    end
    if 10 < r then
	E("too many redirects!  " .. check .. " " .. host .. " -> " .. URL)
	return
    end
    local PU = url.parse(URL, defaultURL)
    D("   " .. PU.scheme .. " ://  " .. check .. " " .. host .. " -> " .. URL)
    if not testing(PU.scheme) then D("not testing " .. PU.scheme .. " " .. host .. " -> " .. URL);  return end
    local hd = {Host = host}
    local p, c, h, s = http.request{method = "HEAD", redirect = false, url = URL, headers = hd}
    if nil == p then E(c .. "!   " .. check .. " " .. host .. " -> " .. URL) else
	l = h.location
	if nil ~= l then
	    local pu = url.parse(l, defaultURL)
	    if l == URL then
		E("redirect loop!  " .. check .. " " .. host .. " -> " .. URL)
	    else
		if nil == pu.host then
		    W("no location host!  " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l)
		    checkURL(host, PU.scheme .. "://" .. PU.host .. l, r + 1)
		else
		    if testing("Protocol") and pu.scheme ~= PU.scheme then
			W("protocol changed during redirect!  " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l)
		    end
		    checkURL(pu.host, l, r + 1)
		end
	    end
	end
    end
end

local checkPaths = function (host, ip, path)
    I("  Checking IP " .. host .. " -> " .. ip .. " " .. path)
    for i, s in pairs(referenceDebs) do
	if testing("http")  then checkURL(host, "http://"  .. ip .. path .. s) end
	if testing("https") then checkURL(host, "https://" .. ip .. path .. s) end
    end

    for i, s in pairs(releases) do
	for j, k in pairs(releaseFiles) do
	    if testing("http")  then checkURL(host, "http://"  .. ip .. path .. "merged/dists/" .. s .. k) end
	    if testing("https") then checkURL(host, "https://" .. ip .. path .. "merged/dists/" .. s .. k) end
	end
    end
end

local execute = function (s)
    D("executing " .. s)
    os.execute(s)
end

forkIP = function (orig, host)
    if nil == host then host = orig end
    local po = url.parse("http://" .. orig, defaultURL)
    local ph = url.parse("http://" .. host, defaultURL)
    gatherIPs(ph.host)
    for k, v in pairs(IP[ph.host]) do
	if v == "A" then
	    if testing("IPv4") then execute("sleep 1; ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. host .. " " .. k .. " &") end
	elseif v == "AAAA" then
	    if testing("IPv6") then execute("sleep 1; ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. host .. " [" .. k .. "] &") end
	elseif v == "CNAME" then
	    forkIP(orig, k)
	end
    end
end

checkIP = function (orig, host, path, ip)
    if nil ~= ip then
	checkPaths(orig, ip, path)
    else
	D("checkIP " .. orig .. " " .. host)
	gatherIPs(host)
	for k, v in pairs(IP[host]) do
	    if v == "A" then
		if testing("IPv4") then checkPaths(orig, k, path) end
	    elseif v == "AAAA" then
		if testing("IPv6") then checkPaths(orig, "[" .. k .. "]", path) end
	    elseif v == "CNAME" then
		checkIP(orig, k, path)
	    end
	end
    end
end

local checkHost = function (host, path, ip) 
    if nil == path then path = "/" else
	if "/" == path:sub(-1, -1) then
	    W("slash at end of BaseURL in mirror_list.txt!  " .. host .. " " .. path)
	else
	    path = path .. "/"
	end
    end
    checkIP(host, host, path, ip)
end

local downloads = function (host, URL, IP)
    if nil == URL then URL = "/" end
    if nil == IP then IP = "" else IP = "-" .. IP end
    local log = " --rejected-log=results/wget-%s_REJECTS-" .. host .. IP .. ".log -a results/wget-%s-" .. host .. IP ..".log "
    I("starting file download commands for " .. host .. " " .. URL)
    local cm = "ionice -c3 " .. download .. log:format("debs", "debs")
    for i, s in pairs(referenceDebs) do
	cm = cm .. " https://" .. host .. URL .. "/" .. s
    end
    for i, s in pairs(releases) do
	execute(cm .. " &")
	cm = "ionice -c3 " .. download .. log:format(s, s)
	for j, k in pairs(releaseFiles) do
	    cm = cm .. " https://" .. host .. URL .. "/merged/dists/" .. s .. k
	end
    end
    execute(cm .. " &")
end

local getMirrors = function ()
    local mirrors = {}
    local host = ""
    local m = {}
    local URL = "https://" .. options.referenceSite.value .. "/mirror_list.txt"
    I("getting mirrors.")
    local p, c, h = http.request(URL)
    if nil == p then E(c .. " fetching " .. URL) else
	for l in p:gmatch("\n*([^\n]+)\n*") do
	    local t, d = l:match("(%a*):%s*(.*)")
	    if "FQDN" == t then
		if "" ~= host then
		    mirrors[host] = m
		    m = {}
		end
		host = d
	    end
	    m[t] = d
	end
	if "" ~= host then
	    mirrors[host] = m
	end
    end
    return mirrors
end


if 0 ~= #args then
    local option = ""
    for i, a in pairs(args) do
	if ("--help" == a) or ("-h" == a) then
	    print("I should write some docs, huh?")
	elseif "--version" == a then
	    print("mirror-checker-lua version 0.1 alpha")
	elseif "-v" == a then
	    verbosity = verbosity + 1
	    sendArgs = sendArgs .. a .. " "
	elseif "-q" == a then
	    verbosity = -1
	    sendArgs = sendArgs .. a .. " "
	elseif "-k" == a then
	    keep = true
	elseif "--" == a:sub(1, 2) then
	    local s, e = a:find("=")
	    if nil == s then e = -1 end
	    option = a:sub(3, e - 1)
	    local o = options[option]
	    if nil == o then
		print("Unknown option --" .. option)
		option = ""
	    else
		option = a
		sendArgs = sendArgs .. a .. " "
		local s, e = a:find("=")
		if nil == s then e = 0 end
		option = a:sub(3, e - 1)
		if "table" == options[option].typ then
		    local result = {}
		    for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do
			local f = t:sub(1, 1)
			local n = t:sub(2, -1)
			if ("+" ~= f) and ("-" ~= f) then
			    table.insert(result, t)
			end
		    end
		    if 0 ~= #result then
			options[option].value = result
		    else
			for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do
			    local f = t:sub(1, 1)
			    local n = t:sub(2, -1)
			    if "+" == f then
				table.insert(options[option].value, n)
			    elseif "-" == f then
				local r = {}
				for i, k in pairs(options[option].value) do
				    if k ~= n then table.insert(r, k) end
				end
				options[option].value = r
			    end
			end
		    end
		else
		    options[option].value = a
		end
		option = ""
	    end
	elseif "-" == a:sub(1, 1) then
	    print("Unknown option " .. a)
	else
	    table.insert(arg, a)
	end
    end
end

--printTable(options.tests.value, "", "tests")

execute("mkdir -p results")

if 0 < #arg then
    if nil ~= arg[2]
	log = io.open ("mirror-checker-lua_" .. arg[1] .. "_" .. arg[2] .. ".log", "a+")
    else
	log = io.open ("mirror-checker-lua_" .. arg[1] .. ".log", "a+" [, mode])
    end
    local pu = url.parse("http://" .. arg[1], defaultURL)
    I("Starting tests for " ..pu.host .. " with these tests - " .. table.concat(options.tests.value, ", "))
    if nil ~= arg[2] then I("  Using IP " .. arg[2]) end
    if testing("Integrity") or testing("Updated") then
	if not keep then execute("rm -fr results/" .. pu.host) end
	downloads(pu.host, pu.path, arg[2])
	checkExes("mirror-checker.lua " .. sendArgs)
	checkExes(download)
    end
    checkHost(pu.host, pu.path, arg[2])
else
    if not keep then os.execute("rm -f results/*.log") end
    log = io.open ("mirror-checker-lua.log", "a+")
    I("Starting tests " .. table.concat(options.tests.value, ", "))
    execute("mkdir -p results")
    local mirrors = getMirrors()
    mirrors[options.referenceSite.value] = nil
--    checkHost(options.referenceSite.value)
    forkIP(options.referenceSite.value)
--    checkHost("deb.devuan.org")
    forkIP("deb.devuan.org")
    for k, m in pairs(mirrors) do
	local pu = url.parse("http://" .. m.BaseURL, defaultURL)
--	checkHost(pu.host)
	forkIP(m.BaseURL)
        checkExes("mirror-checker.lua " .. sendArgs)
        if testing("Integrity") or testing("Updated") then checkExes(download) end
    end
    while 1 <= checkExes("mirror-checker.lua " .. sendArgs) do os.execute("sleep 30") end
    if testing("Integrity") or testing("Updated") then
	while 1 < checkExes(download) do os.execute("sleep 30") end
    end
end