path: root/apt-panopticon.lua

#!/usr/bin/env luajit


local args = {...}

--[[ TODO - What to do about HTTPS://deb.devuan.org/ redirects.
	    Some mirrors give a 404.
		Sledjhamr gives a 404, coz it's not listening on 443 for deb.devuan.org.
	    Some mirrors give a 200.
		They shouldn't have the proper certificate, but are giving a result anyway.
]]

origin = false
verbosity = -1
keep = false
-- TODO - Should actually implement this.
fork = true
options =
{
    referenceSite =
	{
	    typ = "string",
	    help = "",
	    value = "pkgmaster.devuan.org",
	},
    roundRobin =
	{
	    typ = "string",
	    help = "",
	    value = "deb.devuan.org",
	},
    tests =
	{
	    typ = "table",
	    help = "",
	    value =
	    {
	        "IPv4",
	        "IPv6",
--	        "ftp",
	        "http",
	        "https",
--	        "rsync",
--	        "DNS-RR",
	        "Protocol",
--	        "URL-Sanity",
--	        "Integrity",
--	        "Updated",
	    },
	},
}

local defaultURL = {scheme = "http"}
local downloadLock = "flock -n results/wget-"
local download = "wget --timeout=300 -np -N -r -P results "	-- Note wget has a default read timeout of 900 seconds (15 minutes).
local releases = {"jessie", "ascii", "beowulf", "ceres"}
local releaseFiles =
{
    -- Release file.
    "/Release",
    "/InRelease",
    "/main/binary-all/Packages.gz",
    -- Contents files.
    "/main/Contents-all.gz",
    "/main/Contents-amd64.gz",
    "/main/Contents-arm64.gz",
    "-security/main/Contents-all.gz",
    "-security/main/Contents-amd64.gz",
    "-security/main/Contents-arm64.gz",
}
local notExist =
{
    "ceres-security"	-- This will never exist, it's our code name for the testing suite.
}
local referenceDebs =
{
    -- Debian package.
    "merged/pool/DEBIAN/main/d/dash/dash_0.5.8-2.4_amd64.deb",
    -- Debian security package.  NOTE this one should always be redirected?
    "merged/pool/DEBIAN-SECURITY/updates/main/a/apt/apt-transport-https_1.4.9_amd64.deb",
}
local referenceDevs =
{
    -- Devuan package.  NOTE this one should not get redirected, but that's more a warning than an error.
    "merged/pool/DEVUAN/main/d/desktop-base/desktop-base_2.0.3_all.deb",
--    "merged/pool/DEVUAN/main/u/util-linux/util-linux_2.32.1-0.1+devuan2.1_amd64.deb",
}
local arg = {}
local sendArgs = ""
local logFile


local socket = require 'socket'
local ftp = require 'socket.ftp'
local http = require 'socket.http'
local https = require 'ssl.https'	-- See https://github.com/brunoos/luasec/wiki/LuaSec-0.6 for docs.
local url = require 'socket.url'


-- Use this to dump a table to a string.
dumpTable = function (table, space, name)
    local r = ""
    if "" == space then r = r .. space .. name .. " =\n" else r = r .. space .. "[" .. name .. "] =\n" end
    r = r .. space .. "{\n"
    r = r .. dumpTableSub(table, space .. "  ")
    if "" == space then r = r .. space .. "}\n" else r = r .. space .. "},\n" end
    return r
end
dumpTableSub = function (table, space)
    local r = ""
    for k, v in pairs(table) do 
        if type(k) == "string" then k = '"' .. k .. '"' end
	if type(v) == "table" then
	    r = r .. dumpTable(v, space, k)
	elseif type(v) == "string" then
	    r = r .. space .. "[" .. k .. "] = '" .. v .. "';\n"
	elseif type(v) == "function" then
	    r = r .. space .. "[" .. k .. "] = function ();\n"
	elseif type(v) == "userdata" then
	    r = r .. space .. "userdata " .. "[" .. k .. "];\n"
	elseif type(v) == "boolean" then
	    if (v) then
		r = r .. space .. "[" .. k .. "] = true;\n"
	    else
		r = r .. space .. "[" .. k .. "] = false;\n"
	    end
	else
	    r = r .. space .. "[" .. k .. "] = " .. v .. ";\n"
	end
    end
    return r
end

local ip = ""
local results = {}

local log = function(v, t, s, prot, test, host)
    local x = ""
    if nil == prot then prot = "" end
    if nil ~= test then x = x .. test else test = "" end
    if nil ~= host then
	if #x > 0 then x = x .. " " end
	x = x .. host
    end
    if #x > 0 then
	t = t .. "(" .. x ..  ")"
	if "" == test then
	    if v == 0 then results[prot].errors = results[prot].errors + 1 end
	    if v == 1 then results[prot].warnings = results[prot].warnings + 1 end
	else
	    if v == 0 then results[prot][test].errors = results[prot][test].errors + 1 end
	    if v == 1 then results[prot][test].warnings = results[prot][test].warnings + 1 end
	end
    end
    if v <= verbosity then
	if 3 <= verbosity then t = os.date() .. " " .. t end
	print(t .. ": " .. s)
    end
    if nil ~= logFile then
	logFile:write(os.date() .. " " .. t .. ": " .. s .. "\n")
	logFile:flush()
    end
end
local D = function(s) log(3,  "DEBUG   ", s) end
local I = function(s) log(2,  "INFO    ", s) end
local W = function(s, p, t, h) log(1,  "WARNING ", s, p, t, h) end
local E = function(s, p, t, h) log(0,  "ERROR   ", s, p, t, h) end
local C = function(s) log(-1, "CRITICAL", s) end

local mirrors = {}

local testing = function(t, host)
    for i, v in pairs(options.tests.value) do
	if t == v then
	    local h = mirrors[host]
	    if nil == h then return true end
	    if true == h["Protocols"][t] then return true else D("Skipping " .. t .. " checks for " .. host) end
	end
    end
    return false
end

local checkExes = function (exe)
    local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | wc -l'):read("*l")
    D(count .. " " .. exe .. " commands still running.")
    return tonumber(count)
end

local repoExists = function (r)
    r = r:match("([%a-]*)")
    if nil == r then return false end
    for k, v in pairs(notExist) do
	if v == r then return false end
    end
    return true
end

local IP = {}
gatherIPs = function (host)
    if nil == IP[host] then
        local IPs
	local dig = io.popen('dig +keepopen +noall +nottlid +answer ' .. host .. ' A ' .. host .. ' AAAA ' .. host .. ' CNAME ' .. host .. ' SRV | sort -r | uniq')
	repeat
	    IPs = dig:read("*l")
	    if nil ~= IPs then
		for k, t, v in IPs:gmatch("([%w_%-%.]*)%.%s*IN%s*(%a*)%s*(.*)") do
		    if "." == v:sub(-1, -1) then v = v:sub(1, -2) end
		    if nil == IP[k] then IP[k] = {} end
		    IP[k][v] = t
		    D("  DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v)
		    if t == "CNAME" then
			gatherIPs(v)
			IP[k][v] = IP[v]
		    elseif v == "SRV" then
			print("SVR record found, now what do we do?")
		    end
		end
	    end
	until nil == IPs
    end
end

-- Returns FTP directory listing
local nlst = function (u)
    local t = {}
    local p = url.parse(u)
    p.command = "nlst"
    p.sink = ltn12.sink.table(t)
    local r, e = ftp.get(p)
    return r and table.concat(t), e
end

local timeouts = 0;
checkHEAD = function (host, URL, r, retry)
    if nil == r then r = 0 end
    if nil == retry then retry = 0 end
    local check = "Checking file"
    local PU = url.parse(URL, defaultURL)
    local pu = url.parse(PU.scheme .. "://" .. host, defaultURL)
    if 0 < r then
	check = "Redirecting to"
    end
    if 0 < retry then
	os.execute("sleep " .. math.random(1, 4))
	check = "Retry " .. retry .. " " .. check
    end
    if 3 < timeouts then
	E("too many timeouts!  " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
	return
    end
    if 20 < r then
	E("too many redirects!  " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
	return
    end
    if 4 < retry then
	E("too many retries!  " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
	return
    end
    D(PU.scheme .. " ://  " .. check .. " " .. host .. " -> " .. URL)
    if not testing(PU.scheme, host) then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL);  return end
    -- TODO - Perhaps we should try it anyway, and mark it as a warning if it DOES work?
    if "https" == PU.scheme and options.roundRobin.value == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert.");  return end
    local hd = {}
    if pu.host ~= PU.host then hd = {Host = host} end
    local htp = http;
    if PU.scheme == "https" then htp = https end
    -- NOTE - the docs for lua-sec say that redirect isn't supported is version 0.6, no idea if that means it ignores redirections like we want.
    -- TODO - 										find out!
    --		The protocol and options are lua-sec specific arguments.
    local p, c, h, s = htp.request{method = "HEAD", redirect = false, url = URL, headers = hd, protocol = "any", options = "all"}
    if nil == s then s = "" end
    if nil == p then
	E("  " .. c .. " " .. s .. "!   " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
	-- So far the only errors are "timeout", "Network is unreachable", and "closed", and I suspect "closed" is due to saturating my bandwidth.
	if "timeout" == c then timeouts = timeouts + 1 end
	if ("closed" == c) or ("Network is unreachable" == c) or ("timeout" == c) then checkHEAD(host, URL, r, retry + 1, timeouts) end
    else
	if ("4" == tostring(c):sub(1, 1)) or ("5" == tostring(c):sub(1, 1)) then
	    E("  " .. c .. " " .. s .. ".   " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
	else
	    I("  " .. c .. " " .. s .. ".   " .. check .. " " .. host .. " -> " .. URL)
	    timeouts = timeouts - 1	-- Backoff the timeouts count if we managed to get through.
	end
	l = h.location
	if nil ~= l then
	    pu = url.parse(l, defaultURL)
	    if (pu.scheme ~= PU.scheme) then
		if testing("Protocol") then W("    protocol changed during redirect!  " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l, PU.scheme, "Protocol", host) end
		if (pu.host == host) and pu.path == PU.path then D("Not testing protocol change " .. URL .. " -> " .. l);  return end
	    end

	    if l == URL then
		E("    redirect loop!  " .. check .. " " .. host .. " -> " .. URL, PU.scheme, PU.scheme, host)
	    elseif nil == pu.host then
		I("    relative redirect.  " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l)
		checkHEAD(host, PU.scheme .. "://" .. PU.host .. l, r + 1)
	    elseif (PU.host == pu.host) or (host == pu.host) then
		checkHEAD(pu.host, l, r + 1)
	    else
		--[[ The hard part here is that we end up throwing ALL of the test files at the redirected location.
		    Not good for deb.debian.org, which we should only be throwing .debs at.
		    What we do is loop through the DNS entries, and only test the specific protocol & file being tested here.

		    This is what I came up with for checking if we are already testing a specific URL.
		    Still duplicates a tiny bit, but much less than the previous find based method.
		]]
		local file = pu.host .. "://" .. pu.path
		local f = io.popen(string.format('if [ ! -f results/%s.check ] ; then touch results/%s.check; echo -n "check"; fi', file:gsub("/", "_"), file:gsub("/", "_") )):read("*a")
		if (nil == f) or ("check" == f) then
		    I("  Now checking redirected host " .. file)
		    checkHost(pu.host, pu.host, nil, "redir", pu.path)
		else
		    D("  Already checking " .. file)
		end
	    end
	end
    end
end

local checkFiles = function (host, ip, path, file)
    if nil == path then path = "" end
    if nil ~= file then
	if "redir" == ip then ip = host end
	I("  Checking IP for file " .. host .. " -> " .. ip .. " " .. path .. " " .. file)
	if testing("http", host)  then checkHEAD(host, "http://"  .. ip .. path .. "/" .. file) end
	if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/" .. file) end
    else
	I("  Checking IP " .. host .. " -> " .. ip .. " " .. path)
	for i, s in pairs(referenceDevs) do
	    if testing("http", host)  then checkHEAD(host, "http://"  .. ip .. path .. "/" .. s) end
	    if 3 < timeouts then return end
	    if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/" .. s) end
	    if 3 < timeouts then return end
	end

	for i, s in pairs(releases) do
	    for j, k in pairs(releaseFiles) do
		if repoExists(s .. k) then
		    if testing("http", host)  then checkHEAD(host, "http://"  .. ip .. path .. "/merged/dists/" .. s .. k) end
	    if 3 < timeouts then return end
		    if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/merged/dists/" .. s .. k) end
	    if 3 < timeouts then return end
		end
	    end
	end
    end
end

local execute = function (s)
    D("  executing " .. s)
    os.execute(s)
end

checkHost = function (orig, host, path, ip, file)
    if nil == host then host = orig end
    if nil == path then path = "" end
    if nil == file then file = "" end
    local ph = url.parse("http://" .. host)
    if (nil ~= ip) and ("redir" ~= ip) then
	local po = url.parse("http://" .. orig)
	if "" ~= file then
	    D("checking redirected file " .. po.host .. " " .. file)
	    checkFiles(po.host, ip, path, file)
	else
	    checkFiles(po.host, ip, path)
	end
    else
	if orig == host then
	    D("checkHost " .. orig .. "" .. file)
    	    if testing("IPv4") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file .." &") end
	else D("checkHost " .. orig .. " -> " .. host) end
	local h = mirrors[ph.host]
	if nil == h then return end
	for k, v in pairs(h.IPs) do
	    if "table" == type(v) then
		for k1, v1 in pairs(v) do
		    if v1 == "A" then
    			if testing("IPv4") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file .." &") end
		    elseif v1 == "AAAA" then
			if testing("IPv6") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k1 .. "] " .. file .. " &") end
		    end
		end
	    else
		if v == "A" then
    		    if testing("IPv4") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file .." &") end
		elseif v == "AAAA" then
		    if testing("IPv6") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k .. "] " .. file .. " &") end
		end
	    end
	end
    end
end

local downloads = function (cut, host, URL)
    if 0 ~= cut then cd = " --cut-dirs=" .. cut .. " " else cd = "" end
    if nil == URL then URL = "/" end
    local lock = "%s-" .. host .. ".log "
    local log = " --rejected-log=results/wget-%s_REJECTS-" .. host .. ".log -a results/wget-%s-" .. host .. ".log "
    I("starting file download commands for " .. host .. " " .. URL)
    local cm = "ionice -c3 " .. downloadLock .. lock:format("debs") .. download .. log:format("debs", "debs") .. cd
    for i, s in pairs(referenceDevs) do
	cm = cm .. " https://" .. host .. URL .. "/" .. s
    end
    for i, s in pairs(referenceDebs) do
	cm = cm .. " https://" .. host .. URL .. "/" .. s
    end
    for i, s in pairs(releases) do
	execute(cm .. " &")
	cm = "ionice -c3 " .. downloadLock .. lock:format(s) .. download .. log:format(s, s) .. cd
	if repoExists(s .. k) then
	    for j, k in pairs(releaseFiles) do
		cm = cm .. " https://" .. host .. URL .. "/merged/dists/" .. s .. k
	    end
	end
    end
    execute(cm .. " &")
end

local getMirrors = function ()
    local mirrors = {}
    local host = ""
    local m = {}
    local active = true
    local URL = "https://" .. options.referenceSite.value .. "/mirror_list.txt"
    I("getting mirrors.")
    local p, c, h = http.request(URL)
    if nil == p then E(c .. " fetching " .. URL) else
	for l in p:gmatch("\n*([^\n]+)\n*") do
	    local t, d = l:match("(%a*):%s*(.*)")
	    d = string.lower(d)
	    if "FQDN" == t then
		if "" ~= host then
		    if active then mirrors[host] = m end
		    m = {}
		    active = true
		end
		host = d
		m[t] = d
		gatherIPs(host)
		m["IPs"] = IP[host]
	    elseif "Protocols" == t then
		local prot = {}
		for w in d:gmatch("(%w+)") do
		    prot[w] = true;
		end
		m[t] = prot
	    elseif "Active" == t and nil == d:find("yes", 1, true) then
		W("Mirror " .. host .. " is not active - " .. d)
		active = false
-- TODO - Should do some input validation on BaseURL, and everything else.
	    else
		m[t] = d
	    end
	end
	if "" ~= host and active then
	    mirrors[host] = m
	end
    end
    mirrors[options.roundRobin.value] = { ["Protocols"] = { ["http"] = true; ["https"] = true; };  ["FQDN"] = 'deb.devuan.org'; ["Active"] = 'yes'; ["BaseURL"] = 'deb.devuan.org'; }
    gatherIPs(options.roundRobin.value)
    mirrors[options.roundRobin.value].IPs = IP[options.roundRobin.value]
    local file, e = io.open("results/mirrors.lua", "w+")
    if nil == file then C("opening mirrors file - " .. e) else
	file:write(dumpTable(mirrors, "", "mirrors") .. "\nreturn mirrors\n")
        file:close()
    end
    return mirrors
end


if 0 ~= #args then
    local option = ""
    for i, a in pairs(args) do
	if ("--help" == a) or ("-h" == a) then
	    print("I should write some docs, huh?  Read README.md for instructions.")
	elseif "--version" == a then
	    print("apt-panopticon version 0.1 WIP development version")
	elseif "-v" == a then
	    verbosity = verbosity + 1
	    sendArgs = sendArgs .. a .. " "
	elseif "-q" == a then
	    verbosity = -1
	    sendArgs = sendArgs .. a .. " "
	elseif "-k" == a then
	    keep = true
	elseif "-n" == a then
	    fork = false
	elseif "-o" == a then
	    origin = true
	elseif "--" == a:sub(1, 2) then
	    local s, e = a:find("=")
	    if nil == s then e = -1 end
	    option = a:sub(3, e - 1)
	    local o = options[option]
	    if nil == o then
		print("Unknown option --" .. option)
		option = ""
	    else
		option = a
		sendArgs = sendArgs .. a .. " "
		local s, e = a:find("=")
		if nil == s then e = 0 end
		option = a:sub(3, e - 1)
		if "table" == options[option].typ then
		    local result = {}
		    for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do
			local f = t:sub(1, 1)
			local n = t:sub(2, -1)
			if ("+" ~= f) and ("-" ~= f) then
			    table.insert(result, t)
			end
		    end
		    if 0 ~= #result then
			options[option].value = result
		    else
			for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do
			    local f = t:sub(1, 1)
			    local n = t:sub(2, -1)
			    if "+" == f then
				table.insert(options[option].value, n)
			    elseif "-" == f then
				local r = {}
				for i, k in pairs(options[option].value) do
				    if k ~= n then table.insert(r, k) end
				end
				options[option].value = r
			    end
			end
		    end
		else
		    options[option].value = a
		end
		option = ""
	    end
	elseif "-" == a:sub(1, 1) then
	    print("Unknown option " .. a)
	else
	    table.insert(arg, a)
	end
    end
end

--print(dumpTable(options.tests.value, "", "tests"))

execute("mkdir -p results")

if 0 < #arg then
    if "/" == arg[1]:sub(-1, -1) then
	W("slash at end of path!  " .. arg[1])
	arg[1] = arg[1]:sub(1, -2)
    end
    if " " == arg[1]:sub(-1, -1) then
	W("space at end of path!  " .. arg[1])
	arg[1] = arg[1]:sub(1, -2)
    end
    local pu = url.parse("http://" .. arg[1])
    if nil ~= arg[2] then
	logFile, e = io.open("results/" .. pu.host .. "_" .. arg[2] .. ".log", "a+")
    else
	logFile, e = io.open("results/" .. pu.host .. ".log", "a+")
    end
    if nil == logFile then C("opening log file - " .. e); return end
    I("Starting tests for " ..arg[1] .. " with these tests - " .. table.concat(options.tests.value, ", "))
    mirrors = loadfile("results/mirrors.lua")()
    if nil ~= arg[2] then I("  Using IP " .. arg[2]); ip = arg[2] end
    if nil ~= arg[3] then I("  Using file " .. arg[3]); end

    for k, v in pairs{"ftp", "http", "https", "rsync"} do
	if testing(v) then
	    local tests = {errors = 0; warnings = 0}
	    if testing("Integrity")  then tests.Integrity  = {errors = 0; warnings = 0} end
	    if testing("Protocol")   then tests.Protocol   = {errors = 0; warnings = 0} end
	    if testing("Updated")    then tests.Updated    = {errors = 0; warnings = 0} end
	    if testing("URL-Sanity") then tests.URL_Sanity = {errors = 0; warnings = 0} end
	    results[v] = tests
	end
    end

    if testing("Integrity") or testing("Updated") then
	if nil == arg[3] then
	    if not keep then execute("rm -fr results/" .. pu.host) end
	    cut = 0
	    for t in arg[1]:gmatch("(/)") do
		cut = cut + 1
	    end
	    downloads(cut, pu.host, pu.path)
	    checkExes("apt-panopticon.lua " .. sendArgs)
	    checkExes(downloadLock)
	end
    end
    if origin then
	checkFiles(pu.host, pu.host, pu.path);
    else
	checkHost(pu.host, pu.host, pu.path, arg[2], arg[3])
    end
    logFile:close()
    local rfile, e = io.open("results/" .. pu.host .. "_" .. ip .. ".lua", "w+")
    if nil == rfile then C("opening results file - " .. e) else
	rfile:write(dumpTable(results, "", "results") .. "\nreturn results\n")
        rfile:close()
    end
else
    if not keep then os.execute("rm -f results/*.log") end
    os.execute("rm -f results/*.check")
    os.execute("mkdir -p results; touch results/stamp")
    logFile, e = io.open("results/apt-panopticon.log", "a+")
    if nil == logFile then C("opening log file - " .. e); return end
    I("Starting tests " .. table.concat(options.tests.value, ", "))
    execute("mkdir -p results")
    mirrors = getMirrors()
    checkHost(options.referenceSite.value)
    for k, m in pairs(mirrors) do
	if "/" == m.BaseURL:sub(-1, -1) then
	    W("slash at end of BaseURL in mirror_list.txt!  " .. m.BaseURL)
	    m.BaseURL = m.BaseURL:sub(1, -2)
	end
	if " " == m.BaseURL:sub(-1, -1) then
	    W("space at end of BaseURL in mirror_list.txt!  " .. m.BaseURL)
	    m.BaseURL = m.BaseURL:sub(1, -2)
	end
	local pu = url.parse("http://" .. m.BaseURL)
	if options.referenceSite.value ~= pu.host then
	    checkHost(m.BaseURL)
	    checkExes("apt-panopticon.lua " .. sendArgs)
	    if testing("Integrity") or testing("Updated") then checkExes(downloadLock) end
	end
    end
    while 1 <= checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end
    if testing("Integrity") or testing("Updated") then
	while 0 < checkExes(downloadLock) do os.execute("sleep 10") end
    end
    os.execute("rm -f results/*.check")
    logFile:close()
end