From 0f43ca83d630264e42f7875bc6af1d77125b5ebe Mon Sep 17 00:00:00 2001
From: onefang
Date: Wed, 25 Dec 2019 18:47:55 +1000
Subject: Track, store, and report info from the hosts that get redirected to.

---
 apt-panopticon.lua | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 100 insertions(+), 3 deletions(-)

(limited to 'apt-panopticon.lua')

diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index 13adead..b2827b4 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -376,6 +376,8 @@ checkHEAD = function (host, URL, r, retry, sanity)
 		local extraArgs = sendArgs .. ' -o -r '
 		if 'https' == pu.scheme then extraArgs = extraArgs .. ' --tests=-http' end
 		if 'http'  == pu.scheme then extraArgs = extraArgs .. ' --tests=-https' end
+		local pth = path:match('^(.*/pool/).*$')
+		if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E('Odd redirect path ' .. path) end
 		I("  Now checking redirected host " .. u)
 		APT.fork("ionice -c3 nice -n 19 " .. downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file)
 		D('logging to ' .. APT.logName(pu.host, nil, file)[2])
@@ -422,12 +424,10 @@ local checkFiles = function (host, ip, path, file)
 		end
 	    end
 	end
-
 	for i, s in pairs(referenceDebs) do
 	    if checkTimeouts(host, "http",  ip .. path .. "/" .. s) then return end
 	    if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end
 	end
-
 	for i, s in pairs(referenceDevs) do
 	    if checkTimeouts(host, "http",  ip .. path .. "/" .. s) then return end
 	    if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end
@@ -994,7 +994,7 @@ if 0 < #arg then
         rfile:close()
     end
 
-    if APT.origin and APT.options.referenceSite.value ~= pu.host then
+    if APT.origin and (not APT.redir) and (APT.options.referenceSite.value ~= pu.host) then
 os.execute('sleep 1')	-- Wait for things to start up before checking for them.
 	while 0 < APT.checkExes(downloadLock .. "Release-"  .. pu.host .. ".log.txt") do os.execute("sleep 10") end
 	while 0 < APT.checkExes(downloadLock .. "Packages-" .. pu.host .. ".log.txt") do os.execute("sleep 10") end
@@ -1038,6 +1038,103 @@ else
 os.execute('sleep 1')	-- Wait for things to start up before checking for them.
     while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end
 
+    local APT_args    = APT.args
+    local APT_logFile = APT.logFile
+    local debians = {}
+    local srvs = io.popen('ls -1 results/*.lua')
+    for l in srvs:lines() do
+	local hst = l:sub(9, -5)
+	if (l:find('_') == nil) and (nil == APT.mirrors[hst]) then
+	    local ips = loadfile(l)().IPs
+	    if nil ~= ips then
+		debians[hst] = {Country = '', FDQN = hst, Active = 'yes', Rate = '', BaseURL = hst, Protocols = {http = true, https = true}, Bandwidth = '', IPs = ips}
+		local baseFiles = {}
+		local IPfiles = {}
+		for i, a in pairs(ips) do
+		    IPfiles[i] = {}
+		    if type(a) == 'table' then
+			for j, b in pairs(a) do
+			    IPfiles[i][j] = {}
+			end
+		    else
+		    end
+		end
+		local files = io.popen('ls -1 results/LOG_' .. hst .. '_*.html')
+		for ll in files:lines() do
+		    local dn = false
+		    for i, a in pairs(ips) do
+			if type(a) == 'table' then
+			    for j, b in pairs(a) do
+				if nil ~= ll:match('(results/LOG_' .. hst .. '_' .. j .. '_.*)') then
+				    table.insert(IPfiles[i][j], ll)
+				    dn = true
+				end
+			    end
+			else
+			    if nil ~= ll:match('(results/LOG_' .. hst .. '_' .. i .. '_.*)') then
+				table.insert(IPfiles[i], ll)
+				dn = true
+			    end
+			end
+		    end
+		    if not dn then table.insert(baseFiles, ll) end
+		end
+
+		APT.logOpen(hst)
+		APT.logFile:write('<h1>Note log lines will be out of order, this is a bunch of other log files combined.</h1>\n')
+		for i, f in pairs(baseFiles) do
+		    f = f:sub(9, -1)
+		    APT.logFile:write('<hr>\n<hr>\n<h2><a href="' .. f .. '">' .. f .. '</a></h2>\n')
+		    for l in io.lines('results/' .. f) do
+			if l:match('^' .. os.date('%a %b %d ') .. '.*$') then APT.logFile:write(l .. '\n') end
+		    end
+		end
+		APT.logPost()
+		APT.args = APT_args
+		APT.logFile = APT_logFile
+
+		for ip, a in pairs(IPfiles) do
+		    if nil == a[1] then
+			for i, f in pairs(a) do
+			    if not APT.logOpen(hst, i) then print('PROBLEM OPENING LOG FILE ' .. hst .. ' ' .. i) end
+			    APT.logFile:write('<h1>Note log lines will be out of order, this is a bunch of other log files combined.</h1>\n')
+			    for j, g in pairs(f) do
+				g = g:sub(9, -1)
+				APT.logFile:write('<hr>\n<hr>\n<h2><a href="' .. g .. '">' .. g .. '</a></h2>\n')
+				for l in io.lines('results/' .. g) do
+				    if l:match('^' .. os.date('%a %b %d ') .. '.*$') then APT.logFile:write(l .. '\n') end
+				end
+			    end
+			    APT.logPost()
+			    APT.args = APT_args
+			    APT.logFile = APT_logFile
+			end
+		    else
+			if not APT.logOpen(hst, ip) then print('PROBLEM OPENING LOG FILE ' .. hst .. ' ' .. ip) end
+			APT.logFile:write('<h1>Note log lines will be out of order, this is a bunch of other log files combined.</h1>\n')
+			for i, f in pairs(a) do
+			    f = f:sub(9, -1)
+			    APT.logFile:write('<hr>\n<hr>\n<h2><a href="' .. f .. '">' .. f .. '</a></h2>\n')
+			    for l in io.lines('results/' .. f) do
+				if l:match('^' .. os.date('%a %b %d ') .. '.*$') then APT.logFile:write(l .. '\n') end
+			    end
+			end
+			APT.logPost()
+			APT.args = APT_args
+			APT.logFile = APT_logFile
+		    end
+		end
+
+	    end
+	end
+    end
+    local file, e = io.open("results/debians.lua", "w+")
+    if nil == file then C("opening debians file - " .. e) else
+	file:write(APT.dumpTable(debians, "", "debians") .. "\nreturn debians\n")
+        file:close()
+    end
+
+
     for k, v in pairs(APT.mirrors) do
 	local f =  'results/' .. k .. '.lua'
 	if APT.checkFile(f) then
-- 
cgit v1.1