Better handling of redirects to different hosts.

2 files changed, 38 insertions, 30 deletions

diff --git a/apt-panopticommon.lua b/apt-panopticommon.lua
index eab901e..accfcd5 100644
--- a/apt-panopticommon.lua
+++ b/apt-panopticommon.lua
@@ -21,6 +21,7 @@ APT.notExist =
 
 APT.verbosity = -1
 APT.origin = false
+APT.redir = false
 APT.keep = false
 
 APT.options =
@@ -122,6 +123,9 @@ APT.parseArgs = function(args)
             elseif "-o" == a then
                 APT.origin = true
                 -- Not adding to sendArgs.
+            elseif "-r" == a then
+                APT.redir = true
+                sendArgs = sendArgs .. a .. " "
             elseif "--" == a:sub(1, 2) then
                 local s, e = a:find("=")
                 if nil == s then e = -1 end
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index 6668da2..81fd281 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -162,6 +162,8 @@ local url = require 'socket.url'
 local ip = ""
 local cor = nil
 
+local downloadLock = "flock -n results/curl-"
+
 
 local repoExists = function (r)
     r = r:match("([%a-]*)")
@@ -220,6 +222,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
     local check = "Checking file"
     local PU = url.parse(URL, defaultURL)
     local pu = url.parse(PU.scheme .. "://" .. host, defaultURL)
+    if not APT.testing(PU.scheme, host) and APT.redir then I("Not supported, not tested " .. PU.scheme .. " " .. host .. " -> " .. URL, PU.scheme, "", host); return end
     if 0 < r then
         check = "Redirecting to"
     end
@@ -341,7 +344,9 @@ checkHEAD = function (host, URL, r, retry, sanity)
             E("  " .. code .. " " .. cstr .. ".   " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
         end
     else
-    if not APT.testing(PU.scheme, host) then W("Not supported, but works " .. PU.scheme .. " " .. host .. " -> " .. URL, PU.scheme, "", host) end
+    if not APT.testing(PU.scheme, host) then
+        W("Not supported, but works " .. PU.scheme .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
+    end
         I("  " .. code .. " " .. cstr .. ".   " .. check .. " " .. host .. " -> " .. URL)
 --      timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through.
         if nil ~= location then
@@ -363,22 +368,16 @@ checkHEAD = function (host, URL, r, retry, sanity)
                 --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location.
                     Not good for deb.debian.org, which we should only be throwing .debs at.
                     What we do is loop through the DNS entries, and only test the specific protocol & file being tested here.
-
-                    This is what I came up with for checking if we are already testing a specific URL.
-                    Still duplicates a tiny bit, but much less than the previous find based method.
-TODO - maybe we can switch to using flock like we do with the other curl calls?
                 ]]
-C('Commented out code being called in checkHEAD() for redirects that should no longer happen!')
---[[
-                local file = pu.host .. "://" .. pu.path
-                local f = io.popen(string.format('if [ ! -f results/%s.check ] ; then touch results/%s.check; echo -n "check"; fi', file:gsub("/", "_"), file:gsub("/", "_") )):read("*a")
-                if (nil == f) or ("check" == f) then
-                    I("  Now checking redirected host " .. file)
-                    checkHost(pu.host, pu.host, nil, "redir", pu.path)
-                else
-                    D("  Already checking " .. file)
-                end
-]]
+                local u = pu.host .. "://" .. pu.path
+                local file = pu.path:match(".*/([%w%.%+%-_]*)$")        -- Get the filename.
+                local path = pu.path:sub(2, -1 -(#file))
+                local check = u:gsub("/", "_")
+                local extraArgs = sendArgs .. ' -o -r '
+                if 'https' == pu.scheme then extraArgs = ' --tests=-http' end
+                if 'http'  == pu.scheme then extraArgs = ' --tests=-https' end
+                I("  Now checking redirected host " .. u)
+                APT.fork("ionice -c3 nice -n 19 " .. downloadLock .. "REDIR-" .. check .. ".log" .. " ./apt-panopticon.lua " .. extraArgs .. pu.host .. "/" .. path .. " " .. file)
             end
         end
     end
@@ -483,7 +482,6 @@ local postDownload = function(host, r, k)
     end
 end
 
-local downloadLock = "flock -n results/curl-"
 local download = "curl" ..
                  " --connect-timeout " .. APT.options.timeout.value ..
                  " --create-dirs -f -L" ..
@@ -904,13 +902,18 @@ if 0 < #arg then
     end
     local pu = url.parse("http://" .. arg[1])
 
+    if APT.redir and (nil == arg[3])then
+        arg[3] = arg[2]
+        arg[2] = nil
+    end
+
     if APT.testing("Integrity") or APT.testing("Updated") then
         if APT.origin and APT.options.referenceSite.value == pu.host then
 --          if not APT.keep then os.execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end
         end
     end
 
-    if APT.origin then APT.results["IPs"] = gatherIPs(pu.host) end
+    if APT.origin or APT.redir then APT.results["IPs"] = gatherIPs(pu.host) end
 
     if nil ~= arg[2] then
         APT.logFile, e = io.open("results/LOG_" .. pu.host .. "_" .. arg[2] .. ".html", "a+")
@@ -924,8 +927,7 @@ if 0 < #arg then
     if nil ~= arg[3] then I("  Using file " .. arg[3]); end
     APT.results = APT.padResults(APT.results)
 
-    if APT.origin then
-
+    if APT.origin or APT.redir then
         local file = arg[3]
         if nil == file then file = '' end
         local path = pu.path
@@ -949,17 +951,20 @@ if 0 < #arg then
             end
         end
 
-        if APT.testing("Integrity") or APT.testing("Updated") then
-            if APT.origin and (APT.options.roundRobin.value ~= pu.host) then
-                I("Starting file downloads for " .. pu.host)
-                D('*&gt;* About to create coroutine.')
-                cor = coroutine.create(doDownloads)
-                local ok, message = coroutine.resume(cor, pu.host, pu.path, parseStart(pu.host))
-                if not ok then cor = nil; print(message) end
+        if not APT.redir then
+            if APT.testing("Integrity") or APT.testing("Updated") then
+                if APT.origin and (APT.options.roundRobin.value ~= pu.host) then
+                    I("Starting file downloads for " .. pu.host)
+                    D('*&gt;* About to create coroutine.')
+                    cor = coroutine.create(doDownloads)
+                    local ok, message = coroutine.resume(cor, pu.host, pu.path, parseStart(pu.host))
+                    if not ok then cor = nil; print(message) end
+                end
             end
+            checkFiles(pu.host, pu.host, pu.path);
+        else
+            checkFiles(pu.host, pu.host, pu.path:sub(1, -1), file);
         end
-
-        checkFiles(pu.host, pu.host, pu.path);
     else
         checkHost(pu.host, pu.host, pu.path, arg[2], arg[3])
     end
@@ -1004,7 +1009,6 @@ else
     if nil ~= odt then os.execute('                               rm -f results_old; ln -s results_' .. odt .. ' results_old 2>/dev/null') end
     if nil ~= dt  then os.execute('mkdir -p results_' .. dt .. '; rm -f results;     ln -s results_' .. dt  .. ' results     2>/dev/null') end
     os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi;  touch results/stamp')
-    os.execute("rm -f results/*.check  2>/dev/null")
     if not APT.keep then
         os.execute("rm -f results/*.log  2>/dev/null")
         os.execute("rm -f results/*.html 2>/dev/null")