Check redirections on individual files.

1 files changed, 70 insertions, 28 deletions

diff --git a/mirror-checker.lua b/mirror-checker.lua
index 6eb1c1a..d370497 100755
--- a/mirror-checker.lua
+++ b/mirror-checker.lua
@@ -179,7 +179,23 @@ checkURL = function (host, URL, r, retry)
     local check = "Checking file"
     if 0 < r then
         check = "Redirecting to"
---        checkHost(host, host)
+        --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location.
+            Not good for deb.debian.org, which we should only be throwing .debs at.
+            What we should do is loop through the DNS entries, and only test the specific protocol & file being tested here.
+
+            This is what I came up with for checking if we are already testing a specific URL.
+            Still duplicates a tiny bit, but much less than the previous find based method.
+        ]]
+        local pl = url.parse(URL, defaultURL)
+        local file = host .. "://" .. pl.path
+        local f = io.popen(string.format('if [ ! -f results/%s.check ] ; then touch results/%s.check; echo -n "check"; fi', file:gsub("/", "_"), file:gsub("/", "_") )):read("*a")
+        if (nil == f) or ("check" == f) then
+            I("  Now checking redirected host " .. file)
+            checkRedirects(host, host, nil, "redir", pl.path)
+        else
+            D("  Already checking " .. file)
+        end
+
     end
     if 0 < retry then
         check = "Retry " .. retry .. " " .. check
@@ -229,18 +245,25 @@ checkURL = function (host, URL, r, retry)
     end
 end
 
-local checkPaths = function (host, ip, path)
-    I("  Checking IP " .. host .. " -> " .. ip .. " " .. path)
-    for i, s in pairs(referenceDebs) do
-        if testing("http")  then checkURL(host, "http://"  .. ip .. path .. "/" .. s) end
-        if testing("https") then checkURL(host, "https://" .. ip .. path .. "/" .. s) end
-    end
+local checkPaths = function (host, ip, path, file)
+    if nil ~= file then
+        if "redirect" == ip then ip = host end
+        I("  Checking IP for file " .. host .. " -> " .. ip .. " " .. path .. " " .. file)
+        if testing("http")  then checkURL(host, "http://"  .. ip .. path .. "/" .. file) end
+        if testing("https") then checkURL(host, "https://" .. ip .. path .. "/" .. file) end
+    else
+        I("  Checking IP " .. host .. " -> " .. ip .. " " .. path)
+        for i, s in pairs(referenceDebs) do
+            if testing("http")  then checkURL(host, "http://"  .. ip .. path .. "/" .. s) end
+            if testing("https") then checkURL(host, "https://" .. ip .. path .. "/" .. s) end
+        end
 
-    for i, s in pairs(releases) do
-        for j, k in pairs(releaseFiles) do
-            if repoExists(s .. k) then
-                if testing("http")  then checkURL(host, "http://"  .. ip .. path .. "/merged/dists/" .. s .. k) end
-                if testing("https") then checkURL(host, "https://" .. ip .. path .. "/merged/dists/" .. s .. k) end
+        for i, s in pairs(releases) do
+            for j, k in pairs(releaseFiles) do
+                if repoExists(s .. k) then
+                    if testing("http")  then checkURL(host, "http://"  .. ip .. path .. "/merged/dists/" .. s .. k) end
+                    if testing("https") then checkURL(host, "https://" .. ip .. path .. "/merged/dists/" .. s .. k) end
+                end
             end
         end
     end
@@ -251,24 +274,37 @@ local execute = function (s)
     os.execute(s)
 end
 
-checkRedirects = function (orig, host, path, ip)
+checkRedirects = function (orig, host, path, ip, file)
     if nil == host then host = orig end
     if nil == path then path = "" end
+    if nil == file then file = "" end
     local po = url.parse("http://" .. orig)
     local ph = url.parse("http://" .. host)
-    if nil ~= ip then
-        checkPaths(po.host, ip, path)
+    if (nil ~= ip) and ("redir" ~= ip) then
+        if "" ~= file then
+            D("checking redirected file " .. po.host .. " " .. file)
+            checkPaths(po.host, ip, path, file)
+        else
+            checkPaths(po.host, ip, path)
+        end
     else
-        if orig == host then D("checkRedirects " .. orig) else D("checkRedirects " .. orig .. " " .. host) end
+        if orig == host then D("checkRedirects " .. orig .. "" .. file) else D("checkRedirects " .. orig .. " -> " .. host) end
         gatherIPs(ph.host)
         for k, v in pairs(IP[ph.host]) do
             D("  DNS record " .. v .. " " .. ph.host .. " -> " .. k)
             if v == "A" then
-                if testing("IPv4") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " &") end
+                if testing("IPv4") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file .." &") end
             elseif v == "AAAA" then
-                if testing("IPv6") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k .. "] &") end
-            elseif v == "CNAME" then    execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. k .. path .. " &")
-                checkRedirects(orig, k, path)  -- Check the original, with the DNS records from the CNAME, as well as the CNAME, so they both get checked.
+                if testing("IPv6") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k .. "] " .. file .. " &") end
+            elseif v == "CNAME" then
+                if "" == file then
+-- TODO - maybe "ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " &"
+-- ./mirror-checker.lua --tests=-IPv6 -q  deb.roundr.devuan.org &   ->  executing ionice -c3 ./mirror-checker.lua --tests=-IPv6 -q deb.devuan.org deb.roundr.devuan.org & 
+                    execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. k .. path .. " &")
+                else
+                    execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. k .. path .. " redirect " .. file .. " &")
+                end
+                checkRedirects(orig, k, path, ip, file)  -- Check the original, with the DNS records from the CNAME, as well as the CNAME, so they both get checked.
             end
         end
     end
@@ -409,20 +445,25 @@ if 0 < #arg then
     if nil == logFile then C("opening log file - " .. e); return end
     I("Starting tests for " ..arg[1] .. " with these tests - " .. table.concat(options.tests.value, ", "))
     if nil ~= arg[2] then I("  Using IP " .. arg[2]) end
+    if nil ~= arg[3] then I("  Using file " .. arg[3]); end
     if testing("Integrity") or testing("Updated") then
-        if not keep then execute("rm -fr results/" .. pu.host) end
-        cut = 0
-        for t in arg[1]:gmatch("(/)") do
-            cut = cut + 1
+        if nil == arg[3] then 
+            if not keep then execute("rm -fr results/" .. pu.host) end
+            cut = 0
+            for t in arg[1]:gmatch("(/)") do
+                cut = cut + 1
+            end
+            downloads(cut, pu.host, pu.path)
+            checkExes("mirror-checker.lua " .. sendArgs)
+            checkExes(downloadLock)
         end
-        downloads(cut, pu.host, pu.path)
-        checkExes("mirror-checker.lua " .. sendArgs)
-        checkExes(downloadLock)
     end
-    checkRedirects(pu.host, pu.host, pu.path, arg[2])
+    checkRedirects(pu.host, pu.host, pu.path, arg[2], arg[3])
     logFile:close()
 else
     if not keep then os.execute("rm -f results/*.log") end
+    os.execute("rm -f results/*.check")
+    os.execute("mkdir -p results; touch results/stamp")
     logFile, e = io.open("results/mirror-checker-lua.log", "a+")
     if nil == logFile then C("opening log file - " .. e); return end
     I("Starting tests " .. table.concat(options.tests.value, ", "))
@@ -448,5 +489,6 @@ else
     if testing("Integrity") or testing("Updated") then
         while 0 < checkExes(downloadLock) do os.execute("sleep 30") end
     end
+    os.execute("rm -f results/*.check")
     logFile:close()
 end