From 4f817a6c1ff77cf166c860f41adb8b7105a6f283 Mon Sep 17 00:00:00 2001 From: onefang Date: Fri, 28 Jun 2019 18:44:52 +1000 Subject: Check redirections on individual files. --- mirror-checker.lua | 98 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 70 insertions(+), 28 deletions(-) diff --git a/mirror-checker.lua b/mirror-checker.lua index 6eb1c1a..d370497 100755 --- a/mirror-checker.lua +++ b/mirror-checker.lua @@ -179,7 +179,23 @@ checkURL = function (host, URL, r, retry) local check = "Checking file" if 0 < r then check = "Redirecting to" --- checkHost(host, host) + --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location. + Not good for deb.debian.org, which we should only be throwing .debs at. + What we should do is loop through the DNS entries, and only test the specific protocol & file being tested here. + + This is what I came up with for checking if we are already testing a specific URL. + Still duplicates a tiny bit, but much less than the previous find based method. + ]] + local pl = url.parse(URL, defaultURL) + local file = host .. "://" .. pl.path + local f = io.popen(string.format('if [ ! -f results/%s.check ] ; then touch results/%s.check; echo -n "check"; fi', file:gsub("/", "_"), file:gsub("/", "_") )):read("*a") + if (nil == f) or ("check" == f) then + I(" Now checking redirected host " .. file) + checkRedirects(host, host, nil, "redir", pl.path) + else + D(" Already checking " .. file) + end + end if 0 < retry then check = "Retry " .. retry .. " " .. check @@ -229,18 +245,25 @@ checkURL = function (host, URL, r, retry) end end -local checkPaths = function (host, ip, path) - I(" Checking IP " .. host .. " -> " .. ip .. " " .. path) - for i, s in pairs(referenceDebs) do - if testing("http") then checkURL(host, "http://" .. ip .. path .. "/" .. s) end - if testing("https") then checkURL(host, "https://" .. ip .. path .. "/" .. s) end - end +local checkPaths = function (host, ip, path, file) + if nil ~= file then + if "redirect" == ip then ip = host end + I(" Checking IP for file " .. host .. " -> " .. ip .. " " .. path .. " " .. file) + if testing("http") then checkURL(host, "http://" .. ip .. path .. "/" .. file) end + if testing("https") then checkURL(host, "https://" .. ip .. path .. "/" .. file) end + else + I(" Checking IP " .. host .. " -> " .. ip .. " " .. path) + for i, s in pairs(referenceDebs) do + if testing("http") then checkURL(host, "http://" .. ip .. path .. "/" .. s) end + if testing("https") then checkURL(host, "https://" .. ip .. path .. "/" .. s) end + end - for i, s in pairs(releases) do - for j, k in pairs(releaseFiles) do - if repoExists(s .. k) then - if testing("http") then checkURL(host, "http://" .. ip .. path .. "/merged/dists/" .. s .. k) end - if testing("https") then checkURL(host, "https://" .. ip .. path .. "/merged/dists/" .. s .. k) end + for i, s in pairs(releases) do + for j, k in pairs(releaseFiles) do + if repoExists(s .. k) then + if testing("http") then checkURL(host, "http://" .. ip .. path .. "/merged/dists/" .. s .. k) end + if testing("https") then checkURL(host, "https://" .. ip .. path .. "/merged/dists/" .. s .. k) end + end end end end @@ -251,24 +274,37 @@ local execute = function (s) os.execute(s) end -checkRedirects = function (orig, host, path, ip) +checkRedirects = function (orig, host, path, ip, file) if nil == host then host = orig end if nil == path then path = "" end + if nil == file then file = "" end local po = url.parse("http://" .. orig) local ph = url.parse("http://" .. host) - if nil ~= ip then - checkPaths(po.host, ip, path) + if (nil ~= ip) and ("redir" ~= ip) then + if "" ~= file then + D("checking redirected file " .. po.host .. " " .. file) + checkPaths(po.host, ip, path, file) + else + checkPaths(po.host, ip, path) + end else - if orig == host then D("checkRedirects " .. orig) else D("checkRedirects " .. orig .. " " .. host) end + if orig == host then D("checkRedirects " .. orig .. "" .. file) else D("checkRedirects " .. orig .. " -> " .. host) end gatherIPs(ph.host) for k, v in pairs(IP[ph.host]) do D(" DNS record " .. v .. " " .. ph.host .. " -> " .. k) if v == "A" then - if testing("IPv4") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " &") end + if testing("IPv4") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file .." &") end elseif v == "AAAA" then - if testing("IPv6") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k .. "] &") end - elseif v == "CNAME" then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. k .. path .. " &") - checkRedirects(orig, k, path) -- Check the original, with the DNS records from the CNAME, as well as the CNAME, so they both get checked. + if testing("IPv6") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k .. "] " .. file .. " &") end + elseif v == "CNAME" then + if "" == file then +-- TODO - maybe "ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " &" +-- ./mirror-checker.lua --tests=-IPv6 -q deb.roundr.devuan.org & -> executing ionice -c3 ./mirror-checker.lua --tests=-IPv6 -q deb.devuan.org deb.roundr.devuan.org & + execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. k .. path .. " &") + else + execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. k .. path .. " redirect " .. file .. " &") + end + checkRedirects(orig, k, path, ip, file) -- Check the original, with the DNS records from the CNAME, as well as the CNAME, so they both get checked. end end end @@ -409,20 +445,25 @@ if 0 < #arg then if nil == logFile then C("opening log file - " .. e); return end I("Starting tests for " ..arg[1] .. " with these tests - " .. table.concat(options.tests.value, ", ")) if nil ~= arg[2] then I(" Using IP " .. arg[2]) end + if nil ~= arg[3] then I(" Using file " .. arg[3]); end if testing("Integrity") or testing("Updated") then - if not keep then execute("rm -fr results/" .. pu.host) end - cut = 0 - for t in arg[1]:gmatch("(/)") do - cut = cut + 1 + if nil == arg[3] then + if not keep then execute("rm -fr results/" .. pu.host) end + cut = 0 + for t in arg[1]:gmatch("(/)") do + cut = cut + 1 + end + downloads(cut, pu.host, pu.path) + checkExes("mirror-checker.lua " .. sendArgs) + checkExes(downloadLock) end - downloads(cut, pu.host, pu.path) - checkExes("mirror-checker.lua " .. sendArgs) - checkExes(downloadLock) end - checkRedirects(pu.host, pu.host, pu.path, arg[2]) + checkRedirects(pu.host, pu.host, pu.path, arg[2], arg[3]) logFile:close() else if not keep then os.execute("rm -f results/*.log") end + os.execute("rm -f results/*.check") + os.execute("mkdir -p results; touch results/stamp") logFile, e = io.open("results/mirror-checker-lua.log", "a+") if nil == logFile then C("opening log file - " .. e); return end I("Starting tests " .. table.concat(options.tests.value, ", ")) @@ -448,5 +489,6 @@ else if testing("Integrity") or testing("Updated") then while 0 < checkExes(downloadLock) do os.execute("sleep 30") end end + os.execute("rm -f results/*.check") logFile:close() end -- cgit v1.1