From c742660a5d5245e4bae7fa6cb7b022590aad1658 Mon Sep 17 00:00:00 2001 From: onefang Date: Tue, 24 Dec 2019 15:08:02 +1000 Subject: Better handling of redirects to different hosts. --- apt-panopticommon.lua | 4 ++++ apt-panopticon.lua | 64 +++++++++++++++++++++++++++------------------------ 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/apt-panopticommon.lua b/apt-panopticommon.lua index eab901e..accfcd5 100644 --- a/apt-panopticommon.lua +++ b/apt-panopticommon.lua @@ -21,6 +21,7 @@ APT.notExist = APT.verbosity = -1 APT.origin = false +APT.redir = false APT.keep = false APT.options = @@ -122,6 +123,9 @@ APT.parseArgs = function(args) elseif "-o" == a then APT.origin = true -- Not adding to sendArgs. + elseif "-r" == a then + APT.redir = true + sendArgs = sendArgs .. a .. " " elseif "--" == a:sub(1, 2) then local s, e = a:find("=") if nil == s then e = -1 end diff --git a/apt-panopticon.lua b/apt-panopticon.lua index 6668da2..81fd281 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua @@ -162,6 +162,8 @@ local url = require 'socket.url' local ip = "" local cor = nil +local downloadLock = "flock -n results/curl-" + local repoExists = function (r) r = r:match("([%a-]*)") @@ -220,6 +222,7 @@ checkHEAD = function (host, URL, r, retry, sanity) local check = "Checking file" local PU = url.parse(URL, defaultURL) local pu = url.parse(PU.scheme .. "://" .. host, defaultURL) + if not APT.testing(PU.scheme, host) and APT.redir then I("Not supported, not tested " .. PU.scheme .. " " .. host .. " -> " .. URL, PU.scheme, "", host); return end if 0 < r then check = "Redirecting to" end @@ -341,7 +344,9 @@ checkHEAD = function (host, URL, r, retry, sanity) E(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) end else - if not APT.testing(PU.scheme, host) then W("Not supported, but works " .. PU.scheme .. " " .. host .. " -> " .. URL, PU.scheme, "", host) end + if not APT.testing(PU.scheme, host) then + W("Not supported, but works " .. PU.scheme .. " " .. host .. " -> " .. URL, PU.scheme, "", host) + end I(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL) -- timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through. if nil ~= location then @@ -363,22 +368,16 @@ checkHEAD = function (host, URL, r, retry, sanity) --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location. Not good for deb.debian.org, which we should only be throwing .debs at. What we do is loop through the DNS entries, and only test the specific protocol & file being tested here. - - This is what I came up with for checking if we are already testing a specific URL. - Still duplicates a tiny bit, but much less than the previous find based method. -TODO - maybe we can switch to using flock like we do with the other curl calls? ]] -C('Commented out code being called in checkHEAD() for redirects that should no longer happen!') ---[[ - local file = pu.host .. "://" .. pu.path - local f = io.popen(string.format('if [ ! -f results/%s.check ] ; then touch results/%s.check; echo -n "check"; fi', file:gsub("/", "_"), file:gsub("/", "_") )):read("*a") - if (nil == f) or ("check" == f) then - I(" Now checking redirected host " .. file) - checkHost(pu.host, pu.host, nil, "redir", pu.path) - else - D(" Already checking " .. file) - end -]] + local u = pu.host .. "://" .. pu.path + local file = pu.path:match(".*/([%w%.%+%-_]*)$") -- Get the filename. + local path = pu.path:sub(2, -1 -(#file)) + local check = u:gsub("/", "_") + local extraArgs = sendArgs .. ' -o -r ' + if 'https' == pu.scheme then extraArgs = ' --tests=-http' end + if 'http' == pu.scheme then extraArgs = ' --tests=-https' end + I(" Now checking redirected host " .. u) + APT.fork("ionice -c3 nice -n 19 " .. downloadLock .. "REDIR-" .. check .. ".log" .. " ./apt-panopticon.lua " .. extraArgs .. pu.host .. "/" .. path .. " " .. file) end end end @@ -483,7 +482,6 @@ local postDownload = function(host, r, k) end end -local downloadLock = "flock -n results/curl-" local download = "curl" .. " --connect-timeout " .. APT.options.timeout.value .. " --create-dirs -f -L" .. @@ -904,13 +902,18 @@ if 0 < #arg then end local pu = url.parse("http://" .. arg[1]) + if APT.redir and (nil == arg[3])then + arg[3] = arg[2] + arg[2] = nil + end + if APT.testing("Integrity") or APT.testing("Updated") then if APT.origin and APT.options.referenceSite.value == pu.host then -- if not APT.keep then os.execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end end end - if APT.origin then APT.results["IPs"] = gatherIPs(pu.host) end + if APT.origin or APT.redir then APT.results["IPs"] = gatherIPs(pu.host) end if nil ~= arg[2] then APT.logFile, e = io.open("results/LOG_" .. pu.host .. "_" .. arg[2] .. ".html", "a+") @@ -924,8 +927,7 @@ if 0 < #arg then if nil ~= arg[3] then I(" Using file " .. arg[3]); end APT.results = APT.padResults(APT.results) - if APT.origin then - + if APT.origin or APT.redir then local file = arg[3] if nil == file then file = '' end local path = pu.path @@ -949,17 +951,20 @@ if 0 < #arg then end end - if APT.testing("Integrity") or APT.testing("Updated") then - if APT.origin and (APT.options.roundRobin.value ~= pu.host) then - I("Starting file downloads for " .. pu.host) - D('*>* About to create coroutine.') - cor = coroutine.create(doDownloads) - local ok, message = coroutine.resume(cor, pu.host, pu.path, parseStart(pu.host)) - if not ok then cor = nil; print(message) end + if not APT.redir then + if APT.testing("Integrity") or APT.testing("Updated") then + if APT.origin and (APT.options.roundRobin.value ~= pu.host) then + I("Starting file downloads for " .. pu.host) + D('*>* About to create coroutine.') + cor = coroutine.create(doDownloads) + local ok, message = coroutine.resume(cor, pu.host, pu.path, parseStart(pu.host)) + if not ok then cor = nil; print(message) end + end end + checkFiles(pu.host, pu.host, pu.path); + else + checkFiles(pu.host, pu.host, pu.path:sub(1, -1), file); end - - checkFiles(pu.host, pu.host, pu.path); else checkHost(pu.host, pu.host, pu.path, arg[2], arg[3]) end @@ -1004,7 +1009,6 @@ else if nil ~= odt then os.execute(' rm -f results_old; ln -s results_' .. odt .. ' results_old 2>/dev/null') end if nil ~= dt then os.execute('mkdir -p results_' .. dt .. '; rm -f results; ln -s results_' .. dt .. ' results 2>/dev/null') end os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp') - os.execute("rm -f results/*.check 2>/dev/null") if not APT.keep then os.execute("rm -f results/*.log 2>/dev/null") os.execute("rm -f results/*.html 2>/dev/null") -- cgit v1.1