aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--apt-panopticommon.lua4
-rwxr-xr-xapt-panopticon.lua64
2 files changed, 38 insertions, 30 deletions
diff --git a/apt-panopticommon.lua b/apt-panopticommon.lua
index eab901e..accfcd5 100644
--- a/apt-panopticommon.lua
+++ b/apt-panopticommon.lua
@@ -21,6 +21,7 @@ APT.notExist =
21 21
22APT.verbosity = -1 22APT.verbosity = -1
23APT.origin = false 23APT.origin = false
24APT.redir = false
24APT.keep = false 25APT.keep = false
25 26
26APT.options = 27APT.options =
@@ -122,6 +123,9 @@ APT.parseArgs = function(args)
122 elseif "-o" == a then 123 elseif "-o" == a then
123 APT.origin = true 124 APT.origin = true
124 -- Not adding to sendArgs. 125 -- Not adding to sendArgs.
126 elseif "-r" == a then
127 APT.redir = true
128 sendArgs = sendArgs .. a .. " "
125 elseif "--" == a:sub(1, 2) then 129 elseif "--" == a:sub(1, 2) then
126 local s, e = a:find("=") 130 local s, e = a:find("=")
127 if nil == s then e = -1 end 131 if nil == s then e = -1 end
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index 6668da2..81fd281 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -162,6 +162,8 @@ local url = require 'socket.url'
162local ip = "" 162local ip = ""
163local cor = nil 163local cor = nil
164 164
165local downloadLock = "flock -n results/curl-"
166
165 167
166local repoExists = function (r) 168local repoExists = function (r)
167 r = r:match("([%a-]*)") 169 r = r:match("([%a-]*)")
@@ -220,6 +222,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
220 local check = "Checking file" 222 local check = "Checking file"
221 local PU = url.parse(URL, defaultURL) 223 local PU = url.parse(URL, defaultURL)
222 local pu = url.parse(PU.scheme .. "://" .. host, defaultURL) 224 local pu = url.parse(PU.scheme .. "://" .. host, defaultURL)
225 if not APT.testing(PU.scheme, host) and APT.redir then I("Not supported, not tested " .. PU.scheme .. " " .. host .. " -> " .. URL, PU.scheme, "", host); return end
223 if 0 < r then 226 if 0 < r then
224 check = "Redirecting to" 227 check = "Redirecting to"
225 end 228 end
@@ -341,7 +344,9 @@ checkHEAD = function (host, URL, r, retry, sanity)
341 E(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) 344 E(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
342 end 345 end
343 else 346 else
344 if not APT.testing(PU.scheme, host) then W("Not supported, but works " .. PU.scheme .. " " .. host .. " -> " .. URL, PU.scheme, "", host) end 347 if not APT.testing(PU.scheme, host) then
348 W("Not supported, but works " .. PU.scheme .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
349 end
345 I(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL) 350 I(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL)
346-- timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through. 351-- timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through.
347 if nil ~= location then 352 if nil ~= location then
@@ -363,22 +368,16 @@ checkHEAD = function (host, URL, r, retry, sanity)
363 --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location. 368 --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location.
364 Not good for deb.debian.org, which we should only be throwing .debs at. 369 Not good for deb.debian.org, which we should only be throwing .debs at.
365 What we do is loop through the DNS entries, and only test the specific protocol & file being tested here. 370 What we do is loop through the DNS entries, and only test the specific protocol & file being tested here.
366
367 This is what I came up with for checking if we are already testing a specific URL.
368 Still duplicates a tiny bit, but much less than the previous find based method.
369TODO - maybe we can switch to using flock like we do with the other curl calls?
370 ]] 371 ]]
371C('Commented out code being called in checkHEAD() for redirects that should no longer happen!') 372 local u = pu.host .. "://" .. pu.path
372--[[ 373 local file = pu.path:match(".*/([%w%.%+%-_]*)$") -- Get the filename.
373 local file = pu.host .. "://" .. pu.path 374 local path = pu.path:sub(2, -1 -(#file))
374 local f = io.popen(string.format('if [ ! -f results/%s.check ] ; then touch results/%s.check; echo -n "check"; fi', file:gsub("/", "_"), file:gsub("/", "_") )):read("*a") 375 local check = u:gsub("/", "_")
375 if (nil == f) or ("check" == f) then 376 local extraArgs = sendArgs .. ' -o -r '
376 I(" Now checking redirected host " .. file) 377 if 'https' == pu.scheme then extraArgs = ' --tests=-http' end
377 checkHost(pu.host, pu.host, nil, "redir", pu.path) 378 if 'http' == pu.scheme then extraArgs = ' --tests=-https' end
378 else 379 I(" Now checking redirected host " .. u)
379 D(" Already checking " .. file) 380 APT.fork("ionice -c3 nice -n 19 " .. downloadLock .. "REDIR-" .. check .. ".log" .. " ./apt-panopticon.lua " .. extraArgs .. pu.host .. "/" .. path .. " " .. file)
380 end
381]]
382 end 381 end
383 end 382 end
384 end 383 end
@@ -483,7 +482,6 @@ local postDownload = function(host, r, k)
483 end 482 end
484end 483end
485 484
486local downloadLock = "flock -n results/curl-"
487local download = "curl" .. 485local download = "curl" ..
488 " --connect-timeout " .. APT.options.timeout.value .. 486 " --connect-timeout " .. APT.options.timeout.value ..
489 " --create-dirs -f -L" .. 487 " --create-dirs -f -L" ..
@@ -904,13 +902,18 @@ if 0 < #arg then
904 end 902 end
905 local pu = url.parse("http://" .. arg[1]) 903 local pu = url.parse("http://" .. arg[1])
906 904
905 if APT.redir and (nil == arg[3])then
906 arg[3] = arg[2]
907 arg[2] = nil
908 end
909
907 if APT.testing("Integrity") or APT.testing("Updated") then 910 if APT.testing("Integrity") or APT.testing("Updated") then
908 if APT.origin and APT.options.referenceSite.value == pu.host then 911 if APT.origin and APT.options.referenceSite.value == pu.host then
909-- if not APT.keep then os.execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end 912-- if not APT.keep then os.execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end
910 end 913 end
911 end 914 end
912 915
913 if APT.origin then APT.results["IPs"] = gatherIPs(pu.host) end 916 if APT.origin or APT.redir then APT.results["IPs"] = gatherIPs(pu.host) end
914 917
915 if nil ~= arg[2] then 918 if nil ~= arg[2] then
916 APT.logFile, e = io.open("results/LOG_" .. pu.host .. "_" .. arg[2] .. ".html", "a+") 919 APT.logFile, e = io.open("results/LOG_" .. pu.host .. "_" .. arg[2] .. ".html", "a+")
@@ -924,8 +927,7 @@ if 0 < #arg then
924 if nil ~= arg[3] then I(" Using file " .. arg[3]); end 927 if nil ~= arg[3] then I(" Using file " .. arg[3]); end
925 APT.results = APT.padResults(APT.results) 928 APT.results = APT.padResults(APT.results)
926 929
927 if APT.origin then 930 if APT.origin or APT.redir then
928
929 local file = arg[3] 931 local file = arg[3]
930 if nil == file then file = '' end 932 if nil == file then file = '' end
931 local path = pu.path 933 local path = pu.path
@@ -949,17 +951,20 @@ if 0 < #arg then
949 end 951 end
950 end 952 end
951 953
952 if APT.testing("Integrity") or APT.testing("Updated") then 954 if not APT.redir then
953 if APT.origin and (APT.options.roundRobin.value ~= pu.host) then 955 if APT.testing("Integrity") or APT.testing("Updated") then
954 I("Starting file downloads for " .. pu.host) 956 if APT.origin and (APT.options.roundRobin.value ~= pu.host) then
955 D('*&gt;* About to create coroutine.') 957 I("Starting file downloads for " .. pu.host)
956 cor = coroutine.create(doDownloads) 958 D('*&gt;* About to create coroutine.')
957 local ok, message = coroutine.resume(cor, pu.host, pu.path, parseStart(pu.host)) 959 cor = coroutine.create(doDownloads)
958 if not ok then cor = nil; print(message) end 960 local ok, message = coroutine.resume(cor, pu.host, pu.path, parseStart(pu.host))
961 if not ok then cor = nil; print(message) end
962 end
959 end 963 end
964 checkFiles(pu.host, pu.host, pu.path);
965 else
966 checkFiles(pu.host, pu.host, pu.path:sub(1, -1), file);
960 end 967 end
961
962 checkFiles(pu.host, pu.host, pu.path);
963 else 968 else
964 checkHost(pu.host, pu.host, pu.path, arg[2], arg[3]) 969 checkHost(pu.host, pu.host, pu.path, arg[2], arg[3])
965 end 970 end
@@ -1004,7 +1009,6 @@ else
1004 if nil ~= odt then os.execute(' rm -f results_old; ln -s results_' .. odt .. ' results_old 2>/dev/null') end 1009 if nil ~= odt then os.execute(' rm -f results_old; ln -s results_' .. odt .. ' results_old 2>/dev/null') end
1005 if nil ~= dt then os.execute('mkdir -p results_' .. dt .. '; rm -f results; ln -s results_' .. dt .. ' results 2>/dev/null') end 1010 if nil ~= dt then os.execute('mkdir -p results_' .. dt .. '; rm -f results; ln -s results_' .. dt .. ' results 2>/dev/null') end
1006 os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp') 1011 os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp')
1007 os.execute("rm -f results/*.check 2>/dev/null")
1008 if not APT.keep then 1012 if not APT.keep then
1009 os.execute("rm -f results/*.log 2>/dev/null") 1013 os.execute("rm -f results/*.log 2>/dev/null")
1010 os.execute("rm -f results/*.html 2>/dev/null") 1014 os.execute("rm -f results/*.html 2>/dev/null")