aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
authoronefang2019-09-29 14:19:11 +1000
committeronefang2019-09-29 14:19:11 +1000
commitc9fc01638e8a8d62e9057aec70ef8fe40b53c74f (patch)
tree04f1cb61412f27afea5980b4559ca67c07a22d0d
parentcheckRedirects -> checkHost (diff)
downloadapt-panopticon-c9fc01638e8a8d62e9057aec70ef8fe40b53c74f.zip
apt-panopticon-c9fc01638e8a8d62e9057aec70ef8fe40b53c74f.tar.gz
apt-panopticon-c9fc01638e8a8d62e9057aec70ef8fe40b53c74f.tar.bz2
apt-panopticon-c9fc01638e8a8d62e9057aec70ef8fe40b53c74f.tar.xz
Rejig the redirects a little.
-rw-r--r--README.md2
-rwxr-xr-xmirror-checker.lua52
2 files changed, 27 insertions, 27 deletions
diff --git a/README.md b/README.md
index 347049b..bb107a9 100644
--- a/README.md
+++ b/README.md
@@ -113,7 +113,7 @@ The --tests= option can be used to adjust the list of tests performed.
113* https, test HTTPS protocol access. 113* https, test HTTPS protocol access.
114* rsync, test RSYNC protocol access. 114* rsync, test RSYNC protocol access.
115* DNS-RR, ??????????????? 115* DNS-RR, ???????????????
116* Protocol, warn if the protocol changed during a test. 116* Protocol, warn if the protocol changed during a redirect.
117* URL-Sanity, add gratuitous multiple slashes to the URLs. 117* URL-Sanity, add gratuitous multiple slashes to the URLs.
118* Integrity, check PGP signatures. 118* Integrity, check PGP signatures.
119* Updated, check Release dates. 119* Updated, check Release dates.
diff --git a/mirror-checker.lua b/mirror-checker.lua
index 81b32db..34d1023 100755
--- a/mirror-checker.lua
+++ b/mirror-checker.lua
@@ -203,29 +203,12 @@ checkHEAD = function (host, URL, r, retry)
203 local check = "Checking file" 203 local check = "Checking file"
204 if 0 < r then 204 if 0 < r then
205 check = "Redirecting to" 205 check = "Redirecting to"
206 --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location.
207 Not good for deb.debian.org, which we should only be throwing .debs at.
208 What we should do is loop through the DNS entries, and only test the specific protocol & file being tested here.
209
210 This is what I came up with for checking if we are already testing a specific URL.
211 Still duplicates a tiny bit, but much less than the previous find based method.
212 ]]
213 local pl = url.parse(URL, defaultURL)
214 local file = host .. "://" .. pl.path
215 local f = io.popen(string.format('if [ ! -f results/%s.check ] ; then touch results/%s.check; echo -n "check"; fi', file:gsub("/", "_"), file:gsub("/", "_") )):read("*a")
216 if (nil == f) or ("check" == f) then
217 I(" Now checking redirected host " .. file)
218 checkHost(host, host, nil, "redir", pl.path)
219 else
220 D(" Already checking " .. file)
221 end
222
223 end 206 end
224 if 0 < retry then 207 if 0 < retry then
225-- TODO - should have a random sleep here before retrying. 208-- TODO - should have a random sleep here before retrying.
226 check = "Retry " .. retry .. " " .. check 209 check = "Retry " .. retry .. " " .. check
227 end 210 end
228 if 10 < r then 211 if 20 < r then
229 E("too many redirects! " .. check .. " " .. host .. " -> " .. URL) 212 E("too many redirects! " .. check .. " " .. host .. " -> " .. URL)
230 return 213 return
231 end 214 end
@@ -242,6 +225,7 @@ checkHEAD = function (host, URL, r, retry)
242 local htp = http; 225 local htp = http;
243 if PU.scheme == "https" then htp = https end 226 if PU.scheme == "https" then htp = https end
244 -- NOTE - the docs for lua-sec say that redirect isn't supported is version 0.6, no idea if that means it ignores redirections like we want. 227 -- NOTE - the docs for lua-sec say that redirect isn't supported is version 0.6, no idea if that means it ignores redirections like we want.
228 -- TODO - find out!
245 -- The protocol and options are lua-sec specific arguments. 229 -- The protocol and options are lua-sec specific arguments.
246 local p, c, h, s = htp.request{method = "HEAD", redirect = false, url = URL, headers = hd, protocol = "any", options = "all"} 230 local p, c, h, s = htp.request{method = "HEAD", redirect = false, url = URL, headers = hd, protocol = "any", options = "all"}
247 if nil == s then s = "" end 231 if nil == s then s = "" end
@@ -259,17 +243,33 @@ checkHEAD = function (host, URL, r, retry)
259 l = h.location 243 l = h.location
260 if nil ~= l then 244 if nil ~= l then
261 local pu = url.parse(l, defaultURL) 245 local pu = url.parse(l, defaultURL)
262 if pu.scheme ~= PU.scheme then 246 if testing("Protocol") and (pu.scheme ~= PU.scheme) then
263 if testing("Protocol") then 247 W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l)
264 W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l) 248 end
265 end 249
266 elseif l == URL then 250 if l == URL then
267 E(" redirect loop! " .. check .. " " .. host .. " -> " .. URL) 251 E(" redirect loop! " .. check .. " " .. host .. " -> " .. URL)
268 elseif nil == pu.host then 252 elseif nil == pu.host then
269 W(" no location host! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l) 253 I(" relative redirect. " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l)
270 checkHEAD(host, PU.scheme .. "://" .. PU.host .. l, r + 1) 254 checkHEAD(host, PU.scheme .. "://" .. PU.host .. l, r + 1)
271 else 255 elseif PU.host == pu.host then
272 checkHEAD(pu.host, l, r + 1) 256 checkHEAD(pu.host, l, r + 1)
257 else
258 --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location.
259 Not good for deb.debian.org, which we should only be throwing .debs at.
260 What we do is loop through the DNS entries, and only test the specific protocol & file being tested here.
261
262 This is what I came up with for checking if we are already testing a specific URL.
263 Still duplicates a tiny bit, but much less than the previous find based method.
264 ]]
265 local file = pu.host .. "://" .. pu.path
266 local f = io.popen(string.format('if [ ! -f results/%s.check ] ; then touch results/%s.check; echo -n "check"; fi', file:gsub("/", "_"), file:gsub("/", "_") )):read("*a")
267 if (nil == f) or ("check" == f) then
268 I(" Now checking redirected host " .. file)
269 checkHost(pu.host, pu.host, nil, "redir", pu.path)
270 else
271 D(" Already checking " .. file)
272 end
273 end 273 end
274 end 274 end
275 end 275 end
@@ -277,7 +277,7 @@ end
277 277
278local checkFiles = function (host, ip, path, file) 278local checkFiles = function (host, ip, path, file)
279 if nil ~= file then 279 if nil ~= file then
280 if "redirect" == ip then ip = host end 280 if "redir" == ip then ip = host end
281 I(" Checking IP for file " .. host .. " -> " .. ip .. " " .. path .. " " .. file) 281 I(" Checking IP for file " .. host .. " -> " .. ip .. " " .. path .. " " .. file)
282 if testing("http", host) then checkHEAD(host, "http://" .. ip .. path .. "/" .. file) end 282 if testing("http", host) then checkHEAD(host, "http://" .. ip .. path .. "/" .. file) end
283 if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/" .. file) end 283 if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/" .. file) end