diff options
author | onefang | 2019-09-29 14:19:11 +1000 |
---|---|---|
committer | onefang | 2019-09-29 14:19:11 +1000 |
commit | c9fc01638e8a8d62e9057aec70ef8fe40b53c74f (patch) | |
tree | 04f1cb61412f27afea5980b4559ca67c07a22d0d | |
parent | checkRedirects -> checkHost (diff) | |
download | apt-panopticon-c9fc01638e8a8d62e9057aec70ef8fe40b53c74f.zip apt-panopticon-c9fc01638e8a8d62e9057aec70ef8fe40b53c74f.tar.gz apt-panopticon-c9fc01638e8a8d62e9057aec70ef8fe40b53c74f.tar.bz2 apt-panopticon-c9fc01638e8a8d62e9057aec70ef8fe40b53c74f.tar.xz |
Rejig the redirects a little.
-rw-r--r-- | README.md | 2 | ||||
-rwxr-xr-x | mirror-checker.lua | 52 |
2 files changed, 27 insertions, 27 deletions
@@ -113,7 +113,7 @@ The --tests= option can be used to adjust the list of tests performed. | |||
113 | * https, test HTTPS protocol access. | 113 | * https, test HTTPS protocol access. |
114 | * rsync, test RSYNC protocol access. | 114 | * rsync, test RSYNC protocol access. |
115 | * DNS-RR, ??????????????? | 115 | * DNS-RR, ??????????????? |
116 | * Protocol, warn if the protocol changed during a test. | 116 | * Protocol, warn if the protocol changed during a redirect. |
117 | * URL-Sanity, add gratuitous multiple slashes to the URLs. | 117 | * URL-Sanity, add gratuitous multiple slashes to the URLs. |
118 | * Integrity, check PGP signatures. | 118 | * Integrity, check PGP signatures. |
119 | * Updated, check Release dates. | 119 | * Updated, check Release dates. |
diff --git a/mirror-checker.lua b/mirror-checker.lua index 81b32db..34d1023 100755 --- a/mirror-checker.lua +++ b/mirror-checker.lua | |||
@@ -203,29 +203,12 @@ checkHEAD = function (host, URL, r, retry) | |||
203 | local check = "Checking file" | 203 | local check = "Checking file" |
204 | if 0 < r then | 204 | if 0 < r then |
205 | check = "Redirecting to" | 205 | check = "Redirecting to" |
206 | --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location. | ||
207 | Not good for deb.debian.org, which we should only be throwing .debs at. | ||
208 | What we should do is loop through the DNS entries, and only test the specific protocol & file being tested here. | ||
209 | |||
210 | This is what I came up with for checking if we are already testing a specific URL. | ||
211 | Still duplicates a tiny bit, but much less than the previous find based method. | ||
212 | ]] | ||
213 | local pl = url.parse(URL, defaultURL) | ||
214 | local file = host .. "://" .. pl.path | ||
215 | local f = io.popen(string.format('if [ ! -f results/%s.check ] ; then touch results/%s.check; echo -n "check"; fi', file:gsub("/", "_"), file:gsub("/", "_") )):read("*a") | ||
216 | if (nil == f) or ("check" == f) then | ||
217 | I(" Now checking redirected host " .. file) | ||
218 | checkHost(host, host, nil, "redir", pl.path) | ||
219 | else | ||
220 | D(" Already checking " .. file) | ||
221 | end | ||
222 | |||
223 | end | 206 | end |
224 | if 0 < retry then | 207 | if 0 < retry then |
225 | -- TODO - should have a random sleep here before retrying. | 208 | -- TODO - should have a random sleep here before retrying. |
226 | check = "Retry " .. retry .. " " .. check | 209 | check = "Retry " .. retry .. " " .. check |
227 | end | 210 | end |
228 | if 10 < r then | 211 | if 20 < r then |
229 | E("too many redirects! " .. check .. " " .. host .. " -> " .. URL) | 212 | E("too many redirects! " .. check .. " " .. host .. " -> " .. URL) |
230 | return | 213 | return |
231 | end | 214 | end |
@@ -242,6 +225,7 @@ checkHEAD = function (host, URL, r, retry) | |||
242 | local htp = http; | 225 | local htp = http; |
243 | if PU.scheme == "https" then htp = https end | 226 | if PU.scheme == "https" then htp = https end |
244 | -- NOTE - the docs for lua-sec say that redirect isn't supported is version 0.6, no idea if that means it ignores redirections like we want. | 227 | -- NOTE - the docs for lua-sec say that redirect isn't supported is version 0.6, no idea if that means it ignores redirections like we want. |
228 | -- TODO - find out! | ||
245 | -- The protocol and options are lua-sec specific arguments. | 229 | -- The protocol and options are lua-sec specific arguments. |
246 | local p, c, h, s = htp.request{method = "HEAD", redirect = false, url = URL, headers = hd, protocol = "any", options = "all"} | 230 | local p, c, h, s = htp.request{method = "HEAD", redirect = false, url = URL, headers = hd, protocol = "any", options = "all"} |
247 | if nil == s then s = "" end | 231 | if nil == s then s = "" end |
@@ -259,17 +243,33 @@ checkHEAD = function (host, URL, r, retry) | |||
259 | l = h.location | 243 | l = h.location |
260 | if nil ~= l then | 244 | if nil ~= l then |
261 | local pu = url.parse(l, defaultURL) | 245 | local pu = url.parse(l, defaultURL) |
262 | if pu.scheme ~= PU.scheme then | 246 | if testing("Protocol") and (pu.scheme ~= PU.scheme) then |
263 | if testing("Protocol") then | 247 | W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l) |
264 | W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l) | 248 | end |
265 | end | 249 | |
266 | elseif l == URL then | 250 | if l == URL then |
267 | E(" redirect loop! " .. check .. " " .. host .. " -> " .. URL) | 251 | E(" redirect loop! " .. check .. " " .. host .. " -> " .. URL) |
268 | elseif nil == pu.host then | 252 | elseif nil == pu.host then |
269 | W(" no location host! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l) | 253 | I(" relative redirect. " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l) |
270 | checkHEAD(host, PU.scheme .. "://" .. PU.host .. l, r + 1) | 254 | checkHEAD(host, PU.scheme .. "://" .. PU.host .. l, r + 1) |
271 | else | 255 | elseif PU.host == pu.host then |
272 | checkHEAD(pu.host, l, r + 1) | 256 | checkHEAD(pu.host, l, r + 1) |
257 | else | ||
258 | --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location. | ||
259 | Not good for deb.debian.org, which we should only be throwing .debs at. | ||
260 | What we do is loop through the DNS entries, and only test the specific protocol & file being tested here. | ||
261 | |||
262 | This is what I came up with for checking if we are already testing a specific URL. | ||
263 | Still duplicates a tiny bit, but much less than the previous find based method. | ||
264 | ]] | ||
265 | local file = pu.host .. "://" .. pu.path | ||
266 | local f = io.popen(string.format('if [ ! -f results/%s.check ] ; then touch results/%s.check; echo -n "check"; fi', file:gsub("/", "_"), file:gsub("/", "_") )):read("*a") | ||
267 | if (nil == f) or ("check" == f) then | ||
268 | I(" Now checking redirected host " .. file) | ||
269 | checkHost(pu.host, pu.host, nil, "redir", pu.path) | ||
270 | else | ||
271 | D(" Already checking " .. file) | ||
272 | end | ||
273 | end | 273 | end |
274 | end | 274 | end |
275 | end | 275 | end |
@@ -277,7 +277,7 @@ end | |||
277 | 277 | ||
278 | local checkFiles = function (host, ip, path, file) | 278 | local checkFiles = function (host, ip, path, file) |
279 | if nil ~= file then | 279 | if nil ~= file then |
280 | if "redirect" == ip then ip = host end | 280 | if "redir" == ip then ip = host end |
281 | I(" Checking IP for file " .. host .. " -> " .. ip .. " " .. path .. " " .. file) | 281 | I(" Checking IP for file " .. host .. " -> " .. ip .. " " .. path .. " " .. file) |
282 | if testing("http", host) then checkHEAD(host, "http://" .. ip .. path .. "/" .. file) end | 282 | if testing("http", host) then checkHEAD(host, "http://" .. ip .. path .. "/" .. file) end |
283 | if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/" .. file) end | 283 | if testing("https", host) then checkHEAD(host, "https://" .. ip .. path .. "/" .. file) end |