aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
authoronefang2019-09-07 01:38:29 +1000
committeronefang2019-09-07 01:38:29 +1000
commitd1322db549df5444c2a0baa836524c724b14c512 (patch)
tree0475f8658f7ba7e6d16d46f85e94e2945060512a
parentUse the saved mirrors list and the split packages lists. (diff)
downloadapt-panopticon-d1322db549df5444c2a0baa836524c724b14c512.zip
apt-panopticon-d1322db549df5444c2a0baa836524c724b14c512.tar.gz
apt-panopticon-d1322db549df5444c2a0baa836524c724b14c512.tar.bz2
apt-panopticon-d1322db549df5444c2a0baa836524c724b14c512.tar.xz
Various changes to the URL checker.
Don't test protocols on hosts that don't support them. Don't test HTTPS on the deb.devuan.org IPs, they are all mirrors that shouldn't have the deb.devuan.org cert. Actually test HTTPS, coz LuaSocket doesn't actually support it, but doesn't tell you and silently converts them to HTTP. Use LuaSec for that. Neaten up the post checking.
Diffstat (limited to '')
-rwxr-xr-xmirror-checker.lua37
1 files changed, 25 insertions, 12 deletions
diff --git a/mirror-checker.lua b/mirror-checker.lua
index d141f8e..aab62cc 100755
--- a/mirror-checker.lua
+++ b/mirror-checker.lua
@@ -3,6 +3,12 @@
3 3
4local args = {...} 4local args = {...}
5 5
6--[[ TODO - What to do about HTTPS://deb.devuan.org/ redirects.
7 Some mirrors give a 404.
8 Sledjhamr gives a 404, coz it's not listening on 443 for deb.devuan.org.
9 Some mirrors give a 200.
10 They shouldn't have the proper certificate, but are giving a result anyway.
11]]
6 12
7verbosity = 0 13verbosity = 0
8keep = false 14keep = false
@@ -80,6 +86,7 @@ local logFile
80local socket = require 'socket' 86local socket = require 'socket'
81local ftp = require 'socket.ftp' 87local ftp = require 'socket.ftp'
82local http = require 'socket.http' 88local http = require 'socket.http'
89local https = require 'ssl.https' -- See https://github.com/brunoos/luasec/wiki/LuaSec-0.6 for docs.
83local url = require 'socket.url' 90local url = require 'socket.url'
84 91
85 92
@@ -117,6 +124,7 @@ dumpTableSub = function (table, space)
117 return r 124 return r
118end 125end
119 126
127
120local log = function(v, t, s) 128local log = function(v, t, s)
121 if v <= verbosity then 129 if v <= verbosity then
122 if 3 <= verbosity then t = os.date() .. " " .. t end 130 if 3 <= verbosity then t = os.date() .. " " .. t end
@@ -227,9 +235,15 @@ checkURL = function (host, URL, r, retry)
227 end 235 end
228 local PU = url.parse(URL, defaultURL) 236 local PU = url.parse(URL, defaultURL)
229 D(PU.scheme .. " :// " .. check .. " " .. host .. " -> " .. URL) 237 D(PU.scheme .. " :// " .. check .. " " .. host .. " -> " .. URL)
230 if not testing(PU.scheme) then D("not testing " .. PU.scheme .. " " .. host .. " -> " .. URL); return end 238 if not testing(PU.scheme, host) then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL); return end
239 -- TODO - Perhaps we should try it anyway, and mark it as a warning if it DOES work?
240 if "https" == PU.scheme and "deb.devuan.org" == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert."); return end
231 local hd = {Host = host} 241 local hd = {Host = host}
232 local p, c, h, s = http.request{method = "HEAD", redirect = false, url = URL, headers = hd} 242 local htp = http;
243 if PU.scheme == "https" then htp = https end
244 -- NOTE - the docs for lua-sec say that redirect isn't supported is version 0.6, no idea if that means it ignores redirections like we want.
245 -- The protocol and options are lua-sec specific arguments.
246 local p, c, h, s = htp.request{method = "HEAD", redirect = false, url = URL, headers = hd, protocol = "any", options = "all"}
233 if nil == s then s = "" end 247 if nil == s then s = "" end
234 if nil == p then 248 if nil == p then
235 E(" " .. c .. " " .. s .. "! " .. check .. " " .. host .. " -> " .. URL) 249 E(" " .. c .. " " .. s .. "! " .. check .. " " .. host .. " -> " .. URL)
@@ -245,18 +259,17 @@ checkURL = function (host, URL, r, retry)
245 l = h.location 259 l = h.location
246 if nil ~= l then 260 if nil ~= l then
247 local pu = url.parse(l, defaultURL) 261 local pu = url.parse(l, defaultURL)
248 if l == URL then 262 if pu.scheme ~= PU.scheme then
263 if testing("Protocol") then
264 W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l)
265 end
266 elseif l == URL then
249 E(" redirect loop! " .. check .. " " .. host .. " -> " .. URL) 267 E(" redirect loop! " .. check .. " " .. host .. " -> " .. URL)
268 elseif nil == pu.host then
269 W(" no location host! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l)
270 checkURL(host, PU.scheme .. "://" .. PU.host .. l, r + 1)
250 else 271 else
251 if nil == pu.host then 272 checkURL(pu.host, l, r + 1)
252 W(" no location host! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l)
253 checkURL(host, PU.scheme .. "://" .. PU.host .. l, r + 1)
254 else
255 if testing("Protocol") and pu.scheme ~= PU.scheme then
256 W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l)
257 end
258 checkURL(pu.host, l, r + 1, retry)
259 end
260 end 273 end
261 end 274 end
262 end 275 end