From d1322db549df5444c2a0baa836524c724b14c512 Mon Sep 17 00:00:00 2001 From: onefang Date: Sat, 7 Sep 2019 01:38:29 +1000 Subject: Various changes to the URL checker. Don't test protocols on hosts that don't support them. Don't test HTTPS on the deb.devuan.org IPs, they are all mirrors that shouldn't have the deb.devuan.org cert. Actually test HTTPS, coz LuaSocket doesn't actually support it, but doesn't tell you and silently converts them to HTTP. Use LuaSec for that. Neaten up the post checking. --- mirror-checker.lua | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/mirror-checker.lua b/mirror-checker.lua index d141f8e..aab62cc 100755 --- a/mirror-checker.lua +++ b/mirror-checker.lua @@ -3,6 +3,12 @@ local args = {...} +--[[ TODO - What to do about HTTPS://deb.devuan.org/ redirects. + Some mirrors give a 404. + Sledjhamr gives a 404, coz it's not listening on 443 for deb.devuan.org. + Some mirrors give a 200. + They shouldn't have the proper certificate, but are giving a result anyway. +]] verbosity = 0 keep = false @@ -80,6 +86,7 @@ local logFile local socket = require 'socket' local ftp = require 'socket.ftp' local http = require 'socket.http' +local https = require 'ssl.https' -- See https://github.com/brunoos/luasec/wiki/LuaSec-0.6 for docs. local url = require 'socket.url' @@ -117,6 +124,7 @@ dumpTableSub = function (table, space) return r end + local log = function(v, t, s) if v <= verbosity then if 3 <= verbosity then t = os.date() .. " " .. t end @@ -227,9 +235,15 @@ checkURL = function (host, URL, r, retry) end local PU = url.parse(URL, defaultURL) D(PU.scheme .. " :// " .. check .. " " .. host .. " -> " .. URL) - if not testing(PU.scheme) then D("not testing " .. PU.scheme .. " " .. host .. " -> " .. URL); return end + if not testing(PU.scheme, host) then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL); return end + -- TODO - Perhaps we should try it anyway, and mark it as a warning if it DOES work? + if "https" == PU.scheme and "deb.devuan.org" == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert."); return end local hd = {Host = host} - local p, c, h, s = http.request{method = "HEAD", redirect = false, url = URL, headers = hd} + local htp = http; + if PU.scheme == "https" then htp = https end + -- NOTE - the docs for lua-sec say that redirect isn't supported is version 0.6, no idea if that means it ignores redirections like we want. + -- The protocol and options are lua-sec specific arguments. + local p, c, h, s = htp.request{method = "HEAD", redirect = false, url = URL, headers = hd, protocol = "any", options = "all"} if nil == s then s = "" end if nil == p then E(" " .. c .. " " .. s .. "! " .. check .. " " .. host .. " -> " .. URL) @@ -245,18 +259,17 @@ checkURL = function (host, URL, r, retry) l = h.location if nil ~= l then local pu = url.parse(l, defaultURL) - if l == URL then + if pu.scheme ~= PU.scheme then + if testing("Protocol") then + W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l) + end + elseif l == URL then E(" redirect loop! " .. check .. " " .. host .. " -> " .. URL) + elseif nil == pu.host then + W(" no location host! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l) + checkURL(host, PU.scheme .. "://" .. PU.host .. l, r + 1) else - if nil == pu.host then - W(" no location host! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l) - checkURL(host, PU.scheme .. "://" .. PU.host .. l, r + 1) - else - if testing("Protocol") and pu.scheme ~= PU.scheme then - W(" protocol changed during redirect! " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l) - end - checkURL(pu.host, l, r + 1, retry) - end + checkURL(pu.host, l, r + 1) end end end -- cgit v1.1