Various changes to the URL checker.

Don't test protocols on hosts that don't support them.

Don't test HTTPS on the deb.devuan.org IPs, they are all mirrors that
shouldn't have the deb.devuan.org cert.

Actually test HTTPS, coz LuaSocket doesn't actually support it, but
doesn't tell you and silently converts them to HTTP.  Use LuaSec for
that.

Neaten up the post checking.

1 files changed, 25 insertions, 12 deletions

diff --git a/mirror-checker.lua b/mirror-checker.lua
index d141f8e..aab62cc 100755
--- a/mirror-checker.lua
+++ b/mirror-checker.lua
@@ -3,6 +3,12 @@
 
 local args = {...}
 
+--[[ TODO - What to do about HTTPS://deb.devuan.org/ redirects.
+            Some mirrors give a 404.
+                Sledjhamr gives a 404, coz it's not listening on 443 for deb.devuan.org.
+            Some mirrors give a 200.
+                They shouldn't have the proper certificate, but are giving a result anyway.
+]]
 
 verbosity = 0
 keep = false
@@ -80,6 +86,7 @@ local logFile
 local socket = require 'socket'
 local ftp = require 'socket.ftp'
 local http = require 'socket.http'
+local https = require 'ssl.https'       -- See https://github.com/brunoos/luasec/wiki/LuaSec-0.6 for docs.
 local url = require 'socket.url'
 
 
@@ -117,6 +124,7 @@ dumpTableSub = function (table, space)
     return r
 end
 
+
 local log = function(v, t, s)
     if v <= verbosity then
         if 3 <= verbosity then t = os.date() .. " " .. t end
@@ -227,9 +235,15 @@ checkURL = function (host, URL, r, retry)
     end
     local PU = url.parse(URL, defaultURL)
     D(PU.scheme .. " ://  " .. check .. " " .. host .. " -> " .. URL)
-    if not testing(PU.scheme) then D("not testing " .. PU.scheme .. " " .. host .. " -> " .. URL);  return end
+    if not testing(PU.scheme, host) then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL);  return end
+    -- TODO - Perhaps we should try it anyway, and mark it as a warning if it DOES work?
+    if "https" == PU.scheme and "deb.devuan.org" == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert.");  return end
     local hd = {Host = host}
-    local p, c, h, s = http.request{method = "HEAD", redirect = false, url = URL, headers = hd}
+    local htp = http;
+    if PU.scheme == "https" then htp = https end
+    -- NOTE - the docs for lua-sec say that redirect isn't supported is version 0.6, no idea if that means it ignores redirections like we want.
+    --          The protocol and options are lua-sec specific arguments.
+    local p, c, h, s = htp.request{method = "HEAD", redirect = false, url = URL, headers = hd, protocol = "any", options = "all"}
     if nil == s then s = "" end
     if nil == p then
         E("  " .. c .. " " .. s .. "!   " .. check .. " " .. host .. " -> " .. URL)
@@ -245,18 +259,17 @@ checkURL = function (host, URL, r, retry)
         l = h.location
         if nil ~= l then
             local pu = url.parse(l, defaultURL)
-            if l == URL then
+            if pu.scheme ~= PU.scheme then
+                if testing("Protocol") then
+                    W("    protocol changed during redirect!  " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l)
+                end
+            elseif l == URL then
                 E("    redirect loop!  " .. check .. " " .. host .. " -> " .. URL)
+            elseif nil == pu.host then
+                W("    no location host!  " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l)
+                checkURL(host, PU.scheme .. "://" .. PU.host .. l, r + 1)
             else
-                if nil == pu.host then
-                    W("    no location host!  " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l)
-                    checkURL(host, PU.scheme .. "://" .. PU.host .. l, r + 1)
-                else
-                    if testing("Protocol") and pu.scheme ~= PU.scheme then
-                        W("    protocol changed during redirect!  " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l)
-                    end
-                    checkURL(pu.host, l, r + 1, retry)
-                end
+                checkURL(pu.host, l, r + 1)
             end
         end
     end