2 files changed, 76 insertions, 31 deletions

diff --git a/README.md b/README.md
index 94b3c65..5db696e 100644
--- a/README.md
+++ b/README.md
@@ -31,12 +31,12 @@ in someplace like `/usr/local/bin` and make sure it is executable.
 It should run on any recent Linux, you'll need to have the following
 installed -
 
+* curl
 * dig, part of BIND.  On Debian based systems it'll be in the dnsutils package.
 * flock,  on Debian based systems it'll be in the util-linux package.
 * ionice, on Debian based systems it'll be in the util-linux package.
 * luajit
 * LuaSocket, on Debian based systems it'll be in the lua-socket package.
-* LuaSec, on Debian based systems it'll be in the lua-sec package.
 * wget
 
 
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index 227bec1..c536990 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -107,7 +107,6 @@ local logFile
 local socket = require 'socket'
 local ftp = require 'socket.ftp'
 local http = require 'socket.http'
-local https = require 'ssl.https'       -- See https://github.com/brunoos/luasec/wiki/LuaSec-0.6 for docs.
 local url = require 'socket.url'
 
 
@@ -286,42 +285,87 @@ checkHEAD = function (host, URL, r, retry)
     if not testing(PU.scheme, host) then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL);  return end
     -- TODO - Perhaps we should try it anyway, and mark it as a warning if it DOES work?
     if "https" == PU.scheme and options.roundRobin.value == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert.");  return end
-    local hd = {}
-    if pu.host ~= PU.host then hd = {Host = host} end
-    local htp = http;
-    if PU.scheme == "https" then htp = https end
-    -- NOTE - the docs for lua-sec say that redirect isn't supported is version 0.6, no idea if that means it ignores redirections like we want.
-    -- TODO -                                                                           find out!
-    --          The protocol and options are lua-sec specific arguments.
-    local p, c, h, s = htp.request{method = "HEAD", redirect = false, url = URL, headers = hd, protocol = "any", options = "all"}
-    if nil == s then s = "" end
-    if nil == p then
-        E("  " .. c .. " " .. s .. "!   " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
-        -- So far the only errors are "timeout", "Network is unreachable", and "closed", and I suspect "closed" is due to saturating my bandwidth.
-        if "timeout" == c then timeouts = timeouts + 1 end
-        if ("closed" == c) or ("Network is unreachable" == c) or ("timeout" == c) then checkHEAD(host, URL, r, retry + 1, timeouts) end
-    else
-        if ("4" == tostring(c):sub(1, 1)) or ("5" == tostring(c):sub(1, 1)) then
-            E("  " .. c .. " " .. s .. ".   " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
-        else
-            I("  " .. c .. " " .. s .. ".   " .. check .. " " .. host .. " -> " .. URL)
-            timeouts = timeouts - 1     -- Backoff the timeouts count if we managed to get through.
+
+    --[[ Using curl command line -
+        -I - HEAD
+        --connect-to IP - connect to IP, but use SNI from URL.
+        -header "" - add extra headers.
+        -L - DO follow redirects.
+        --max-redirs n - set maximum redirects, default is 50, -1 = unlimited.
+        --retry n - maximum retries, default is 0, no retries.
+        -o file - write to file instead of stdout.
+        --path-as-is - https://curl.haxx.se/libcurl/c/CURLOPT_PATH_AS_IS.html might be useful for URL_sanity.
+        -s silent - don't output progress or error messages.
+        --connect-timeout n - timeout in seconds.
+            Should return with error code 28 on a timeout?
+        -D file - write the received headers to a file.  This includes the status code and string.
+    ]]
+    local fname = host .. "_" .. PU.host .. "_" .. PU.path:gsub("/", "_") .. ".txt"
+    local hdr = ""
+    local IP = ""
+    if pu.host ~= PU.host then
+        if "http" == PU.scheme then
+            hdr = '-H "Host: ' .. host .. '"'
         end
-        l = h.location
-        if nil ~= l then
-            pu = url.parse(l, defaultURL)
+        IP = '--connect-to ' .. PU.host
+    end
+    local cmd = 'curl -I --retry 0 -s --path-as-is --connect-timeout 30 --max-redirs 0 ' .. IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' ..
+                hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"'
+    local rslt, status = execute(cmd)
+    os.execute("sleep 2")
+    os.execute('cat results/"HEADERS_' .. fname .. '" >>results/"STATUS_' .. fname .. '"; rm results/"HEADERS_' .. fname .. '"')
+    if "exit" == rslt then
+        if 28 == status then
+            E("  TIMEOUT!", PU.scheme, "", host)
+            timeouts = timeouts + 1
+            checkHEAD(host, URL, r, retry + 1, timeouts)
+            return
+        elseif 0 ~= status then
+            E("  The curl command return an error code of " .. status .. ", consult the curl manual for what this means.", PU.scheme, "", host)
+            checkHEAD(host, URL, r, retry + 1, timeouts)
+            return
+        end
+    elseif "signal" == rslt then
+            E("  The curl command was interupted by signal " .. status)
+            return
+    end
+    local rfile, e = io.open("results/STATUS_" .. fname, "r")
+    local code = "000"
+    local cstr = ""
+    local location = nil
+    if nil == rfile then W("opening results/STATUS_" .. fname .. " file - " .. e) else
+        for line in rfile:lines("*l") do
+            if "#" == line:sub(1, 1) then
+                code = line:sub(2, 4)
+                if ("https" == PU.scheme) and ("0" ~= line:sub(6, 6)) then E("  The certificate is invalid.", PU.scheme, "https", host) end
+            elseif "http" == line:sub(1, 4):lower() then
+                -- -2 coz the headers file gets a \r at the end.
+                cstr = line:sub(14, -2)
+            elseif "location" == line:sub(1, 8):lower() then
+                location = line:sub(11, -2)
+            end
+        end
+    end
+    os.execute('rm results/"STATUS_' .. fname .. '"')
+    if ("4" == tostring(code):sub(1, 1)) or ("5" == tostring(code):sub(1, 1)) then
+        E("  " .. code .. " " .. cstr .. ".   " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
+    else
+        I("  " .. code .. " " .. cstr .. ".   " .. check .. " " .. host .. " -> " .. URL)
+        timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through.
+        if nil ~= location then
+            pu = url.parse(location, defaultURL)
             if (pu.scheme ~= PU.scheme) then
-                if testing("Protocol") then W("    protocol changed during redirect!  " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l, PU.scheme, "Protocol", host) end
-                if (pu.host == host) and pu.path == PU.path then D("Not testing protocol change " .. URL .. " -> " .. l);  return end
+                if testing("Protocol") then W("    protocol changed during redirect!  " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host) end
+                if (pu.host == host) and pu.path == PU.path then D("Not testing protocol change " .. URL .. " -> " .. location);  return end
             end
 
-            if l == URL then
+            if location == URL then
                 E("    redirect loop!  " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
             elseif nil == pu.host then
-                I("    relative redirect.  " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l)
-                checkHEAD(host, PU.scheme .. "://" .. PU.host .. l, r + 1)
+                I("    relative redirect.  " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location)
+                checkHEAD(host, PU.scheme .. "://" .. PU.host .. location, r + 1)
             elseif (PU.host == pu.host) or (host == pu.host) then
-                checkHEAD(pu.host, l, r + 1)
+                checkHEAD(pu.host, location, r + 1)
             else
                 --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location.
                     Not good for deb.debian.org, which we should only be throwing .debs at.
@@ -448,6 +492,7 @@ local getMirrors = function ()
     I("getting mirrors.")
     local p, c, h = http.request(URL)
     if nil == p then E(c .. " fetching " .. URL) else
+
         for l in p:gmatch("\n*([^\n]+)\n*") do
             local t, d = l:match("(%a*):%s*(.*)")
             d = string.lower(d)