From f0b2ddcf274e5dc125fe4b07a651ee07bba6ad5c Mon Sep 17 00:00:00 2001
From: onefang
Date: Tue, 12 Nov 2019 02:39:02 +1000
Subject: Rewrite the lua-sec checks using command line curl instead.

Works better with SNI.
---
 apt-panopticon.lua | 105 ++++++++++++++++++++++++++++++++++++++---------------
 1 file changed, 75 insertions(+), 30 deletions(-)

(limited to 'apt-panopticon.lua')

diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index 227bec1..c536990 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -107,7 +107,6 @@ local logFile
 local socket = require 'socket'
 local ftp = require 'socket.ftp'
 local http = require 'socket.http'
-local https = require 'ssl.https'	-- See https://github.com/brunoos/luasec/wiki/LuaSec-0.6 for docs.
 local url = require 'socket.url'
 
 
@@ -286,42 +285,87 @@ checkHEAD = function (host, URL, r, retry)
     if not testing(PU.scheme, host) then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL);  return end
     -- TODO - Perhaps we should try it anyway, and mark it as a warning if it DOES work?
     if "https" == PU.scheme and options.roundRobin.value == host then D("Not testing " .. PU.scheme .. " " .. host .. " -> " .. URL .. " mirrors shouldn't have the correct cert.");  return end
-    local hd = {}
-    if pu.host ~= PU.host then hd = {Host = host} end
-    local htp = http;
-    if PU.scheme == "https" then htp = https end
-    -- NOTE - the docs for lua-sec say that redirect isn't supported is version 0.6, no idea if that means it ignores redirections like we want.
-    -- TODO - 										find out!
-    --		The protocol and options are lua-sec specific arguments.
-    local p, c, h, s = htp.request{method = "HEAD", redirect = false, url = URL, headers = hd, protocol = "any", options = "all"}
-    if nil == s then s = "" end
-    if nil == p then
-	E("  " .. c .. " " .. s .. "!   " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
-	-- So far the only errors are "timeout", "Network is unreachable", and "closed", and I suspect "closed" is due to saturating my bandwidth.
-	if "timeout" == c then timeouts = timeouts + 1 end
-	if ("closed" == c) or ("Network is unreachable" == c) or ("timeout" == c) then checkHEAD(host, URL, r, retry + 1, timeouts) end
-    else
-	if ("4" == tostring(c):sub(1, 1)) or ("5" == tostring(c):sub(1, 1)) then
-	    E("  " .. c .. " " .. s .. ".   " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
-	else
-	    I("  " .. c .. " " .. s .. ".   " .. check .. " " .. host .. " -> " .. URL)
-	    timeouts = timeouts - 1	-- Backoff the timeouts count if we managed to get through.
+
+    --[[ Using curl command line -
+	-I - HEAD
+	--connect-to IP - connect to IP, but use SNI from URL.
+	-header "" - add extra headers.
+	-L - DO follow redirects.
+	--max-redirs n - set maximum redirects, default is 50, -1 = unlimited.
+	--retry n - maximum retries, default is 0, no retries.
+	-o file - write to file instead of stdout.
+	--path-as-is - https://curl.haxx.se/libcurl/c/CURLOPT_PATH_AS_IS.html might be useful for URL_sanity.
+	-s silent - don't output progress or error messages.
+	--connect-timeout n - timeout in seconds.
+	    Should return with error code 28 on a timeout?
+	-D file - write the received headers to a file.  This includes the status code and string.
+    ]]
+    local fname = host .. "_" .. PU.host .. "_" .. PU.path:gsub("/", "_") .. ".txt"
+    local hdr = ""
+    local IP = ""
+    if pu.host ~= PU.host then
+	if "http" == PU.scheme then
+	    hdr = '-H "Host: ' .. host .. '"'
 	end
-	l = h.location
-	if nil ~= l then
-	    pu = url.parse(l, defaultURL)
+	IP = '--connect-to ' .. PU.host
+    end
+    local cmd = 'curl -I --retry 0 -s --path-as-is --connect-timeout 30 --max-redirs 0 ' .. IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' ..
+		hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"'
+    local rslt, status = execute(cmd)
+    os.execute("sleep 2")
+    os.execute('cat results/"HEADERS_' .. fname .. '" >>results/"STATUS_' .. fname .. '"; rm results/"HEADERS_' .. fname .. '"')
+    if "exit" == rslt then
+	if 28 == status then
+	    E("  TIMEOUT!", PU.scheme, "", host)
+	    timeouts = timeouts + 1
+	    checkHEAD(host, URL, r, retry + 1, timeouts)
+	    return
+	elseif 0 ~= status then
+	    E("  The curl command return an error code of " .. status .. ", consult the curl manual for what this means.", PU.scheme, "", host)
+	    checkHEAD(host, URL, r, retry + 1, timeouts)
+	    return
+	end
+    elseif "signal" == rslt then
+	    E("  The curl command was interupted by signal " .. status)
+	    return
+    end
+    local rfile, e = io.open("results/STATUS_" .. fname, "r")
+    local code = "000"
+    local cstr = ""
+    local location = nil
+    if nil == rfile then W("opening results/STATUS_" .. fname .. " file - " .. e) else
+	for line in rfile:lines("*l") do
+	    if "#" == line:sub(1, 1) then
+		code = line:sub(2, 4)
+		if ("https" == PU.scheme) and ("0" ~= line:sub(6, 6)) then E("  The certificate is invalid.", PU.scheme, "https", host) end
+	    elseif "http" == line:sub(1, 4):lower() then
+		-- -2 coz the headers file gets a \r at the end.
+		cstr = line:sub(14, -2)
+	    elseif "location" == line:sub(1, 8):lower() then
+		location = line:sub(11, -2)
+	    end
+	end
+    end
+    os.execute('rm results/"STATUS_' .. fname .. '"')
+    if ("4" == tostring(code):sub(1, 1)) or ("5" == tostring(code):sub(1, 1)) then
+	E("  " .. code .. " " .. cstr .. ".   " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
+    else
+	I("  " .. code .. " " .. cstr .. ".   " .. check .. " " .. host .. " -> " .. URL)
+	timeouts = timeouts - 1	-- Backoff the timeouts count if we managed to get through.
+	if nil ~= location then
+	    pu = url.parse(location, defaultURL)
 	    if (pu.scheme ~= PU.scheme) then
-		if testing("Protocol") then W("    protocol changed during redirect!  " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l, PU.scheme, "Protocol", host) end
-		if (pu.host == host) and pu.path == PU.path then D("Not testing protocol change " .. URL .. " -> " .. l);  return end
+		if testing("Protocol") then W("    protocol changed during redirect!  " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location, PU.scheme, "Protocol", host) end
+		if (pu.host == host) and pu.path == PU.path then D("Not testing protocol change " .. URL .. " -> " .. location);  return end
 	    end
 
-	    if l == URL then
+	    if location == URL then
 		E("    redirect loop!  " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
 	    elseif nil == pu.host then
-		I("    relative redirect.  " .. check .. " " .. host .. " -> " .. URL .. " -> " .. l)
-		checkHEAD(host, PU.scheme .. "://" .. PU.host .. l, r + 1)
+		I("    relative redirect.  " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location)
+		checkHEAD(host, PU.scheme .. "://" .. PU.host .. location, r + 1)
 	    elseif (PU.host == pu.host) or (host == pu.host) then
-		checkHEAD(pu.host, l, r + 1)
+		checkHEAD(pu.host, location, r + 1)
 	    else
 		--[[ The hard part here is that we end up throwing ALL of the test files at the redirected location.
 		    Not good for deb.debian.org, which we should only be throwing .debs at.
@@ -448,6 +492,7 @@ local getMirrors = function ()
     I("getting mirrors.")
     local p, c, h = http.request(URL)
     if nil == p then E(c .. " fetching " .. URL) else
+
 	for l in p:gmatch("\n*([^\n]+)\n*") do
 	    local t, d = l:match("(%a*):%s*(.*)")
 	    d = string.lower(d)
-- 
cgit v1.1