From 36a706c4ae8e46f88c1f7193b2686df8509a080a Mon Sep 17 00:00:00 2001
From: onefang
Date: Mon, 2 Dec 2019 22:22:16 +1000
Subject: Implement URLSanity test.

---
 apt-panopticon.lua | 38 ++++++++++++++++++++++++++++++--------
 1 file changed, 30 insertions(+), 8 deletions(-)

(limited to 'apt-panopticon.lua')

diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index ade9de3..21e7599 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -43,7 +43,7 @@ options =
 --	        "rsync",
 	        "DNSRR",
 	        "Protocol",
---	        "URLSanity",
+	        "URLSanity",
 	        "Integrity",
 	        "Updated",
 	    },
@@ -404,9 +404,10 @@ end
 
 local timeouts = 0;
 local totalTimeouts = 0
-checkHEAD = function (host, URL, r, retry)
+checkHEAD = function (host, URL, r, retry, sanity)
     if nil == r then r = 0 end
     if nil == retry then retry = 0 end
+    if nil == sanity then sanity = false end
     local check = "Checking file"
     local PU = url.parse(URL, defaultURL)
     local pu = url.parse(PU.scheme .. "://" .. host, defaultURL)
@@ -469,12 +470,20 @@ checkHEAD = function (host, URL, r, retry)
     if "0" ~= status then
 	local msg = curlStatus[0 + status]
 	if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end
-	E("  The curl command return an error code of " .. status .. " - " .. msg, PU.scheme, "", host)
+	if sanity then
+	    E("  The curl command return an error code of " .. status .. " - " .. msg, PU.scheme, "URLSanity", host)
+	else
+	    E("  The curl command return an error code of " .. status .. " - " .. msg, PU.scheme, "", host)
+	end
 	if ("28" == status) or ("7" == status) then
-	    E("  TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "", host)
+	    if sanity then
+		E("  TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "URLSanity", host)
+	    else
+		E("  TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "", host)
+	    end
 	    timeouts = timeouts + 1
 	end
-	checkHEAD(host, URL, r, retry + 1, timeouts)
+	checkHEAD(host, URL, r, retry + 1, sanity)
 	return
     end
     local rfile, e = io.open("results/STATUS_" .. fname, "r")
@@ -496,7 +505,11 @@ checkHEAD = function (host, URL, r, retry)
     end
     os.execute('rm results/"STATUS_' .. fname .. '" 2>/dev/null')
     if ("4" == tostring(code):sub(1, 1)) or ("5" == tostring(code):sub(1, 1)) then
-	E("  " .. code .. " " .. cstr .. ".   " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
+	if sanity then
+	    E("  " .. code .. " " .. cstr .. ".   " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "URLSanity", host)
+	else
+	    E("  " .. code .. " " .. cstr .. ".   " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
+	end
     else
 	I("  " .. code .. " " .. cstr .. ".   " .. check .. " " .. host .. " -> " .. URL)
 --	timeouts = timeouts - 1	-- Backoff the timeouts count if we managed to get through.
@@ -511,9 +524,9 @@ checkHEAD = function (host, URL, r, retry)
 		E("    redirect loop!  " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
 	    elseif nil == pu.host then
 		I("    relative redirect.  " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location)
-		checkHEAD(host, PU.scheme .. "://" .. PU.host .. location, r + 1)
+		checkHEAD(host, PU.scheme .. "://" .. PU.host .. location, r + 1, retry, sanity)
 	    elseif (PU.host == pu.host) or (host == pu.host) then
-		checkHEAD(pu.host, location, r + 1)
+		checkHEAD(pu.host, location, r + 1, retry, sanity)
 	    else
 		--[[ The hard part here is that we end up throwing ALL of the test files at the redirected location.
 		    Not good for deb.debian.org, which we should only be throwing .debs at.
@@ -544,6 +557,15 @@ local checkTimeouts = function(host, scheme, URL)
 	    return true
 	end
     end
+    if testing("URLSanity") then
+	URL = URL:gsub("merged/", "merged///")
+	totalTimeouts = totalTimeouts + timeouts; timeouts = 0
+	checkHEAD(host, scheme .. "://" .. URL, 0, 0, true)
+	if 4 <= (totalTimeouts) then
+	    E("Way too many timeouts!", scheme, "URLSanity", host)
+	    return true
+	end
+    end
     return false
 end
 
-- 
cgit v1.1