From 36a706c4ae8e46f88c1f7193b2686df8509a080a Mon Sep 17 00:00:00 2001
From: onefang
Date: Mon, 2 Dec 2019 22:22:16 +1000
Subject: Implement URLSanity test.
---
apt-panopticon-report-email-web.lua | 4 ++--
apt-panopticon.lua | 38 +++++++++++++++++++++++++++++--------
2 files changed, 32 insertions(+), 10 deletions(-)
diff --git a/apt-panopticon-report-email-web.lua b/apt-panopticon-report-email-web.lua
index 60027d0..fc1f345 100755
--- a/apt-panopticon-report-email-web.lua
+++ b/apt-panopticon-report-email-web.lua
@@ -326,7 +326,7 @@ if nil == email then C("opening mirrors file - " .. e) else
local rsync = "[skip]"
local dns = ""
local protocol = status(k, results, "Protocol")
- local sanity = "[skip]"
+ local sanity = status(k, results, "URLSanity")
local integrity = status(k, results, "Integrity")
local updated = status(k, results, "Updated")
@@ -434,7 +434,7 @@ if nil == web then C("opening mirrors file - " .. e) else
local rsync = "[skip]"
local dns = ""
local protocol = status(k, results, "Protocol")
- local sanity = "[skip]"
+ local sanity = status(k, results, "URLSanity")
local integrity = status(k, results, "Integrity")
local updated = status(k, results, "Updated")
local rate = v.Rate
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index ade9de3..21e7599 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -43,7 +43,7 @@ options =
-- "rsync",
"DNSRR",
"Protocol",
--- "URLSanity",
+ "URLSanity",
"Integrity",
"Updated",
},
@@ -404,9 +404,10 @@ end
local timeouts = 0;
local totalTimeouts = 0
-checkHEAD = function (host, URL, r, retry)
+checkHEAD = function (host, URL, r, retry, sanity)
if nil == r then r = 0 end
if nil == retry then retry = 0 end
+ if nil == sanity then sanity = false end
local check = "Checking file"
local PU = url.parse(URL, defaultURL)
local pu = url.parse(PU.scheme .. "://" .. host, defaultURL)
@@ -469,12 +470,20 @@ checkHEAD = function (host, URL, r, retry)
if "0" ~= status then
local msg = curlStatus[0 + status]
if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end
- E(" The curl command return an error code of " .. status .. " - " .. msg, PU.scheme, "", host)
+ if sanity then
+ E(" The curl command return an error code of " .. status .. " - " .. msg, PU.scheme, "URLSanity", host)
+ else
+ E(" The curl command return an error code of " .. status .. " - " .. msg, PU.scheme, "", host)
+ end
if ("28" == status) or ("7" == status) then
- E(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "", host)
+ if sanity then
+ E(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "URLSanity", host)
+ else
+ E(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "", host)
+ end
timeouts = timeouts + 1
end
- checkHEAD(host, URL, r, retry + 1, timeouts)
+ checkHEAD(host, URL, r, retry + 1, sanity)
return
end
local rfile, e = io.open("results/STATUS_" .. fname, "r")
@@ -496,7 +505,11 @@ checkHEAD = function (host, URL, r, retry)
end
os.execute('rm results/"STATUS_' .. fname .. '" 2>/dev/null')
if ("4" == tostring(code):sub(1, 1)) or ("5" == tostring(code):sub(1, 1)) then
- E(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
+ if sanity then
+ E(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "URLSanity", host)
+ else
+ E(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
+ end
else
I(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL)
-- timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through.
@@ -511,9 +524,9 @@ checkHEAD = function (host, URL, r, retry)
E(" redirect loop! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
elseif nil == pu.host then
I(" relative redirect. " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location)
- checkHEAD(host, PU.scheme .. "://" .. PU.host .. location, r + 1)
+ checkHEAD(host, PU.scheme .. "://" .. PU.host .. location, r + 1, retry, sanity)
elseif (PU.host == pu.host) or (host == pu.host) then
- checkHEAD(pu.host, location, r + 1)
+ checkHEAD(pu.host, location, r + 1, retry, sanity)
else
--[[ The hard part here is that we end up throwing ALL of the test files at the redirected location.
Not good for deb.debian.org, which we should only be throwing .debs at.
@@ -544,6 +557,15 @@ local checkTimeouts = function(host, scheme, URL)
return true
end
end
+ if testing("URLSanity") then
+ URL = URL:gsub("merged/", "merged///")
+ totalTimeouts = totalTimeouts + timeouts; timeouts = 0
+ checkHEAD(host, scheme .. "://" .. URL, 0, 0, true)
+ if 4 <= (totalTimeouts) then
+ E("Way too many timeouts!", scheme, "URLSanity", host)
+ return true
+ end
+ end
return false
end
--
cgit v1.1