Implement URLSanity test.

2 files changed, 32 insertions, 10 deletions

diff --git a/apt-panopticon-report-email-web.lua b/apt-panopticon-report-email-web.lua
index 60027d0..fc1f345 100755
--- a/apt-panopticon-report-email-web.lua
+++ b/apt-panopticon-report-email-web.lua
@@ -326,7 +326,7 @@ if nil == email then C("opening mirrors file - " .. e) else
         local rsync     = "[skip]"
         local dns       = ""
         local protocol  = status(k, results, "Protocol")
-        local sanity    = "[skip]"
+        local sanity    = status(k, results, "URLSanity")
         local integrity = status(k, results, "Integrity")
         local updated   = status(k, results, "Updated")
 
@@ -434,7 +434,7 @@ if nil == web then C("opening mirrors file - " .. e) else
         local rsync     = "[<font color='grey'><b>skip</b></font>]"
         local dns       = ""
         local protocol  = status(k, results, "Protocol")
-        local sanity    = "[<font color='grey'><b>skip</b></font>]"
+        local sanity    = status(k, results, "URLSanity")
         local integrity = status(k, results, "Integrity")
         local updated   = status(k, results, "Updated")
         local rate = v.Rate
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index ade9de3..21e7599 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -43,7 +43,7 @@ options =
 --              "rsync",
                 "DNSRR",
                 "Protocol",
---              "URLSanity",
+                "URLSanity",
                 "Integrity",
                 "Updated",
             },
@@ -404,9 +404,10 @@ end
 
 local timeouts = 0;
 local totalTimeouts = 0
-checkHEAD = function (host, URL, r, retry)
+checkHEAD = function (host, URL, r, retry, sanity)
     if nil == r then r = 0 end
     if nil == retry then retry = 0 end
+    if nil == sanity then sanity = false end
     local check = "Checking file"
     local PU = url.parse(URL, defaultURL)
     local pu = url.parse(PU.scheme .. "://" .. host, defaultURL)
@@ -469,12 +470,20 @@ checkHEAD = function (host, URL, r, retry)
     if "0" ~= status then
         local msg = curlStatus[0 + status]
         if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end
-        E("  The curl command return an error code of " .. status .. " - " .. msg, PU.scheme, "", host)
+        if sanity then
+            E("  The curl command return an error code of " .. status .. " - " .. msg, PU.scheme, "URLSanity", host)
+        else
+            E("  The curl command return an error code of " .. status .. " - " .. msg, PU.scheme, "", host)
+        end
         if ("28" == status) or ("7" == status) then
-            E("  TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "", host)
+            if sanity then
+                E("  TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "URLSanity", host)
+            else
+                E("  TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "", host)
+            end
             timeouts = timeouts + 1
         end
-        checkHEAD(host, URL, r, retry + 1, timeouts)
+        checkHEAD(host, URL, r, retry + 1, sanity)
         return
     end
     local rfile, e = io.open("results/STATUS_" .. fname, "r")
@@ -496,7 +505,11 @@ checkHEAD = function (host, URL, r, retry)
     end
     os.execute('rm results/"STATUS_' .. fname .. '" 2>/dev/null')
     if ("4" == tostring(code):sub(1, 1)) or ("5" == tostring(code):sub(1, 1)) then
-        E("  " .. code .. " " .. cstr .. ".   " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
+        if sanity then
+            E("  " .. code .. " " .. cstr .. ".   " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "URLSanity", host)
+        else
+            E("  " .. code .. " " .. cstr .. ".   " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
+        end
     else
         I("  " .. code .. " " .. cstr .. ".   " .. check .. " " .. host .. " -> " .. URL)
 --      timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through.
@@ -511,9 +524,9 @@ checkHEAD = function (host, URL, r, retry)
                 E("    redirect loop!  " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
             elseif nil == pu.host then
                 I("    relative redirect.  " .. check .. " " .. host .. " -> " .. URL .. " -> " .. location)
-                checkHEAD(host, PU.scheme .. "://" .. PU.host .. location, r + 1)
+                checkHEAD(host, PU.scheme .. "://" .. PU.host .. location, r + 1, retry, sanity)
             elseif (PU.host == pu.host) or (host == pu.host) then
-                checkHEAD(pu.host, location, r + 1)
+                checkHEAD(pu.host, location, r + 1, retry, sanity)
             else
                 --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location.
                     Not good for deb.debian.org, which we should only be throwing .debs at.
@@ -544,6 +557,15 @@ local checkTimeouts = function(host, scheme, URL)
             return true
         end
     end
+    if testing("URLSanity") then
+        URL = URL:gsub("merged/", "merged///")
+        totalTimeouts = totalTimeouts + timeouts; timeouts = 0
+        checkHEAD(host, scheme .. "://" .. URL, 0, 0, true)
+        if 4 <= (totalTimeouts) then
+            E("Way too many timeouts!", scheme, "URLSanity", host)
+            return true
+        end
+    end
     return false
 end