From 0560dc5975ff29be6d63885925aa1942135f0043 Mon Sep 17 00:00:00 2001
From: onefang
Date: Mon, 2 Dec 2019 05:50:58 +1000
Subject: Add more report stuff.
TIMEOUT, report inactive mirrors and why it's not active, add a * for
things tested but not supported.
---
apt-panopticon-report-email.lua | 8 ++-
apt-panopticon-report-web.lua | 36 +++++++---
apt-panopticon.lua | 155 ++++++++++++++++++++++------------------
3 files changed, 117 insertions(+), 82 deletions(-)
diff --git a/apt-panopticon-report-email.lua b/apt-panopticon-report-email.lua
index c102b81..53db16f 100755
--- a/apt-panopticon-report-email.lua
+++ b/apt-panopticon-report-email.lua
@@ -234,8 +234,10 @@ if nil == file then C("opening mirrors file - " .. e) else
results = collate(k, i, results)
end
end
+ local ftp = "[skip]"
local http = status(k, results, "http")
local https = status(k, results, "https")
+ local rsync = "[skip]"
local dns = ""
local protocol = status(k, results, "Protocol")
local sanity = "[skip]"
@@ -275,8 +277,10 @@ if nil == file then C("opening mirrors file - " .. e) else
dns = " DNS-RR: " .. dns
end
- file:write( " http: " .. http .. " https: " .. https .. dns .. "\n" ..
- " Protocol: " .. protocol .. " URL-sanity: " .. sanity .. " Integrity: " .. integrity .. " Updated: " .. updated .. "\n")
+ file:write( " ftp: " .. ftp .. " http: " .. http .. " https: " .. https .." rsync: " .. rsync .. "\n" ..
+ " " .. dns .. "\n" ..
+ " Protocol: " .. protocol .. " URL-sanity: " .. sanity .. " Integrity: " .. integrity .. "\n" ..
+ " Updated: " .. updated .. "\n")
end
file:write( "\n==== faulty mirrors: ====\n" .. faulty)
file:write( "\n---- END MIRROR-STATUS ----\n\n" ..
diff --git a/apt-panopticon-report-web.lua b/apt-panopticon-report-web.lua
index 61102c8..fb41341 100755
--- a/apt-panopticon-report-web.lua
+++ b/apt-panopticon-report-web.lua
@@ -155,24 +155,27 @@ local status = function(host, results, typ)
local e = 0
local w = 0
local s = nil ~= mirrors[host].Protocols[typ]
+ local to = false
if ('http' ~= typ) and ('https' ~= typ) and ('ftp' ~= typ) and ('rsync' ~= typ) then s = true end
if nil ~= results[typ] then
e = results[typ].errors
w = results[typ].warnings
+--[[
for k, v in pairs(results[typ]) do
if "table" == type(v) then
- e = e + v.errors
- w = w + v.warnings
+ if 0 <= v.errors then e = e + v.errors else to = true end
+ if 0 <= v.warnings then w = w + v.warnings else to = true end
end
end
+]]
else
for k, v in pairs(results) do
if "table" == type(v) then
for i, u in pairs(v) do
if "table" == type(u) then
if typ == i then
- e = e + u.errors
- w = w + u.warnings
+ if 0 <= u.errors then e = e + u.errors else to = true end
+ if 0 <= u.warnings then w = w + u.warnings else to = true end
end
end
end
@@ -180,18 +183,24 @@ local status = function(host, results, typ)
end
end
- if 0 < e then
+ if to then
+ if s then
+ result = "[TIMEOUT]"
+ else
+ result = "[TIMEOUT *]"
+ end
+ elseif 0 < e then
if s then
result = "[FAILED]"
else
- result = "[FAILED]"
+ result = "[FAILED *]"
end
faulty = faulty .. host .. " (" .. typ .. ")
\n"
else
if s then
result = "[OK]"
else
- result = "[OK]"
+ result = "[OK *]"
end
end
return result .. plurals(e, w)
@@ -258,7 +267,7 @@ end
local file, e = io.open("results/Report-web.html", "w+")
if nil == file then C("opening mirrors file - " .. e) else
file:write( "apt-panopticon results\n" ..
- '' ..
+ '' ..
"Welcome to the apt-panopticon results page.
\n" ..
"This is the status of the mirror servers in the Devuan package mirror network.
\n" ..
"The full list of Devuan package mirrors is available at the URL: " ..
@@ -269,21 +278,25 @@ if nil == file then C("opening mirrors file - " .. e) else
"
[FAILED] or [OK]" ..
" means the tested thing is supported for that mirror.
\n" ..
- "[FAILED] or [OK]" ..
+ "
[FAILED *] or [OK *]" ..
" means the tested thing is unsupported for that mirror, but might have been tested anyway.
\n" ..
+ "[TIMEOUT] or [TIMEOUT]" ..
+ " means the server had too many timeouts, and tests where aborted, so there is no result for this test.
" ..
"The DNS round robin (DNS-RR) column shows the IPs for that mirror, or [no] if it isn't part of the DNS-RR. " ..
"The IPs link to the testing log for that IP accessed via the DNS-RR. " ..
"deb.devuan.org is the DNS-RR itself, so it doesn't get tested directly.
\n" ..
"The time in the Updated column is how often the mirror updates itself.
" ..
- "Mirrors with a grey background are not active.
\n" ..
+ "Mirrors with a grey background are not active (though may be usable as part of the DNS-RR).
\n" ..
"[skip] means that the test hasn't been written yet.
\n" ..
" | FTP | HTTP | HTTPS | RSYNC | DNS round robin | Protocol | URL sanity | Integrity | Updated |
|---|
\n"
)
for k, v in orderedPairs(mirrors) do
local results = loadfile("results/" .. k .. ".lua")()
+ local active = ""
if "yes" == v.Active then
file:write(" | " .. k .. " | ")
else
+ if nil == v.Active then active = 'nil' else active = v.Active end
file:write("
|---|
| " .. k .. " | ")
end
local IPs = v.IPs
@@ -343,6 +356,9 @@ if nil == file then C("opening mirrors file - " .. e) else
file:write("" .. ftp .. " | " .. http .. " | " .. https .. " | " .. rsync .. " | " .. dns ..
" | " .. protocol .. " | " .. sanity ..
" | " .. integrity .. " | " .. updated .. " |
|---|
\n")
+ if "" ~= v.Active then
+ file:write("| " .. active .. " |
\n")
+ end
end
file:write( "\n
\n==== faulty mirrors: ====
\n" .. faulty)
file:write( "
\n
\n==== DNS and logs: ====
\n")
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index d66e32d..5d9ed82 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -756,7 +756,7 @@ local getMirrors = function ()
local host = ""
local m = {}
local active = true
- local URL = "https://" .. options.referenceSite.value .. "/mirror_list.txt"
+ local URL = "http://" .. options.referenceSite.value .. "/mirror_list.txt"
I("getting mirrors.")
local p, c, h = http.request(URL)
if nil == p then E(c .. " fetching " .. URL) else
@@ -766,7 +766,8 @@ local getMirrors = function ()
d = string.lower(d)
if "FQDN" == t then
if "" ~= host then
- if active then mirrors[host] = m end
+-- if active then mirrors[host] = m end
+ mirrors[host] = m
m = {}
active = true
end
@@ -780,15 +781,16 @@ local getMirrors = function ()
prot[w] = true;
end
m[t] = prot
- elseif "Active" == t and nil == d:find("yes", 1, true) then
+ elseif "Active" == t and nil == d:sub(1, 3):find("yes", 1, true) then
W("Mirror " .. host .. " is not active - " .. d, "", "", host)
active = false
+ m[t] = d
-- TODO - Should do some input validation on BaseURL, and everything else.
else
m[t] = d
end
end
- if "" ~= host and active then
+ if "" ~= host --[[and active]] then
mirrors[host] = m
end
end
@@ -933,88 +935,90 @@ if 0 < #arg then
end
if testing("Integrity") or testing("Updated") then
- if origin and (options.roundRobin.value ~= pu.host) then
- while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
- os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock")
- for i, n in pairs(releases) do
- for l, o in pairs(releaseFiles) do
- if repoExists(i .. o) then
- postDownload(pu.host, n, o)
+ if 4 > (totalTimeouts) then
+ if origin and (options.roundRobin.value ~= pu.host) then
+ while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
+ os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock")
+ for i, n in pairs(releases) do
+ for l, o in pairs(releaseFiles) do
+ if repoExists(i .. o) then
+ postDownload(pu.host, n, o)
+ end
end
- end
- os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED')
- if options.referenceSite.value == pu.host then
- execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
- 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
- '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt')
- else
+ os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED')
+ if options.referenceSite.value == pu.host then
+ execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
+ 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
+ '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt')
+ else
-- TODO - compare to the pkgmaster copy.
- end
+ end
- local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r")
- if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else
- local diff = dfile:read("*a")
- if "" ~= diff then
- downloads(pu.host, pu.path, n, diff)
+ local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r")
+ if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else
+ local diff = dfile:read("*a")
+ if "" ~= diff then
+ downloads(pu.host, pu.path, n, diff)
+ end
end
- end
- end
+ end
- downloads(pu.host, pu.path, "", "")
- while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
- os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock")
+ downloads(pu.host, pu.path, "", "")
+ while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
+ os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock")
- for i, n in pairs(releases) do
- local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r")
- if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else
- local diff = dfile:read("*a")
- for l in diff:gmatch("\n*([^\n]+)\n*") do
- postDownload(pu.host, n, "/" .. l)
+ for i, n in pairs(releases) do
+ local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r")
+ if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else
+ local diff = dfile:read("*a")
+ for l in diff:gmatch("\n*([^\n]+)\n*") do
+ postDownload(pu.host, n, "/" .. l)
+ end
+ end
+ if options.referenceSite.value == pu.host then
+ -- In case it wasn't dealt with already.
+ os.execute('touch results/NEW_Packages_' .. n .. '.test.txt')
end
end
- if options.referenceSite.value == pu.host then
- -- In case it wasn't dealt with already.
- os.execute('touch results/NEW_Packages_' .. n .. '.test.txt')
- end
- end
- for i, n in pairs(releases) do
- local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
- if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
- for l in nfile:lines() do
- local p = l:match('(pool/.*%.deb)')
- if nil ~= p then
- downloads(pu.host, pu.path, nil, p)
+ for i, n in pairs(releases) do
+ local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
+ if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
+ for l in nfile:lines() do
+ local p = l:match('(pool/.*%.deb)')
+ if nil ~= p then
+ downloads(pu.host, pu.path, nil, p)
+ end
end
end
end
- end
- downloads(pu.host, pu.path, nil, "")
- while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
- for i, n in pairs(releases) do
- local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
- if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
- for l in nfile:lines() do
- local v, p, sz, m, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) | (%x.+) |')
- if nil ~= p then
- local status, fsz = execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5')
- if testing("Integrity") then
- if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end.
- E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host)
- print('|' .. sz .. '~=' .. fsz:sub(2, -2) .. '|')
- else
- local status, fm = execute('md5sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1')
- if m ~= fm:sub(2, -2) then E('Package MD5 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end
- local status, fsha = execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1')
- if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end
+ downloads(pu.host, pu.path, nil, "")
+ while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
+ for i, n in pairs(releases) do
+ local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
+ if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
+ for l in nfile:lines() do
+ local v, p, sz, m, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) | (%x.+) |')
+ if nil ~= p then
+ local status, fsz = execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5')
+ if testing("Integrity") then
+ if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end.
+ E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host)
+ print('|' .. sz .. '~=' .. fsz:sub(2, -2) .. '|')
+ else
+ local status, fm = execute('md5sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1')
+ if m ~= fm:sub(2, -2) then E('Package MD5 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end
+ local status, fsha = execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1')
+ if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end
+ end
end
- end
- if testing("Updated") then
- if sz ~= fsz:sub(2, -2) then
- E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host)
+ if testing("Updated") then
+ if sz ~= fsz:sub(2, -2) then
+ E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host)
+ end
end
end
end
@@ -1022,6 +1026,17 @@ if 0 < #arg then
end
end
+ else
+ for k, v in pairs{"ftp", "http", "https", "rsync"} do
+ if testing(v) then
+ local tests = results[v]
+ if testing("Integrity") then tests.Integrity = {errors = -1; warnings = -1} end
+ if testing("Protocol") then tests.Protocol = {errors = -1; warnings = -1} end
+ if testing("Updated") then tests.Updated = {errors = -1; warnings = -1} end
+ if testing("URLSanity") then tests.URLSanity = {errors = -1; warnings = -1} end
+ results[v] = tests
+ end
+ end
end
end
--
cgit v1.1