From 0f6c928194683f3bb85cedb1e92ed3f0cd67c84a Mon Sep 17 00:00:00 2001
From: onefang
Date: Sun, 1 Dec 2019 01:41:16 +1000
Subject: Much work on the Updated tests.
Plus keep a history of results.
---
apt-panopticon-report-email.lua | 2 +-
apt-panopticon-report-web.lua | 2 +-
apt-panopticon.lua | 243 +++++++++++++++++++++++++++++++++-------
3 files changed, 206 insertions(+), 41 deletions(-)
diff --git a/apt-panopticon-report-email.lua b/apt-panopticon-report-email.lua
index b9c1b30..eb2a0a7 100755
--- a/apt-panopticon-report-email.lua
+++ b/apt-panopticon-report-email.lua
@@ -240,7 +240,7 @@ if nil == file then C("opening mirrors file - " .. e) else
local protocol = status(k, results, "Protocol")
local sanity = "[skip]"
local integrity = status(k, results, "Integrity")
- local updated = "[skip]"
+ local updated = status(k, results, "Updated")
-- DNS-RR test.
if ("deb.devuan.org" ~= k) and (nil ~= mirrors["deb.devuan.org"]) then
diff --git a/apt-panopticon-report-web.lua b/apt-panopticon-report-web.lua
index 9a90860..753a23c 100755
--- a/apt-panopticon-report-web.lua
+++ b/apt-panopticon-report-web.lua
@@ -278,7 +278,7 @@ if nil == file then C("opening mirrors file - " .. e) else
local protocol = status(k, results, "Protocol")
local sanity = "[skip]"
local integrity = status(k, results, "Integrity")
- local updated = "[skip]"
+ local updated = status(k, results, "Updated")
-- DNS-RR test.
if ("deb.devuan.org" ~= k) and (nil ~= mirrors["deb.devuan.org"]) then
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index d40a74c..d66e32d 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -45,7 +45,7 @@ options =
"Protocol",
-- "URLSanity",
"Integrity",
--- "Updated",
+ "Updated",
},
},
timeout =
@@ -100,7 +100,7 @@ local referenceDebs =
local referenceDevs =
{
-- Devuan package. NOTE this one should not get redirected, but that's more a warning than an error.
- "merged/pool/DEVUAN/main/d/desktop-base/desktop-base_2.0.3_all.deb",
+ "merged/pool/DEVUAN/main/d/desktop-base/desktop-base_3.0_all.deb",
"merged/pool/DEVUAN/main/u/util-linux/util-linux_2.32.1-0.1+devuan2.1_amd64.deb",
}
local arg = {}
@@ -354,6 +354,11 @@ local checkExes = function (exe)
return tonumber(count)
end
+local checkFile = function(f)
+ local h, e = io.open(f, "r")
+ if nil == h then return false else h:close(); return true end
+end
+
local repoExists = function (r)
r = r:match("([%a-]*)")
if nil == r then return false end
@@ -610,34 +615,106 @@ end
local addDownload = function(host, URL, f, r, k)
+ local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename.
+ local o, e = io.open("results/" .. host .. "/merged/dists/" .. r .. k, "r")
+ if nil ~= o then
+ o:close()
+ -- Curls "check timestamp and overwrite file" stuff sucks.
+ -- -R means the destination file gets the timestamp of the remote file.
+ -- Can only do ONE timestamp check per command.
+ -- This doesn't work either. All downloads get all these headers. Pffft
+-- local status, ts = execute('TZ="GMT" ls -l --time-style="+%a, %d %b %Y %T %Z" results/' .. host .. "/merged/dists/" .. r .. k .. ' | cut -d " " -f 6-11')
+-- f:write('header "If-Modified-Since: ' .. ts:sub(2, -2) .. '"\n')
+ -- Curl will DELETE the existing file if the timestamp fails to download a new one, unless we change directory first,
+ -- which wont work with multiple files in multiple directories. WTF?
+ os.execute(" mv results/" .. host .. "/merged/dists/" .. r .. k ..
+ " results/" .. host .. "/merged/dists/" .. r .. k .. ".old")
+ end
+
+ D('Downloading http://' .. host .. URL .. '/merged/dists/' .. r .. k)
f:write('url "' .. 'http://' .. host .. URL .. '/merged/dists/' .. r .. k .. '"\n')
f:write('output "results/' .. host .. '/merged/dists/' .. r .. k .. '"\n')
- -- Curls "check timestamp and overwrite file" stuff sucks.
- -- Can only do ONE timestamp check per command.
- -- Will DELETE the existing file if the timestamp fails to download a new one, unless we change directory first, which wont work here.
- os.execute("if [ -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then mv" ..
- " results/" .. host .. "/merged/dists/" .. r .. k ..
- " results/" .. host .. "/merged/dists/" .. r .. k .. ".old; fi")
end
local postDownload = function(host, r, k)
+ local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename.
+ local dir = k:sub(1, 0 - (#file + 1))
os.execute("if [ -f results/" .. host .. "/merged/dists/" .. r .. k .. ".old ]" ..
- " && [ ! -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then cp" ..
+ " && [ ! -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then cp -a" ..
" results/" .. host .. "/merged/dists/" .. r .. k .. ".old" ..
" results/" .. host .. "/merged/dists/" .. r .. k .. "; fi")
+ if ".gz" == k:sub(-3, -1) then execute("gzip -dfk results/" .. host .. "/merged/dists/" .. r .. k) end
+ if ".xz" == k:sub(-3, -1) then execute("xz -dfk results/" .. host .. "/merged/dists/" .. r .. k .. " 2>/dev/null") end
if testing("Integrity") then
if ".gpg" == k:sub(-4, -1) then
local status, out = execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. r .. k ..
" results/" .. host .. "/merged/dists/" .. r .. k:sub(1, -5) .. " 2>/dev/null")
if "0" ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. r .. k, "http", "Integrity", host) end
end
+-- TODO - should check the PGP sig of InRelease as well.
+ end
+ if testing("Integrity") or testing("Updated") then
+ if options.referenceSite.value == host then
+ if "Packages." == file:sub(1, 9) then
+-- TODO - compare the SHA256 sums in pkgmaster's Release for both the packed and unpacked versions.
+ local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed', "w+")
+ if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed' .. ' file - ' .. e) else
+ local pp = {}
+ if checkFile('results_old/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages') then
+ for l in io.lines('results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages') do
+ if "Package: " == l:sub(1, 9) then
+ if 0 ~= #pp then
+ Pp:write(r .. " | ")
+ for i = 1, 6 do
+ if nil == pp[i] then print(host .. " " .. r .. " " .. dir .. " " .. i) else Pp:write(pp[i] .. " | ") end
+ end
+ Pp:write("\n")
+ end
+ pp = {}
+ pp[1] = l:sub(10, -1)
+ elseif "Version: " == l:sub(1, 9) then
+ pp[2] = l:sub(10, -1)
+ elseif "Filename: " == l:sub(1, 10) then
+ pp[3] = l:sub(11, -1)
+ elseif "Size: " == l:sub(1, 6) then
+ pp[4] = l:sub(7, -1)
+ elseif "MD5sum: " == l:sub(1, 8) then
+ pp[5] = l:sub(9, -1)
+ elseif "SHA256: " == l:sub(1, 8) then
+ pp[6] = l:sub(9, -1)
+ end
+ end
+ else
+ W("Can't find file results_old/pkgmaster.devuan.org/merged/dists/" .. r .. dir .. "Packages")
+ end
+ Pp:close()
+ os.execute('sort results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed >results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed-sorted')
+ os.execute('rm results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed')
+ if checkFile('results_old/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages.parsed-sorted') then
+ execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages.parsed-sorted ' ..
+ 'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages.parsed-sorted ' ..
+ ' | grep -E "^-" | grep -Ev "^\\+\\+\\+|^---" >>results/OLD_PACKAGES_' .. r .. '.txt')
+ execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages.parsed-sorted ' ..
+ 'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages.parsed-sorted ' ..
+ ' | grep -E "^\\+" | grep -Ev "^\\+\\+\\+|^---" >>results/NEW_Packages_' .. r .. '.txt')
+ -- Find the smallest new package for each release.
+ os.execute('sort -b -k 9,9 -n results/NEW_Packages_' .. r .. '.txt >results/NEW_Packages_' .. r .. '.sorted.txt')
+ os.execute('grep -s " | pool/DEBIAN/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >results/NEW_Packages_' .. r .. '.test.txt')
+ os.execute('grep -s " | pool/DEBIAN-SECURITY/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >>results/NEW_Packages_' .. r .. '.test.txt')
+ os.execute('grep -s " | pool/DEVUAN/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >>results/NEW_Packages_' .. r .. '.test.txt')
+ else
+ W("Can't find file results_old/pkgmaster.devuan.org/merged/dists/" .. r .. dir .. "Packages.parsed-sorted")
+ end
+ end
+ end
+ end
end
end
local downloadLock = "flock -n results/curl-"
local download = "curl --connect-timeout " .. options.timeout.value .. " --create-dirs -L -z 'results/stamp.old' -v -R "
local downloads = function(host, URL, release, list)
- if nil == URL then URL = "/" end
+ if nil == URL then URL = "" end
local lock = "%s-" .. host .. ".lock"
local log = " --stderr results/curl-%s_" .. host .. ".log"
local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock:format("META") .. " " .. download .. log:format("META") .. " -K results/" .. host .. ".curl"
@@ -648,24 +725,19 @@ local downloads = function(host, URL, release, list)
if nil ~= list then
if "" ~= list then
- for l in list:gmatch("\n*([^\n]+)\n*") do
- addDownload(host, URL, f, release, "/" .. l)
+ if nil ~= release then
+ for l in list:gmatch("\n*([^\n]+)\n*") do
+ addDownload(host, URL, f, release, "/" .. l)
+ end
+ else
+ D('Downlaading http://' .. host .. URL .. '/merged/' .. list)
+ f:write('url "' .. 'http://' .. host .. URL .. '/merged/' .. list .. '"\n')
+ f:write('output "results/' .. host .. '/merged/' .. list .. '"\n')
end
f:close()
return
end
else
-
---[[
- for i, s in pairs(referenceDevs) do
- cm = cm .. " https://" .. host .. URL .. "/" .. s
- end
- for i, s in pairs(referenceDebs) do
- cm = cm .. " https://" .. host .. URL .. "/" .. s
- end
- execute(cm)
-]]
-
for i, s in pairs(releases) do
for j, k in pairs(releaseFiles) do
if repoExists(s .. k) then
@@ -678,6 +750,7 @@ local downloads = function(host, URL, release, list)
fork(cm)
end
+
local getMirrors = function ()
local mirrors = {}
local host = ""
@@ -818,6 +891,13 @@ if 0 < #arg then
arg[1] = arg[1]:sub(1, -2)
end
local pu = url.parse("http://" .. arg[1])
+
+ if testing("Integrity") or testing("Updated") then
+ if origin and options.referenceSite.value == pu.host then
+-- if not keep then execute("rm -fr results/" .. pu.host) end
+ end
+ end
+
if nil ~= arg[2] then
logFile, e = io.open("results/LOG_" .. pu.host .. "_" .. arg[2] .. ".html", "a+")
else
@@ -840,25 +920,22 @@ if 0 < #arg then
results[v] = tests
end
end
- if testing("Integrity") or testing("Updated") then
- if nil == arg[2] then
- I("Starting file downloads for " .. pu.host)
--- if not keep then execute("rm -fr results/" .. pu.host) end
- downloads(pu.host, pu.path)
--- checkExes("apt-panopticon.lua " .. sendArgs)
--- checkExes(downloadLock)
- end
- end
if origin then
+ if testing("Integrity") or testing("Updated") then
+ if origin and (options.roundRobin.value ~= pu.host) then
+ I("Starting file downloads for " .. pu.host)
+ downloads(pu.host, pu.path)
+ end
+ end
checkFiles(pu.host, pu.host, pu.path);
else
checkHost(pu.host, pu.host, pu.path, arg[2], arg[3])
end
if testing("Integrity") or testing("Updated") then
- if nil == arg[2] then
+ if origin and (options.roundRobin.value ~= pu.host) then
while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
- os.execute("rm -f results/" .. pu.host .. ".curl")
+ os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock")
for i, n in pairs(releases) do
for l, o in pairs(releaseFiles) do
if repoExists(i .. o) then
@@ -866,7 +943,85 @@ if 0 < #arg then
end
end
+ os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED')
+ if options.referenceSite.value == pu.host then
+ execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
+ 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
+ '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt')
+ else
+-- TODO - compare to the pkgmaster copy.
+ end
+
+ local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r")
+ if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else
+ local diff = dfile:read("*a")
+ if "" ~= diff then
+ downloads(pu.host, pu.path, n, diff)
+ end
+ end
+
+ end
+
+ downloads(pu.host, pu.path, "", "")
+ while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
+ os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock")
+
+ for i, n in pairs(releases) do
+ local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r")
+ if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else
+ local diff = dfile:read("*a")
+ for l in diff:gmatch("\n*([^\n]+)\n*") do
+ postDownload(pu.host, n, "/" .. l)
+ end
+ end
+ if options.referenceSite.value == pu.host then
+ -- In case it wasn't dealt with already.
+ os.execute('touch results/NEW_Packages_' .. n .. '.test.txt')
+ end
+ end
+
+
+ for i, n in pairs(releases) do
+ local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
+ if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
+ for l in nfile:lines() do
+ local p = l:match('(pool/.*%.deb)')
+ if nil ~= p then
+ downloads(pu.host, pu.path, nil, p)
+ end
+ end
+ end
end
+ downloads(pu.host, pu.path, nil, "")
+ while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
+ for i, n in pairs(releases) do
+ local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
+ if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
+ for l in nfile:lines() do
+ local v, p, sz, m, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) | (%x.+) |')
+ if nil ~= p then
+ local status, fsz = execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5')
+ if testing("Integrity") then
+ if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end.
+ E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host)
+ print('|' .. sz .. '~=' .. fsz:sub(2, -2) .. '|')
+ else
+ local status, fm = execute('md5sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1')
+ if m ~= fm:sub(2, -2) then E('Package MD5 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end
+ local status, fsha = execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1')
+ if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end
+ end
+ end
+ if testing("Updated") then
+ if sz ~= fsz:sub(2, -2) then
+ E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host)
+ end
+ end
+ end
+ end
+ end
+ end
+
end
end
@@ -880,14 +1035,21 @@ if 0 < #arg then
logPost()
logFile:close()
else
- os.execute("mkdir -p results; if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp")
+ local dt = os.date('!%Y-%m-%d-%H-%M')
+ local fodt = io.popen('TZ="GMT" date -r results/stamp +%Y-%m-%d-%H-%M', 'r')
+ odt = fodt:read('*l')
+ fodt:close()
+ os.execute(' rm results_old; ln -s results_' .. odt .. ' results_old')
+ os.execute('mkdir -p results_' .. dt .. '; rm results; ln -s results_' .. dt .. ' results')
+ os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp')
+ os.execute("rm -f results/*.check")
if not keep then
os.execute("rm -f results/*.curl")
os.execute("rm -f results/*.log")
os.execute("rm -f results/*.html")
os.execute("rm -f results/*.txt")
end
- os.execute("rm -f results/*.check")
+
logFile, e = io.open("results/LOG_apt-panopticon.html", "a+")
if nil == logFile then C("opening log file - " .. e); return end
logPre()
@@ -895,6 +1057,10 @@ else
os.execute("mkdir -p results")
mirrors = getMirrors()
checkHost(options.referenceSite.value)
+ for i, n in pairs(releases) do
+ while not checkFile('results/NEW_Packages_' .. n .. '.test.txt') do os.execute("sleep 10") end
+ end
+
for k, m in pairs(mirrors) do
if "/" == m.BaseURL:sub(-1, -1) then
W("slash at end of BaseURL in mirror_list.txt! " .. m.BaseURL, "", "", m.FQDN)
@@ -911,10 +1077,9 @@ else
if testing("Integrity") or testing("Updated") then checkExes(downloadLock) end
end
end
+
while 1 <= checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end
- if testing("Integrity") or testing("Updated") then
- while 0 < checkExes(downloadLock) do os.execute("sleep 10") end
- end
+
os.execute("rm -f results/*.check; rm -f results/*.lock")
-- Create the reports.
--
cgit v1.1