From 0f6c928194683f3bb85cedb1e92ed3f0cd67c84a Mon Sep 17 00:00:00 2001 From: onefang Date: Sun, 1 Dec 2019 01:41:16 +1000 Subject: Much work on the Updated tests. Plus keep a history of results. --- apt-panopticon.lua | 243 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 204 insertions(+), 39 deletions(-) (limited to 'apt-panopticon.lua') diff --git a/apt-panopticon.lua b/apt-panopticon.lua index d40a74c..d66e32d 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua @@ -45,7 +45,7 @@ options = "Protocol", -- "URLSanity", "Integrity", --- "Updated", + "Updated", }, }, timeout = @@ -100,7 +100,7 @@ local referenceDebs = local referenceDevs = { -- Devuan package. NOTE this one should not get redirected, but that's more a warning than an error. - "merged/pool/DEVUAN/main/d/desktop-base/desktop-base_2.0.3_all.deb", + "merged/pool/DEVUAN/main/d/desktop-base/desktop-base_3.0_all.deb", "merged/pool/DEVUAN/main/u/util-linux/util-linux_2.32.1-0.1+devuan2.1_amd64.deb", } local arg = {} @@ -354,6 +354,11 @@ local checkExes = function (exe) return tonumber(count) end +local checkFile = function(f) + local h, e = io.open(f, "r") + if nil == h then return false else h:close(); return true end +end + local repoExists = function (r) r = r:match("([%a-]*)") if nil == r then return false end @@ -610,34 +615,106 @@ end local addDownload = function(host, URL, f, r, k) + local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename. + local o, e = io.open("results/" .. host .. "/merged/dists/" .. r .. k, "r") + if nil ~= o then + o:close() + -- Curls "check timestamp and overwrite file" stuff sucks. + -- -R means the destination file gets the timestamp of the remote file. + -- Can only do ONE timestamp check per command. + -- This doesn't work either. All downloads get all these headers. Pffft +-- local status, ts = execute('TZ="GMT" ls -l --time-style="+%a, %d %b %Y %T %Z" results/' .. host .. "/merged/dists/" .. r .. k .. ' | cut -d " " -f 6-11') +-- f:write('header "If-Modified-Since: ' .. ts:sub(2, -2) .. '"\n') + -- Curl will DELETE the existing file if the timestamp fails to download a new one, unless we change directory first, + -- which wont work with multiple files in multiple directories. WTF? + os.execute(" mv results/" .. host .. "/merged/dists/" .. r .. k .. + " results/" .. host .. "/merged/dists/" .. r .. k .. ".old") + end + + D('Downloading http://' .. host .. URL .. '/merged/dists/' .. r .. k) f:write('url "' .. 'http://' .. host .. URL .. '/merged/dists/' .. r .. k .. '"\n') f:write('output "results/' .. host .. '/merged/dists/' .. r .. k .. '"\n') - -- Curls "check timestamp and overwrite file" stuff sucks. - -- Can only do ONE timestamp check per command. - -- Will DELETE the existing file if the timestamp fails to download a new one, unless we change directory first, which wont work here. - os.execute("if [ -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then mv" .. - " results/" .. host .. "/merged/dists/" .. r .. k .. - " results/" .. host .. "/merged/dists/" .. r .. k .. ".old; fi") end local postDownload = function(host, r, k) + local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename. + local dir = k:sub(1, 0 - (#file + 1)) os.execute("if [ -f results/" .. host .. "/merged/dists/" .. r .. k .. ".old ]" .. - " && [ ! -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then cp" .. + " && [ ! -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then cp -a" .. " results/" .. host .. "/merged/dists/" .. r .. k .. ".old" .. " results/" .. host .. "/merged/dists/" .. r .. k .. "; fi") + if ".gz" == k:sub(-3, -1) then execute("gzip -dfk results/" .. host .. "/merged/dists/" .. r .. k) end + if ".xz" == k:sub(-3, -1) then execute("xz -dfk results/" .. host .. "/merged/dists/" .. r .. k .. " 2>/dev/null") end if testing("Integrity") then if ".gpg" == k:sub(-4, -1) then local status, out = execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. r .. k .. " results/" .. host .. "/merged/dists/" .. r .. k:sub(1, -5) .. " 2>/dev/null") if "0" ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. r .. k, "http", "Integrity", host) end end +-- TODO - should check the PGP sig of InRelease as well. + end + if testing("Integrity") or testing("Updated") then + if options.referenceSite.value == host then + if "Packages." == file:sub(1, 9) then +-- TODO - compare the SHA256 sums in pkgmaster's Release for both the packed and unpacked versions. + local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed', "w+") + if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed' .. ' file - ' .. e) else + local pp = {} + if checkFile('results_old/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages') then + for l in io.lines('results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages') do + if "Package: " == l:sub(1, 9) then + if 0 ~= #pp then + Pp:write(r .. " | ") + for i = 1, 6 do + if nil == pp[i] then print(host .. " " .. r .. " " .. dir .. " " .. i) else Pp:write(pp[i] .. " | ") end + end + Pp:write("\n") + end + pp = {} + pp[1] = l:sub(10, -1) + elseif "Version: " == l:sub(1, 9) then + pp[2] = l:sub(10, -1) + elseif "Filename: " == l:sub(1, 10) then + pp[3] = l:sub(11, -1) + elseif "Size: " == l:sub(1, 6) then + pp[4] = l:sub(7, -1) + elseif "MD5sum: " == l:sub(1, 8) then + pp[5] = l:sub(9, -1) + elseif "SHA256: " == l:sub(1, 8) then + pp[6] = l:sub(9, -1) + end + end + else + W("Can't find file results_old/pkgmaster.devuan.org/merged/dists/" .. r .. dir .. "Packages") + end + Pp:close() + os.execute('sort results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed >results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed-sorted') + os.execute('rm results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed') + if checkFile('results_old/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages.parsed-sorted') then + execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages.parsed-sorted ' .. + 'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages.parsed-sorted ' .. + ' | grep -E "^-" | grep -Ev "^\\+\\+\\+|^---" >>results/OLD_PACKAGES_' .. r .. '.txt') + execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages.parsed-sorted ' .. + 'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages.parsed-sorted ' .. + ' | grep -E "^\\+" | grep -Ev "^\\+\\+\\+|^---" >>results/NEW_Packages_' .. r .. '.txt') + -- Find the smallest new package for each release. + os.execute('sort -b -k 9,9 -n results/NEW_Packages_' .. r .. '.txt >results/NEW_Packages_' .. r .. '.sorted.txt') + os.execute('grep -s " | pool/DEBIAN/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >results/NEW_Packages_' .. r .. '.test.txt') + os.execute('grep -s " | pool/DEBIAN-SECURITY/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >>results/NEW_Packages_' .. r .. '.test.txt') + os.execute('grep -s " | pool/DEVUAN/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >>results/NEW_Packages_' .. r .. '.test.txt') + else + W("Can't find file results_old/pkgmaster.devuan.org/merged/dists/" .. r .. dir .. "Packages.parsed-sorted") + end + end + end + end end end local downloadLock = "flock -n results/curl-" local download = "curl --connect-timeout " .. options.timeout.value .. " --create-dirs -L -z 'results/stamp.old' -v -R " local downloads = function(host, URL, release, list) - if nil == URL then URL = "/" end + if nil == URL then URL = "" end local lock = "%s-" .. host .. ".lock" local log = " --stderr results/curl-%s_" .. host .. ".log" local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock:format("META") .. " " .. download .. log:format("META") .. " -K results/" .. host .. ".curl" @@ -648,24 +725,19 @@ local downloads = function(host, URL, release, list) if nil ~= list then if "" ~= list then - for l in list:gmatch("\n*([^\n]+)\n*") do - addDownload(host, URL, f, release, "/" .. l) + if nil ~= release then + for l in list:gmatch("\n*([^\n]+)\n*") do + addDownload(host, URL, f, release, "/" .. l) + end + else + D('Downlaading http://' .. host .. URL .. '/merged/' .. list) + f:write('url "' .. 'http://' .. host .. URL .. '/merged/' .. list .. '"\n') + f:write('output "results/' .. host .. '/merged/' .. list .. '"\n') end f:close() return end else - ---[[ - for i, s in pairs(referenceDevs) do - cm = cm .. " https://" .. host .. URL .. "/" .. s - end - for i, s in pairs(referenceDebs) do - cm = cm .. " https://" .. host .. URL .. "/" .. s - end - execute(cm) -]] - for i, s in pairs(releases) do for j, k in pairs(releaseFiles) do if repoExists(s .. k) then @@ -678,6 +750,7 @@ local downloads = function(host, URL, release, list) fork(cm) end + local getMirrors = function () local mirrors = {} local host = "" @@ -818,6 +891,13 @@ if 0 < #arg then arg[1] = arg[1]:sub(1, -2) end local pu = url.parse("http://" .. arg[1]) + + if testing("Integrity") or testing("Updated") then + if origin and options.referenceSite.value == pu.host then +-- if not keep then execute("rm -fr results/" .. pu.host) end + end + end + if nil ~= arg[2] then logFile, e = io.open("results/LOG_" .. pu.host .. "_" .. arg[2] .. ".html", "a+") else @@ -840,25 +920,22 @@ if 0 < #arg then results[v] = tests end end - if testing("Integrity") or testing("Updated") then - if nil == arg[2] then - I("Starting file downloads for " .. pu.host) --- if not keep then execute("rm -fr results/" .. pu.host) end - downloads(pu.host, pu.path) --- checkExes("apt-panopticon.lua " .. sendArgs) --- checkExes(downloadLock) - end - end if origin then + if testing("Integrity") or testing("Updated") then + if origin and (options.roundRobin.value ~= pu.host) then + I("Starting file downloads for " .. pu.host) + downloads(pu.host, pu.path) + end + end checkFiles(pu.host, pu.host, pu.path); else checkHost(pu.host, pu.host, pu.path, arg[2], arg[3]) end if testing("Integrity") or testing("Updated") then - if nil == arg[2] then + if origin and (options.roundRobin.value ~= pu.host) then while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end - os.execute("rm -f results/" .. pu.host .. ".curl") + os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock") for i, n in pairs(releases) do for l, o in pairs(releaseFiles) do if repoExists(i .. o) then @@ -866,7 +943,85 @@ if 0 < #arg then end end + os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED') + if options.referenceSite.value == pu.host then + execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. + 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. + '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt') + else +-- TODO - compare to the pkgmaster copy. + end + + local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") + if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else + local diff = dfile:read("*a") + if "" ~= diff then + downloads(pu.host, pu.path, n, diff) + end + end + + end + + downloads(pu.host, pu.path, "", "") + while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end + os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock") + + for i, n in pairs(releases) do + local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") + if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else + local diff = dfile:read("*a") + for l in diff:gmatch("\n*([^\n]+)\n*") do + postDownload(pu.host, n, "/" .. l) + end + end + if options.referenceSite.value == pu.host then + -- In case it wasn't dealt with already. + os.execute('touch results/NEW_Packages_' .. n .. '.test.txt') + end + end + + + for i, n in pairs(releases) do + local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") + if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else + for l in nfile:lines() do + local p = l:match('(pool/.*%.deb)') + if nil ~= p then + downloads(pu.host, pu.path, nil, p) + end + end + end end + downloads(pu.host, pu.path, nil, "") + while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end + for i, n in pairs(releases) do + local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") + if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else + for l in nfile:lines() do + local v, p, sz, m, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) | (%x.+) |') + if nil ~= p then + local status, fsz = execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5') + if testing("Integrity") then + if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. + E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) + print('|' .. sz .. '~=' .. fsz:sub(2, -2) .. '|') + else + local status, fm = execute('md5sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') + if m ~= fm:sub(2, -2) then E('Package MD5 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end + local status, fsha = execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') + if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end + end + end + if testing("Updated") then + if sz ~= fsz:sub(2, -2) then + E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) + end + end + end + end + end + end + end end @@ -880,14 +1035,21 @@ if 0 < #arg then logPost() logFile:close() else - os.execute("mkdir -p results; if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp") + local dt = os.date('!%Y-%m-%d-%H-%M') + local fodt = io.popen('TZ="GMT" date -r results/stamp +%Y-%m-%d-%H-%M', 'r') + odt = fodt:read('*l') + fodt:close() + os.execute(' rm results_old; ln -s results_' .. odt .. ' results_old') + os.execute('mkdir -p results_' .. dt .. '; rm results; ln -s results_' .. dt .. ' results') + os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp') + os.execute("rm -f results/*.check") if not keep then os.execute("rm -f results/*.curl") os.execute("rm -f results/*.log") os.execute("rm -f results/*.html") os.execute("rm -f results/*.txt") end - os.execute("rm -f results/*.check") + logFile, e = io.open("results/LOG_apt-panopticon.html", "a+") if nil == logFile then C("opening log file - " .. e); return end logPre() @@ -895,6 +1057,10 @@ else os.execute("mkdir -p results") mirrors = getMirrors() checkHost(options.referenceSite.value) + for i, n in pairs(releases) do + while not checkFile('results/NEW_Packages_' .. n .. '.test.txt') do os.execute("sleep 10") end + end + for k, m in pairs(mirrors) do if "/" == m.BaseURL:sub(-1, -1) then W("slash at end of BaseURL in mirror_list.txt! " .. m.BaseURL, "", "", m.FQDN) @@ -911,10 +1077,9 @@ else if testing("Integrity") or testing("Updated") then checkExes(downloadLock) end end end + while 1 <= checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end - if testing("Integrity") or testing("Updated") then - while 0 < checkExes(downloadLock) do os.execute("sleep 10") end - end + os.execute("rm -f results/*.check; rm -f results/*.lock") -- Create the reports. -- cgit v1.1