From 0560dc5975ff29be6d63885925aa1942135f0043 Mon Sep 17 00:00:00 2001 From: onefang Date: Mon, 2 Dec 2019 05:50:58 +1000 Subject: Add more report stuff. TIMEOUT, report inactive mirrors and why it's not active, add a * for things tested but not supported. --- apt-panopticon.lua | 155 +++++++++++++++++++++++++++++------------------------ 1 file changed, 85 insertions(+), 70 deletions(-) (limited to 'apt-panopticon.lua') diff --git a/apt-panopticon.lua b/apt-panopticon.lua index d66e32d..5d9ed82 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua @@ -756,7 +756,7 @@ local getMirrors = function () local host = "" local m = {} local active = true - local URL = "https://" .. options.referenceSite.value .. "/mirror_list.txt" + local URL = "http://" .. options.referenceSite.value .. "/mirror_list.txt" I("getting mirrors.") local p, c, h = http.request(URL) if nil == p then E(c .. " fetching " .. URL) else @@ -766,7 +766,8 @@ local getMirrors = function () d = string.lower(d) if "FQDN" == t then if "" ~= host then - if active then mirrors[host] = m end +-- if active then mirrors[host] = m end + mirrors[host] = m m = {} active = true end @@ -780,15 +781,16 @@ local getMirrors = function () prot[w] = true; end m[t] = prot - elseif "Active" == t and nil == d:find("yes", 1, true) then + elseif "Active" == t and nil == d:sub(1, 3):find("yes", 1, true) then W("Mirror " .. host .. " is not active - " .. d, "", "", host) active = false + m[t] = d -- TODO - Should do some input validation on BaseURL, and everything else. else m[t] = d end end - if "" ~= host and active then + if "" ~= host --[[and active]] then mirrors[host] = m end end @@ -933,88 +935,90 @@ if 0 < #arg then end if testing("Integrity") or testing("Updated") then - if origin and (options.roundRobin.value ~= pu.host) then - while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end - os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock") - for i, n in pairs(releases) do - for l, o in pairs(releaseFiles) do - if repoExists(i .. o) then - postDownload(pu.host, n, o) + if 4 > (totalTimeouts) then + if origin and (options.roundRobin.value ~= pu.host) then + while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end + os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock") + for i, n in pairs(releases) do + for l, o in pairs(releaseFiles) do + if repoExists(i .. o) then + postDownload(pu.host, n, o) + end end - end - os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED') - if options.referenceSite.value == pu.host then - execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. - 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. - '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt') - else + os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED') + if options.referenceSite.value == pu.host then + execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. + 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. + '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt') + else -- TODO - compare to the pkgmaster copy. - end + end - local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") - if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else - local diff = dfile:read("*a") - if "" ~= diff then - downloads(pu.host, pu.path, n, diff) + local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") + if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else + local diff = dfile:read("*a") + if "" ~= diff then + downloads(pu.host, pu.path, n, diff) + end end - end - end + end - downloads(pu.host, pu.path, "", "") - while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end - os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock") + downloads(pu.host, pu.path, "", "") + while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end + os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock") - for i, n in pairs(releases) do - local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") - if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else - local diff = dfile:read("*a") - for l in diff:gmatch("\n*([^\n]+)\n*") do - postDownload(pu.host, n, "/" .. l) + for i, n in pairs(releases) do + local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") + if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else + local diff = dfile:read("*a") + for l in diff:gmatch("\n*([^\n]+)\n*") do + postDownload(pu.host, n, "/" .. l) + end + end + if options.referenceSite.value == pu.host then + -- In case it wasn't dealt with already. + os.execute('touch results/NEW_Packages_' .. n .. '.test.txt') end end - if options.referenceSite.value == pu.host then - -- In case it wasn't dealt with already. - os.execute('touch results/NEW_Packages_' .. n .. '.test.txt') - end - end - for i, n in pairs(releases) do - local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") - if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else - for l in nfile:lines() do - local p = l:match('(pool/.*%.deb)') - if nil ~= p then - downloads(pu.host, pu.path, nil, p) + for i, n in pairs(releases) do + local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") + if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else + for l in nfile:lines() do + local p = l:match('(pool/.*%.deb)') + if nil ~= p then + downloads(pu.host, pu.path, nil, p) + end end end end - end - downloads(pu.host, pu.path, nil, "") - while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end - for i, n in pairs(releases) do - local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") - if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else - for l in nfile:lines() do - local v, p, sz, m, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) | (%x.+) |') - if nil ~= p then - local status, fsz = execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5') - if testing("Integrity") then - if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. - E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) - print('|' .. sz .. '~=' .. fsz:sub(2, -2) .. '|') - else - local status, fm = execute('md5sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') - if m ~= fm:sub(2, -2) then E('Package MD5 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end - local status, fsha = execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') - if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end + downloads(pu.host, pu.path, nil, "") + while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end + for i, n in pairs(releases) do + local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") + if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else + for l in nfile:lines() do + local v, p, sz, m, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) | (%x.+) |') + if nil ~= p then + local status, fsz = execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5') + if testing("Integrity") then + if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. + E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) + print('|' .. sz .. '~=' .. fsz:sub(2, -2) .. '|') + else + local status, fm = execute('md5sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') + if m ~= fm:sub(2, -2) then E('Package MD5 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end + local status, fsha = execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') + if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end + end end - end - if testing("Updated") then - if sz ~= fsz:sub(2, -2) then - E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) + if testing("Updated") then + if sz ~= fsz:sub(2, -2) then + E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) + end end end end @@ -1022,6 +1026,17 @@ if 0 < #arg then end end + else + for k, v in pairs{"ftp", "http", "https", "rsync"} do + if testing(v) then + local tests = results[v] + if testing("Integrity") then tests.Integrity = {errors = -1; warnings = -1} end + if testing("Protocol") then tests.Protocol = {errors = -1; warnings = -1} end + if testing("Updated") then tests.Updated = {errors = -1; warnings = -1} end + if testing("URLSanity") then tests.URLSanity = {errors = -1; warnings = -1} end + results[v] = tests + end + end end end -- cgit v1.1