From bc3635178c333491f9c0a5ccdf2cd4b2054f4072 Mon Sep 17 00:00:00 2001 From: onefang Date: Wed, 4 Dec 2019 03:56:49 +1000 Subject: Try to keep the old results directories small and clean. --- apt-panopticon.lua | 71 +++++++++++++++++++++++++++--------------------------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/apt-panopticon.lua b/apt-panopticon.lua index 8566dad..7d263c7 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua @@ -682,49 +682,43 @@ local postDownload = function(host, r, k) -- TODO - should check the PGP sig of InRelease as well. end if testing("Integrity") or testing("Updated") then - if options.referenceSite.value == host then - if "Packages." == file:sub(1, 9) then + if "Packages." == file:sub(1, 9) then -- TODO - compare the SHA256 sums in pkgmaster's Release for both the packed and unpacked versions. -- Also note that this might get only a partial download due to maxtime. + if options.referenceSite.value == host then local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed', "w+") if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed' .. ' file - ' .. e) else local pp = {} - if checkFile('results_old/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages') then - for l in io.lines('results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages') do - if "Package: " == l:sub(1, 9) then - if 0 ~= #pp then - Pp:write(r .. " | ") - for i = 1, 6 do - if nil == pp[i] then print(host .. " " .. r .. " " .. dir .. " " .. i) else Pp:write(pp[i] .. " | ") end - end - Pp:write("\n") + for l in io.lines('results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages') do + if "Package: " == l:sub(1, 9) then + if 0 ~= #pp then + for i = 1, 6 do + if nil == pp[i] then print(host .. " " .. r .. " " .. dir .. " " .. i) else Pp:write(pp[i] .. " | ") end end - pp = {} - pp[1] = l:sub(10, -1) - elseif "Version: " == l:sub(1, 9) then - pp[2] = l:sub(10, -1) - elseif "Filename: " == l:sub(1, 10) then - pp[3] = l:sub(11, -1) - elseif "Size: " == l:sub(1, 6) then - pp[4] = l:sub(7, -1) - elseif "MD5sum: " == l:sub(1, 8) then - pp[5] = l:sub(9, -1) - elseif "SHA256: " == l:sub(1, 8) then - pp[6] = l:sub(9, -1) + Pp:write("\n") end + pp = {} + pp[1] = l:sub(10, -1) + elseif "Version: " == l:sub(1, 9) then + pp[2] = l:sub(10, -1) + elseif "Filename: " == l:sub(1, 10) then + pp[3] = l:sub(11, -1) + elseif "Size: " == l:sub(1, 6) then + pp[4] = l:sub(7, -1) + elseif "MD5sum: " == l:sub(1, 8) then + pp[5] = l:sub(9, -1) + elseif "SHA256: " == l:sub(1, 8) then + pp[6] = l:sub(9, -1) end - else - W("Can't find file results_old/pkgmaster.devuan.org/merged/dists/" .. r .. dir .. "Packages") end Pp:close() - os.execute('sort results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed >results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed-sorted') - os.execute('rm results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed') - if checkFile('results_old/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages.parsed-sorted') then - execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages.parsed-sorted ' .. - 'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages.parsed-sorted ' .. + os.execute('sort results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed >results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages_parsed-sorted') + if checkFile('results_old/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages_parsed-sorted') then + execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages_parsed-sorted ' .. + 'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages_parsed-sorted ' .. ' | grep -E "^-" | grep -Ev "^\\+\\+\\+|^---" >>results/OLD_PACKAGES_' .. r .. '.txt') - execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages.parsed-sorted ' .. - 'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages.parsed-sorted ' .. + execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages_parsed-sorted ' .. + 'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages_parsed-sorted ' .. ' | grep -E "^\\+" | grep -Ev "^\\+\\+\\+|^---" >>results/NEW_Packages_' .. r .. '.txt') -- Find the smallest new package for each release. os.execute('sort -b -k 9,9 -n results/NEW_Packages_' .. r .. '.txt >results/NEW_Packages_' .. r .. '.sorted.txt') @@ -732,11 +726,12 @@ local postDownload = function(host, r, k) os.execute('grep -s " | pool/DEBIAN-SECURITY/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >>results/NEW_Packages_' .. r .. '.test.txt') os.execute('grep -s " | pool/DEVUAN/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >>results/NEW_Packages_' .. r .. '.test.txt') else - W("Can't find file results_old/pkgmaster.devuan.org/merged/dists/" .. r .. dir .. "Packages.parsed-sorted") + C("Can't find file results_old/pkgmaster.devuan.org/merged/dists/" .. r .. dir .. "Packages_parsed-sorted") end end - os.execute('rm results/' .. host .. '/merged/dists/' .. r .. dir .. 'Packages') end + os.execute('rm results/' .. host .. '/merged/dists/' .. r .. dir .. 'Packages') + os.execute('rm results/' .. host .. '/merged/dists/' .. r .. dir .. 'Packages.*') end end end @@ -796,7 +791,6 @@ local getMirrors = function () d = string.lower(d) if "FQDN" == t then if "" ~= host then --- if active then mirrors[host] = m end mirrors[host] = m m = {} active = true @@ -982,6 +976,8 @@ if 0 < #arg then execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt') + os.execute('rm results/' .. pu.host .. '/merged/dists/' .. n .. '/Release') +-- os.execute('rm results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED') -- TODO - Maybe check the date in Release, though since they are updated daily, is there any point? Perhaps it's for checking amprolla got run? else -- TODO - compare to the pkgmaster copy. @@ -1074,6 +1070,10 @@ if 0 < #arg then end end + if origin and options.referenceSite.value ~= pu.host then + if not keep then os.execute("rm -fr results/" .. pu.host) end + os.execute('rm STATUS_' .. pu.host .. '* 2>/dev/null') + end local f = pu.host if "" ~= ip then f = f .. "_" .. ip end local rfile, e = io.open("results/" .. f .. ".lua", "w+") @@ -1142,7 +1142,6 @@ else end end --- os.execute("rm -f results/*.lua") logPost() logFile:close() end -- cgit v1.1