From 19fe4bce053bd0e6be67a3ea4e1a35360e2d70cc Mon Sep 17 00:00:00 2001 From: onefang Date: Thu, 12 Dec 2019 18:47:45 +1000 Subject: Speed up things by being clever. Don't wait for pkgmaster to be completely finished, wait for the things we need to wait for. Knocks about a minute off the run time. Could knock another 30 seconds off by runing the IP lookups in parallel somehow. Maybe. --- apt-panopticon.lua | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/apt-panopticon.lua b/apt-panopticon.lua index 9786b8b..ff39269 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua @@ -515,15 +515,18 @@ local postDownload = function(host, r, k) ' | grep -E "^\\+" | grep -Ev "^\\+\\+\\+|^---" >>results/NEW_Packages_' .. r .. '.txt') -- Find the smallest new package for each release. os.execute('sort -b -k 9,9 -n results/NEW_Packages_' .. r .. '.txt >results/NEW_Packages_' .. r .. '.sorted.txt') - os.execute('grep -s " | pool/DEBIAN/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >results/NEW_Packages_' .. r .. '.test.txt') - os.execute('grep -s " | pool/DEBIAN-SECURITY/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >>results/NEW_Packages_' .. r .. '.test.txt') - os.execute('grep -s " | pool/DEVUAN/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >>results/NEW_Packages_' .. r .. '.test.txt') + os.execute('grep -s " | pool/DEBIAN/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >results/NEW_Packages_' .. r .. '.test.tmp') + os.execute('grep -s " | pool/DEBIAN-SECURITY/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >>results/NEW_Packages_' .. r .. '.test.tmp') + os.execute('grep -s " | pool/DEVUAN/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >>results/NEW_Packages_' .. r .. '.test.tmp') + os.execute('mv results/NEW_Release_' .. n .. '.test.tmp results/NEW_Release_' .. n .. '.test.txt') else C("Can't find file Packages/" .. r .. dir .. "Packages_parsed-sorted") end os.execute('mkdir -p Packages/' .. r .. dir) os.execute('mv -f results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages_parsed-sorted Packages/' .. r .. dir .. 'Packages_parsed-sorted') end + else + while not APT.checkFile('results/NEW_Release_' .. n .. '.test.txt') do os.execute('sleep 10') end end os.execute('rm -f results/' .. host .. '/merged/dists/' .. r .. dir .. 'Packages 2>/dev/null') os.execute('rm -f results/' .. host .. '/merged/dists/' .. r .. dir .. 'Packages.* 2>/dev/null') @@ -687,11 +690,13 @@ if 0 < #arg then if APT.options.referenceSite.value == pu.host then os.execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. - '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt') + '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.tmp') -- TODO - Maybe check the date in Release, though since they are updated daily, is there any point? Perhaps it's for checking amprolla got run? - os.execute('rm -f results/' .. pu.host .. '/merged/dists/' .. n .. '/Release 2>/dev/null') + os.execute('rm -f results/' .. pu.host .. '/merged/dists/' .. n .. '/Release 2>/dev/null; ' .. + 'mv results/NEW_Release_' .. n .. '.tmp results/NEW_Release_' .. n .. '.txt') else -- TODO - compare to the pkgmaster copy. + while not APT.checkFile('results/NEW_Release_' .. n .. '.txt') do os.execute('sleep 10') end end local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") @@ -855,9 +860,6 @@ else os.execute("mkdir -p results") APT.mirrors = getMirrors() checkHost(APT.options.referenceSite.value) - for i, n in pairs(releases) do - while not APT.checkFile('results/NEW_Packages_' .. n .. '.test.txt') do os.execute("sleep 10") end - end for k, m in pairs(APT.mirrors) do if "/" == m.BaseURL:sub(-1, -1) then -- cgit v1.1