From 1ba723ffeb3f987b8ed8fd7a14c9cd7fc60caef5 Mon Sep 17 00:00:00 2001 From: onefang Date: Mon, 23 Dec 2019 13:13:27 +1000 Subject: Update script now updates everything, and sends progress reports. Also locks out the normal checks while it's updating. --- apt-panopticon-update-data.lua | 77 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 69 insertions(+), 8 deletions(-) diff --git a/apt-panopticon-update-data.lua b/apt-panopticon-update-data.lua index 8adad4d..82a6ba5 100755 --- a/apt-panopticon-update-data.lua +++ b/apt-panopticon-update-data.lua @@ -9,15 +9,64 @@ local E = APT.E local C = APT.C local arg, sendArgs = APT.parseArgs({...}) +local oldest = 0 +local lock = 'apt-panopticon.lock' +while APT.checkFile(lock) do os.execute('sleep 10') end +os.execute('touch ' .. lock) + +if -1 < APT.verbosity then print('\nUpacking tarballs and cleaning them.') end local files = io.popen('ls -1 results_*.tar.xz') for l in files:lines() do - print('tar -xf ' .. l) + if 0 == oldest then + local status, whn = APT.execute('TZ="GMT" ls -d1 --time-style="+%s" ' .. l .. ' | cut -d " " -f 6-6') + whn = whn:sub(2, -2) + local status, new = APT.execute('TZ="GMT" date -d "' .. whn:sub(9, 18) .. ' ' .. whn:sub(20, 21) .. ':' .. whn:sub(23, 24) .. '" "+%s"') + oldest = tonumber("0" .. new:sub(2, -2)) - (60 * 60) + if -1 < APT.verbosity then print('\nOldest time is ' .. oldest) end + end + if -1 < APT.verbosity then io.stdout:write('\r' .. 'tar -xf ' .. l); io.stdout:flush() end os.execute('tar -xf ' .. l) + os.execute('rm -f ' .. l:sub(1, -8) .. '/*.curl') + os.execute('rm -f ' .. l:sub(1, -8) .. '/STATUS_*') + if APT.checkFile(l:sub(1, -8) .. '/mirrors.lua') then + APT.mirrors = loadfile(l:sub(1, -8) .. "/mirrors.lua")() + for k, v in pairs(APT.mirrors) do + if APT.options.referenceSite.value ~= k then + if APT.checkFile(l:sub(1, -8) .. '/' .. k) then + os.execute('rm -fr ' .. l:sub(1, -8) .. '/' .. k) + end + else + for i, n in pairs(APT.releases) do + os.execute( 'rm -f ' .. l:sub(1, -8) .. '/' .. k .. '/merged/dists/' .. n .. '/Release;' .. + 'rm -f ' .. l:sub(1, -8) .. '/' .. k .. '/merged/dists/' .. n .. '/Release.gpg;' .. + 'rm -fr ' .. l:sub(1, -8) .. '/' .. k .. '/merged/dists/' .. n .. '/contrib 2>/dev/null;' .. + 'rm -fr ' .. l:sub(1, -8) .. '/' .. k .. '/merged/dists/' .. n .. '/main 2>/dev/null;' .. + 'rm -fr ' .. l:sub(1, -8) .. '/' .. k .. '/merged/dists/' .. n .. '/non-free 2>/dev/null') + end + os.execute('rm -fr ' .. l:sub(1, -8) .. '/' .. k .. '/merged/pool') + end + local results = {} + local f = l:sub(1, -8) .. "/" .. k .. ".lua" + if APT.checkFile(f) then + results = loadfile(f)() + results = APT.padResults(results) + results['IPs'] = v.IPs + local rfile, e = io.open(f, "w+") + if nil == rfile then C("opening results file - " .. e) else + rfile:write(APT.dumpTable(results, "", "results") .. "\nreturn results\n") + rfile:close() + end + end + end + end end + +if -1 < APT.verbosity then print('\nDeleting old RRD data.') end os.execute('rm -fr rrd') +if -1 < APT.verbosity then print('\nRecreating RRD data.') end local files = io.popen('ls -d1 results_*') for l in files:lines() do if ('results_old' ~= l) and ('.tar.xz' ~= l:sub(25, -1)) then @@ -27,22 +76,34 @@ for l in files:lines() do local status, new = APT.execute('TZ="GMT" date -d "' .. whn:sub(9, 18) .. ' ' .. whn:sub(20, 21) .. ':' .. whn:sub(23, 24) .. '" "+%s"') APT.now = tonumber("0" .. new:sub(2, -2)) if 0 ~= APT.now then - print(l .. ' -> ' .. APT.now) - APT.mirrors = loadfile(l .. "/mirrors.lua")() - for k, v in pairs(APT.mirrors) do - APT.doRRD(l, k, v) + if APT.checkFile(l .. '/mirrors.lua') then + if -1 < APT.verbosity then io.stdout:write('\r' .. l .. ' -> ' .. APT.now); io.stdout:flush() end + APT.mirrors = loadfile(l .. "/mirrors.lua")() + for k, v in pairs(APT.mirrors) do + APT.doRRD(l, k, v, oldest) + end + else + if -1 < APT.verbosity then io.stdout:write('\r' .. l); io.stdout:flush() end end else - print(l) + if -1 < APT.verbosity then io.stdout:write('\r' .. l); io.stdout:flush() end end else - print(l) + if -1 < APT.verbosity then io.stdout:write('\r' .. l); io.stdout:flush() end end end end + +os.execute('rm ' .. lock) + + +if -1 < APT.verbosity then print('\nRecreating tarballs.') end local files = io.popen('ls -1 results_*.tar.xz') for l in files:lines() do - print('tar -c --xz ' .. l:sub(1, 24) .. ' -f ' .. l .. '; rm -fr ' .. l:sub(1, 24)) + if -1 < APT.verbosity then io.stdout:write('\r' .. 'tar -c --xz ' .. l:sub(1, 24) .. ' -f ' .. l .. '; rm -fr ' .. l:sub(1, 24)); io.stdout:flush() end os.execute('tar -c --xz ' .. l:sub(1, 24) .. ' -f ' .. l .. '; rm -fr ' .. l:sub(1, 24)) end + + +if -1 < APT.verbosity then print('\nFinished updating data.') end -- cgit v1.1