From 1ba723ffeb3f987b8ed8fd7a14c9cd7fc60caef5 Mon Sep 17 00:00:00 2001
From: onefang
Date: Mon, 23 Dec 2019 13:13:27 +1000
Subject: Update script now updates everything, and sends progress reports.

Also locks out the normal checks while it's updating.
---
 apt-panopticon-update-data.lua | 77 +++++++++++++++++++++++++++++++++++++-----
 1 file changed, 69 insertions(+), 8 deletions(-)

(limited to 'apt-panopticon-update-data.lua')

diff --git a/apt-panopticon-update-data.lua b/apt-panopticon-update-data.lua
index 8adad4d..82a6ba5 100755
--- a/apt-panopticon-update-data.lua
+++ b/apt-panopticon-update-data.lua
@@ -9,15 +9,64 @@ local E = APT.E
 local C = APT.C
 local arg, sendArgs = APT.parseArgs({...})
 
+local oldest = 0
 
+local lock = 'apt-panopticon.lock'
+while APT.checkFile(lock) do os.execute('sleep 10') end
+os.execute('touch ' .. lock)
 
+
+if -1 < APT.verbosity then print('\nUpacking tarballs and cleaning them.') end
 local files = io.popen('ls -1 results_*.tar.xz')
 for l in files:lines() do
-    print('tar -xf ' .. l)
+    if 0 == oldest then
+	local status, whn = APT.execute('TZ="GMT" ls -d1 --time-style="+%s" ' .. l .. ' | cut -d " " -f 6-6')
+	whn = whn:sub(2, -2)
+	local status, new = APT.execute('TZ="GMT" date -d "' .. whn:sub(9, 18) .. ' ' .. whn:sub(20, 21) .. ':' .. whn:sub(23, 24) .. '" "+%s"')
+	oldest = tonumber("0" .. new:sub(2, -2)) - (60 * 60)
+	if -1 < APT.verbosity then print('\nOldest time is ' .. oldest) end
+    end
+    if -1 < APT.verbosity then io.stdout:write('\r' .. 'tar -xf ' .. l);  io.stdout:flush() end
     os.execute('tar -xf ' .. l)
+    os.execute('rm -f ' .. l:sub(1, -8) .. '/*.curl')
+    os.execute('rm -f ' .. l:sub(1, -8) .. '/STATUS_*')
+    if APT.checkFile(l:sub(1, -8) .. '/mirrors.lua') then
+	APT.mirrors = loadfile(l:sub(1, -8) .. "/mirrors.lua")()
+	for k, v in pairs(APT.mirrors) do
+	    if APT.options.referenceSite.value ~= k then
+		if APT.checkFile(l:sub(1, -8) .. '/' .. k) then
+		    os.execute('rm -fr ' .. l:sub(1, -8) .. '/' .. k)
+		end
+	    else
+		for i, n in pairs(APT.releases) do
+		    os.execute( 'rm -f  ' .. l:sub(1, -8) .. '/' .. k .. '/merged/dists/' .. n .. '/Release;' ..
+				'rm -f  ' .. l:sub(1, -8) .. '/' .. k .. '/merged/dists/' .. n .. '/Release.gpg;' ..
+				'rm -fr ' .. l:sub(1, -8) .. '/' .. k .. '/merged/dists/' .. n .. '/contrib 2>/dev/null;' ..
+				'rm -fr ' .. l:sub(1, -8) .. '/' .. k .. '/merged/dists/' .. n .. '/main 2>/dev/null;' ..
+				'rm -fr ' .. l:sub(1, -8) .. '/' .. k .. '/merged/dists/' .. n .. '/non-free 2>/dev/null')
+		end
+		os.execute('rm -fr ' .. l:sub(1, -8) .. '/' .. k .. '/merged/pool')
+	    end
+	    local results = {}
+	    local f = l:sub(1, -8) .. "/" .. k .. ".lua"
+	    if APT.checkFile(f) then
+		results = loadfile(f)()
+		results = APT.padResults(results)
+		results['IPs'] = v.IPs
+		local rfile, e = io.open(f, "w+")
+		if nil == rfile then C("opening results file - " .. e) else
+		    rfile:write(APT.dumpTable(results, "", "results") .. "\nreturn results\n")
+		    rfile:close()
+		end
+	    end
+	end
+    end
 end
 
+
+if -1 < APT.verbosity then print('\nDeleting old RRD data.') end
 os.execute('rm -fr rrd')
+if -1 < APT.verbosity then print('\nRecreating RRD data.') end
 local files = io.popen('ls -d1 results_*')
 for l in files:lines() do
     if ('results_old' ~= l) and ('.tar.xz' ~= l:sub(25, -1)) then
@@ -27,22 +76,34 @@ for l in files:lines() do
 	    local status, new = APT.execute('TZ="GMT" date -d "' .. whn:sub(9, 18) .. ' ' .. whn:sub(20, 21) .. ':' .. whn:sub(23, 24) .. '" "+%s"')
 	    APT.now = tonumber("0" .. new:sub(2, -2))
 	    if 0 ~= APT.now then
-		print(l .. ' -> ' .. APT.now)
-		APT.mirrors = loadfile(l .. "/mirrors.lua")()
-		for k, v in pairs(APT.mirrors) do
-		    APT.doRRD(l, k, v)
+		if APT.checkFile(l .. '/mirrors.lua') then
+		if -1 < APT.verbosity then io.stdout:write('\r' .. l .. ' -> ' .. APT.now);  io.stdout:flush() end
+		    APT.mirrors = loadfile(l .. "/mirrors.lua")()
+		    for k, v in pairs(APT.mirrors) do
+			APT.doRRD(l, k, v, oldest)
+		    end
+		else
+		    if -1 < APT.verbosity then io.stdout:write('\r' .. l);  io.stdout:flush() end
 		end
 	    else
-		print(l)
+		if -1 < APT.verbosity then io.stdout:write('\r' .. l);  io.stdout:flush() end
 	    end
 	else
-	    print(l)
+	    if -1 < APT.verbosity then io.stdout:write('\r' .. l);  io.stdout:flush() end
 	end
     end
 end
 
+
+os.execute('rm ' .. lock)
+
+
+if -1 < APT.verbosity then print('\nRecreating tarballs.') end
 local files = io.popen('ls -1 results_*.tar.xz')
 for l in files:lines() do
-    print('tar -c --xz  ' .. l:sub(1, 24) .. ' -f ' .. l .. '; rm -fr ' .. l:sub(1, 24))
+    if -1 < APT.verbosity then io.stdout:write('\r' .. 'tar -c --xz  ' .. l:sub(1, 24) .. ' -f ' .. l .. '; rm -fr ' .. l:sub(1, 24));  io.stdout:flush() end
     os.execute('tar -c --xz  ' .. l:sub(1, 24) .. ' -f ' .. l .. '; rm -fr ' .. l:sub(1, 24))
 end
+
+
+if -1 < APT.verbosity then print('\nFinished updating data.') end
-- 
cgit v1.1