aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/apt-panopticon-update-data.lua
blob: 8251c9d9c10fb62ca0b12b4d1a6bcf1847ff86e8 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
#!/usr/bin/env luajit

local APT = require 'apt-panopticommon'
local D = APT.D
local I = APT.I
local T = APT.T
local W = APT.W
local E = APT.E
local C = APT.C
local arg, sendArgs = APT.parseArgs({...})

local oldest = os.date('!%s') - (60 * 60)


local lock = 'apt-panopticon.lock'
while APT.checkFile(lock) do os.execute('sleep 10') end
os.execute('touch ' .. lock)


if -1 < APT.verbosity then print('\nUpacking tarballs and cleaning them.') end
local files = io.popen('ls -1 results_*/results_*.tar.xz 2>/dev/null; ls -1 results_*.tar.xz 2>/dev/null')
for l in files:lines() do
    local lsub = l:sub(1, -8)

    local dt = os.date('!%s', os.time{year = l:sub(-23, -20), month = l:sub(-18, -17), day = l:sub(-15, -14), hour = l:sub(-12, -11), min = l:sub(-9, -8)}) - (60 * 60)
    if dt < oldest then oldest = dt; print('\nOldest time is ' .. oldest) end

    if -1 < APT.verbosity then io.stdout:write('\r' .. 'tar -xf ' .. l);  io.stdout:flush() end
    os.execute('tar -xf ' .. l)

    os.execute('rm -f ' .. lsub .. '/*.curl')
    os.execute('rm -f ' .. lsub .. '/STATUS_*')

    if APT.checkFile(lsub .. '/mirrors.lua') then
	APT.mirrors = loadfile(lsub .. "/mirrors.lua")()
	for k, v in pairs(APT.mirrors) do
	    if APT.options.referenceSite.value ~= k then
		if APT.checkFile(lsub .. '/' .. k) then
		    os.execute('rm -fr ' .. lsub .. '/' .. k)
		end
	    else
		for i, n in pairs(APT.releases) do
		    os.execute( 'rm -f  ' .. lsub .. '/' .. k .. '/merged/dists/' .. n .. '/Release;' ..
				'rm -f  ' .. lsub .. '/' .. k .. '/merged/dists/' .. n .. '/Release.gpg;' ..
				'rm -fr ' .. lsub .. '/' .. k .. '/merged/dists/' .. n .. '/contrib 2>/dev/null;' ..
				'rm -fr ' .. lsub .. '/' .. k .. '/merged/dists/' .. n .. '/main 2>/dev/null;' ..
				'rm -fr ' .. lsub .. '/' .. k .. '/merged/dists/' .. n .. '/non-free 2>/dev/null')
		end
		os.execute('rm -fr ' .. lsub .. '/' .. k .. '/merged/pool')
	    end

	    local results = {}
	    local f = l:sub(1, -8) .. "/" .. k .. ".lua"
	    if APT.checkFile(f) then
		results = loadfile(f)()
		results['IPs'] = v.IPs
		local rfile, e = io.open(f, "w+")
		if nil == rfile then C("opening results file - " .. e) else
		    rfile:write(APT.dumpTable(results, "results") .. "\nreturn results\n")
		    rfile:close()
		end
	    end

	end
    end

    local fls = io.popen('ls -1 ' .. lsub .. '/*.lua 2>/dev/null')
    for k in fls:lines() do
	if APT.checkFile(k) then
	    local results = loadfile(k)()
	    results = APT.padResults(results)
	    for i, w in pairs(APT.protocols) do
		tests = results[w]
		if nil ~= tests.Redirects then
		    tests.redirects = tests.Redirects
		    tests.Redirects = nil
		elseif nil == tests.redirects then
		    tests.redirects = {}
		end
		results[w] = tests
	    end
	    if nil ~= results.Redirects then
		for i, w in pairs(APT.protocols) do
		    results[w].redirects  = results.Redirects
		end
		results.Redirects = nil
	    end
	    local rfile, e = io.open(k, "w+")
	    if nil == rfile then C("opening results file - " .. e) else
		rfile:write(APT.dumpTable(results, "results") .. "\nreturn results\n")
		rfile:close()
	    end
	end
    end
end


if -1 < APT.verbosity then print('\nDeleting old RRD data.') end
os.execute('rm -fr rrd')
if -1 < APT.verbosity then print('\nRecreating RRD data.') end
local files = io.popen('ls -d1 results_*')
for l in files:lines() do
    if ('results_old' ~= l) and ('.tar.xz' ~= l:sub(25, -1)) then
	if APT.checkFile(l .. '/stamp') then
	    APT.now = os.date('!%s', os.time{year = l:sub(9, 12), month = l:sub(14, 15), day = l:sub(17, 18), hour = l:sub(20, 21), min = l:sub(23, 24)})
	    if 0 ~= APT.now then
		if APT.checkFile(l .. '/mirrors.lua') then
		if -1 < APT.verbosity then io.stdout:write('\r' .. l .. ' -> ' .. APT.now);  io.stdout:flush() end
		    APT.mirrors = loadfile(l .. "/mirrors.lua")()
		    for k, v in pairs(APT.mirrors) do
			APT.doRRD(l, k, v, oldest)
		    end
		else
		    if -1 < APT.verbosity then io.stdout:write('\r' .. l);  io.stdout:flush() end
		end
	    else
		if -1 < APT.verbosity then io.stdout:write('\r' .. l);  io.stdout:flush() end
	    end
	else
	    if -1 < APT.verbosity then io.stdout:write('\r' .. l);  io.stdout:flush() end
	end
    end
end


os.execute('rm ' .. lock)


if -1 < APT.verbosity then print('\nRecreating tarballs.') end

local files = io.popen('ls -1 results_*/results_*.tar.xz 2>/dev/null')
for l in files:lines() do
    if -1 < APT.verbosity then io.stdout:write('\r' .. 'tar -c --xz  ' .. l:sub(20, 43) .. ' -f ' .. l);  io.stdout:flush() end
    os.execute('mkdir -p ' .. l:sub(1, 18) .. '; tar -c --xz  ' .. l:sub(20, 43) .. ' -f ' .. l .. '; rm -fr ' .. l:sub(20, 43))
end
local files = io.popen('ls -1 results_*.tar.xz 2>/dev/null')
for l in files:lines() do
    if -1 < APT.verbosity then io.stdout:write('\r' .. 'tar -c --xz  ' .. l:sub(1, 24) .. ' -f ' .. l:sub(1, 18) .. '/' .. l);  io.stdout:flush() end
    os.execute('mkdir -p ' .. l:sub(1, 18) .. '; tar -c --xz  ' .. l:sub(1, 24) .. ' -f ' .. l:sub(1, 18) .. '/' .. l .. '; rm -fr ' .. l:sub(1, 24) .. '; rm -f ' .. l)
end

local rslt  = APT.exe( 'ls -dl results | cut -d " " -f 11'):Do().result:sub(2, -2)
local rslto = APT.exe( 'ls -dl results_old | cut -d " " -f 11'):Do().result:sub(2, -2)
local files = io.popen('ls -d1 results_*-*-*-*-* 2>/dev/null')
for l in files:lines() do
    if (rslt ~= l) and (rslto ~= l) then
	if -1 < APT.verbosity then io.stdout:write('\r' .. 'tar -c --xz  ' .. l:sub(1, 24) .. ' -f ' .. l:sub(1, 18) .. '/' .. l .. '.tar.xz');  io.stdout:flush() end
	os.execute('mkdir -p ' .. l:sub(1, 18) .. '; tar -c --xz  ' .. l:sub(1, 24) .. ' -f ' .. l:sub(1, 18) .. '/' .. l .. '; rm -fr ' .. l:sub(1, 24) .. '; rm -f ' .. l)
    end
end


if -1 < APT.verbosity then print('\nFinished updating data.') end