aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/apt-panopticon-update-data.lua
blob: 105a1c7ae9e88255e895a1cefe343db1dbe4ab84 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
#!/usr/bin/env luajit

local APT = require 'apt-panopticommon'
local D = APT.D
local I = APT.I
local T = APT.T
local W = APT.W
local E = APT.E
local C = APT.C
local arg, sendArgs = APT.parseArgs({...})

local oldest = os.date('!%s') - (60 * 60)

local fixResults = function(f)
    if APT.checkFile(f) then
	local results = loadfile(f)()
	results = APT.padResults(results)
        for i, w in pairs(APT.protocols) do
	    tests = results[w]
	    if nil ~= tests.Redirects then
		tests.redirects = tests.Redirects
		tests.Redirects = nil
	    elseif nil == tests.redirects then
		tests.redirects = {}
	    end
	    results[w] = tests
	end
	if nil ~= results.Redirects then
	    for i, w in pairs(APT.protocols) do
		results[w].redirects  = results.Redirects
	    end
	    results.Redirects = nil
	end
	local rfile, e = io.open(f, "w+")
	if nil == rfile then C("opening results file - " .. e) else
	    rfile:write(APT.dumpTable(results, "results") .. "\nreturn results\n")
	    rfile:close()
	end
    end
end


local lock = 'apt-panopticon.lock'
while APT.checkFile(lock) do os.execute('sleep 10') end
os.execute('touch ' .. lock)


if -1 < APT.verbosity then print('\nUpacking tarballs and cleaning them.') end
local files = io.popen('ls -1 results_*/results_*.tar.xz 2>/dev/null; ls -1 results_*.tar.xz 2>/dev/null')
for l in files:lines() do
    local lsub = l:sub(1, -8)

    local dt = os.date('!%s', os.time{year = l:sub(-23, -20), month = l:sub(-18, -17), day = l:sub(-15, -14), hour = l:sub(-12, -11), minute = l:sub(-9, -8)}) - (60 * 60)
    if dt < oldest then oldest = dt; print('\nOldest time is ' .. oldest) end

    if -1 < APT.verbosity then io.stdout:write('\r' .. 'tar -xf ' .. l);  io.stdout:flush() end
    os.execute('tar -xf ' .. l)

    os.execute('rm -f ' .. lsub .. '/*.curl')
    os.execute('rm -f ' .. lsub .. '/STATUS_*')

    if APT.checkFile(lsub .. '/mirrors.lua') then
	APT.mirrors = loadfile(lsub .. "/mirrors.lua")()
	for k, v in pairs(APT.mirrors) do
	    if APT.options.referenceSite.value ~= k then
		if APT.checkFile(lsub .. '/' .. k) then
		    os.execute('rm -fr ' .. lsub .. '/' .. k)
		end
	    else
		for i, n in pairs(APT.releases) do
		    os.execute( 'rm -f  ' .. lsub .. '/' .. k .. '/merged/dists/' .. n .. '/Release;' ..
				'rm -f  ' .. lsub .. '/' .. k .. '/merged/dists/' .. n .. '/Release.gpg;' ..
				'rm -fr ' .. lsub .. '/' .. k .. '/merged/dists/' .. n .. '/contrib 2>/dev/null;' ..
				'rm -fr ' .. lsub .. '/' .. k .. '/merged/dists/' .. n .. '/main 2>/dev/null;' ..
				'rm -fr ' .. lsub .. '/' .. k .. '/merged/dists/' .. n .. '/non-free 2>/dev/null')
		end
		os.execute('rm -fr ' .. lsub .. '/' .. k .. '/merged/pool')
	    end

	    local results = {}
	    local f = l:sub(1, -8) .. "/" .. k .. ".lua"
	    if APT.checkFile(f) then
		results = loadfile(f)()
		results['IPs'] = v.IPs
		local rfile, e = io.open(f, "w+")
		if nil == rfile then C("opening results file - " .. e) else
		    rfile:write(APT.dumpTable(results, "results") .. "\nreturn results\n")
		    rfile:close()
		end
	    end

	end
    end

    local fls = io.popen('ls -1 ' .. lsub .. '/*.lua 2>/dev/null')
    for k in fls:lines() do
	fixResults(k)
    end
end


if -1 < APT.verbosity then print('\nDeleting old RRD data.') end
os.execute('rm -fr rrd')
if -1 < APT.verbosity then print('\nRecreating RRD data.') end
local files = io.popen('ls -d1 results_*')
for l in files:lines() do
    if ('results_old' ~= l) and ('.tar.xz' ~= l:sub(25, -1)) then
	if APT.checkFile(l .. '/stamp') then
--	    local status, whn = APT.execute('TZ="GMT" ls -d1 --time-style="+%s" ' .. l .. ' | cut -d " " -f 6-6')
	    local whn = APT.exe('TZ="GMT" ls -d1 --time-style="+%s" ' .. l .. ' | cut -d " " -f 6-6'):Do()
	    whn = whn:sub(2, -2)
--	    local status, new = APT.execute('TZ="GMT" date -d "' .. whn:sub(9, 18) .. ' ' .. whn:sub(20, 21) .. ':' .. whn:sub(23, 24) .. '" "+%s"')
	    local new = APT.exe('TZ="GMT" date -d "' .. whn:sub(9, 18) .. ' ' .. whn:sub(20, 21) .. ':' .. whn:sub(23, 24) .. '" "+%s"'):Do()
	    APT.now = tonumber("0" .. new:sub(2, -2))
	    if 0 ~= APT.now then
		if APT.checkFile(l .. '/mirrors.lua') then
		if -1 < APT.verbosity then io.stdout:write('\r' .. l .. ' -> ' .. APT.now);  io.stdout:flush() end
		    APT.mirrors = loadfile(l .. "/mirrors.lua")()
		    for k, v in pairs(APT.mirrors) do
			APT.doRRD(l, k, v, oldest)
		    end
		else
		    if -1 < APT.verbosity then io.stdout:write('\r' .. l);  io.stdout:flush() end
		end
	    else
		if -1 < APT.verbosity then io.stdout:write('\r' .. l);  io.stdout:flush() end
	    end
	else
	    if -1 < APT.verbosity then io.stdout:write('\r' .. l);  io.stdout:flush() end
	end
    end
end


os.execute('rm ' .. lock)


if -1 < APT.verbosity then print('\nRecreating tarballs.') end

local files = io.popen('ls -1 results_*/results_*.tar.xz 2>/dev/null')
for l in files:lines() do
    if -1 < APT.verbosity then io.stdout:write('\r' .. 'tar -c --xz  ' .. l:sub(20, 43) .. ' -f ' .. l);  io.stdout:flush() end
    os.execute('mkdir -p ' .. l:sub(1, 18) .. '; tar -c --xz  ' .. l:sub(20, 43) .. ' -f ' .. l .. '; rm -fr ' .. l:sub(20, 43))
end
local files = io.popen('ls -1 results_*.tar.xz 2>/dev/null')
for l in files:lines() do
    if -1 < APT.verbosity then io.stdout:write('\r' .. 'tar -c --xz  ' .. l:sub(1, 24) .. ' -f ' .. l:sub(1, 18) .. '/' .. l);  io.stdout:flush() end
    os.execute('mkdir -p ' .. l:sub(1, 18) .. '; tar -c --xz  ' .. l:sub(1, 24) .. ' -f ' .. l:sub(1, 18) .. '/' .. l .. '; rm -fr ' .. l:sub(1, 24) .. '; rm -f ' .. l)
end


if -1 < APT.verbosity then print('\nFinished updating data.') end