diff options
Diffstat (limited to 'apt-panopticon-report-web.lua')
-rwxr-xr-x | apt-panopticon-report-web.lua | 406 |
1 files changed, 0 insertions, 406 deletions
diff --git a/apt-panopticon-report-web.lua b/apt-panopticon-report-web.lua deleted file mode 100755 index fb41341..0000000 --- a/apt-panopticon-report-web.lua +++ /dev/null | |||
@@ -1,406 +0,0 @@ | |||
1 | #!/usr/bin/env luajit | ||
2 | |||
3 | local args = {...} | ||
4 | |||
5 | verbosity = -1 | ||
6 | local logFile | ||
7 | |||
8 | |||
9 | --[[ Ordered table iterator, allow to iterate on the natural order of the keys of a table. | ||
10 | From http://lua-users.org/wiki/SortedIteration | ||
11 | ]] | ||
12 | function __genOrderedIndex( t ) | ||
13 | local orderedIndex = {} | ||
14 | for key in pairs(t) do | ||
15 | table.insert( orderedIndex, key ) | ||
16 | end | ||
17 | table.sort( orderedIndex ) | ||
18 | return orderedIndex | ||
19 | end | ||
20 | function orderedNext(t, state) | ||
21 | -- Equivalent of the next function, but returns the keys in the alphabetic | ||
22 | -- order. We use a temporary ordered key table that is stored in the | ||
23 | -- table being iterated. | ||
24 | |||
25 | local key = nil | ||
26 | --print("orderedNext: state = "..tostring(state) ) | ||
27 | if state == nil then | ||
28 | -- the first time, generate the index | ||
29 | t.__orderedIndex = __genOrderedIndex( t ) | ||
30 | key = t.__orderedIndex[1] | ||
31 | else | ||
32 | -- fetch the next value | ||
33 | for i = 1,table.getn(t.__orderedIndex) do | ||
34 | if t.__orderedIndex[i] == state then | ||
35 | key = t.__orderedIndex[i+1] | ||
36 | end | ||
37 | end | ||
38 | end | ||
39 | |||
40 | if key then | ||
41 | return key, t[key] | ||
42 | end | ||
43 | |||
44 | -- no more value to return, cleanup | ||
45 | t.__orderedIndex = nil | ||
46 | return | ||
47 | end | ||
48 | function orderedPairs(t) | ||
49 | -- Equivalent of the pairs() function on tables. Allows to iterate | ||
50 | -- in order | ||
51 | return orderedNext, t, nil | ||
52 | end | ||
53 | |||
54 | -- Use this to dump a table to a string, with HTML. | ||
55 | dumpTableHTML = function (table, space, name) | ||
56 | local r = name .. "\n" | ||
57 | r = r .. dumpTableHTMLSub(table, space .. " ") | ||
58 | r = r .. space .. "" | ||
59 | return r | ||
60 | end | ||
61 | dumpTableHTMLSub = function (table, space) | ||
62 | local r = "" | ||
63 | for k, v in orderedPairs(table) do | ||
64 | if type(v) == "table" then | ||
65 | if " " == space then | ||
66 | r = r .. space .. dumpTableHTML(v, space, k .. "<ul>") .. "</ul>\n" | ||
67 | else | ||
68 | r = r .. "<li>" .. space .. dumpTableHTML(v, space, k .. "<ul>") .. "</ul></li>\n" | ||
69 | end | ||
70 | else | ||
71 | r = r .. space .. "<li>" .. k .. "</li>\n" | ||
72 | end | ||
73 | end | ||
74 | return r | ||
75 | end | ||
76 | |||
77 | local plurals = function(e, w) | ||
78 | local result = "" | ||
79 | if 1 == e then | ||
80 | result = e .. " <font color='red'><b>error</b></font>" | ||
81 | elseif e ~= 0 then | ||
82 | result = e .. " <font color='red'><b>errors</b></font>" | ||
83 | end | ||
84 | if 0 < w then | ||
85 | if 0 < e then result = result .. ", " end | ||
86 | if 1 == w then | ||
87 | result = result .. w .. " <font color='yellow'><b>warning</b></font>" | ||
88 | else | ||
89 | result = result .. w .. " <font color='yellow'><b>warnings</b></font>" | ||
90 | end | ||
91 | end | ||
92 | if "" ~= result then result = " (" .. result .. ")" end | ||
93 | return result | ||
94 | end | ||
95 | |||
96 | local results = {} | ||
97 | |||
98 | local log = function(v, t, s, prot, test, host) | ||
99 | local x = "" | ||
100 | if nil == prot then prot = "" end | ||
101 | if nil ~= test then x = x .. test else test = "" end | ||
102 | if nil ~= host then | ||
103 | if #x > 0 then x = x .. " " end | ||
104 | x = x .. host | ||
105 | end | ||
106 | if #x > 0 then | ||
107 | t = t .. "(" .. x .. ")" | ||
108 | end | ||
109 | if v <= verbosity then | ||
110 | if 3 <= verbosity then t = os.date() .. " " .. t end | ||
111 | print(t .. ": " .. s) | ||
112 | end | ||
113 | if nil ~= logFile then | ||
114 | logFile:write(os.date() .. " " .. t .. ": " .. s .. "\n") | ||
115 | logFile:flush() | ||
116 | end | ||
117 | end | ||
118 | local D = function(s) log(3, "DEBUG ", s) end | ||
119 | local I = function(s) log(2, "INFO ", s) end | ||
120 | local W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end | ||
121 | local E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end | ||
122 | local C = function(s) log(-1, "CRITICAL", s) end | ||
123 | |||
124 | local mirrors = loadfile("results/mirrors.lua")() | ||
125 | |||
126 | local revDNS = function(dom, IP) | ||
127 | if "deb.devuan.org" ~= dom then | ||
128 | if nil ~= mirrors["deb.devuan.org"] then | ||
129 | if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][IP] then | ||
130 | return "<font color='purple'><b>DNS-RR</b></font>" | ||
131 | end | ||
132 | end | ||
133 | else | ||
134 | for k, v in pairs(mirrors) do | ||
135 | if "deb.devuan.org" ~= k then | ||
136 | local IPs = v.IPs | ||
137 | for i, u in pairs(IPs) do | ||
138 | if "table" == type(u) then | ||
139 | for h, t in pairs(u) do | ||
140 | if IP == h then return k end | ||
141 | end | ||
142 | else | ||
143 | if IP == i then return k end | ||
144 | end | ||
145 | end | ||
146 | end | ||
147 | end | ||
148 | end | ||
149 | return "" | ||
150 | end | ||
151 | |||
152 | local faulty = "" | ||
153 | local status = function(host, results, typ) | ||
154 | local result = "" | ||
155 | local e = 0 | ||
156 | local w = 0 | ||
157 | local s = nil ~= mirrors[host].Protocols[typ] | ||
158 | local to = false | ||
159 | if ('http' ~= typ) and ('https' ~= typ) and ('ftp' ~= typ) and ('rsync' ~= typ) then s = true end | ||
160 | if nil ~= results[typ] then | ||
161 | e = results[typ].errors | ||
162 | w = results[typ].warnings | ||
163 | --[[ | ||
164 | for k, v in pairs(results[typ]) do | ||
165 | if "table" == type(v) then | ||
166 | if 0 <= v.errors then e = e + v.errors else to = true end | ||
167 | if 0 <= v.warnings then w = w + v.warnings else to = true end | ||
168 | end | ||
169 | end | ||
170 | ]] | ||
171 | else | ||
172 | for k, v in pairs(results) do | ||
173 | if "table" == type(v) then | ||
174 | for i, u in pairs(v) do | ||
175 | if "table" == type(u) then | ||
176 | if typ == i then | ||
177 | if 0 <= u.errors then e = e + u.errors else to = true end | ||
178 | if 0 <= u.warnings then w = w + u.warnings else to = true end | ||
179 | end | ||
180 | end | ||
181 | end | ||
182 | end | ||
183 | end | ||
184 | end | ||
185 | |||
186 | if to then | ||
187 | if s then | ||
188 | result = "[<font color='blue'><b>TIMEOUT</b></font>]" | ||
189 | else | ||
190 | result = "[<font color='darkblue'><b>TIMEOUT *</b></font>]" | ||
191 | end | ||
192 | elseif 0 < e then | ||
193 | if s then | ||
194 | result = "[<font color='red'><b>FAILED</b></font>]" | ||
195 | else | ||
196 | result = "[<font color='darkred'><b>FAILED *</b></font>]" | ||
197 | end | ||
198 | faulty = faulty .. host .. " (" .. typ .. ")<br>\n" | ||
199 | else | ||
200 | if s then | ||
201 | result = "[<font color='lime'><b>OK</b></font>]" | ||
202 | else | ||
203 | result = "[<font color='darkgreen'><b>OK *</b></font>]" | ||
204 | end | ||
205 | end | ||
206 | return result .. plurals(e, w) | ||
207 | end | ||
208 | |||
209 | local collate = function(host, ip, results) | ||
210 | local f = "results/" .. host .. "_" .. ip .. ".lua" | ||
211 | local rfile, e = io.open(f, "r") | ||
212 | if nil == rfile then I("opening " .. f .. " file - " .. e) else | ||
213 | rfile:close() | ||
214 | local rs = loadfile(f)() | ||
215 | for k, v in pairs(rs) do | ||
216 | if "table" == type(v) then | ||
217 | for i, u in pairs(v) do | ||
218 | if "table" == type(u) then | ||
219 | for h, t in pairs(u) do | ||
220 | local a = results[k][h] | ||
221 | if nil == a then a = 0 end | ||
222 | results[k][h] = a + t | ||
223 | end | ||
224 | else | ||
225 | local a = results[k][i] | ||
226 | if nil == a then a = 0 end | ||
227 | results[k][i] = a + u | ||
228 | end | ||
229 | end | ||
230 | else | ||
231 | local a = results[k] | ||
232 | if nil == a then a = 0 end | ||
233 | results[k] = a + v | ||
234 | end | ||
235 | end | ||
236 | end | ||
237 | return results | ||
238 | end | ||
239 | |||
240 | local m = {} | ||
241 | |||
242 | local logCount = function(domain, ip) | ||
243 | local nm = "LOG_" .. domain | ||
244 | local log = "" | ||
245 | local extra = "" | ||
246 | if nil ~= ip then nm = nm .. "_" .. ip end | ||
247 | nm = nm .. ".html" | ||
248 | local rfile, e = io.open("results/" .. nm, "r") | ||
249 | if nil ~= rfile then | ||
250 | local errors = 0 | ||
251 | local warnings = 0 | ||
252 | for l in rfile:lines() do | ||
253 | if nil ~= l:match("><b>ERROR ") then errors = errors + 1 end | ||
254 | if nil ~= l:match("><b>WARNING ") then warnings = warnings + 1 end | ||
255 | end | ||
256 | rfile:close() | ||
257 | if nil == ip then | ||
258 | log = "<a href='" .. nm .. "'>" .. domain .. "</a>" | ||
259 | else | ||
260 | log = "<a href='" .. nm .. "'>" .. ip .. "</a>" | ||
261 | end | ||
262 | log = log .. plurals(errors, warnings) | ||
263 | end | ||
264 | return log | ||
265 | end | ||
266 | |||
267 | local file, e = io.open("results/Report-web.html", "w+") | ||
268 | if nil == file then C("opening mirrors file - " .. e) else | ||
269 | file:write( "<html><head><title>apt-panopticon results</title>\n" .. | ||
270 | '</head><body bgcolor="black" text="white">' .. | ||
271 | "<h1>Welcome to the apt-panopticon results page.</h1>\n" .. | ||
272 | "<p>This is the status of the mirror servers in the Devuan package mirror network.</p>\n" .. | ||
273 | "<p>The full list of Devuan package mirrors is available at the URL: " .. | ||
274 | "<a href='https://pkgmaster.devuan.org/mirror_list.txt'>https://pkgmaster.devuan.org/mirror_list.txt</a></p>\n" .. | ||
275 | "<p>Due to the nature of the tests, some errors or warnings will be counted several times. " .. | ||
276 | "The links in the table and DNS list go to the detailed testing logs.</p>\n\n" .. | ||
277 | "<hr>\n<h2>==== package mirror status " .. os.date("!%Y-%m-%d %H:%M") .. " GMT: ====</h2>\n<table>\n" .. | ||
278 | |||
279 | "<p>[<font color='red'><b>FAILED</b></font>] or [<font color='lime'><b>OK</b></font>]" .. | ||
280 | " means the tested thing is supported for that mirror.</p>\n" .. | ||
281 | "<p>[<font color='darkred'><b>FAILED *</b></font>] or [<font color='darkgreen'><b>OK *</b></font>]" .. | ||
282 | " means the tested thing is unsupported for that mirror, but might have been tested anyway.</p>\n" .. | ||
283 | "<p>[<font color='blue'><b>TIMEOUT</b></font>] or [<font color='darkblue'><b>TIMEOUT</b></font>]" .. | ||
284 | " means the server had too many timeouts, and tests where aborted, so there is no result for this test.</p>" .. | ||
285 | "<p>The DNS round robin (DNS-RR) column shows the IPs for that mirror, or [<font color='grey'><b>no</b></font>] if it isn't part of the DNS-RR. " .. | ||
286 | "The IPs link to the testing log for that IP accessed via the DNS-RR. " .. | ||
287 | "deb.devuan.org is the DNS-RR itself, so it doesn't get tested directly.</p>\n" .. | ||
288 | "<p>The time in the Updated column is how often the mirror updates itself.</p>" .. | ||
289 | "<p>Mirrors with a <font style='background-color:dimgrey'>grey background</font> are not active (though may be usable as part of the DNS-RR).</p>\n" .. | ||
290 | "<p>[<font color='grey'><b>skip</b></font>] means that the test hasn't been written yet.</p>\n" .. | ||
291 | "<tr><th></th><th>FTP</th><th>HTTP</th><th>HTTPS</th><th>RSYNC</th><th>DNS round robin</th><th>Protocol</th><th>URL sanity</th><th>Integrity</th><th>Updated</th></tr>\n" | ||
292 | ) | ||
293 | for k, v in orderedPairs(mirrors) do | ||
294 | local results = loadfile("results/" .. k .. ".lua")() | ||
295 | local active = "" | ||
296 | if "yes" == v.Active then | ||
297 | file:write(" <tr><th>" .. k .. "</th> ") | ||
298 | else | ||
299 | if nil == v.Active then active = 'nil' else active = v.Active end | ||
300 | file:write(" <tr style='background-color:dimgrey'><th>" .. k .. "</th> ") | ||
301 | end | ||
302 | local IPs = v.IPs | ||
303 | for i, u in pairs(IPs) do | ||
304 | if "table" == type(u) then | ||
305 | for h, t in pairs(u) do | ||
306 | results = collate(k, h, results) | ||
307 | end | ||
308 | else | ||
309 | results = collate(k, i, results) | ||
310 | end | ||
311 | end | ||
312 | local ftp = "[<font color='grey'><b>skip</b></font>]" | ||
313 | local http = status(k, results, "http") | ||
314 | local https = status(k, results, "https") | ||
315 | local rsync = "[<font color='grey'><b>skip</b></font>]" | ||
316 | local dns = "" | ||
317 | local protocol = status(k, results, "Protocol") | ||
318 | local sanity = "[<font color='grey'><b>skip</b></font>]" | ||
319 | local integrity = status(k, results, "Integrity") | ||
320 | local updated = status(k, results, "Updated") | ||
321 | local rate = v.Rate | ||
322 | if nil ~= rate then updated = updated .. ' ' .. rate end | ||
323 | |||
324 | -- DNS-RR test. | ||
325 | if ("deb.devuan.org" ~= k) and (nil ~= mirrors["deb.devuan.org"]) then | ||
326 | for l, w in pairs(mirrors[k].IPs) do | ||
327 | if type(w) == "table" then | ||
328 | for i, u in pairs(w) do | ||
329 | if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then | ||
330 | local log = logCount("deb.devuan.org", i) | ||
331 | if "" ~= log then | ||
332 | if "" == dns then dns = " " else dns = dns .. " " end | ||
333 | dns = dns .. logCount("deb.devuan.org", i) | ||
334 | else | ||
335 | if "" == dns then dns = " " else dns = dns .. " " end | ||
336 | dns = dns .. "<font color='maroon'><b>" .. i .. "</b></font>" | ||
337 | end | ||
338 | end | ||
339 | end | ||
340 | else | ||
341 | if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then | ||
342 | local log = logCount("deb.devuan.org", l) | ||
343 | if "" ~= log then | ||
344 | if "" == dns then dns = " " else dns = dns .. " " end | ||
345 | dns = dns .. log | ||
346 | else | ||
347 | if "" == dns then dns = " " else dns = dns .. " " end | ||
348 | dns = dns .. "<font color='maroon'><b>" .. l .. "</b></font>" | ||
349 | end | ||
350 | end | ||
351 | end | ||
352 | end | ||
353 | if "" == dns then dns = "[<font color='grey'><b>no</b></font>]" end | ||
354 | end | ||
355 | |||
356 | file:write("<td>" .. ftp .. " </td><td>" .. http .. " </td><td>" .. https .. " </td><td>" .. rsync .. " </td><td>" .. dns .. | ||
357 | " </td><td>" .. protocol .. " </td><td>" .. sanity .. | ||
358 | " </td><td>" .. integrity .. " </td><td>" .. updated .. "</td></tr>\n") | ||
359 | if "" ~= v.Active then | ||
360 | file:write("<tr><td style='background-color:dimgrey'>" .. active .. "</td></tr>\n") | ||
361 | end | ||
362 | end | ||
363 | file:write( "</table>\n<br>\n<h2>==== faulty mirrors: ====</h2>\n" .. faulty) | ||
364 | file:write( "<br>\n<br>\n<h2>==== DNS and logs: ====</h2>\n") | ||
365 | |||
366 | for k, v in pairs(mirrors) do | ||
367 | local log = k | ||
368 | local n = {} | ||
369 | log = logCount(k) | ||
370 | mirrors[k].Protocols = nil | ||
371 | mirrors[k].FQDN = nil | ||
372 | mirrors[k].Active = nil | ||
373 | mirrors[k].Rate = nil | ||
374 | mirrors[k].BaseURL = nil | ||
375 | mirrors[k].Country = nil | ||
376 | mirrors[k].Bandwidth = nil | ||
377 | for l, w in pairs(mirrors[k].IPs) do | ||
378 | if type(w) == "table" then | ||
379 | n[l] = {} | ||
380 | for i, u in pairs(w) do | ||
381 | local log = logCount(k, i) | ||
382 | if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(k, i)] = u end | ||
383 | end | ||
384 | else | ||
385 | local log = logCount(k, l) | ||
386 | if "" == log then n[l] = w else n[log .. " " .. revDNS(k, l)] = w end | ||
387 | end | ||
388 | end | ||
389 | m[log .. " DNS entries -"] = n | ||
390 | end | ||
391 | file:write( "<p>This lists each mirror, and the DNS entries for that mirror. " .. | ||
392 | "The links point to the testing log files for " .. logCount("apt-panopticon") .. " for each domain name / IP combination that was tested. " .. | ||
393 | "If a mirror has a CNAME, that CNAME is listed along with that CNAMEs DNS entries. " .. | ||
394 | "deb.devuan.org is the DNS round robin, which points to the mirrors that are part of the DNS-RR. " .. | ||
395 | "If an IP is part of the DNS-RR, it is marked with '<font color='purple'><b>DNS-RR</b></font>' " .. | ||
396 | "pkgmaster.devuan.org is the master mirror, all the others sync to it. " .. | ||
397 | "</p>\n" | ||
398 | ) | ||
399 | file:write(dumpTableHTML(m, "", "")) | ||
400 | file:write( "\n<br>\n<hr>\n\n" .. | ||
401 | "<p>The <a href='Report-email.txt'>email report</a>. " .. | ||
402 | "All <a href='../results'>the logs and other output</a>. " .. | ||
403 | "You can get the <a href='https://sledjhamr.org/cgit/apt-panopticon/about/'>source code here</a>.</p>" .. | ||
404 | "</body></html>\n") | ||
405 | file:close() | ||
406 | end | ||