aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
authoronefang2019-12-23 13:01:05 +1000
committeronefang2019-12-23 13:01:05 +1000
commit5e8b0adf2a39debac8354fd98ca7d5b729580ff7 (patch)
tree73b9c3666a89bbf52adb714c712209a09d48deb0
parentMove the declaration of the APT.tests table to where APT.protocols is. (diff)
downloadapt-panopticon-5e8b0adf2a39debac8354fd98ca7d5b729580ff7.zip
apt-panopticon-5e8b0adf2a39debac8354fd98ca7d5b729580ff7.tar.gz
apt-panopticon-5e8b0adf2a39debac8354fd98ca7d5b729580ff7.tar.bz2
apt-panopticon-5e8b0adf2a39debac8354fd98ca7d5b729580ff7.tar.xz
Major refactor, especially of the downloading and processing code.
Make the code more readable, less scattered. Use a coroutine to multitask better. Plugable functions for parsing the download results, and figuring out what to download next. Track timeouts at a finer level. Dig for IPs in the forked apt-panopticons, not all at once at the beginning. Various cleanups and tweaks.
Diffstat (limited to '')
-rw-r--r--apt-panopticommon.lua56
-rwxr-xr-xapt-panopticon-report-RRD.lua1
-rwxr-xr-xapt-panopticon-report-email-web.lua31
-rwxr-xr-xapt-panopticon-update-data.lua2
-rwxr-xr-xapt-panopticon.lua680
5 files changed, 471 insertions, 299 deletions
diff --git a/apt-panopticommon.lua b/apt-panopticommon.lua
index 8d6de05..4b0be8b 100644
--- a/apt-panopticommon.lua
+++ b/apt-panopticommon.lua
@@ -53,7 +53,19 @@ APT.options =
53 { 53 {
54 typ = "number", 54 typ = "number",
55 help = "", 55 help = "",
56 value = 15, 56 value = 5,
57 },
58 timeouts =
59 {
60 typ = "number",
61 help = "",
62 value = 3,
63 },
64 retries =
65 {
66 typ = "number",
67 help = "",
68 value = 3,
57 }, 69 },
58 reports = 70 reports =
59 { 71 {
@@ -290,14 +302,16 @@ local log = function(v, t, s, prot, test, host)
290 t = t .. "(" .. x .. ")" 302 t = t .. "(" .. x .. ")"
291 if "" ~= prot then 303 if "" ~= prot then
292 if "" == test then 304 if "" == test then
293 if nil == APT.results[prot] then APT.results[prot] = {errors = 0; warnings = 0} end 305 if nil == APT.results[prot] then APT.results[prot] = {errors = 0; warnings = 0; timeouts = 0} end
294 if v == 0 then APT.results[prot].errors = APT.results[prot].errors + 1 end 306 if v == 0 then APT.results[prot].errors = APT.results[prot].errors + 1 end
295 if v == 1 then APT.results[prot].warnings = APT.results[prot].warnings + 1 end 307 if v == 1 then APT.results[prot].warnings = APT.results[prot].warnings + 1 end
308 if v == 2 then APT.results[prot].timeouts = APT.results[prot].timeouts + 1 end
296 else 309 else
297 if nil == APT.results[prot] then APT.results[prot] = {errors = 0; warnings = 0} end 310 if nil == APT.results[prot] then APT.results[prot] = {errors = 0; warnings = 0; timeouts = 0} end
298 if nil == APT.results[prot][test] then APT.results[prot][test] = {errors = 0; warnings = 0} end 311 if nil == APT.results[prot][test] then APT.results[prot][test] = {errors = 0; warnings = 0; timeouts = 0} end
299 if v == 0 then APT.results[prot][test].errors = APT.results[prot][test].errors + 1 end 312 if v == 0 then APT.results[prot][test].errors = APT.results[prot][test].errors + 1 end
300 if v == 1 then APT.results[prot][test].warnings = APT.results[prot][test].warnings + 1 end 313 if v == 1 then APT.results[prot][test].warnings = APT.results[prot][test].warnings + 1 end
314 if v == 2 then APT.results[prot][test].timeouts = APT.results[prot][test].timeouts + 1 end
301 end 315 end
302 end 316 end
303 end 317 end
@@ -311,8 +325,9 @@ local log = function(v, t, s, prot, test, host)
311 if -1 == v then colour = "fuchsia" end -- CRITICAL 325 if -1 == v then colour = "fuchsia" end -- CRITICAL
312 if 0 == v then colour = "red " end -- ERROR 326 if 0 == v then colour = "red " end -- ERROR
313 if 1 == v then colour = "yellow " end -- WARNING 327 if 1 == v then colour = "yellow " end -- WARNING
314 if 2 == v then colour = "white " end -- INFO 328 if 2 == v then colour = "blue " end -- TIMEOUT
315 if 3 == v then colour = "gray " end -- DEBUG 329 if 3 == v then colour = "white " end -- INFO
330 if 4 == v then colour = "gray " end -- DEBUG
316 APT.logFile:write(os.date() .. " <font color='" .. colour .. "'><b>" .. t .. "</b></font>: " .. s .. "</br>\n") 331 APT.logFile:write(os.date() .. " <font color='" .. colour .. "'><b>" .. t .. "</b></font>: " .. s .. "</br>\n")
317 else 332 else
318 APT.logFile:write(os.date() .. " " .. t .. ": " .. s .. "\n") 333 APT.logFile:write(os.date() .. " " .. t .. ": " .. s .. "\n")
@@ -320,13 +335,15 @@ local log = function(v, t, s, prot, test, host)
320 APT.logFile:flush() 335 APT.logFile:flush()
321 end 336 end
322end 337end
323APT.D = function(s) log(3, "DEBUG ", s) end 338APT.D = function(s) log(4, "DEBUG ", s) end
324APT.I = function(s) log(2, "INFO ", s) end 339APT.I = function(s) log(3, "INFO ", s) end
340APT.T = function(s, p, t, h) log(2, "TIMEOUT ", s, p, t, h) end
325APT.W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end 341APT.W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end
326APT.E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end 342APT.E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end
327APT.C = function(s) log(-1, "CRITICAL", s) end 343APT.C = function(s) log(-1, "CRITICAL", s) end
328local D = APT.D 344local D = APT.D
329local I = APT.I 345local I = APT.I
346local T = APT.T
330local W = APT.W 347local W = APT.W
331local E = APT.E 348local E = APT.E
332local C = APT.C 349local C = APT.C
@@ -380,7 +397,7 @@ APT.checkFile = function(f)
380 if nil == h then return false else h:close(); return true end 397 if nil == h then return false else h:close(); return true end
381end 398end
382 399
383APT.plurals = function(e, w) 400APT.plurals = function(e, w, t)
384 local result = "" 401 local result = ""
385 if 1 == e then 402 if 1 == e then
386 result = e .. " error" 403 result = e .. " error"
@@ -397,18 +414,29 @@ APT.plurals = function(e, w)
397 end 414 end
398 if ("" ~= result) and APT.html then result = "<font color='yellow'><b>" .. result .. "</b></font>" end 415 if ("" ~= result) and APT.html then result = "<font color='yellow'><b>" .. result .. "</b></font>" end
399 end 416 end
417 if 0 < t then
418 if (0 < e) or (0 < w) then result = result .. ", " end
419 if 1 == t then
420 result = result .. t .. " timeout"
421 else
422 result = result .. t .. " timeouts"
423 end
424 if ("" ~= result) and APT.html then result = "<font color='blue'><b>" .. result .. "</b></font>" end
425 end
400 if "" ~= result then result = " (" .. result .. ")" end 426 if "" ~= result then result = " (" .. result .. ")" end
401 return result 427 return result
402end 428end
403 429
404APT.padResults = function(results) 430APT.padResults = function(results)
431 local c = 0
432 if nil == results then results = {}; c = 999 end
405 for k, v in pairs(APT.protocols) do 433 for k, v in pairs(APT.protocols) do
406 tests = results[v] 434 tests = results[v]
407 if nil == tests then tests = {errors = 0; warnings = 0} end 435 if nil == tests then tests = {errors = c; warnings = c; timeouts = c} end
408 if nil == tests.Integrity then tests.Integrity = {errors = 0; warnings = 0} end 436 if nil == tests.Integrity then tests.Integrity = {errors = c; warnings = c; timeouts = c} end
409 if nil == tests.Protocol then tests.Protocol = {errors = 0; warnings = 0} end 437 if nil == tests.Protocol then tests.Protocol = {errors = c; warnings = c; timeouts = c} end
410 if nil == tests.Updated then tests.Updated = {errors = 0; warnings = 0} end 438 if nil == tests.Updated then tests.Updated = {errors = c; warnings = c; timeouts = c} end
411 if nil == tests.URLSanity then tests.URLSanity = {errors = 0; warnings = 0} end 439 if nil == tests.URLSanity then tests.URLSanity = {errors = c; warnings = c; timeouts = c} end
412 results[v] = tests 440 results[v] = tests
413 end 441 end
414 if nil == results.timeout then results.timeout = false end 442 if nil == results.timeout then results.timeout = false end
diff --git a/apt-panopticon-report-RRD.lua b/apt-panopticon-report-RRD.lua
index dfe21d2..e703659 100755
--- a/apt-panopticon-report-RRD.lua
+++ b/apt-panopticon-report-RRD.lua
@@ -3,6 +3,7 @@
3local APT = require 'apt-panopticommon' 3local APT = require 'apt-panopticommon'
4local D = APT.D 4local D = APT.D
5local I = APT.I 5local I = APT.I
6local T = APT.T
6local W = APT.W 7local W = APT.W
7local E = APT.E 8local E = APT.E
8local C = APT.C 9local C = APT.C
diff --git a/apt-panopticon-report-email-web.lua b/apt-panopticon-report-email-web.lua
index 8a5404a..c52b6c0 100755
--- a/apt-panopticon-report-email-web.lua
+++ b/apt-panopticon-report-email-web.lua
@@ -3,6 +3,7 @@
3local APT = require 'apt-panopticommon' 3local APT = require 'apt-panopticommon'
4local D = APT.D 4local D = APT.D
5local I = APT.I 5local I = APT.I
6local T = APT.T
6local W = APT.W 7local W = APT.W
7local E = APT.E 8local E = APT.E
8local C = APT.C 9local C = APT.C
@@ -47,16 +48,19 @@ local status = function(host, results, typ)
47 local result = "" 48 local result = ""
48 local e = 0 49 local e = 0
49 local w = 0 50 local w = 0
51 local t = 0
50 local s = nil ~= APT.mirrors[host].Protocols[typ] 52 local s = nil ~= APT.mirrors[host].Protocols[typ]
51 local to = results.timeout 53 local to = results.timeout
52 if not APT.search(APT.protocols, typ) then s = true end 54 if not APT.search(APT.protocols, typ) then s = true end
53 if nil ~= results[typ] then 55 if nil ~= results[typ] then
54 e = results[typ].errors 56 e = results[typ].errors
55 w = results[typ].warnings 57 w = results[typ].warnings
58 t = results[typ].timeouts
56 for k, v in pairs(results[typ]) do 59 for k, v in pairs(results[typ]) do
57 if "table" == type(v) then 60 if "table" == type(v) then
58 if 0 <= v.errors then e = e + v.errors else to = true end 61 if 0 <= v.errors then e = e + v.errors else to = true end
59 if 0 <= v.warnings then w = w + v.warnings else to = true end 62 if 0 <= v.warnings then w = w + v.warnings else to = true end
63 if 0 <= v.timeouts then t = t + v.timeouts else to = true end
60 end 64 end
61 end 65 end
62 else 66 else
@@ -67,6 +71,7 @@ local status = function(host, results, typ)
67 if typ == i then 71 if typ == i then
68 if 0 <= u.errors then e = e + u.errors end 72 if 0 <= u.errors then e = e + u.errors end
69 if 0 <= u.warnings then w = w + u.warnings end 73 if 0 <= u.warnings then w = w + u.warnings end
74 if 0 <= u.timeouts then t = t + u.timeouts end
70 end 75 end
71 end 76 end
72 end 77 end
@@ -99,6 +104,21 @@ local status = function(host, results, typ)
99 else 104 else
100 faulty = faulty .. host .. " (" .. typ .. ")\n" 105 faulty = faulty .. host .. " (" .. typ .. ")\n"
101 end 106 end
107 elseif 0 < t then
108 result = "[timeout"
109 if not s then result = result .. "*" end
110 if APT.html then
111 if s then
112 result = "[<font color='blue'><b>timeout</b></font>"
113 else
114 result = "[<font color='darkblue'><b>timeout*</b></font>"
115 end
116 end
117 if APT.html then
118 faulty = faulty .. host .. " (" .. typ .. ")<br>\n"
119 else
120 faulty = faulty .. host .. " (" .. typ .. ")\n"
121 end
102 else 122 else
103 result = "[OK" 123 result = "[OK"
104 if not s then result = result .. "*" end 124 if not s then result = result .. "*" end
@@ -110,7 +130,7 @@ local status = function(host, results, typ)
110 end 130 end
111 end 131 end
112 end 132 end
113 return result .. APT.plurals(e, w) .. "]" 133 return result .. APT.plurals(e, w, t) .. "]"
114end 134end
115 135
116local m = {} 136local m = {}
@@ -125,9 +145,11 @@ local logCount = function(domain, ip)
125 if nil ~= rfile then 145 if nil ~= rfile then
126 local errors = 0 146 local errors = 0
127 local warnings = 0 147 local warnings = 0
148 local timeouts = 0
128 for l in rfile:lines() do 149 for l in rfile:lines() do
129 if nil ~= l:match("><b>ERROR ") then errors = errors + 1 end 150 if nil ~= l:match("><b>ERROR ") then errors = errors + 1 end
130 if nil ~= l:match("><b>WARNING ") then warnings = warnings + 1 end 151 if nil ~= l:match("><b>WARNING ") then warnings = warnings + 1 end
152 if nil ~= l:match("><b>TIMEOUT ") then timeouts = timeouts + 1 end
131 end 153 end
132 rfile:close() 154 rfile:close()
133 if APT.html then 155 if APT.html then
@@ -137,7 +159,7 @@ local logCount = function(domain, ip)
137 log = "<a href='" .. nm .. "'>" .. ip .. "</a>" 159 log = "<a href='" .. nm .. "'>" .. ip .. "</a>"
138 end 160 end
139 end 161 end
140 log = log .. APT.plurals(errors, warnings) 162 log = log .. APT.plurals(errors, warnings, timeouts)
141 end 163 end
142 return log 164 return log
143end 165end
@@ -303,8 +325,11 @@ if nil == web then C("opening mirrors file - " .. e) else
303 " means the tested thing is supported for that mirror.</p>\n" .. 325 " means the tested thing is supported for that mirror.</p>\n" ..
304 "<p>[<font color='darkred'><b>FAILED*</b></font>] or [<font color='darkgreen'><b>OK*</b></font>]" .. 326 "<p>[<font color='darkred'><b>FAILED*</b></font>] or [<font color='darkgreen'><b>OK*</b></font>]" ..
305 " means the tested thing is unsupported for that mirror, but might have been tested anyway.</p>\n" .. 327 " means the tested thing is unsupported for that mirror, but might have been tested anyway.</p>\n" ..
328 "<p>[<font color='blue'><b>timeout</b></font>] or [<font color='darkblue'><b>timeout</b></font>]" ..
329 " means the mirror had some timeouts, and tests where not yet aborted. The darker colour means unsupported by the mirror, but tested anyway.</p>" ..
306 "<p>[<font color='blue'><b>TIMEOUT</b></font>] or [<font color='darkblue'><b>TIMEOUT</b></font>]" .. 330 "<p>[<font color='blue'><b>TIMEOUT</b></font>] or [<font color='darkblue'><b>TIMEOUT</b></font>]" ..
307 " means the server had too many timeouts, and tests where aborted, so there is no result for this test.</p>" .. 331 " means the mirror had too many timeouts, and tests where aborted, so there is no result for this test.</p>" ..
332 "<p>NOTE: timeouts may be due to a problem on the testing computer.</p>" ..
308 "<p>The DNS round robin (DNS-RR) column shows the IPs for that mirror, or [<font color='grey'><b>no</b></font>] if it isn't part of the DNS-RR. &nbsp; " .. 333 "<p>The DNS round robin (DNS-RR) column shows the IPs for that mirror, or [<font color='grey'><b>no</b></font>] if it isn't part of the DNS-RR. &nbsp; " ..
309 "The IPs link to the testing log for that IP accessed via the DNS-RR. &nbsp; " .. 334 "The IPs link to the testing log for that IP accessed via the DNS-RR. &nbsp; " ..
310 "deb.devuan.org is the DNS-RR itself, so it doesn't get tested directly.</p>\n" .. 335 "deb.devuan.org is the DNS-RR itself, so it doesn't get tested directly.</p>\n" ..
diff --git a/apt-panopticon-update-data.lua b/apt-panopticon-update-data.lua
index 98c909a..8adad4d 100755
--- a/apt-panopticon-update-data.lua
+++ b/apt-panopticon-update-data.lua
@@ -3,7 +3,9 @@
3local APT = require 'apt-panopticommon' 3local APT = require 'apt-panopticommon'
4local D = APT.D 4local D = APT.D
5local I = APT.I 5local I = APT.I
6local T = APT.T
6local W = APT.W 7local W = APT.W
8local E = APT.E
7local C = APT.C 9local C = APT.C
8local arg, sendArgs = APT.parseArgs({...}) 10local arg, sendArgs = APT.parseArgs({...})
9 11
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index 6bec6a1..4e4be94 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -5,6 +5,7 @@ local now = os.time()
5local APT = require 'apt-panopticommon' 5local APT = require 'apt-panopticommon'
6local D = APT.D 6local D = APT.D
7local I = APT.I 7local I = APT.I
8local T = APT.T
8local W = APT.W 9local W = APT.W
9local E = APT.E 10local E = APT.E
10local C = APT.C 11local C = APT.C
@@ -23,14 +24,14 @@ local releases = {"jessie", "ascii", "beowulf", "ceres"}
23local releaseFiles = 24local releaseFiles =
24{ 25{
25 -- Release file. 26 -- Release file.
26 "/Release", -- 3.7 MB 27 "Release", -- 3.7 MB
27 "/Release.gpg", -- 28 "Release.gpg", --
28-- "/InRelease", -- 3.7 MB 29-- "InRelease", -- 3.7 MB
29-- "/main/binary-all/Packages.xz", -- 2.6 GB for all that changed recently. 30-- "main/binary-all/Packages.xz", -- 2.6 GB for all that changed recently.
30 -- Contents files. -- 3.3 GB 31 -- Contents files. -- 3.3 GB
31-- "/main/Contents-all.xz", 32-- "main/Contents-all.xz",
32-- "/main/Contents-amd64.xz", 33-- "main/Contents-amd64.xz",
33-- "/main/Contents-arm64.xz", 34-- "main/Contents-arm64.xz",
34-- "-security/main/Contents-all.xz", 35-- "-security/main/Contents-all.xz",
35-- "-security/main/Contents-amd64.xz", 36-- "-security/main/Contents-amd64.xz",
36-- "-security/main/Contents-arm64.xz", 37-- "-security/main/Contents-arm64.xz",
@@ -160,6 +161,7 @@ local url = require 'socket.url'
160 161
161 162
162local ip = "" 163local ip = ""
164local cor = nil
163 165
164 166
165local repoExists = function (r) 167local repoExists = function (r)
@@ -197,6 +199,7 @@ gatherIPs = function (host)
197 end 199 end
198 until nil == IPs 200 until nil == IPs
199 end 201 end
202 return IP[host]
200end 203end
201 204
202-- Returns FTP directory listing 205-- Returns FTP directory listing
@@ -229,7 +232,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
229 E("too many timeouts! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) 232 E("too many timeouts! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
230 return 233 return
231 end 234 end
232 if 4 <= (totalTimeouts) then 235 if APT.options.timeouts.value <= (totalTimeouts) then
233 E("Way too many timeouts!", PU.scheme, "", host) 236 E("Way too many timeouts!", PU.scheme, "", host)
234 return 237 return
235 end 238 end
@@ -237,7 +240,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
237 E("too many redirects! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) 240 E("too many redirects! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
238 return 241 return
239 end 242 end
240 if 4 <= retry then 243 if APT.options.retries.value <= retry then
241 E("too many retries! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host) 244 E("too many retries! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
242 return 245 return
243 end 246 end
@@ -343,6 +346,8 @@ checkHEAD = function (host, URL, r, retry, sanity)
343 Still duplicates a tiny bit, but much less than the previous find based method. 346 Still duplicates a tiny bit, but much less than the previous find based method.
344TODO - maybe we can switch to using flock like we do with the other curl calls? 347TODO - maybe we can switch to using flock like we do with the other curl calls?
345 ]] 348 ]]
349C('Commented out code being called in checkHEAD() for redirects that should no longer happen!')
350--[[
346 local file = pu.host .. "://" .. pu.path 351 local file = pu.host .. "://" .. pu.path
347 local f = io.popen(string.format('if [ ! -f results/%s.check ] ; then touch results/%s.check; echo -n "check"; fi', file:gsub("/", "_"), file:gsub("/", "_") )):read("*a") 352 local f = io.popen(string.format('if [ ! -f results/%s.check ] ; then touch results/%s.check; echo -n "check"; fi', file:gsub("/", "_"), file:gsub("/", "_") )):read("*a")
348 if (nil == f) or ("check" == f) then 353 if (nil == f) or ("check" == f) then
@@ -351,28 +356,27 @@ TODO - maybe we can switch to using flock like we do with the other curl calls?
351 else 356 else
352 D(" Already checking " .. file) 357 D(" Already checking " .. file)
353 end 358 end
359]]
354 end 360 end
355 end 361 end
356 end 362 end
357end 363end
358 364
359local checkTimeouts = function(host, scheme, URL) 365local checkTimeouts = function(host, scheme, URL)
360 if APT.testing(scheme) then 366 totalTimeouts = totalTimeouts + timeouts; timeouts = 0
361 totalTimeouts = totalTimeouts + timeouts; timeouts = 0 367 checkHEAD(host, scheme .. "://" .. URL)
362 checkHEAD(host, scheme .. "://" .. URL)
363 if 4 <= (totalTimeouts) then
364 E("Way too many timeouts!", scheme, "", host)
365 return true
366 end
367 end
368 if APT.testing("URLSanity") then 368 if APT.testing("URLSanity") then
369 URL = URL:gsub("merged/", "merged///") 369 URL = URL:gsub("merged/", "merged///")
370 totalTimeouts = totalTimeouts + timeouts; timeouts = 0
371 checkHEAD(host, scheme .. "://" .. URL, 0, 0, true) 370 checkHEAD(host, scheme .. "://" .. URL, 0, 0, true)
372 if 4 <= (totalTimeouts) then 371 end
373 E("Way too many timeouts!", scheme, "URLSanity", host) 372 if nil ~= cor then
374 return true 373 D('*&gt;* About to resume coroutine after checkHEAD(' .. host .. ' , ' .. scheme .. ' :// ' .. URL .. ')')
375 end 374 local ok, message = coroutine.resume(cor)
375 if not ok then cor = nil; print(message) end
376 end
377 if APT.options.timeouts.value <= (totalTimeouts) then
378 E("Way too many timeouts!", scheme, "URLSanity", host)
379 return true
376 end 380 end
377 return false 381 return false
378end 382end
@@ -387,19 +391,18 @@ local checkFiles = function (host, ip, path, file)
387 if checkTimeouts(host, "https", ip .. path .. "/" .. file) then return end 391 if checkTimeouts(host, "https", ip .. path .. "/" .. file) then return end
388 else 392 else
389 I(" Checking IP " .. host .. " -> " .. ip .. " " .. path) 393 I(" Checking IP " .. host .. " -> " .. ip .. " " .. path)
390 for i, s in pairs(referenceDevs) do
391 if checkTimeouts(host, "http", ip .. path .. "/" .. s) then return end
392 if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end
393 end
394
395 for i, s in pairs(releases) do 394 for i, s in pairs(releases) do
396 for j, k in pairs(releaseFiles) do 395 for j, k in pairs(releaseFiles) do
397 if repoExists(s .. k) then 396 if repoExists(s .. k) then
398 if checkTimeouts(host, "http", ip .. path .. "/merged/dists/" .. s .. k) then return end 397 if checkTimeouts(host, "http", ip .. path .. "/merged/dists/" .. s .. '/' .. k) then return end
399 if checkTimeouts(host, "https", ip .. path .. "/merged/dists/" .. s .. k) then return end 398 if checkTimeouts(host, "https", ip .. path .. "/merged/dists/" .. s .. '/' .. k) then return end
400 end 399 end
401 end 400 end
402 end 401 end
402 for i, s in pairs(referenceDevs) do
403 if checkTimeouts(host, "http", ip .. path .. "/" .. s) then return end
404 if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end
405 end
403 end 406 end
404end 407end
405 408
@@ -421,127 +424,51 @@ checkHost = function (orig, host, path, ip, file)
421 D("checkHost " .. orig .. "" .. file) 424 D("checkHost " .. orig .. "" .. file)
422 if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file) end 425 if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file) end
423 else D("checkHost " .. orig .. " -> " .. host) end 426 else D("checkHost " .. orig .. " -> " .. host) end
424 local h = APT.mirrors[ph.host]
425 if nil == h then return end
426 for k, v in pairs(h.IPs) do
427 if "table" == type(v) then
428 for k1, v1 in pairs(v) do
429 if v1 == "A" then
430 if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end
431 elseif v1 == "AAAA" then
432 if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end
433 end
434 end
435 else
436 if v == "A" then
437 if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end
438 elseif v == "AAAA" then
439 if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end
440 end
441 end
442 end
443 end 427 end
444end 428end
445 429
446 430
447local addDownload = function(host, URL, f, r, k) 431local addDownload = function(host, URL, f, r, k)
448 local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename. 432 local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename.
449 if APT.checkFile("results/" .. host .. "/merged/dists/" .. r .. k) then 433 if APT.checkFile("results/" .. host .. "/merged/dists/" .. r .. '/' .. k) then
450 -- Curls "check timestamp and overwrite file" stuff sucks. 434 -- Curls "check timestamp and overwrite file" stuff sucks.
451 -- -R means the destination file gets the timestamp of the remote file. 435 -- -R means the destination file gets the timestamp of the remote file.
452 -- Can only do ONE timestamp check per command. 436 -- Can only do ONE timestamp check per command.
453 -- This doesn't work either. All downloads get all these headers. Pffft 437 -- This doesn't work either. All downloads get all these headers. Pffft
454-- local status, ts = APT.execute('TZ="GMT" ls -l --time-style="+%a, %d %b %Y %T %Z" results/' .. host .. "/merged/dists/" .. r .. k .. ' | cut -d " " -f 6-11') 438-- local status, ts = APT.execute('TZ="GMT" ls -l --time-style="+%a, %d %b %Y %T %Z" results/' .. host .. "/merged/dists/" .. r .. '/' .. k .. ' | cut -d " " -f 6-11')
455-- f:write('header "If-Modified-Since: ' .. ts:sub(2, -2) .. '"\n') 439-- f:write('header "If-Modified-Since: ' .. ts:sub(2, -2) .. '"\n')
456 -- Curl will DELETE the existing file if the timestamp fails to download a new one, unless we change directory first, 440 -- Curl will DELETE the existing file if the timestamp fails to download a new one, unless we change directory first,
457 -- which wont work with multiple files in multiple directories. WTF? 441 -- which wont work with multiple files in multiple directories. WTF?
458 os.execute(" mv results/" .. host .. "/merged/dists/" .. r .. k .. 442 os.execute(" mv results/" .. host .. "/merged/dists/" .. r .. '/' .. k ..
459 " results/" .. host .. "/merged/dists/" .. r .. k .. ".old") 443 " results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. ".old")
460 end 444 end
461 445
462 D('Downloading http://' .. host .. URL .. '/merged/dists/' .. r .. k) 446 D('Downloading http://' .. host .. URL .. '/merged/dists/' .. r .. '/' .. k)
463 f:write('url "' .. 'http://' .. host .. URL .. '/merged/dists/' .. r .. k .. '"\n') 447 f:write('url "' .. 'http://' .. host .. URL .. '/merged/dists/' .. r .. '/' .. k .. '"\n')
464 f:write('output "results/' .. host .. '/merged/dists/' .. r .. k .. '"\n') 448 f:write('output "results/' .. host .. '/merged/dists/' .. r .. '/' .. k .. '"\n')
465end 449end
466 450
467local postDownload = function(host, r, k) 451local postDownload = function(host, r, k)
468 local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename. 452 local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename.
469 local dir = k:sub(1, 0 - (#file + 1)) 453 if nil == file then file = k end
470 os.execute("if [ -f results/" .. host .. "/merged/dists/" .. r .. k .. ".old ]" .. 454 os.execute("if [ -f results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. ".old ]" ..
471 " && [ ! -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then cp -a" .. 455 " && [ ! -f results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. " ]; then cp -a" ..
472 " results/" .. host .. "/merged/dists/" .. r .. k .. ".old" .. 456 " results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. ".old" ..
473 " results/" .. host .. "/merged/dists/" .. r .. k .. "; fi") 457 " results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. "; fi")
474 if ".gz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 gzip -dfk results/" .. host .. "/merged/dists/" .. r .. k) end 458 if APT.checkFile('results/' .. host .. '/merged/dists/' .. r .. '/' .. k) then
475 if ".xz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 xz -dfk results/" .. host .. "/merged/dists/" .. r .. k .. " 2>/dev/null") end 459 if ".gz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 gzip -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k) end
476 if APT.testing("Integrity") then 460 if ".xz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 xz -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. " 2>/dev/null") end
477 if ".gpg" == k:sub(-4, -1) then
478 local status, out = APT.execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. r .. k ..
479 " results/" .. host .. "/merged/dists/" .. r .. k:sub(1, -5) .. " 2>/dev/null")
480 if "0" ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. r .. k, "http", "Integrity", host) end
481 os.execute('rm results/' .. host .. '/merged/dists/' .. r .. k)
482 end
483-- TODO - should check the PGP sig of InRelease as well.
484 end
485 if APT.testing("Integrity") or APT.testing("Updated") then
486 if "Packages." == file:sub(1, 9) then
487-- TODO - compare the SHA256 sums in pkgmaster's Release for both the packed and unpacked versions.
488-- Also note that this might get only a partial download due to maxtime.
489 if APT.options.referenceSite.value == host then
490 local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed', "w+")
491 if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed' .. ' file - ' .. e) else
492 local pp = {}
493 for l in io.lines('results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages') do
494 if "Package: " == l:sub(1, 9) then
495 if 0 ~= #pp then
496 for i = 1, 5 do
497 if nil == pp[i] then print(host .. " " .. r .. " " .. dir .. " " .. i) else Pp:write(pp[i] .. " | ") end
498 end
499 Pp:write("\n")
500 end
501 pp = {}
502 pp[1] = l:sub(10, -1)
503 elseif "Version: " == l:sub(1, 9) then
504 pp[2] = l:sub(10, -1)
505 elseif "Filename: " == l:sub(1, 10) then
506 pp[3] = l:sub(11, -1)
507 elseif "Size: " == l:sub(1, 6) then
508 pp[4] = l:sub(7, -1)
509 elseif "SHA256: " == l:sub(1, 8) then
510 pp[5] = l:sub(9, -1)
511 end
512 end
513 Pp:close()
514 os.execute('sort results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed >results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages_parsed-sorted')
515 if APT.checkFile('Packages/' .. r .. dir .. 'Packages_parsed-sorted') then
516 os.execute('diff -U 0 Packages/' .. r .. dir .. 'Packages_parsed-sorted ' ..
517 'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages_parsed-sorted ' ..
518 ' | grep -E "^-" | grep -Ev "^\\+\\+\\+|^---" >>results/OLD_PACKAGES_' .. r .. '.txt')
519 os.execute('diff -U 0 Packages/' .. r .. dir .. 'Packages_parsed-sorted ' ..
520 'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages_parsed-sorted ' ..
521 ' | grep -E "^\\+" | grep -Ev "^\\+\\+\\+|^---" >>results/NEW_Packages_' .. r .. '.txt')
522 -- Find the smallest new package for each release.
523 os.execute('sort -b -k 9,9 -n results/NEW_Packages_' .. r .. '.txt >results/NEW_Packages_' .. r .. '.sorted.txt')
524 os.execute('grep -s " | pool/DEBIAN/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >results/NEW_Packages_' .. r .. '.test.tmp')
525 os.execute('grep -s " | pool/DEBIAN-SECURITY/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >>results/NEW_Packages_' .. r .. '.test.tmp')
526 os.execute('grep -s " | pool/DEVUAN/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >>results/NEW_Packages_' .. r .. '.test.tmp')
527 os.execute('mv results/NEW_Packages_' .. r .. '.test.tmp results/NEW_Packages_' .. r .. '.test.txt')
528 else
529 C("Can't find file Packages/" .. r .. dir .. "Packages_parsed-sorted")
530 end
531 os.execute('mkdir -p Packages/' .. r .. dir)
532 os.execute('mv -f results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages_parsed-sorted Packages/' .. r .. dir .. 'Packages_parsed-sorted')
533 end
534 else
535 while not APT.checkFile('results/NEW_Packages_' .. r .. '.test.txt') do os.execute('sleep 10') end
536 end
537 os.execute('rm -f results/' .. host .. '/merged/dists/' .. r .. dir .. 'Packages 2>/dev/null')
538 os.execute('rm -f results/' .. host .. '/merged/dists/' .. r .. dir .. 'Packages.* 2>/dev/null')
539 end
540 end 461 end
541end 462end
542 463
543local downloadLock = "flock -n results/curl-" 464local downloadLock = "flock -n results/curl-"
544local download = "curl --connect-timeout " .. APT.options.timeout.value .. " --create-dirs -f -L --max-time " .. APT.options.maxtime.value .. " -z 'results/stamp.old' -v -R " 465local download = "curl" ..
466 " --connect-timeout " .. APT.options.timeout.value ..
467 " --create-dirs -f -L" ..
468 " --fail-early" ..
469 " --max-time " .. APT.options.maxtime.value ..
470 " --retry " .. APT.options.retries.value ..
471 " -R -v -z 'results/stamp.old' "
545local downloads = function(host, URL, meta, release, list) 472local downloads = function(host, URL, meta, release, list)
546 if nil == URL then URL = "" end 473 if nil == URL then URL = "" end
547 local lock = meta .. "-" .. host .. ".log" 474 local lock = meta .. "-" .. host .. ".log"
@@ -600,6 +527,7 @@ local validateURL = function(m)
600 m.BaseURL = p.authority .. p.path 527 m.BaseURL = p.authority .. p.path
601 return m 528 return m
602end 529end
530
603local getMirrors = function () 531local getMirrors = function ()
604 local mirrors = {} 532 local mirrors = {}
605 local host = "" 533 local host = ""
@@ -621,8 +549,6 @@ local getMirrors = function ()
621 end 549 end
622 host = d 550 host = d
623 m[t] = d 551 m[t] = d
624 gatherIPs(host)
625 m["IPs"] = IP[host]
626 elseif "Protocols" == t then 552 elseif "Protocols" == t then
627 local prot = {} 553 local prot = {}
628 for w in d:gmatch("(%w+)") do 554 for w in d:gmatch("(%w+)") do
@@ -643,19 +569,305 @@ local getMirrors = function ()
643 end 569 end
644 end 570 end
645 if APT.testing("DNSRR") then 571 if APT.testing("DNSRR") then
646 mirrors[APT.options.roundRobin.value] = { ["Protocols"] = { ["http"] = true; ["https"] = true; }; ["FQDN"] = 'deb.devuan.org'; ["Active"] = 'yes'; ["BaseURL"] = 'deb.devuan.org'; } 572 mirrors[APT.options.roundRobin.value] = { ["Protocols"] = { ["http"] = true; ["https"] = true; };
647 gatherIPs(APT.options.roundRobin.value) 573 ["FQDN"] = APT.options.roundRobin.value; ["Active"] = 'yes'; ["BaseURL"] = APT.options.roundRobin.value; }
648 mirrors[APT.options.roundRobin.value].IPs = IP[APT.options.roundRobin.value]
649 end
650 local file, e = io.open("results/mirrors.lua", "w+")
651 if nil == file then C("opening mirrors file - " .. e) else
652 file:write(APT.dumpTable(mirrors, "", "mirrors") .. "\nreturn mirrors\n")
653 file:close()
654 end 574 end
655 return mirrors 575 return mirrors
656end 576end
657 577
658 578
579local postParse = function(host, list)
580 if APT.options.referenceSite.value == host then
581 if nil ~= list then
582 local sem = 'results/NEW_' .. list.out .. '_%s.txt'
583 for i, n in pairs(releases) do
584 local f = sem:format(n)
585 if APT.checkFile(f .. '.tmp') then
586 os.execute('mv ' .. f .. '.tmp ' .. f)
587 else
588 os.execute('touch ' .. f)
589 end
590 end
591 end
592 end
593end
594
595local parseDebs = function(host)
596 for i, n in pairs(releases) do
597 local inFile = 'results/NEW_debs_' .. n .. '.txt'
598 local nfile, e = io.open(inFile, "r")
599 if nil == nfile then W("opening " .. inFile .. " file - " .. e) else
600 for l in nfile:lines() do
601 local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |')
602 if nil ~= p then
603 if APT.checkFile('results/' .. host .. "/merged/" .. p) then
604 local status, fsz = APT.execute('ls -l results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 5-5')
605 if APT.testing("Integrity") then
606 if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end.
607 E('Package size mismatch - results/' .. host .. "/merged/" .. p .. ' should be ' .. sz .. ', but is ' .. fsz:sub(2, -2) .. '.', 'http', 'Integrity', host)
608 else
609 local status, fsha = APT.execute('sha256sum results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 1')
610 if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. host .. "/merged/" .. p, 'http', 'Integrity', host) end
611-- TODO - maybe check the PGP key, though packages are mostly not signed.
612 end
613 end
614 if APT.testing("Updated") then
615 if sz ~= fsz:sub(2, -2) then
616 E('Package size mismatch - results/' .. host .. "/merged/" .. p, 'http', 'Updated', host)
617 end
618 end
619 os.execute('rm -f results/' .. host .. "/merged/" .. p)
620 else
621 E('Failed to download - results/' .. host .. "/merged/" .. p, 'http', 'Updated', host)
622 end
623 end
624 end
625 end
626 end
627 return nil
628end
629
630local parsePackages = function(host)
631 local list = {inf = 'Packages', parser = parseDebs, out = 'debs', files = {}, nextf = ''}
632 for i, n in pairs(releases) do
633 local inFile = 'results/NEW_' .. list.inf .. '_' .. n .. '.txt'
634 local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt'
635 if APT.options.referenceSite.value == host then
636 outFile = outFile .. '.tmp'
637 end
638 local dFile, e = io.open(inFile, "r")
639 if nil == dFile then W("opening " .. inFile .. " file - " .. e) else
640 for l in dFile:lines() do
641 postDownload(host, n, l)
642
643 l = '/' .. l
644 local file = l:match(".*/([%w%.%+%-_]*)$") -- Get the filename.
645 local dir = l:sub(1, 0 - (#file + 1))
646
647 if "Packages." == file:sub(1, 9) then
648-- TODO - compare the SHA256 sums in pkgmaster's Release for both the packed and unpacked versions.
649-- Also note that this might get only a partial download due to maxtime.
650 if APT.options.referenceSite.value == host then
651 local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed', "w+")
652 if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed' .. ' file - ' .. e) else
653 local pp = {}
654 for l in io.lines('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages') do
655 if "Package: " == l:sub(1, 9) then
656 if 0 ~= #pp then
657 for i = 1, 5 do
658 if nil == pp[i] then print(host .. " " .. n .. " " .. dir .. " " .. i) else Pp:write(pp[i] .. " | ") end
659 end
660 Pp:write("\n")
661 end
662 pp = {}
663 pp[1] = l:sub(10, -1)
664 elseif "Version: " == l:sub(1, 9) then
665 pp[2] = l:sub(10, -1)
666 elseif "Filename: " == l:sub(1, 10) then
667 pp[3] = l:sub(11, -1)
668 elseif "Size: " == l:sub(1, 6) then
669 pp[4] = l:sub(7, -1)
670 elseif "SHA256: " == l:sub(1, 8) then
671 pp[5] = l:sub(9, -1)
672 end
673 end
674 Pp:close()
675 os.execute('sort results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed >results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages_parsed-sorted')
676 if APT.checkFile('Packages/' .. n .. dir .. 'Packages_parsed-sorted') then
677 os.execute('diff -U 0 Packages/' .. n .. dir .. 'Packages_parsed-sorted ' ..
678 'results/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. dir .. 'Packages_parsed-sorted ' ..
679 ' | grep -E "^-" | grep -Ev "^\\+\\+\\+|^---" >>results/OLD_' .. list.out .. '_' .. n .. '.txt')
680 os.execute('diff -U 0 Packages/' .. n .. dir .. 'Packages_parsed-sorted ' ..
681 'results/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. dir .. 'Packages_parsed-sorted ' ..
682 ' | grep -E "^\\+" | grep -Ev "^\\+\\+\\+|^---" >>results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt')
683 else
684 C("Can't find file Packages/" .. n .. dir .. "Packages_parsed-sorted")
685 end
686 os.execute('mkdir -p Packages/' .. n .. dir)
687 os.execute('mv -f results/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. dir .. 'Packages_parsed-sorted Packages/' .. n .. dir .. 'Packages_parsed-sorted')
688 end
689 else
690 end
691 os.execute('rm -fr results/' .. host .. '/merged/dists/' .. n .. dir .. ' 2>/dev/null')
692 end
693 end
694
695 if APT.checkFile('results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt') then
696 -- Sort by size.
697 os.execute('sort -b -k 9,9 -n results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt >results/NEW_' .. list.out .. '_' .. n .. '.sorted.txt')
698 os.execute('grep -s " | pool/DEBIAN/" results/NEW_' .. list.out .. '_' .. n .. '.sorted.txt 2>/dev/null | head -n 1 >' .. outFile)
699 os.execute('grep -s " | pool/DEBIAN-SECURITY/" results/NEW_' .. list.out .. '_' .. n .. '.sorted.txt 2>/dev/null | head -n 1 >>' .. outFile)
700 os.execute('grep -s " | pool/DEVUAN/" results/NEW_' .. list.out .. '_' .. n .. '.sorted.txt 2>/dev/null | head -n 1 >>' .. outFile)
701 os.execute('rm -f results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt')
702 end
703
704 end
705
706 local nfile, e = io.open(outFile, "r")
707 if nil ~= nfile then
708-- for l in nfile:lines() do
709 local l = nfile:read('*l')
710 if nil ~= l then
711 local p = l:match('(pool/.*%.deb)')
712 if nil ~= p then
713 table.insert(list.files, p)
714 end
715 end
716-- end
717 end
718 end
719
720 postParse(host, list)
721 return list
722end
723
724local parseRelease = function(host)
725 local list = {inf = 'Release', parser = parsePackages, out = 'Packages', files = {}, nextf = 'debs'}
726 for i, n in pairs(releases) do
727 for l, o in pairs(releaseFiles) do
728 if repoExists(i .. o) then
729 postDownload(host, n, o)
730 if (".gpg" == o:sub(-4, -1)) and (APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/' .. o)) then
731 if APT.testing("Integrity") then
732 local status, out = APT.execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o ..
733 " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5) .. " 2>/dev/null")
734 if "0" ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end
735-- TODO - should check the PGP sig of InRelease as well.
736 end
737 os.execute('rm results/' .. host .. '/merged/dists/' .. n .. '/' .. o)
738 end
739 end
740 end
741
742 if APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/Release') then
743 os.execute('sort -k 3 results/' .. host .. '/merged/dists/' .. n .. '/Release >results/' .. host .. '/merged/dists/' .. n .. '/Release.SORTED')
744 local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt'
745 if APT.checkFile('results_old/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. '/Release.SORTED') then
746 if APT.options.referenceSite.value == host then
747 outFile = outFile .. '.tmp'
748 os.execute('diff -U 0 results_old/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. '/Release.SORTED ' ..
749 'results/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. '/Release.SORTED ' ..
750 '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >' .. outFile)
751-- TODO - Maybe check the date in Release, though since they are updated daily, is there any point? Perhaps it's for checking amprolla got run?
752-- Also check if that date is in the future, apt recently got a check for that, though not sure why.
753 os.execute('rm -f results/' .. host .. '/merged/dists/' .. n .. '/Release 2>/dev/null; ')
754 else
755-- TODO - compare to the pkgmaster copy.
756 end
757
758-- TODO - if it's not Integrity and not reference, then just do a HEAD check and compare file times?
759 if APT.testing("Integrity") or (APT.options.referenceSite.value == host) then
760 local dfile, e = io.open(outFile, "r")
761 if nil == dfile then W("opening " .. outFile .. " file - " .. e) else
762 for l in dfile:lines() do
763 table.insert(list.files, 'dists/' .. n .. '/' .. l)
764 end
765 end
766 end
767 end
768 end
769
770 end
771 postParse(host, list)
772 return list
773end
774
775local parseStart = function(host)
776 local list = {inf = '', parser = parseRelease, out = 'Release', files = {}, nextf = 'Packages'}
777 for i, n in pairs(releases) do
778 local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt'
779 for l, o in pairs(releaseFiles) do
780 if repoExists(n .. o) then
781 if APT.options.referenceSite.value == host then
782 local dfile, e = io.open(outFile .. '.tmp', "a+")
783 if nil == dfile then W("opening " .. outFile .. ".tmp file - " .. e) else
784 dfile:write(o .. '\n')
785 end
786 end
787 table.insert(list.files, 'dists/' .. n .. '/' .. o)
788 end
789 end
790 end
791 postParse(host, list)
792 return list
793end
794
795local doDownloads = function(host, path, list)
796 while nil ~= list do
797 if 0 ~= #(list.files) then
798 for j, f in pairs(list.files) do
799 downloads(host, path, list.out, nil, f)
800 end
801 downloads(host, path, list.out, nil, '')
802--[[ I've seen flock & curl act oddly. Perhaps flock didn't have time to start up?
803 /var/www/html/apt-panopticon/apt-panopticon/results_2019-12-22-15-00
804
805Mon Dec 23 01:02:54 2019 DEBUG : forking
806
807ionice -c3 nice -n 19 flock -n results/curl-debs-pkgmaster.devuan.org.log curl --connect-timeout 5 --create-dirs -f -L --fail-early --max-time 300 --retry 3 -R -v -z 'results/stamp.old' --stderr results/curl-debs-pkgmaster.devuan.org.log -K results/curl-debs-pkgmaster.devuan.org.files
808
809
810Mon Dec 23 01:02:54 2019 DEBUG : 0 flock -n results/curl-debs-pkgmaster.devuan.org.log commands still running.
811Mon Dec 23 01:02:54 2019 DEBUG : *>* Resumed coroutine NO LONGER waiting on - 0 < APT.checkExes(flock -n results/curl-debs-pkgmaster.devuan.org.log
812Mon Dec 23 01:02:54 2019 DEBUG : *** Doing list.parser() for debs
813Mon Dec 23 01:02:54 2019 ERROR (http Updated pkgmaster.devuan.org): Failed to download - results/pkgmaster.devuan.org/merged/pool/DEBIAN/main/a/aptly/aptly_1.3.0+ds1-4_amd64.deb
814
815drwxr-x--- 2 www-data www-data 4096 2019-12-23 01:02:57.000000000 +1000 aptly
816-rw-r--r-- 1 www-data www-data 7129 2019-12-23 01:03:54.000000000 +1000 curl-debs-pkgmaster.devuan.org.log
817
818]]
819os.execute('sleep 2')
820 while 0 < APT.checkExes(downloadLock .. list.out .. "-" .. host .. ".log") do
821 D('*&lt;* About to yield coroutine while waiting on - 0 < APT.checkExes(' .. downloadLock .. list.out .. '-' .. host .. '.log')
822 coroutine.yield()
823 D('*&gt;* Resumed coroutine while waiting on - 0 < APT.checkExes(' .. downloadLock .. list.out .. '-' .. host .. '.log')
824 end
825 D('*&gt;* Resumed coroutine NO LONGER waiting on - 0 < APT.checkExes(' .. downloadLock .. list.out .. '-' .. host .. '.log')
826
827 local min, max, spd = 999999999999, 0
828 if APT.checkFile("results/curl-" .. list.out .. "-" .. host .. ".log") then
829 for l in io.lines("results/curl-" .. list.out .. "-" .. host .. ".log") do
830 local speed, crrnt = l:match('^%c *%d+ +%d+k? +%d+ +%d+k? +%d+ +%d+ +(%d+k?) +%d+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +(%d+k?)')
831 if nil ~= speed then
832 if 'k' == speed:sub(-1, -1) then speed = speed:sub(1, -2) .. '000' end
833 if 'k' == crrnt:sub(-1, -1) then crrnt = crrnt:sub(1, -2) .. '000' end
834 speed = tonumber(speed)
835 crrnt = tonumber(crrnt)
836 if speed < min and speed ~= 0 then min = speed end
837 if speed > max then max = speed end
838 if crrnt < min and crrnt ~= 0 then min = crrnt end
839 if crrnt > max then max = crrnt end
840 end
841 if l:find('timed out') ~= nil then
842 E(" TIMEOUT " .. timeouts + 1 .. ', details in curl-' .. list.out .. '-' .. host .. '.log', 'http', '', host)
843 timeouts = timeouts + 1
844 APT.results["timeout"] = true
845 end
846 end
847 end
848 APT.results["speed"] = {min = min, max = max}
849 end
850
851 if (APT.options.referenceSite.value ~= host) and ('' ~= list.nextf) then
852 local sem = 'results/NEW_' .. list.nextf .. '_%s.txt'
853 for i, n in pairs(releases) do
854 local f = sem:format(n)
855 while not APT.checkFile(f) do
856 D('*&lt;* About to yield coroutine while waiting on - not APT.checkFile(' .. f .. ')')
857 coroutine.yield()
858 D('*&gt;* Resumed coroutine while waiting on - not APT.checkFile(' .. f .. ')')
859 end
860 end
861 end
862 D('*** Doing list.parser() for ' .. list.out)
863 list = list.parser(host)
864 if APT.options.timeouts.value <= (totalTimeouts) then break end
865 end
866 D('*&lt;&lt;* About to end coroutine.')
867 cor = nil
868end
869
870
659if 0 < #arg then 871if 0 < #arg then
660 if "/" == arg[1]:sub(-1, -1) then 872 if "/" == arg[1]:sub(-1, -1) then
661 W("slash at end of path! " .. arg[1]) 873 W("slash at end of path! " .. arg[1])
@@ -673,6 +885,8 @@ if 0 < #arg then
673 end 885 end
674 end 886 end
675 887
888 if APT.origin then APT.results["IPs"] = gatherIPs(pu.host) end
889
676 if nil ~= arg[2] then 890 if nil ~= arg[2] then
677 APT.logFile, e = io.open("results/LOG_" .. pu.host .. "_" .. arg[2] .. ".html", "a+") 891 APT.logFile, e = io.open("results/LOG_" .. pu.host .. "_" .. arg[2] .. ".html", "a+")
678 else 892 else
@@ -681,165 +895,57 @@ if 0 < #arg then
681 if nil == APT.logFile then C("opening log file - " .. e); return end 895 if nil == APT.logFile then C("opening log file - " .. e); return end
682 APT.logPre() 896 APT.logPre()
683 I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", ")) 897 I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", "))
684 APT.mirrors = loadfile("results/mirrors.lua")()
685 if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end 898 if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end
686 if nil ~= arg[3] then I(" Using file " .. arg[3]); end 899 if nil ~= arg[3] then I(" Using file " .. arg[3]); end
687 APT.results = APT.padResults(APT.results) 900 APT.results = APT.padResults(APT.results)
901
688 if APT.origin then 902 if APT.origin then
903
904 local file = arg[3]
905 if nil == file then file = '' end
906 local path = pu.path
907 if nil == path then path = '' end
908 local ips = APT.results["IPs"]
909 for k, v in pairs(ips) do
910 if "table" == type(v) then
911 for k1, v1 in pairs(v) do
912 if v1 == "A" then
913 if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file) end
914 elseif v1 == "AAAA" then
915 if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file) end
916 end
917 end
918 else
919 if v == "A" then
920 if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file) end
921 elseif v == "AAAA" then
922 if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file) end
923 end
924 end
925 end
926
689 if APT.testing("Integrity") or APT.testing("Updated") then 927 if APT.testing("Integrity") or APT.testing("Updated") then
690 if APT.origin and (APT.options.roundRobin.value ~= pu.host) then 928 if APT.origin and (APT.options.roundRobin.value ~= pu.host) then
691 I("Starting file downloads for " .. pu.host) 929 I("Starting file downloads for " .. pu.host)
692 downloads(pu.host, pu.path, 'Release') 930 D('*&gt;* About to create coroutine.')
931 cor = coroutine.create(doDownloads)
932 local ok, message = coroutine.resume(cor, pu.host, pu.path, parseStart(pu.host))
933 if not ok then cor = nil; print(message) end
693 end 934 end
694 end 935 end
936
695 checkFiles(pu.host, pu.host, pu.path); 937 checkFiles(pu.host, pu.host, pu.path);
696 else 938 else
697 checkHost(pu.host, pu.host, pu.path, arg[2], arg[3]) 939 checkHost(pu.host, pu.host, pu.path, arg[2], arg[3])
698 end 940 end
699 941
700 if APT.testing("Integrity") or APT.testing("Updated") then 942 while nil ~= cor do
701 if 4 > (totalTimeouts) then 943 os.execute('sleep 10')
702 if APT.origin and (APT.options.roundRobin.value ~= pu.host) then 944 D('*&gt;* About to resume coroutine before writing results.')
703 while 0 < APT.checkExes(downloadLock .. "Release-" .. pu.host .. ".log") do os.execute("sleep 10") end
704
705 local fcount = 0
706 for i, n in pairs(releases) do
707 for l, o in pairs(releaseFiles) do
708 if repoExists(i .. o) then
709 postDownload(pu.host, n, o)
710 end
711 end
712
713 if APT.checkFile('results/' .. pu.host .. '/merged/dists/' .. n .. '/Release') then
714 os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED')
715 if APT.checkFile('results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED') then
716 if APT.options.referenceSite.value == pu.host then
717 os.execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
718 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
719 '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.tmp')
720-- TODO - Maybe check the date in Release, though since they are updated daily, is there any point? Perhaps it's for checking amprolla got run?
721 os.execute('rm -f results/' .. pu.host .. '/merged/dists/' .. n .. '/Release 2>/dev/null; ' ..
722 'mv results/NEW_Release_' .. n .. '.tmp results/NEW_Release_' .. n .. '.txt')
723 else
724-- TODO - compare to the pkgmaster copy.
725 while not APT.checkFile('results/NEW_Release_' .. n .. '.txt') do os.execute('sleep 10') end
726 end
727
728 local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r")
729 if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else
730 local diff = dfile:read("*a")
731 if "" ~= diff then
732 downloads(pu.host, pu.path, 'Packages', n, diff)
733 fcount = fcount + 1
734 end
735 end
736 end
737 end
738
739 end
740
741 if 0 ~= fcount then
742 downloads(pu.host, pu.path, 'Packages', "", "")
743 while 0 < APT.checkExes(downloadLock .. "Packages-" .. pu.host .. ".log") do os.execute("sleep 10") end
744
745 for i, n in pairs(releases) do
746 local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r")
747 if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else
748 local diff = dfile:read("*a")
749 for l in diff:gmatch("\n*([^\n]+)\n*") do
750 postDownload(pu.host, n, "/" .. l)
751 end
752 end
753 if APT.options.referenceSite.value == pu.host then
754 -- In case it wasn't dealt with already.
755 os.execute('touch results/NEW_Packages_' .. n .. '.test.txt')
756 end
757 end
758
759 fcount = 0
760 for i, n in pairs(releases) do
761 local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
762 if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
763 for l in nfile:lines() do
764 local p = l:match('(pool/.*%.deb)')
765 if nil ~= p then
766 downloads(pu.host, pu.path, 'package', nil, p)
767 fcount = fcount + 1
768 end
769 end
770 end
771 end
772
773 if 0 ~= fcount then
774 downloads(pu.host, pu.path, 'package', nil, "")
775 while 0 < APT.checkExes(downloadLock .. "package-" .. pu.host .. ".log") do os.execute("sleep 10") end
776 for i, n in pairs(releases) do
777 local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
778 if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
779 for l in nfile:lines() do
780 local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |')
781 if nil ~= p then
782 if APT.checkFile('results/' .. pu.host .. "/merged/" .. p) then
783 local status, fsz = APT.execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5')
784 if APT.testing("Integrity") then
785 if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end.
786 E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p .. ' should be ' .. sz .. ', but is ' .. fsz:sub(2, -2) .. '.', 'http', 'Integrity', pu.host)
787 else
788 local status, fsha = APT.execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1')
789 if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end
790-- TODO - maybe check the PGP key, though packages are mostly not signed.
791 end
792 end
793 if APT.testing("Updated") then
794 if sz ~= fsz:sub(2, -2) then
795 E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host)
796 end
797 end
798 else
799 E('Failed to download - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host)
800 end
801 end
802 end
803 end
804 end
805 end
806 else
807 for i, n in pairs(releases) do
808 os.execute('touch results/NEW_Packages_' .. n .. '.test.txt')
809 end
810 end
811 end
812 945
813 APT.results["timeout"] = false 946 local ok, message = coroutine.resume(cor)
814 else 947 if not ok then cor = nil; print(message); break end
815 APT.results["timeout"] = true
816 end
817 end
818
819 local min, max, spd = 999999999999, 0
820 for i, mt in pairs({'Release', 'Packages', 'package'}) do
821 if APT.checkFile("results/curl-" .. mt .. "-" .. pu.host .. ".log") then
822 for l in io.lines("results/curl-" .. mt .. "-" .. pu.host .. ".log") do
823 local speed, crrnt = l:match('^%c *%d+ +%d+k? +%d+ +%d+k? +%d+ +%d+ +(%d+k?) +%d+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +(%d+k?)')
824 if nil ~= speed then
825 if 'k' == speed:sub(-1, -1) then speed = speed:sub(1, -2) .. '000' end
826 if 'k' == crrnt:sub(-1, -1) then crrnt = crrnt:sub(1, -2) .. '000' end
827 speed = tonumber(speed)
828 crrnt = tonumber(crrnt)
829 if speed < min and speed ~= 0 then min = speed end
830 if speed > max then max = speed end
831 if crrnt < min and crrnt ~= 0 then min = crrnt end
832 if crrnt > max then max = crrnt end
833 end
834 if l:find('timed out') ~= nil then
835 E(" TIMEOUT " .. timeouts + 1 .. ', details in curl-' .. mt .. '-' .. pu.host .. '.log', 'http', '', pu.host)
836 timeouts = timeouts + 1
837 APT.results["timeout"] = true
838 end
839 end
840 end
841 end 948 end
842 APT.results["speed"] = {min = min, max = max}
843 949
844 local f = pu.host 950 local f = pu.host
845 if "" ~= ip then f = f .. "_" .. ip end 951 if "" ~= ip then f = f .. "_" .. ip end
@@ -891,13 +997,23 @@ else
891 local pu = url.parse("http://" .. m.BaseURL) 997 local pu = url.parse("http://" .. m.BaseURL)
892 if APT.options.referenceSite.value ~= pu.host then 998 if APT.options.referenceSite.value ~= pu.host then
893 checkHost(m.BaseURL) 999 checkHost(m.BaseURL)
894 APT.checkExes("apt-panopticon.lua " .. sendArgs)
895 if APT.testing("Integrity") or APT.testing("Updated") then APT.checkExes(downloadLock) end
896 end 1000 end
897 end 1001 end
898 1002
899 while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end 1003 while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end
900 os.execute('rm -f results/*.check; rm STATUS_* 2>/dev/null') 1004 os.execute('rm -f results/*.check; rm STATUS_* 2>/dev/null')
1005 for k, v in pairs(APT.mirrors) do
1006 local f = 'results/' .. k .. '.lua'
1007 if APT.checkFile(f) then
1008 results = loadfile(f)()
1009 APT.mirrors[k]['IPs'] = results.IPs
1010 end
1011 end
1012 local file, e = io.open("results/mirrors.lua", "w+")
1013 if nil == file then C("opening mirrors file - " .. e) else
1014 file:write(APT.dumpTable(APT.mirrors, "", "mirrors") .. "\nreturn mirrors\n")
1015 file:close()
1016 end
901 1017
902 -- Create the reports. 1018 -- Create the reports.
903 for n, r in pairs(APT.options.reports.value) do 1019 for n, r in pairs(APT.options.reports.value) do
generated by cgit v1.1 at 2024-11-23 11:55:05 +0000