diff options
author | onefang | 2019-11-05 15:38:24 +1000 |
---|---|---|
committer | onefang | 2019-11-05 15:38:24 +1000 |
commit | cc872d3eba68f08ee365b8771d8504c28da9020d (patch) | |
tree | 2083ec34d871381a04f7fb26c5ce9ec7062ae39f | |
parent | Create a persistant table with the results. (diff) | |
download | apt-panopticon-cc872d3eba68f08ee365b8771d8504c28da9020d.zip apt-panopticon-cc872d3eba68f08ee365b8771d8504c28da9020d.tar.gz apt-panopticon-cc872d3eba68f08ee365b8771d8504c28da9020d.tar.bz2 apt-panopticon-cc872d3eba68f08ee365b8771d8504c28da9020d.tar.xz |
Rename project to apt-panopticon.
It used to be mirror-checker-lua.
-rw-r--r-- | README.md | 81 | ||||
-rwxr-xr-x | apt-panopticon.lua (renamed from mirror-checker.lua) | 20 |
2 files changed, 51 insertions, 50 deletions
@@ -4,11 +4,11 @@ This is currently under development, not everything has been written yet. | |||
4 | Some of this document mentions some of the things that are not written | 4 | Some of this document mentions some of the things that are not written |
5 | yet. | 5 | yet. |
6 | 6 | ||
7 | mirror-checker-lua is a Lua script used by the Devuan mirror admins | 7 | apt-panopticon is a Lua script used by the Devuan mirror admins (maybe, |
8 | (maybe, if they like it) to check the health of Devuan Linux package | 8 | if they like it) to check the health of Devuan Linux package mirrors. |
9 | mirrors. Originally there was bash scripts for this job, then Evilham | 9 | Originally there was bash scripts for this job, then Evilham wrote some |
10 | wrote some Python scripts, now onefang has written it in Lua. We all | 10 | Python scripts, now onefang has written it in Lua. We all have different |
11 | have different tastes in languages. lol | 11 | tastes in languages. lol |
12 | 12 | ||
13 | The main difference is that this Lua version tries to do everything, and | 13 | The main difference is that this Lua version tries to do everything, and |
14 | will be maintained. Currently the shell scripts and Python scripts are | 14 | will be maintained. Currently the shell scripts and Python scripts are |
@@ -17,7 +17,7 @@ badgered him about his Python scripts. It should also be much easier to | |||
17 | use, the previous scripts needed some work before you could run them, | 17 | use, the previous scripts needed some work before you could run them, |
18 | this one you just download and run. | 18 | this one you just download and run. |
19 | 19 | ||
20 | The source code is at [https://sledjhamr.org/cgit/mirror-checker-lua/](https://sledjhamr.org/cgit/mirror-checker-lua/) | 20 | The source code is at [https://sledjhamr.org/cgit/apt-panopticon/](https://sledjhamr.org/cgit/apt-panopticon/) |
21 | 21 | ||
22 | The issue tracker is at [https://sledjhamr.org/mantisbt/project_page.php?project_id=13](https://sledjhamr.org/mantisbt/project_page.php?project_id=13) | 22 | The issue tracker is at [https://sledjhamr.org/mantisbt/project_page.php?project_id=13](https://sledjhamr.org/mantisbt/project_page.php?project_id=13) |
23 | 23 | ||
@@ -25,7 +25,7 @@ The issue tracker is at [https://sledjhamr.org/mantisbt/project_page.php?project | |||
25 | Installation. | 25 | Installation. |
26 | ------------- | 26 | ------------- |
27 | 27 | ||
28 | Download the source. You may want to put the mirror-checker.lua script | 28 | Download the source. You may want to put the apt-panopticon.lua script |
29 | in someplace like `/usr/local/bin` and make sure it is executable. | 29 | in someplace like `/usr/local/bin` and make sure it is executable. |
30 | 30 | ||
31 | It should run on any recent Linux, you'll need to have the following | 31 | It should run on any recent Linux, you'll need to have the following |
@@ -52,57 +52,58 @@ and `results/web` directories, with the notification emails and web pages | |||
52 | Note that unlike typical commands, you can't run single character options | 52 | Note that unlike typical commands, you can't run single character options |
53 | together, so this is wrong - | 53 | together, so this is wrong - |
54 | 54 | ||
55 | $ ./mirror-checker.lua -vvv | 55 | $ ./apt-panopticon.lua -vvv |
56 | 56 | ||
57 | Instead do this - | 57 | Instead do this - |
58 | 58 | ||
59 | $ ./mirror-checker.lua -v -v -v | 59 | $ ./apt-panopticon.lua -v -v -v |
60 | 60 | ||
61 | Just run the script to do all of the tests - | 61 | Just run the script to do all of the tests - |
62 | 62 | ||
63 | $ ./mirror-checker.lua | 63 | $ ./apt-panopticon.lua |
64 | 64 | ||
65 | Which will print any errors. If you don't want to see errors - | 65 | Which will print any errors. If you don't want to see errors - |
66 | 66 | ||
67 | $ ./mirror-checker.lua -q | 67 | $ ./apt-panopticon.lua -q |
68 | 68 | ||
69 | If you want to see warnings as well (as usual, the more `-v` options, the more | 69 | If you want to see warnings as well (as usual, the more `-v` options, the more |
70 | details) - | 70 | details) - |
71 | 71 | ||
72 | $ ./mirror-checker.lua -v | 72 | $ ./apt-panopticon.lua -v |
73 | 73 | ||
74 | Or use the usual options for the help and version number (not written yet) - | 74 | Or use the usual options for the help and version number (not written yet) - |
75 | 75 | ||
76 | $ ./mirror-checker.lua -h | 76 | $ ./apt-panopticon.lua -h |
77 | $ ./mirror-checker.lua --help | 77 | $ ./apt-panopticon.lua --help |
78 | $ ./mirror-checker.lua --version | 78 | $ ./apt-panopticon.lua --version |
79 | 79 | ||
80 | To run the tests on a specific mirror, for example pkgmaster.devuan.org - | 80 | To run the tests on a specific mirror, for example pkgmaster.devuan.org - |
81 | 81 | ||
82 | $ ./mirror-checker.lua pkgmaster.devuan.org | 82 | $ ./apt-panopticon.lua pkgmaster.devuan.org |
83 | 83 | ||
84 | You can use the `--tests` option to tune which tests are run, for example | 84 | You can use the `--tests` option to tune which tests are run, for example |
85 | to stop IPv6 tests, coz you don't have IPv6 - | 85 | to stop IPv6 tests, coz you don't have IPv6 - |
86 | 86 | ||
87 | $ ./mirror-checker.lua --tests=-IPv6 | 87 | $ ./apt-panopticon.lua --tests=-IPv6 |
88 | 88 | ||
89 | To do the same, but not run the HTTPS tests either - | 89 | To do the same, but not run the HTTPS tests either - |
90 | 90 | ||
91 | $ ./mirror-checker.lua --tests=-IPv6,-https | 91 | $ ./apt-panopticon.lua --tests=-IPv6,-https |
92 | 92 | ||
93 | To only run the HTTP integrity tests, only on IPv6 - | 93 | To only run the HTTP integrity tests, only on IPv6 - |
94 | 94 | ||
95 | $ ./mirror-checker.lua --tests=http,Integrity,IPv6 | 95 | $ ./apt-panopticon.lua --tests=http,Integrity,IPv6 |
96 | 96 | ||
97 | 97 | ||
98 | The tests. | 98 | The tests. |
99 | ---------- | 99 | ---------- |
100 | 100 | ||
101 | The basic test is to find all the IPs for a mirror, including any CNAMES, | 101 | The basic test is to find all the IPs for a mirror, including any CNAMES, |
102 | then send HTTP HEAD requests to those IPs, with HOST headers for that mirror, | 102 | then send HTTP HEAD requests to those IPs, with HOST headers for that |
103 | and follow any redirections, doing the same for those redirections. | 103 | mirror, and follow any redirections, doing the same for those |
104 | Unless a specific mirror is given on the command line, the mirror_list.txt | 104 | redirections. Unless a specific mirror is given on the command line, the |
105 | file from pkgmaster.devuan.org is used to select mirrors to test. | 105 | mirror_list.txt file from pkgmaster.devuan.org is used to select mirrors |
106 | to test. | ||
106 | 107 | ||
107 | The --tests= option can be used to adjust the list of tests performed. | 108 | The --tests= option can be used to adjust the list of tests performed. |
108 | 109 | ||
@@ -147,7 +148,8 @@ negative argument deselects a test. Examples are given above. | |||
147 | 148 | ||
148 | --referenceSite | 149 | --referenceSite |
149 | 150 | ||
150 | The mirror to use as a reference for the tests, the default is pkgmaster.devuan.org. | 151 | The mirror to use as a reference for the tests, the default is |
152 | pkgmaster.devuan.org. | ||
151 | 153 | ||
152 | --roundRobin | 154 | --roundRobin |
153 | 155 | ||
@@ -187,7 +189,7 @@ Delete results/*.check. | |||
187 | 189 | ||
188 | touch results/stamp | 190 | touch results/stamp |
189 | 191 | ||
190 | Open results/mirror-checker-lua.log for message logging. | 192 | Open results/apt-panopticon.log for message logging. |
191 | 193 | ||
192 | Download mirror_list.txt from the reference site. Build a table of | 194 | Download mirror_list.txt from the reference site. Build a table of |
193 | Active mirrors keyed by the FDQN, include the listed Protocols as a sub | 195 | Active mirrors keyed by the FDQN, include the listed Protocols as a sub |
@@ -221,12 +223,12 @@ that host. | |||
221 | For each IPv4 and IPv6 address, fork a copy of the script something like | 223 | For each IPv4 and IPv6 address, fork a copy of the script something like |
222 | this (including any arguments originally provided to the script) - | 224 | this (including any arguments originally provided to the script) - |
223 | 225 | ||
224 | ionice -c3 ./mirror-checker.lua example.com/path x.x.x.x & | 226 | ionice -c3 ./apt-panopticon.lua example.com/path x.x.x.x & |
225 | 227 | ||
226 | ionice -c3 ./mirror-checker.lua example.com/path [x:x:x:x:x:x] & | 228 | ionice -c3 ./apt-panopticon.lua example.com/path [x:x:x:x:x:x] & |
227 | 229 | ||
228 | For each CNAME, it checkHost() the host, but with the CNAME as a | 230 | For each CNAME, it checkHost() the host, but with the CNAME as a second |
229 | second argument. | 231 | argument. |
230 | 232 | ||
231 | SRV reconds don't do anything yet, coz I have yet to see one from my test | 233 | SRV reconds don't do anything yet, coz I have yet to see one from my test |
232 | environment, so can't test it. | 234 | environment, so can't test it. |
@@ -234,8 +236,7 @@ environment, so can't test it. | |||
234 | 236 | ||
235 | Each forked call of the script from above does this - | 237 | Each forked call of the script from above does this - |
236 | 238 | ||
237 | Open results/mirror-checker-lua_example.com_x.x.x.x.log for message | 239 | Open results/example.com_x.x.x.x.log for message logging. |
238 | logging. | ||
239 | 240 | ||
240 | Loads the mirrors table from results/mirrors.lua. | 241 | Loads the mirrors table from results/mirrors.lua. |
241 | 242 | ||
@@ -245,18 +246,18 @@ actually perform the Integrity and Updated tests now, those haven't been | |||
245 | written yet. Note that currently this downloads 4GB per mirror. | 246 | written yet. Note that currently this downloads 4GB per mirror. |
246 | 247 | ||
247 | Calls checkHost() with the host as first and second arguments, and | 248 | Calls checkHost() with the host as first and second arguments, and |
248 | includes the IP this time. The inclusion of the IP causes | 249 | includes the IP this time. The inclusion of the IP causes checkHost() to |
249 | checkHost() to call checkFiles(). | 250 | call checkFiles(). |
250 | 251 | ||
251 | 252 | ||
252 | checkFiles() will call checkHEAD() for each of the reference files. | 253 | checkFiles() will call checkHEAD() for each of the reference files. |
253 | 254 | ||
254 | 255 | ||
255 | checkHEAD() uses LuaSocket (or LuaSec for HTTPS) to send a HEAD request to | 256 | checkHEAD() uses LuaSocket (or LuaSec for HTTPS) to send a HEAD request |
256 | the IP, with a Host header set to the original host name. Redirects will | 257 | to the IP, with a Host header set to the original host name. Redirects |
257 | not be followed by that request. If the request returns a redirect, then | 258 | will not be followed by that request. If the request returns a redirect, |
258 | checkHEAD() is called recursively. If the redirect is to some host we are | 259 | then checkHEAD() is called recursively. If the redirect is to some host |
259 | not already checking, we call checkHost() on it, with an IP of | 260 | we are not already checking, we call checkHost() on it, with an IP of |
260 | "redir". This causes checkHost() to bypass the test that would | 261 | "redir". This causes checkHost() to bypass the test that would otherwise |
261 | otherwise call checkFiles(), instead gathering the IPs and fork as usual. | 262 | call checkFiles(), instead gathering the IPs and fork as usual. |
262 | 263 | ||
diff --git a/mirror-checker.lua b/apt-panopticon.lua index 01082d9..c68ec48 100755 --- a/mirror-checker.lua +++ b/apt-panopticon.lua | |||
@@ -367,7 +367,7 @@ checkHost = function (orig, host, path, ip, file) | |||
367 | else | 367 | else |
368 | if orig == host then | 368 | if orig == host then |
369 | D("checkHost " .. orig .. "" .. file) | 369 | D("checkHost " .. orig .. "" .. file) |
370 | if testing("IPv4") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file .." &") end | 370 | if testing("IPv4") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file .." &") end |
371 | else D("checkHost " .. orig .. " -> " .. host) end | 371 | else D("checkHost " .. orig .. " -> " .. host) end |
372 | local h = mirrors[ph.host] | 372 | local h = mirrors[ph.host] |
373 | if nil == h then return end | 373 | if nil == h then return end |
@@ -375,16 +375,16 @@ checkHost = function (orig, host, path, ip, file) | |||
375 | if "table" == type(v) then | 375 | if "table" == type(v) then |
376 | for k1, v1 in pairs(v) do | 376 | for k1, v1 in pairs(v) do |
377 | if v1 == "A" then | 377 | if v1 == "A" then |
378 | if testing("IPv4") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file .." &") end | 378 | if testing("IPv4") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file .." &") end |
379 | elseif v1 == "AAAA" then | 379 | elseif v1 == "AAAA" then |
380 | if testing("IPv6") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k1 .. "] " .. file .. " &") end | 380 | if testing("IPv6") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k1 .. "] " .. file .. " &") end |
381 | end | 381 | end |
382 | end | 382 | end |
383 | else | 383 | else |
384 | if v == "A" then | 384 | if v == "A" then |
385 | if testing("IPv4") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file .." &") end | 385 | if testing("IPv4") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file .." &") end |
386 | elseif v == "AAAA" then | 386 | elseif v == "AAAA" then |
387 | if testing("IPv6") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k .. "] " .. file .. " &") end | 387 | if testing("IPv6") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k .. "] " .. file .. " &") end |
388 | end | 388 | end |
389 | end | 389 | end |
390 | end | 390 | end |
@@ -474,7 +474,7 @@ if 0 ~= #args then | |||
474 | if ("--help" == a) or ("-h" == a) then | 474 | if ("--help" == a) or ("-h" == a) then |
475 | print("I should write some docs, huh? Read README.md for instructions.") | 475 | print("I should write some docs, huh? Read README.md for instructions.") |
476 | elseif "--version" == a then | 476 | elseif "--version" == a then |
477 | print("mirror-checker-lua version 0.1 WIP development version") | 477 | print("apt-panopticon version 0.1 WIP development version") |
478 | elseif "-v" == a then | 478 | elseif "-v" == a then |
479 | verbosity = verbosity + 1 | 479 | verbosity = verbosity + 1 |
480 | sendArgs = sendArgs .. a .. " " | 480 | sendArgs = sendArgs .. a .. " " |
@@ -584,7 +584,7 @@ if 0 < #arg then | |||
584 | cut = cut + 1 | 584 | cut = cut + 1 |
585 | end | 585 | end |
586 | downloads(cut, pu.host, pu.path) | 586 | downloads(cut, pu.host, pu.path) |
587 | checkExes("mirror-checker.lua " .. sendArgs) | 587 | checkExes("apt-panopticon.lua " .. sendArgs) |
588 | checkExes(downloadLock) | 588 | checkExes(downloadLock) |
589 | end | 589 | end |
590 | end | 590 | end |
@@ -603,7 +603,7 @@ else | |||
603 | if not keep then os.execute("rm -f results/*.log") end | 603 | if not keep then os.execute("rm -f results/*.log") end |
604 | os.execute("rm -f results/*.check") | 604 | os.execute("rm -f results/*.check") |
605 | os.execute("mkdir -p results; touch results/stamp") | 605 | os.execute("mkdir -p results; touch results/stamp") |
606 | logFile, e = io.open("results/mirror-checker-lua.log", "a+") | 606 | logFile, e = io.open("results/apt-panopticon.log", "a+") |
607 | if nil == logFile then C("opening log file - " .. e); return end | 607 | if nil == logFile then C("opening log file - " .. e); return end |
608 | I("Starting tests " .. table.concat(options.tests.value, ", ")) | 608 | I("Starting tests " .. table.concat(options.tests.value, ", ")) |
609 | execute("mkdir -p results") | 609 | execute("mkdir -p results") |
@@ -621,11 +621,11 @@ else | |||
621 | local pu = url.parse("http://" .. m.BaseURL) | 621 | local pu = url.parse("http://" .. m.BaseURL) |
622 | if options.referenceSite.value ~= pu.host then | 622 | if options.referenceSite.value ~= pu.host then |
623 | checkHost(m.BaseURL) | 623 | checkHost(m.BaseURL) |
624 | checkExes("mirror-checker.lua " .. sendArgs) | 624 | checkExes("apt-panopticon.lua " .. sendArgs) |
625 | if testing("Integrity") or testing("Updated") then checkExes(downloadLock) end | 625 | if testing("Integrity") or testing("Updated") then checkExes(downloadLock) end |
626 | end | 626 | end |
627 | end | 627 | end |
628 | while 1 <= checkExes("mirror-checker.lua " .. sendArgs) do os.execute("sleep 10") end | 628 | while 1 <= checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end |
629 | if testing("Integrity") or testing("Updated") then | 629 | if testing("Integrity") or testing("Updated") then |
630 | while 0 < checkExes(downloadLock) do os.execute("sleep 10") end | 630 | while 0 < checkExes(downloadLock) do os.execute("sleep 10") end |
631 | end | 631 | end |