diff options
| -rw-r--r-- | README.md | 81 | ||||
| -rwxr-xr-x | apt-panopticon.lua (renamed from mirror-checker.lua) | 20 |
2 files changed, 51 insertions, 50 deletions
| @@ -4,11 +4,11 @@ This is currently under development, not everything has been written yet. | |||
| 4 | Some of this document mentions some of the things that are not written | 4 | Some of this document mentions some of the things that are not written |
| 5 | yet. | 5 | yet. |
| 6 | 6 | ||
| 7 | mirror-checker-lua is a Lua script used by the Devuan mirror admins | 7 | apt-panopticon is a Lua script used by the Devuan mirror admins (maybe, |
| 8 | (maybe, if they like it) to check the health of Devuan Linux package | 8 | if they like it) to check the health of Devuan Linux package mirrors. |
| 9 | mirrors. Originally there was bash scripts for this job, then Evilham | 9 | Originally there was bash scripts for this job, then Evilham wrote some |
| 10 | wrote some Python scripts, now onefang has written it in Lua. We all | 10 | Python scripts, now onefang has written it in Lua. We all have different |
| 11 | have different tastes in languages. lol | 11 | tastes in languages. lol |
| 12 | 12 | ||
| 13 | The main difference is that this Lua version tries to do everything, and | 13 | The main difference is that this Lua version tries to do everything, and |
| 14 | will be maintained. Currently the shell scripts and Python scripts are | 14 | will be maintained. Currently the shell scripts and Python scripts are |
| @@ -17,7 +17,7 @@ badgered him about his Python scripts. It should also be much easier to | |||
| 17 | use, the previous scripts needed some work before you could run them, | 17 | use, the previous scripts needed some work before you could run them, |
| 18 | this one you just download and run. | 18 | this one you just download and run. |
| 19 | 19 | ||
| 20 | The source code is at [https://sledjhamr.org/cgit/mirror-checker-lua/](https://sledjhamr.org/cgit/mirror-checker-lua/) | 20 | The source code is at [https://sledjhamr.org/cgit/apt-panopticon/](https://sledjhamr.org/cgit/apt-panopticon/) |
| 21 | 21 | ||
| 22 | The issue tracker is at [https://sledjhamr.org/mantisbt/project_page.php?project_id=13](https://sledjhamr.org/mantisbt/project_page.php?project_id=13) | 22 | The issue tracker is at [https://sledjhamr.org/mantisbt/project_page.php?project_id=13](https://sledjhamr.org/mantisbt/project_page.php?project_id=13) |
| 23 | 23 | ||
| @@ -25,7 +25,7 @@ The issue tracker is at [https://sledjhamr.org/mantisbt/project_page.php?project | |||
| 25 | Installation. | 25 | Installation. |
| 26 | ------------- | 26 | ------------- |
| 27 | 27 | ||
| 28 | Download the source. You may want to put the mirror-checker.lua script | 28 | Download the source. You may want to put the apt-panopticon.lua script |
| 29 | in someplace like `/usr/local/bin` and make sure it is executable. | 29 | in someplace like `/usr/local/bin` and make sure it is executable. |
| 30 | 30 | ||
| 31 | It should run on any recent Linux, you'll need to have the following | 31 | It should run on any recent Linux, you'll need to have the following |
| @@ -52,57 +52,58 @@ and `results/web` directories, with the notification emails and web pages | |||
| 52 | Note that unlike typical commands, you can't run single character options | 52 | Note that unlike typical commands, you can't run single character options |
| 53 | together, so this is wrong - | 53 | together, so this is wrong - |
| 54 | 54 | ||
| 55 | $ ./mirror-checker.lua -vvv | 55 | $ ./apt-panopticon.lua -vvv |
| 56 | 56 | ||
| 57 | Instead do this - | 57 | Instead do this - |
| 58 | 58 | ||
| 59 | $ ./mirror-checker.lua -v -v -v | 59 | $ ./apt-panopticon.lua -v -v -v |
| 60 | 60 | ||
| 61 | Just run the script to do all of the tests - | 61 | Just run the script to do all of the tests - |
| 62 | 62 | ||
| 63 | $ ./mirror-checker.lua | 63 | $ ./apt-panopticon.lua |
| 64 | 64 | ||
| 65 | Which will print any errors. If you don't want to see errors - | 65 | Which will print any errors. If you don't want to see errors - |
| 66 | 66 | ||
| 67 | $ ./mirror-checker.lua -q | 67 | $ ./apt-panopticon.lua -q |
| 68 | 68 | ||
| 69 | If you want to see warnings as well (as usual, the more `-v` options, the more | 69 | If you want to see warnings as well (as usual, the more `-v` options, the more |
| 70 | details) - | 70 | details) - |
| 71 | 71 | ||
| 72 | $ ./mirror-checker.lua -v | 72 | $ ./apt-panopticon.lua -v |
| 73 | 73 | ||
| 74 | Or use the usual options for the help and version number (not written yet) - | 74 | Or use the usual options for the help and version number (not written yet) - |
| 75 | 75 | ||
| 76 | $ ./mirror-checker.lua -h | 76 | $ ./apt-panopticon.lua -h |
| 77 | $ ./mirror-checker.lua --help | 77 | $ ./apt-panopticon.lua --help |
| 78 | $ ./mirror-checker.lua --version | 78 | $ ./apt-panopticon.lua --version |
| 79 | 79 | ||
| 80 | To run the tests on a specific mirror, for example pkgmaster.devuan.org - | 80 | To run the tests on a specific mirror, for example pkgmaster.devuan.org - |
| 81 | 81 | ||
| 82 | $ ./mirror-checker.lua pkgmaster.devuan.org | 82 | $ ./apt-panopticon.lua pkgmaster.devuan.org |
| 83 | 83 | ||
| 84 | You can use the `--tests` option to tune which tests are run, for example | 84 | You can use the `--tests` option to tune which tests are run, for example |
| 85 | to stop IPv6 tests, coz you don't have IPv6 - | 85 | to stop IPv6 tests, coz you don't have IPv6 - |
| 86 | 86 | ||
| 87 | $ ./mirror-checker.lua --tests=-IPv6 | 87 | $ ./apt-panopticon.lua --tests=-IPv6 |
| 88 | 88 | ||
| 89 | To do the same, but not run the HTTPS tests either - | 89 | To do the same, but not run the HTTPS tests either - |
| 90 | 90 | ||
| 91 | $ ./mirror-checker.lua --tests=-IPv6,-https | 91 | $ ./apt-panopticon.lua --tests=-IPv6,-https |
| 92 | 92 | ||
| 93 | To only run the HTTP integrity tests, only on IPv6 - | 93 | To only run the HTTP integrity tests, only on IPv6 - |
| 94 | 94 | ||
| 95 | $ ./mirror-checker.lua --tests=http,Integrity,IPv6 | 95 | $ ./apt-panopticon.lua --tests=http,Integrity,IPv6 |
| 96 | 96 | ||
| 97 | 97 | ||
| 98 | The tests. | 98 | The tests. |
| 99 | ---------- | 99 | ---------- |
| 100 | 100 | ||
| 101 | The basic test is to find all the IPs for a mirror, including any CNAMES, | 101 | The basic test is to find all the IPs for a mirror, including any CNAMES, |
| 102 | then send HTTP HEAD requests to those IPs, with HOST headers for that mirror, | 102 | then send HTTP HEAD requests to those IPs, with HOST headers for that |
| 103 | and follow any redirections, doing the same for those redirections. | 103 | mirror, and follow any redirections, doing the same for those |
| 104 | Unless a specific mirror is given on the command line, the mirror_list.txt | 104 | redirections. Unless a specific mirror is given on the command line, the |
| 105 | file from pkgmaster.devuan.org is used to select mirrors to test. | 105 | mirror_list.txt file from pkgmaster.devuan.org is used to select mirrors |
| 106 | to test. | ||
| 106 | 107 | ||
| 107 | The --tests= option can be used to adjust the list of tests performed. | 108 | The --tests= option can be used to adjust the list of tests performed. |
| 108 | 109 | ||
| @@ -147,7 +148,8 @@ negative argument deselects a test. Examples are given above. | |||
| 147 | 148 | ||
| 148 | --referenceSite | 149 | --referenceSite |
| 149 | 150 | ||
| 150 | The mirror to use as a reference for the tests, the default is pkgmaster.devuan.org. | 151 | The mirror to use as a reference for the tests, the default is |
| 152 | pkgmaster.devuan.org. | ||
| 151 | 153 | ||
| 152 | --roundRobin | 154 | --roundRobin |
| 153 | 155 | ||
| @@ -187,7 +189,7 @@ Delete results/*.check. | |||
| 187 | 189 | ||
| 188 | touch results/stamp | 190 | touch results/stamp |
| 189 | 191 | ||
| 190 | Open results/mirror-checker-lua.log for message logging. | 192 | Open results/apt-panopticon.log for message logging. |
| 191 | 193 | ||
| 192 | Download mirror_list.txt from the reference site. Build a table of | 194 | Download mirror_list.txt from the reference site. Build a table of |
| 193 | Active mirrors keyed by the FDQN, include the listed Protocols as a sub | 195 | Active mirrors keyed by the FDQN, include the listed Protocols as a sub |
| @@ -221,12 +223,12 @@ that host. | |||
| 221 | For each IPv4 and IPv6 address, fork a copy of the script something like | 223 | For each IPv4 and IPv6 address, fork a copy of the script something like |
| 222 | this (including any arguments originally provided to the script) - | 224 | this (including any arguments originally provided to the script) - |
| 223 | 225 | ||
| 224 | ionice -c3 ./mirror-checker.lua example.com/path x.x.x.x & | 226 | ionice -c3 ./apt-panopticon.lua example.com/path x.x.x.x & |
| 225 | 227 | ||
| 226 | ionice -c3 ./mirror-checker.lua example.com/path [x:x:x:x:x:x] & | 228 | ionice -c3 ./apt-panopticon.lua example.com/path [x:x:x:x:x:x] & |
| 227 | 229 | ||
| 228 | For each CNAME, it checkHost() the host, but with the CNAME as a | 230 | For each CNAME, it checkHost() the host, but with the CNAME as a second |
| 229 | second argument. | 231 | argument. |
| 230 | 232 | ||
| 231 | SRV reconds don't do anything yet, coz I have yet to see one from my test | 233 | SRV reconds don't do anything yet, coz I have yet to see one from my test |
| 232 | environment, so can't test it. | 234 | environment, so can't test it. |
| @@ -234,8 +236,7 @@ environment, so can't test it. | |||
| 234 | 236 | ||
| 235 | Each forked call of the script from above does this - | 237 | Each forked call of the script from above does this - |
| 236 | 238 | ||
| 237 | Open results/mirror-checker-lua_example.com_x.x.x.x.log for message | 239 | Open results/example.com_x.x.x.x.log for message logging. |
| 238 | logging. | ||
| 239 | 240 | ||
| 240 | Loads the mirrors table from results/mirrors.lua. | 241 | Loads the mirrors table from results/mirrors.lua. |
| 241 | 242 | ||
| @@ -245,18 +246,18 @@ actually perform the Integrity and Updated tests now, those haven't been | |||
| 245 | written yet. Note that currently this downloads 4GB per mirror. | 246 | written yet. Note that currently this downloads 4GB per mirror. |
| 246 | 247 | ||
| 247 | Calls checkHost() with the host as first and second arguments, and | 248 | Calls checkHost() with the host as first and second arguments, and |
| 248 | includes the IP this time. The inclusion of the IP causes | 249 | includes the IP this time. The inclusion of the IP causes checkHost() to |
| 249 | checkHost() to call checkFiles(). | 250 | call checkFiles(). |
| 250 | 251 | ||
| 251 | 252 | ||
| 252 | checkFiles() will call checkHEAD() for each of the reference files. | 253 | checkFiles() will call checkHEAD() for each of the reference files. |
| 253 | 254 | ||
| 254 | 255 | ||
| 255 | checkHEAD() uses LuaSocket (or LuaSec for HTTPS) to send a HEAD request to | 256 | checkHEAD() uses LuaSocket (or LuaSec for HTTPS) to send a HEAD request |
| 256 | the IP, with a Host header set to the original host name. Redirects will | 257 | to the IP, with a Host header set to the original host name. Redirects |
| 257 | not be followed by that request. If the request returns a redirect, then | 258 | will not be followed by that request. If the request returns a redirect, |
| 258 | checkHEAD() is called recursively. If the redirect is to some host we are | 259 | then checkHEAD() is called recursively. If the redirect is to some host |
| 259 | not already checking, we call checkHost() on it, with an IP of | 260 | we are not already checking, we call checkHost() on it, with an IP of |
| 260 | "redir". This causes checkHost() to bypass the test that would | 261 | "redir". This causes checkHost() to bypass the test that would otherwise |
| 261 | otherwise call checkFiles(), instead gathering the IPs and fork as usual. | 262 | call checkFiles(), instead gathering the IPs and fork as usual. |
| 262 | 263 | ||
diff --git a/mirror-checker.lua b/apt-panopticon.lua index 01082d9..c68ec48 100755 --- a/mirror-checker.lua +++ b/apt-panopticon.lua | |||
| @@ -367,7 +367,7 @@ checkHost = function (orig, host, path, ip, file) | |||
| 367 | else | 367 | else |
| 368 | if orig == host then | 368 | if orig == host then |
| 369 | D("checkHost " .. orig .. "" .. file) | 369 | D("checkHost " .. orig .. "" .. file) |
| 370 | if testing("IPv4") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file .." &") end | 370 | if testing("IPv4") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file .." &") end |
| 371 | else D("checkHost " .. orig .. " -> " .. host) end | 371 | else D("checkHost " .. orig .. " -> " .. host) end |
| 372 | local h = mirrors[ph.host] | 372 | local h = mirrors[ph.host] |
| 373 | if nil == h then return end | 373 | if nil == h then return end |
| @@ -375,16 +375,16 @@ checkHost = function (orig, host, path, ip, file) | |||
| 375 | if "table" == type(v) then | 375 | if "table" == type(v) then |
| 376 | for k1, v1 in pairs(v) do | 376 | for k1, v1 in pairs(v) do |
| 377 | if v1 == "A" then | 377 | if v1 == "A" then |
| 378 | if testing("IPv4") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file .." &") end | 378 | if testing("IPv4") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file .." &") end |
| 379 | elseif v1 == "AAAA" then | 379 | elseif v1 == "AAAA" then |
| 380 | if testing("IPv6") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k1 .. "] " .. file .. " &") end | 380 | if testing("IPv6") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k1 .. "] " .. file .. " &") end |
| 381 | end | 381 | end |
| 382 | end | 382 | end |
| 383 | else | 383 | else |
| 384 | if v == "A" then | 384 | if v == "A" then |
| 385 | if testing("IPv4") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file .." &") end | 385 | if testing("IPv4") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file .." &") end |
| 386 | elseif v == "AAAA" then | 386 | elseif v == "AAAA" then |
| 387 | if testing("IPv6") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k .. "] " .. file .. " &") end | 387 | if testing("IPv6") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k .. "] " .. file .. " &") end |
| 388 | end | 388 | end |
| 389 | end | 389 | end |
| 390 | end | 390 | end |
| @@ -474,7 +474,7 @@ if 0 ~= #args then | |||
| 474 | if ("--help" == a) or ("-h" == a) then | 474 | if ("--help" == a) or ("-h" == a) then |
| 475 | print("I should write some docs, huh? Read README.md for instructions.") | 475 | print("I should write some docs, huh? Read README.md for instructions.") |
| 476 | elseif "--version" == a then | 476 | elseif "--version" == a then |
| 477 | print("mirror-checker-lua version 0.1 WIP development version") | 477 | print("apt-panopticon version 0.1 WIP development version") |
| 478 | elseif "-v" == a then | 478 | elseif "-v" == a then |
| 479 | verbosity = verbosity + 1 | 479 | verbosity = verbosity + 1 |
| 480 | sendArgs = sendArgs .. a .. " " | 480 | sendArgs = sendArgs .. a .. " " |
| @@ -584,7 +584,7 @@ if 0 < #arg then | |||
| 584 | cut = cut + 1 | 584 | cut = cut + 1 |
| 585 | end | 585 | end |
| 586 | downloads(cut, pu.host, pu.path) | 586 | downloads(cut, pu.host, pu.path) |
| 587 | checkExes("mirror-checker.lua " .. sendArgs) | 587 | checkExes("apt-panopticon.lua " .. sendArgs) |
| 588 | checkExes(downloadLock) | 588 | checkExes(downloadLock) |
| 589 | end | 589 | end |
| 590 | end | 590 | end |
| @@ -603,7 +603,7 @@ else | |||
| 603 | if not keep then os.execute("rm -f results/*.log") end | 603 | if not keep then os.execute("rm -f results/*.log") end |
| 604 | os.execute("rm -f results/*.check") | 604 | os.execute("rm -f results/*.check") |
| 605 | os.execute("mkdir -p results; touch results/stamp") | 605 | os.execute("mkdir -p results; touch results/stamp") |
| 606 | logFile, e = io.open("results/mirror-checker-lua.log", "a+") | 606 | logFile, e = io.open("results/apt-panopticon.log", "a+") |
| 607 | if nil == logFile then C("opening log file - " .. e); return end | 607 | if nil == logFile then C("opening log file - " .. e); return end |
| 608 | I("Starting tests " .. table.concat(options.tests.value, ", ")) | 608 | I("Starting tests " .. table.concat(options.tests.value, ", ")) |
| 609 | execute("mkdir -p results") | 609 | execute("mkdir -p results") |
| @@ -621,11 +621,11 @@ else | |||
| 621 | local pu = url.parse("http://" .. m.BaseURL) | 621 | local pu = url.parse("http://" .. m.BaseURL) |
| 622 | if options.referenceSite.value ~= pu.host then | 622 | if options.referenceSite.value ~= pu.host then |
| 623 | checkHost(m.BaseURL) | 623 | checkHost(m.BaseURL) |
| 624 | checkExes("mirror-checker.lua " .. sendArgs) | 624 | checkExes("apt-panopticon.lua " .. sendArgs) |
| 625 | if testing("Integrity") or testing("Updated") then checkExes(downloadLock) end | 625 | if testing("Integrity") or testing("Updated") then checkExes(downloadLock) end |
| 626 | end | 626 | end |
| 627 | end | 627 | end |
| 628 | while 1 <= checkExes("mirror-checker.lua " .. sendArgs) do os.execute("sleep 10") end | 628 | while 1 <= checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end |
| 629 | if testing("Integrity") or testing("Updated") then | 629 | if testing("Integrity") or testing("Updated") then |
| 630 | while 0 < checkExes(downloadLock) do os.execute("sleep 10") end | 630 | while 0 < checkExes(downloadLock) do os.execute("sleep 10") end |
| 631 | end | 631 | end |
