aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
authoronefang2019-11-05 15:38:24 +1000
committeronefang2019-11-05 15:38:24 +1000
commitcc872d3eba68f08ee365b8771d8504c28da9020d (patch)
tree2083ec34d871381a04f7fb26c5ce9ec7062ae39f
parentCreate a persistant table with the results. (diff)
downloadapt-panopticon-cc872d3eba68f08ee365b8771d8504c28da9020d.zip
apt-panopticon-cc872d3eba68f08ee365b8771d8504c28da9020d.tar.gz
apt-panopticon-cc872d3eba68f08ee365b8771d8504c28da9020d.tar.bz2
apt-panopticon-cc872d3eba68f08ee365b8771d8504c28da9020d.tar.xz
Rename project to apt-panopticon.
It used to be mirror-checker-lua.
-rw-r--r--README.md81
-rwxr-xr-xapt-panopticon.lua (renamed from mirror-checker.lua)20
2 files changed, 51 insertions, 50 deletions
diff --git a/README.md b/README.md
index 5ac7f9b..94b3c65 100644
--- a/README.md
+++ b/README.md
@@ -4,11 +4,11 @@ This is currently under development, not everything has been written yet.
4Some of this document mentions some of the things that are not written 4Some of this document mentions some of the things that are not written
5yet. 5yet.
6 6
7mirror-checker-lua is a Lua script used by the Devuan mirror admins 7apt-panopticon is a Lua script used by the Devuan mirror admins (maybe,
8(maybe, if they like it) to check the health of Devuan Linux package 8if they like it) to check the health of Devuan Linux package mirrors.
9mirrors. Originally there was bash scripts for this job, then Evilham 9Originally there was bash scripts for this job, then Evilham wrote some
10wrote some Python scripts, now onefang has written it in Lua. We all 10Python scripts, now onefang has written it in Lua. We all have different
11have different tastes in languages. lol 11tastes in languages. lol
12 12
13The main difference is that this Lua version tries to do everything, and 13The main difference is that this Lua version tries to do everything, and
14will be maintained. Currently the shell scripts and Python scripts are 14will be maintained. Currently the shell scripts and Python scripts are
@@ -17,7 +17,7 @@ badgered him about his Python scripts. It should also be much easier to
17use, the previous scripts needed some work before you could run them, 17use, the previous scripts needed some work before you could run them,
18this one you just download and run. 18this one you just download and run.
19 19
20The source code is at [https://sledjhamr.org/cgit/mirror-checker-lua/](https://sledjhamr.org/cgit/mirror-checker-lua/) 20The source code is at [https://sledjhamr.org/cgit/apt-panopticon/](https://sledjhamr.org/cgit/apt-panopticon/)
21 21
22The issue tracker is at [https://sledjhamr.org/mantisbt/project_page.php?project_id=13](https://sledjhamr.org/mantisbt/project_page.php?project_id=13) 22The issue tracker is at [https://sledjhamr.org/mantisbt/project_page.php?project_id=13](https://sledjhamr.org/mantisbt/project_page.php?project_id=13)
23 23
@@ -25,7 +25,7 @@ The issue tracker is at [https://sledjhamr.org/mantisbt/project_page.php?project
25Installation. 25Installation.
26------------- 26-------------
27 27
28Download the source. You may want to put the mirror-checker.lua script 28Download the source. You may want to put the apt-panopticon.lua script
29in someplace like `/usr/local/bin` and make sure it is executable. 29in someplace like `/usr/local/bin` and make sure it is executable.
30 30
31It should run on any recent Linux, you'll need to have the following 31It should run on any recent Linux, you'll need to have the following
@@ -52,57 +52,58 @@ and `results/web` directories, with the notification emails and web pages
52Note that unlike typical commands, you can't run single character options 52Note that unlike typical commands, you can't run single character options
53together, so this is wrong - 53together, so this is wrong -
54 54
55 $ ./mirror-checker.lua -vvv 55 $ ./apt-panopticon.lua -vvv
56 56
57Instead do this - 57Instead do this -
58 58
59 $ ./mirror-checker.lua -v -v -v 59 $ ./apt-panopticon.lua -v -v -v
60 60
61Just run the script to do all of the tests - 61Just run the script to do all of the tests -
62 62
63 $ ./mirror-checker.lua 63 $ ./apt-panopticon.lua
64 64
65Which will print any errors. If you don't want to see errors - 65Which will print any errors. If you don't want to see errors -
66 66
67 $ ./mirror-checker.lua -q 67 $ ./apt-panopticon.lua -q
68 68
69If you want to see warnings as well (as usual, the more `-v` options, the more 69If you want to see warnings as well (as usual, the more `-v` options, the more
70details) - 70details) -
71 71
72 $ ./mirror-checker.lua -v 72 $ ./apt-panopticon.lua -v
73 73
74Or use the usual options for the help and version number (not written yet) - 74Or use the usual options for the help and version number (not written yet) -
75 75
76 $ ./mirror-checker.lua -h 76 $ ./apt-panopticon.lua -h
77 $ ./mirror-checker.lua --help 77 $ ./apt-panopticon.lua --help
78 $ ./mirror-checker.lua --version 78 $ ./apt-panopticon.lua --version
79 79
80To run the tests on a specific mirror, for example pkgmaster.devuan.org - 80To run the tests on a specific mirror, for example pkgmaster.devuan.org -
81 81
82 $ ./mirror-checker.lua pkgmaster.devuan.org 82 $ ./apt-panopticon.lua pkgmaster.devuan.org
83 83
84You can use the `--tests` option to tune which tests are run, for example 84You can use the `--tests` option to tune which tests are run, for example
85to stop IPv6 tests, coz you don't have IPv6 - 85to stop IPv6 tests, coz you don't have IPv6 -
86 86
87 $ ./mirror-checker.lua --tests=-IPv6 87 $ ./apt-panopticon.lua --tests=-IPv6
88 88
89To do the same, but not run the HTTPS tests either - 89To do the same, but not run the HTTPS tests either -
90 90
91 $ ./mirror-checker.lua --tests=-IPv6,-https 91 $ ./apt-panopticon.lua --tests=-IPv6,-https
92 92
93To only run the HTTP integrity tests, only on IPv6 - 93To only run the HTTP integrity tests, only on IPv6 -
94 94
95 $ ./mirror-checker.lua --tests=http,Integrity,IPv6 95 $ ./apt-panopticon.lua --tests=http,Integrity,IPv6
96 96
97 97
98The tests. 98The tests.
99---------- 99----------
100 100
101The basic test is to find all the IPs for a mirror, including any CNAMES, 101The basic test is to find all the IPs for a mirror, including any CNAMES,
102then send HTTP HEAD requests to those IPs, with HOST headers for that mirror, 102then send HTTP HEAD requests to those IPs, with HOST headers for that
103and follow any redirections, doing the same for those redirections. 103mirror, and follow any redirections, doing the same for those
104Unless a specific mirror is given on the command line, the mirror_list.txt 104redirections. Unless a specific mirror is given on the command line, the
105file from pkgmaster.devuan.org is used to select mirrors to test. 105mirror_list.txt file from pkgmaster.devuan.org is used to select mirrors
106to test.
106 107
107The --tests= option can be used to adjust the list of tests performed. 108The --tests= option can be used to adjust the list of tests performed.
108 109
@@ -147,7 +148,8 @@ negative argument deselects a test. Examples are given above.
147 148
148--referenceSite 149--referenceSite
149 150
150The mirror to use as a reference for the tests, the default is pkgmaster.devuan.org. 151The mirror to use as a reference for the tests, the default is
152pkgmaster.devuan.org.
151 153
152--roundRobin 154--roundRobin
153 155
@@ -187,7 +189,7 @@ Delete results/*.check.
187 189
188touch results/stamp 190touch results/stamp
189 191
190Open results/mirror-checker-lua.log for message logging. 192Open results/apt-panopticon.log for message logging.
191 193
192Download mirror_list.txt from the reference site. Build a table of 194Download mirror_list.txt from the reference site. Build a table of
193Active mirrors keyed by the FDQN, include the listed Protocols as a sub 195Active mirrors keyed by the FDQN, include the listed Protocols as a sub
@@ -221,12 +223,12 @@ that host.
221For each IPv4 and IPv6 address, fork a copy of the script something like 223For each IPv4 and IPv6 address, fork a copy of the script something like
222this (including any arguments originally provided to the script) - 224this (including any arguments originally provided to the script) -
223 225
224ionice -c3 ./mirror-checker.lua example.com/path x.x.x.x & 226ionice -c3 ./apt-panopticon.lua example.com/path x.x.x.x &
225 227
226ionice -c3 ./mirror-checker.lua example.com/path [x:x:x:x:x:x] & 228ionice -c3 ./apt-panopticon.lua example.com/path [x:x:x:x:x:x] &
227 229
228For each CNAME, it checkHost() the host, but with the CNAME as a 230For each CNAME, it checkHost() the host, but with the CNAME as a second
229second argument. 231argument.
230 232
231SRV reconds don't do anything yet, coz I have yet to see one from my test 233SRV reconds don't do anything yet, coz I have yet to see one from my test
232environment, so can't test it. 234environment, so can't test it.
@@ -234,8 +236,7 @@ environment, so can't test it.
234 236
235Each forked call of the script from above does this - 237Each forked call of the script from above does this -
236 238
237Open results/mirror-checker-lua_example.com_x.x.x.x.log for message 239Open results/example.com_x.x.x.x.log for message logging.
238logging.
239 240
240Loads the mirrors table from results/mirrors.lua. 241Loads the mirrors table from results/mirrors.lua.
241 242
@@ -245,18 +246,18 @@ actually perform the Integrity and Updated tests now, those haven't been
245written yet. Note that currently this downloads 4GB per mirror. 246written yet. Note that currently this downloads 4GB per mirror.
246 247
247Calls checkHost() with the host as first and second arguments, and 248Calls checkHost() with the host as first and second arguments, and
248includes the IP this time. The inclusion of the IP causes 249includes the IP this time. The inclusion of the IP causes checkHost() to
249checkHost() to call checkFiles(). 250call checkFiles().
250 251
251 252
252checkFiles() will call checkHEAD() for each of the reference files. 253checkFiles() will call checkHEAD() for each of the reference files.
253 254
254 255
255checkHEAD() uses LuaSocket (or LuaSec for HTTPS) to send a HEAD request to 256checkHEAD() uses LuaSocket (or LuaSec for HTTPS) to send a HEAD request
256the IP, with a Host header set to the original host name. Redirects will 257to the IP, with a Host header set to the original host name. Redirects
257not be followed by that request. If the request returns a redirect, then 258will not be followed by that request. If the request returns a redirect,
258checkHEAD() is called recursively. If the redirect is to some host we are 259then checkHEAD() is called recursively. If the redirect is to some host
259not already checking, we call checkHost() on it, with an IP of 260we are not already checking, we call checkHost() on it, with an IP of
260"redir". This causes checkHost() to bypass the test that would 261"redir". This causes checkHost() to bypass the test that would otherwise
261otherwise call checkFiles(), instead gathering the IPs and fork as usual. 262call checkFiles(), instead gathering the IPs and fork as usual.
262 263
diff --git a/mirror-checker.lua b/apt-panopticon.lua
index 01082d9..c68ec48 100755
--- a/mirror-checker.lua
+++ b/apt-panopticon.lua
@@ -367,7 +367,7 @@ checkHost = function (orig, host, path, ip, file)
367 else 367 else
368 if orig == host then 368 if orig == host then
369 D("checkHost " .. orig .. "" .. file) 369 D("checkHost " .. orig .. "" .. file)
370 if testing("IPv4") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file .." &") end 370 if testing("IPv4") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file .." &") end
371 else D("checkHost " .. orig .. " -> " .. host) end 371 else D("checkHost " .. orig .. " -> " .. host) end
372 local h = mirrors[ph.host] 372 local h = mirrors[ph.host]
373 if nil == h then return end 373 if nil == h then return end
@@ -375,16 +375,16 @@ checkHost = function (orig, host, path, ip, file)
375 if "table" == type(v) then 375 if "table" == type(v) then
376 for k1, v1 in pairs(v) do 376 for k1, v1 in pairs(v) do
377 if v1 == "A" then 377 if v1 == "A" then
378 if testing("IPv4") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file .." &") end 378 if testing("IPv4") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file .." &") end
379 elseif v1 == "AAAA" then 379 elseif v1 == "AAAA" then
380 if testing("IPv6") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k1 .. "] " .. file .. " &") end 380 if testing("IPv6") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k1 .. "] " .. file .. " &") end
381 end 381 end
382 end 382 end
383 else 383 else
384 if v == "A" then 384 if v == "A" then
385 if testing("IPv4") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file .." &") end 385 if testing("IPv4") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file .." &") end
386 elseif v == "AAAA" then 386 elseif v == "AAAA" then
387 if testing("IPv6") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k .. "] " .. file .. " &") end 387 if testing("IPv6") then execute("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k .. "] " .. file .. " &") end
388 end 388 end
389 end 389 end
390 end 390 end
@@ -474,7 +474,7 @@ if 0 ~= #args then
474 if ("--help" == a) or ("-h" == a) then 474 if ("--help" == a) or ("-h" == a) then
475 print("I should write some docs, huh? Read README.md for instructions.") 475 print("I should write some docs, huh? Read README.md for instructions.")
476 elseif "--version" == a then 476 elseif "--version" == a then
477 print("mirror-checker-lua version 0.1 WIP development version") 477 print("apt-panopticon version 0.1 WIP development version")
478 elseif "-v" == a then 478 elseif "-v" == a then
479 verbosity = verbosity + 1 479 verbosity = verbosity + 1
480 sendArgs = sendArgs .. a .. " " 480 sendArgs = sendArgs .. a .. " "
@@ -584,7 +584,7 @@ if 0 < #arg then
584 cut = cut + 1 584 cut = cut + 1
585 end 585 end
586 downloads(cut, pu.host, pu.path) 586 downloads(cut, pu.host, pu.path)
587 checkExes("mirror-checker.lua " .. sendArgs) 587 checkExes("apt-panopticon.lua " .. sendArgs)
588 checkExes(downloadLock) 588 checkExes(downloadLock)
589 end 589 end
590 end 590 end
@@ -603,7 +603,7 @@ else
603 if not keep then os.execute("rm -f results/*.log") end 603 if not keep then os.execute("rm -f results/*.log") end
604 os.execute("rm -f results/*.check") 604 os.execute("rm -f results/*.check")
605 os.execute("mkdir -p results; touch results/stamp") 605 os.execute("mkdir -p results; touch results/stamp")
606 logFile, e = io.open("results/mirror-checker-lua.log", "a+") 606 logFile, e = io.open("results/apt-panopticon.log", "a+")
607 if nil == logFile then C("opening log file - " .. e); return end 607 if nil == logFile then C("opening log file - " .. e); return end
608 I("Starting tests " .. table.concat(options.tests.value, ", ")) 608 I("Starting tests " .. table.concat(options.tests.value, ", "))
609 execute("mkdir -p results") 609 execute("mkdir -p results")
@@ -621,11 +621,11 @@ else
621 local pu = url.parse("http://" .. m.BaseURL) 621 local pu = url.parse("http://" .. m.BaseURL)
622 if options.referenceSite.value ~= pu.host then 622 if options.referenceSite.value ~= pu.host then
623 checkHost(m.BaseURL) 623 checkHost(m.BaseURL)
624 checkExes("mirror-checker.lua " .. sendArgs) 624 checkExes("apt-panopticon.lua " .. sendArgs)
625 if testing("Integrity") or testing("Updated") then checkExes(downloadLock) end 625 if testing("Integrity") or testing("Updated") then checkExes(downloadLock) end
626 end 626 end
627 end 627 end
628 while 1 <= checkExes("mirror-checker.lua " .. sendArgs) do os.execute("sleep 10") end 628 while 1 <= checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end
629 if testing("Integrity") or testing("Updated") then 629 if testing("Integrity") or testing("Updated") then
630 while 0 < checkExes(downloadLock) do os.execute("sleep 10") end 630 while 0 < checkExes(downloadLock) do os.execute("sleep 10") end
631 end 631 end