aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
Diffstat (limited to '')
-rwxr-xr-xmirror-checker.lua98
1 files changed, 70 insertions, 28 deletions
diff --git a/mirror-checker.lua b/mirror-checker.lua
index 6eb1c1a..d370497 100755
--- a/mirror-checker.lua
+++ b/mirror-checker.lua
@@ -179,7 +179,23 @@ checkURL = function (host, URL, r, retry)
179 local check = "Checking file" 179 local check = "Checking file"
180 if 0 < r then 180 if 0 < r then
181 check = "Redirecting to" 181 check = "Redirecting to"
182-- checkHost(host, host) 182 --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location.
183 Not good for deb.debian.org, which we should only be throwing .debs at.
184 What we should do is loop through the DNS entries, and only test the specific protocol & file being tested here.
185
186 This is what I came up with for checking if we are already testing a specific URL.
187 Still duplicates a tiny bit, but much less than the previous find based method.
188 ]]
189 local pl = url.parse(URL, defaultURL)
190 local file = host .. "://" .. pl.path
191 local f = io.popen(string.format('if [ ! -f results/%s.check ] ; then touch results/%s.check; echo -n "check"; fi', file:gsub("/", "_"), file:gsub("/", "_") )):read("*a")
192 if (nil == f) or ("check" == f) then
193 I(" Now checking redirected host " .. file)
194 checkRedirects(host, host, nil, "redir", pl.path)
195 else
196 D(" Already checking " .. file)
197 end
198
183 end 199 end
184 if 0 < retry then 200 if 0 < retry then
185 check = "Retry " .. retry .. " " .. check 201 check = "Retry " .. retry .. " " .. check
@@ -229,18 +245,25 @@ checkURL = function (host, URL, r, retry)
229 end 245 end
230end 246end
231 247
232local checkPaths = function (host, ip, path) 248local checkPaths = function (host, ip, path, file)
233 I(" Checking IP " .. host .. " -> " .. ip .. " " .. path) 249 if nil ~= file then
234 for i, s in pairs(referenceDebs) do 250 if "redirect" == ip then ip = host end
235 if testing("http") then checkURL(host, "http://" .. ip .. path .. "/" .. s) end 251 I(" Checking IP for file " .. host .. " -> " .. ip .. " " .. path .. " " .. file)
236 if testing("https") then checkURL(host, "https://" .. ip .. path .. "/" .. s) end 252 if testing("http") then checkURL(host, "http://" .. ip .. path .. "/" .. file) end
237 end 253 if testing("https") then checkURL(host, "https://" .. ip .. path .. "/" .. file) end
254 else
255 I(" Checking IP " .. host .. " -> " .. ip .. " " .. path)
256 for i, s in pairs(referenceDebs) do
257 if testing("http") then checkURL(host, "http://" .. ip .. path .. "/" .. s) end
258 if testing("https") then checkURL(host, "https://" .. ip .. path .. "/" .. s) end
259 end
238 260
239 for i, s in pairs(releases) do 261 for i, s in pairs(releases) do
240 for j, k in pairs(releaseFiles) do 262 for j, k in pairs(releaseFiles) do
241 if repoExists(s .. k) then 263 if repoExists(s .. k) then
242 if testing("http") then checkURL(host, "http://" .. ip .. path .. "/merged/dists/" .. s .. k) end 264 if testing("http") then checkURL(host, "http://" .. ip .. path .. "/merged/dists/" .. s .. k) end
243 if testing("https") then checkURL(host, "https://" .. ip .. path .. "/merged/dists/" .. s .. k) end 265 if testing("https") then checkURL(host, "https://" .. ip .. path .. "/merged/dists/" .. s .. k) end
266 end
244 end 267 end
245 end 268 end
246 end 269 end
@@ -251,24 +274,37 @@ local execute = function (s)
251 os.execute(s) 274 os.execute(s)
252end 275end
253 276
254checkRedirects = function (orig, host, path, ip) 277checkRedirects = function (orig, host, path, ip, file)
255 if nil == host then host = orig end 278 if nil == host then host = orig end
256 if nil == path then path = "" end 279 if nil == path then path = "" end
280 if nil == file then file = "" end
257 local po = url.parse("http://" .. orig) 281 local po = url.parse("http://" .. orig)
258 local ph = url.parse("http://" .. host) 282 local ph = url.parse("http://" .. host)
259 if nil ~= ip then 283 if (nil ~= ip) and ("redir" ~= ip) then
260 checkPaths(po.host, ip, path) 284 if "" ~= file then
285 D("checking redirected file " .. po.host .. " " .. file)
286 checkPaths(po.host, ip, path, file)
287 else
288 checkPaths(po.host, ip, path)
289 end
261 else 290 else
262 if orig == host then D("checkRedirects " .. orig) else D("checkRedirects " .. orig .. " " .. host) end 291 if orig == host then D("checkRedirects " .. orig .. "" .. file) else D("checkRedirects " .. orig .. " -> " .. host) end
263 gatherIPs(ph.host) 292 gatherIPs(ph.host)
264 for k, v in pairs(IP[ph.host]) do 293 for k, v in pairs(IP[ph.host]) do
265 D(" DNS record " .. v .. " " .. ph.host .. " -> " .. k) 294 D(" DNS record " .. v .. " " .. ph.host .. " -> " .. k)
266 if v == "A" then 295 if v == "A" then
267 if testing("IPv4") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " &") end 296 if testing("IPv4") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file .." &") end
268 elseif v == "AAAA" then 297 elseif v == "AAAA" then
269 if testing("IPv6") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k .. "] &") end 298 if testing("IPv6") then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " [" .. k .. "] " .. file .. " &") end
270 elseif v == "CNAME" then execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. k .. path .. " &") 299 elseif v == "CNAME" then
271 checkRedirects(orig, k, path) -- Check the original, with the DNS records from the CNAME, as well as the CNAME, so they both get checked. 300 if "" == file then
301-- TODO - maybe "ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " &"
302-- ./mirror-checker.lua --tests=-IPv6 -q deb.roundr.devuan.org & -> executing ionice -c3 ./mirror-checker.lua --tests=-IPv6 -q deb.devuan.org deb.roundr.devuan.org &
303 execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. k .. path .. " &")
304 else
305 execute("ionice -c3 ./mirror-checker.lua " .. sendArgs .. " " .. k .. path .. " redirect " .. file .. " &")
306 end
307 checkRedirects(orig, k, path, ip, file) -- Check the original, with the DNS records from the CNAME, as well as the CNAME, so they both get checked.
272 end 308 end
273 end 309 end
274 end 310 end
@@ -409,20 +445,25 @@ if 0 < #arg then
409 if nil == logFile then C("opening log file - " .. e); return end 445 if nil == logFile then C("opening log file - " .. e); return end
410 I("Starting tests for " ..arg[1] .. " with these tests - " .. table.concat(options.tests.value, ", ")) 446 I("Starting tests for " ..arg[1] .. " with these tests - " .. table.concat(options.tests.value, ", "))
411 if nil ~= arg[2] then I(" Using IP " .. arg[2]) end 447 if nil ~= arg[2] then I(" Using IP " .. arg[2]) end
448 if nil ~= arg[3] then I(" Using file " .. arg[3]); end
412 if testing("Integrity") or testing("Updated") then 449 if testing("Integrity") or testing("Updated") then
413 if not keep then execute("rm -fr results/" .. pu.host) end 450 if nil == arg[3] then
414 cut = 0 451 if not keep then execute("rm -fr results/" .. pu.host) end
415 for t in arg[1]:gmatch("(/)") do 452 cut = 0
416 cut = cut + 1 453 for t in arg[1]:gmatch("(/)") do
454 cut = cut + 1
455 end
456 downloads(cut, pu.host, pu.path)
457 checkExes("mirror-checker.lua " .. sendArgs)
458 checkExes(downloadLock)
417 end 459 end
418 downloads(cut, pu.host, pu.path)
419 checkExes("mirror-checker.lua " .. sendArgs)
420 checkExes(downloadLock)
421 end 460 end
422 checkRedirects(pu.host, pu.host, pu.path, arg[2]) 461 checkRedirects(pu.host, pu.host, pu.path, arg[2], arg[3])
423 logFile:close() 462 logFile:close()
424else 463else
425 if not keep then os.execute("rm -f results/*.log") end 464 if not keep then os.execute("rm -f results/*.log") end
465 os.execute("rm -f results/*.check")
466 os.execute("mkdir -p results; touch results/stamp")
426 logFile, e = io.open("results/mirror-checker-lua.log", "a+") 467 logFile, e = io.open("results/mirror-checker-lua.log", "a+")
427 if nil == logFile then C("opening log file - " .. e); return end 468 if nil == logFile then C("opening log file - " .. e); return end
428 I("Starting tests " .. table.concat(options.tests.value, ", ")) 469 I("Starting tests " .. table.concat(options.tests.value, ", "))
@@ -448,5 +489,6 @@ else
448 if testing("Integrity") or testing("Updated") then 489 if testing("Integrity") or testing("Updated") then
449 while 0 < checkExes(downloadLock) do os.execute("sleep 30") end 490 while 0 < checkExes(downloadLock) do os.execute("sleep 30") end
450 end 491 end
492 os.execute("rm -f results/*.check")
451 logFile:close() 493 logFile:close()
452end 494end