From 8d94dd543e5a6e3db69db044520bef65fc441f0e Mon Sep 17 00:00:00 2001 From: onefang Date: Thu, 12 Dec 2019 17:19:45 +1000 Subject: Do the curl log file stuff differently. Use the log file as the flock lock file. Name the .curl files similar to the .log files, and don't delete them. Don't bother dealing with the stuff we decide to not download, stops spurious log files with errors in them. --- apt-panopticon.lua | 132 ++++++++++++++++++++++++++++------------------------- 1 file changed, 70 insertions(+), 62 deletions(-) diff --git a/apt-panopticon.lua b/apt-panopticon.lua index cd3b38e..5dc2570 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua @@ -532,15 +532,15 @@ end local downloadLock = "flock -n results/curl-" local download = "curl --connect-timeout " .. APT.options.timeout.value .. " --create-dirs -f -L --max-time " .. APT.options.maxtime.value .. " -z 'results/stamp.old' -v -R " -local downloads = function(host, URL, release, list) +local downloads = function(host, URL, meta, release, list) if nil == URL then URL = "" end - local lock = "META-" .. host .. ".lock" - local log = " --stderr results/curl-META-" .. host .. ".log" - local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock .. " " .. download .. log .. " -K results/" .. host .. ".curl" + local lock = meta .. "-" .. host .. ".log" + local log = " --stderr results/curl-" .. meta .. "-" .. host .. ".log" + local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock .. " " .. download .. log .. " -K results/curl-" .. meta .. '-' .. host .. ".files" if APT.testing("IPv4") and (not APT.testing("IPv6")) then cm = cm .. ' -4' end if (not APT.testing("IPv4")) and APT.testing("IPv6") then cm = cm .. ' -6' end - f, e = io.open("results/" .. host .. ".curl", "a+") - if nil == f then C("opening curl file - " .. e); return end + f, e = io.open("results/curl-" .. meta .. '-' .. host .. ".files", "a+") + if nil == f then C("opening curl downloads list file - " .. e); return end if nil ~= list then if "" ~= list then @@ -659,7 +659,7 @@ if 0 < #arg then if APT.testing("Integrity") or APT.testing("Updated") then if APT.origin and (APT.options.roundRobin.value ~= pu.host) then I("Starting file downloads for " .. pu.host) - downloads(pu.host, pu.path) + downloads(pu.host, pu.path, 'Release') end end checkFiles(pu.host, pu.host, pu.path); @@ -670,9 +670,9 @@ if 0 < #arg then if APT.testing("Integrity") or APT.testing("Updated") then if 4 > (totalTimeouts) then if APT.origin and (APT.options.roundRobin.value ~= pu.host) then - while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end - os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " .. - "mv results/curl-META-" .. pu.host .. ".log results/curl-Release-" .. pu.host .. ".log") + while 0 < APT.checkExes(downloadLock .. "Release-" .. pu.host .. ".log") do os.execute("sleep 10") end + + local fcount = 0 for i, n in pairs(releases) do for l, o in pairs(releaseFiles) do if repoExists(i .. o) then @@ -697,75 +697,84 @@ if 0 < #arg then if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else local diff = dfile:read("*a") if "" ~= diff then - downloads(pu.host, pu.path, n, diff) - end + downloads(pu.host, pu.path, 'Packages', n, diff) + fcount = fcount + 1 + end end end end end - downloads(pu.host, pu.path, "", "") - while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end - os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " .. - "mv results/curl-META-" .. pu.host .. ".log results/curl-Packages-" .. pu.host .. ".log") + if 0 ~= fcount then + downloads(pu.host, pu.path, 'Packages', "", "") + while 0 < APT.checkExes(downloadLock .. "Packages-" .. pu.host .. ".log") do os.execute("sleep 10") end - for i, n in pairs(releases) do - local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") - if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else - local diff = dfile:read("*a") - for l in diff:gmatch("\n*([^\n]+)\n*") do - postDownload(pu.host, n, "/" .. l) + for i, n in pairs(releases) do + local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") + if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else + local diff = dfile:read("*a") + for l in diff:gmatch("\n*([^\n]+)\n*") do + postDownload(pu.host, n, "/" .. l) + end + end + if APT.options.referenceSite.value == pu.host then + -- In case it wasn't dealt with already. + os.execute('touch results/NEW_Packages_' .. n .. '.test.txt') end end - if APT.options.referenceSite.value == pu.host then - -- In case it wasn't dealt with already. - os.execute('touch results/NEW_Packages_' .. n .. '.test.txt') - end - end - - for i, n in pairs(releases) do - local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") - if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else - for l in nfile:lines() do - local p = l:match('(pool/.*%.deb)') - if nil ~= p then - downloads(pu.host, pu.path, nil, p) + fcount = 0 + for i, n in pairs(releases) do + local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") + if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else + for l in nfile:lines() do + local p = l:match('(pool/.*%.deb)') + if nil ~= p then + downloads(pu.host, pu.path, 'package', nil, p) + fcount = fcount + 1 + end end end end - end - downloads(pu.host, pu.path, nil, "") - while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end - for i, n in pairs(releases) do - local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") - if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else - for l in nfile:lines() do - local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |') - if nil ~= p then - if APT.checkFile('results/' .. pu.host .. "/merged/" .. p) then - local status, fsz = APT.execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5') - if APT.testing("Integrity") then - if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. - E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p .. ' should be ' .. sz .. ', but is ' .. fsz:sub(2, -2) .. '.', 'http', 'Integrity', pu.host) - else - local status, fsha = APT.execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') - if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end + + if 0 ~= fcount then + downloads(pu.host, pu.path, 'package', nil, "") + while 0 < APT.checkExes(downloadLock .. "package-" .. pu.host .. ".log") do os.execute("sleep 10") end + for i, n in pairs(releases) do + local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") + if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else + for l in nfile:lines() do + local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |') + if nil ~= p then + if APT.checkFile('results/' .. pu.host .. "/merged/" .. p) then + local status, fsz = APT.execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5') + if APT.testing("Integrity") then + if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. + E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p .. ' should be ' .. sz .. ', but is ' .. fsz:sub(2, -2) .. '.', 'http', 'Integrity', pu.host) + else + local status, fsha = APT.execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') + if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end -- TODO - maybe check the PGP key, though packages are mostly not signed. + end + end + if APT.testing("Updated") then + if sz ~= fsz:sub(2, -2) then + E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) + end + end + else + E('Failed to download - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) end end - if APT.testing("Updated") then - if sz ~= fsz:sub(2, -2) then - E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) - end - end - else - E('Failed to download - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) end end end end + else + for i, n in pairs(releases) do + os.execute('touch results/NEW_Packages_' .. n .. '.test.txt') + end end end @@ -781,7 +790,7 @@ if 0 < #arg then end local min, max, spd = 999999999999, 0 - for i, mt in pairs({'Release', 'Packages', 'META'}) do + for i, mt in pairs({'Release', 'Packages', 'package'}) do if APT.checkFile("results/curl-" .. mt .. "-" .. pu.host .. ".log") then for l in io.lines("results/curl-" .. mt .. "-" .. pu.host .. ".log") do local speed, crrnt = l:match('^%c *%d+ +%d+k? +%d+ +%d+k? +%d+ +%d+ +(%d+k?) +%d+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +(%d+k?)') @@ -827,7 +836,6 @@ else os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp') os.execute("rm -f results/*.check 2>/dev/null") if not APT.keep then - os.execute("rm -f results/*.curl 2>/dev/null") os.execute("rm -f results/*.log 2>/dev/null") os.execute("rm -f results/*.html 2>/dev/null") os.execute("rm -f results/*.txt 2>/dev/null") @@ -863,7 +871,7 @@ else while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end - os.execute("rm -f results/*.check; rm -f results/*.lock 2>/dev/null") + os.execute("rm -f results/*.check") -- Create the reports. for n, r in pairs(APT.options.reports.value) do -- cgit v1.1