aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
authoronefang2019-12-12 17:19:45 +1000
committeronefang2019-12-12 17:19:45 +1000
commit8d94dd543e5a6e3db69db044520bef65fc441f0e (patch)
tree7519dda394022a390c39564cdcef625c87d5119e
parentSome refactoring of the results data handling. (diff)
downloadapt-panopticon-8d94dd543e5a6e3db69db044520bef65fc441f0e.zip
apt-panopticon-8d94dd543e5a6e3db69db044520bef65fc441f0e.tar.gz
apt-panopticon-8d94dd543e5a6e3db69db044520bef65fc441f0e.tar.bz2
apt-panopticon-8d94dd543e5a6e3db69db044520bef65fc441f0e.tar.xz
Do the curl log file stuff differently.
Use the log file as the flock lock file. Name the .curl files similar to the .log files, and don't delete them. Don't bother dealing with the stuff we decide to not download, stops spurious log files with errors in them.
-rwxr-xr-xapt-panopticon.lua132
1 files changed, 70 insertions, 62 deletions
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index cd3b38e..5dc2570 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -532,15 +532,15 @@ end
532 532
533local downloadLock = "flock -n results/curl-" 533local downloadLock = "flock -n results/curl-"
534local download = "curl --connect-timeout " .. APT.options.timeout.value .. " --create-dirs -f -L --max-time " .. APT.options.maxtime.value .. " -z 'results/stamp.old' -v -R " 534local download = "curl --connect-timeout " .. APT.options.timeout.value .. " --create-dirs -f -L --max-time " .. APT.options.maxtime.value .. " -z 'results/stamp.old' -v -R "
535local downloads = function(host, URL, release, list) 535local downloads = function(host, URL, meta, release, list)
536 if nil == URL then URL = "" end 536 if nil == URL then URL = "" end
537 local lock = "META-" .. host .. ".lock" 537 local lock = meta .. "-" .. host .. ".log"
538 local log = " --stderr results/curl-META-" .. host .. ".log" 538 local log = " --stderr results/curl-" .. meta .. "-" .. host .. ".log"
539 local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock .. " " .. download .. log .. " -K results/" .. host .. ".curl" 539 local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock .. " " .. download .. log .. " -K results/curl-" .. meta .. '-' .. host .. ".files"
540 if APT.testing("IPv4") and (not APT.testing("IPv6")) then cm = cm .. ' -4' end 540 if APT.testing("IPv4") and (not APT.testing("IPv6")) then cm = cm .. ' -4' end
541 if (not APT.testing("IPv4")) and APT.testing("IPv6") then cm = cm .. ' -6' end 541 if (not APT.testing("IPv4")) and APT.testing("IPv6") then cm = cm .. ' -6' end
542 f, e = io.open("results/" .. host .. ".curl", "a+") 542 f, e = io.open("results/curl-" .. meta .. '-' .. host .. ".files", "a+")
543 if nil == f then C("opening curl file - " .. e); return end 543 if nil == f then C("opening curl downloads list file - " .. e); return end
544 544
545 if nil ~= list then 545 if nil ~= list then
546 if "" ~= list then 546 if "" ~= list then
@@ -659,7 +659,7 @@ if 0 < #arg then
659 if APT.testing("Integrity") or APT.testing("Updated") then 659 if APT.testing("Integrity") or APT.testing("Updated") then
660 if APT.origin and (APT.options.roundRobin.value ~= pu.host) then 660 if APT.origin and (APT.options.roundRobin.value ~= pu.host) then
661 I("Starting file downloads for " .. pu.host) 661 I("Starting file downloads for " .. pu.host)
662 downloads(pu.host, pu.path) 662 downloads(pu.host, pu.path, 'Release')
663 end 663 end
664 end 664 end
665 checkFiles(pu.host, pu.host, pu.path); 665 checkFiles(pu.host, pu.host, pu.path);
@@ -670,9 +670,9 @@ if 0 < #arg then
670 if APT.testing("Integrity") or APT.testing("Updated") then 670 if APT.testing("Integrity") or APT.testing("Updated") then
671 if 4 > (totalTimeouts) then 671 if 4 > (totalTimeouts) then
672 if APT.origin and (APT.options.roundRobin.value ~= pu.host) then 672 if APT.origin and (APT.options.roundRobin.value ~= pu.host) then
673 while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end 673 while 0 < APT.checkExes(downloadLock .. "Release-" .. pu.host .. ".log") do os.execute("sleep 10") end
674 os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " .. 674
675 "mv results/curl-META-" .. pu.host .. ".log results/curl-Release-" .. pu.host .. ".log") 675 local fcount = 0
676 for i, n in pairs(releases) do 676 for i, n in pairs(releases) do
677 for l, o in pairs(releaseFiles) do 677 for l, o in pairs(releaseFiles) do
678 if repoExists(i .. o) then 678 if repoExists(i .. o) then
@@ -697,75 +697,84 @@ if 0 < #arg then
697 if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else 697 if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else
698 local diff = dfile:read("*a") 698 local diff = dfile:read("*a")
699 if "" ~= diff then 699 if "" ~= diff then
700 downloads(pu.host, pu.path, n, diff) 700 downloads(pu.host, pu.path, 'Packages', n, diff)
701 end 701 fcount = fcount + 1
702 end
702 end 703 end
703 end 704 end
704 end 705 end
705 706
706 end 707 end
707 708
708 downloads(pu.host, pu.path, "", "") 709 if 0 ~= fcount then
709 while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end 710 downloads(pu.host, pu.path, 'Packages', "", "")
710 os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " .. 711 while 0 < APT.checkExes(downloadLock .. "Packages-" .. pu.host .. ".log") do os.execute("sleep 10") end
711 "mv results/curl-META-" .. pu.host .. ".log results/curl-Packages-" .. pu.host .. ".log")
712 712
713 for i, n in pairs(releases) do 713 for i, n in pairs(releases) do
714 local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") 714 local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r")
715 if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else 715 if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else
716 local diff = dfile:read("*a") 716 local diff = dfile:read("*a")
717 for l in diff:gmatch("\n*([^\n]+)\n*") do 717 for l in diff:gmatch("\n*([^\n]+)\n*") do
718 postDownload(pu.host, n, "/" .. l) 718 postDownload(pu.host, n, "/" .. l)
719 end
720 end
721 if APT.options.referenceSite.value == pu.host then
722 -- In case it wasn't dealt with already.
723 os.execute('touch results/NEW_Packages_' .. n .. '.test.txt')
719 end 724 end
720 end 725 end
721 if APT.options.referenceSite.value == pu.host then
722 -- In case it wasn't dealt with already.
723 os.execute('touch results/NEW_Packages_' .. n .. '.test.txt')
724 end
725 end
726
727 726
728 for i, n in pairs(releases) do 727 fcount = 0
729 local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") 728 for i, n in pairs(releases) do
730 if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else 729 local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
731 for l in nfile:lines() do 730 if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
732 local p = l:match('(pool/.*%.deb)') 731 for l in nfile:lines() do
733 if nil ~= p then 732 local p = l:match('(pool/.*%.deb)')
734 downloads(pu.host, pu.path, nil, p) 733 if nil ~= p then
734 downloads(pu.host, pu.path, 'package', nil, p)
735 fcount = fcount + 1
736 end
735 end 737 end
736 end 738 end
737 end 739 end
738 end 740
739 downloads(pu.host, pu.path, nil, "") 741 if 0 ~= fcount then
740 while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end 742 downloads(pu.host, pu.path, 'package', nil, "")
741 for i, n in pairs(releases) do 743 while 0 < APT.checkExes(downloadLock .. "package-" .. pu.host .. ".log") do os.execute("sleep 10") end
742 local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") 744 for i, n in pairs(releases) do
743 if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else 745 local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
744 for l in nfile:lines() do 746 if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
745 local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |') 747 for l in nfile:lines() do
746 if nil ~= p then 748 local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |')
747 if APT.checkFile('results/' .. pu.host .. "/merged/" .. p) then 749 if nil ~= p then
748 local status, fsz = APT.execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5') 750 if APT.checkFile('results/' .. pu.host .. "/merged/" .. p) then
749 if APT.testing("Integrity") then 751 local status, fsz = APT.execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5')
750 if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. 752 if APT.testing("Integrity") then
751 E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p .. ' should be ' .. sz .. ', but is ' .. fsz:sub(2, -2) .. '.', 'http', 'Integrity', pu.host) 753 if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end.
752 else 754 E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p .. ' should be ' .. sz .. ', but is ' .. fsz:sub(2, -2) .. '.', 'http', 'Integrity', pu.host)
753 local status, fsha = APT.execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') 755 else
754 if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end 756 local status, fsha = APT.execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1')
757 if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end
755-- TODO - maybe check the PGP key, though packages are mostly not signed. 758-- TODO - maybe check the PGP key, though packages are mostly not signed.
759 end
760 end
761 if APT.testing("Updated") then
762 if sz ~= fsz:sub(2, -2) then
763 E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host)
764 end
765 end
766 else
767 E('Failed to download - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host)
756 end 768 end
757 end 769 end
758 if APT.testing("Updated") then
759 if sz ~= fsz:sub(2, -2) then
760 E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host)
761 end
762 end
763 else
764 E('Failed to download - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host)
765 end 770 end
766 end 771 end
767 end 772 end
768 end 773 end
774 else
775 for i, n in pairs(releases) do
776 os.execute('touch results/NEW_Packages_' .. n .. '.test.txt')
777 end
769 end 778 end
770 end 779 end
771 780
@@ -781,7 +790,7 @@ if 0 < #arg then
781 end 790 end
782 791
783 local min, max, spd = 999999999999, 0 792 local min, max, spd = 999999999999, 0
784 for i, mt in pairs({'Release', 'Packages', 'META'}) do 793 for i, mt in pairs({'Release', 'Packages', 'package'}) do
785 if APT.checkFile("results/curl-" .. mt .. "-" .. pu.host .. ".log") then 794 if APT.checkFile("results/curl-" .. mt .. "-" .. pu.host .. ".log") then
786 for l in io.lines("results/curl-" .. mt .. "-" .. pu.host .. ".log") do 795 for l in io.lines("results/curl-" .. mt .. "-" .. pu.host .. ".log") do
787 local speed, crrnt = l:match('^%c *%d+ +%d+k? +%d+ +%d+k? +%d+ +%d+ +(%d+k?) +%d+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +(%d+k?)') 796 local speed, crrnt = l:match('^%c *%d+ +%d+k? +%d+ +%d+k? +%d+ +%d+ +(%d+k?) +%d+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +(%d+k?)')
@@ -827,7 +836,6 @@ else
827 os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp') 836 os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp')
828 os.execute("rm -f results/*.check 2>/dev/null") 837 os.execute("rm -f results/*.check 2>/dev/null")
829 if not APT.keep then 838 if not APT.keep then
830 os.execute("rm -f results/*.curl 2>/dev/null")
831 os.execute("rm -f results/*.log 2>/dev/null") 839 os.execute("rm -f results/*.log 2>/dev/null")
832 os.execute("rm -f results/*.html 2>/dev/null") 840 os.execute("rm -f results/*.html 2>/dev/null")
833 os.execute("rm -f results/*.txt 2>/dev/null") 841 os.execute("rm -f results/*.txt 2>/dev/null")
@@ -863,7 +871,7 @@ else
863 871
864 while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end 872 while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end
865 873
866 os.execute("rm -f results/*.check; rm -f results/*.lock 2>/dev/null") 874 os.execute("rm -f results/*.check")
867 875
868 -- Create the reports. 876 -- Create the reports.
869 for n, r in pairs(APT.options.reports.value) do 877 for n, r in pairs(APT.options.reports.value) do