aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/apt-panopticon.lua
diff options
context:
space:
mode:
Diffstat (limited to 'apt-panopticon.lua')
-rwxr-xr-xapt-panopticon.lua132
1 files changed, 70 insertions, 62 deletions
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index cd3b38e..5dc2570 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -532,15 +532,15 @@ end
532 532
533local downloadLock = "flock -n results/curl-" 533local downloadLock = "flock -n results/curl-"
534local download = "curl --connect-timeout " .. APT.options.timeout.value .. " --create-dirs -f -L --max-time " .. APT.options.maxtime.value .. " -z 'results/stamp.old' -v -R " 534local download = "curl --connect-timeout " .. APT.options.timeout.value .. " --create-dirs -f -L --max-time " .. APT.options.maxtime.value .. " -z 'results/stamp.old' -v -R "
535local downloads = function(host, URL, release, list) 535local downloads = function(host, URL, meta, release, list)
536 if nil == URL then URL = "" end 536 if nil == URL then URL = "" end
537 local lock = "META-" .. host .. ".lock" 537 local lock = meta .. "-" .. host .. ".log"
538 local log = " --stderr results/curl-META-" .. host .. ".log" 538 local log = " --stderr results/curl-" .. meta .. "-" .. host .. ".log"
539 local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock .. " " .. download .. log .. " -K results/" .. host .. ".curl" 539 local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock .. " " .. download .. log .. " -K results/curl-" .. meta .. '-' .. host .. ".files"
540 if APT.testing("IPv4") and (not APT.testing("IPv6")) then cm = cm .. ' -4' end 540 if APT.testing("IPv4") and (not APT.testing("IPv6")) then cm = cm .. ' -4' end
541 if (not APT.testing("IPv4")) and APT.testing("IPv6") then cm = cm .. ' -6' end 541 if (not APT.testing("IPv4")) and APT.testing("IPv6") then cm = cm .. ' -6' end
542 f, e = io.open("results/" .. host .. ".curl", "a+") 542 f, e = io.open("results/curl-" .. meta .. '-' .. host .. ".files", "a+")
543 if nil == f then C("opening curl file - " .. e); return end 543 if nil == f then C("opening curl downloads list file - " .. e); return end
544 544
545 if nil ~= list then 545 if nil ~= list then
546 if "" ~= list then 546 if "" ~= list then
@@ -659,7 +659,7 @@ if 0 < #arg then
659 if APT.testing("Integrity") or APT.testing("Updated") then 659 if APT.testing("Integrity") or APT.testing("Updated") then
660 if APT.origin and (APT.options.roundRobin.value ~= pu.host) then 660 if APT.origin and (APT.options.roundRobin.value ~= pu.host) then
661 I("Starting file downloads for " .. pu.host) 661 I("Starting file downloads for " .. pu.host)
662 downloads(pu.host, pu.path) 662 downloads(pu.host, pu.path, 'Release')
663 end 663 end
664 end 664 end
665 checkFiles(pu.host, pu.host, pu.path); 665 checkFiles(pu.host, pu.host, pu.path);
@@ -670,9 +670,9 @@ if 0 < #arg then
670 if APT.testing("Integrity") or APT.testing("Updated") then 670 if APT.testing("Integrity") or APT.testing("Updated") then
671 if 4 > (totalTimeouts) then 671 if 4 > (totalTimeouts) then
672 if APT.origin and (APT.options.roundRobin.value ~= pu.host) then 672 if APT.origin and (APT.options.roundRobin.value ~= pu.host) then
673 while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end 673 while 0 < APT.checkExes(downloadLock .. "Release-" .. pu.host .. ".log") do os.execute("sleep 10") end
674 os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " .. 674
675 "mv results/curl-META-" .. pu.host .. ".log results/curl-Release-" .. pu.host .. ".log") 675 local fcount = 0
676 for i, n in pairs(releases) do 676 for i, n in pairs(releases) do
677 for l, o in pairs(releaseFiles) do 677 for l, o in pairs(releaseFiles) do
678 if repoExists(i .. o) then 678 if repoExists(i .. o) then
@@ -697,75 +697,84 @@ if 0 < #arg then
697 if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else 697 if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else
698 local diff = dfile:read("*a") 698 local diff = dfile:read("*a")
699 if "" ~= diff then 699 if "" ~= diff then
700 downloads(pu.host, pu.path, n, diff) 700 downloads(pu.host, pu.path, 'Packages', n, diff)
701 end 701 fcount = fcount + 1
702 end
702 end 703 end
703 end 704 end
704 end 705 end
705 706
706 end 707 end
707 708
708 downloads(pu.host, pu.path, "", "") 709 if 0 ~= fcount then
709 while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end 710 downloads(pu.host, pu.path, 'Packages', "", "")
710 os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " .. 711 while 0 < APT.checkExes(downloadLock .. "Packages-" .. pu.host .. ".log") do os.execute("sleep 10") end
711 "mv results/curl-META-" .. pu.host .. ".log results/curl-Packages-" .. pu.host .. ".log")
712 712
713 for i, n in pairs(releases) do 713 for i, n in pairs(releases) do
714 local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") 714 local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r")
715 if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else 715 if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else
716 local diff = dfile:read("*a") 716 local diff = dfile:read("*a")
717 for l in diff:gmatch("\n*([^\n]+)\n*") do 717 for l in diff:gmatch("\n*([^\n]+)\n*") do
718 postDownload(pu.host, n, "/" .. l) 718 postDownload(pu.host, n, "/" .. l)
719 end
720 end
721 if APT.options.referenceSite.value == pu.host then
722 -- In case it wasn't dealt with already.
723 os.execute('touch results/NEW_Packages_' .. n .. '.test.txt')
719 end 724 end
720 end 725 end
721 if APT.options.referenceSite.value == pu.host then
722 -- In case it wasn't dealt with already.
723 os.execute('touch results/NEW_Packages_' .. n .. '.test.txt')
724 end
725 end
726
727 726
728 for i, n in pairs(releases) do 727 fcount = 0
729 local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") 728 for i, n in pairs(releases) do
730 if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else 729 local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
731 for l in nfile:lines() do 730 if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
732 local p = l:match('(pool/.*%.deb)') 731 for l in nfile:lines() do
733 if nil ~= p then 732 local p = l:match('(pool/.*%.deb)')
734 downloads(pu.host, pu.path, nil, p) 733 if nil ~= p then
734 downloads(pu.host, pu.path, 'package', nil, p)
735 fcount = fcount + 1
736 end
735 end 737 end
736 end 738 end
737 end 739 end
738 end 740
739 downloads(pu.host, pu.path, nil, "") 741 if 0 ~= fcount then
740 while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end 742 downloads(pu.host, pu.path, 'package', nil, "")
741 for i, n in pairs(releases) do 743 while 0 < APT.checkExes(downloadLock .. "package-" .. pu.host .. ".log") do os.execute("sleep 10") end
742 local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") 744 for i, n in pairs(releases) do
743 if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else 745 local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
744 for l in nfile:lines() do 746 if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
745 local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |') 747 for l in nfile:lines() do
746 if nil ~= p then 748 local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |')
747 if APT.checkFile('results/' .. pu.host .. "/merged/" .. p) then 749 if nil ~= p then
748 local status, fsz = APT.execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5') 750 if APT.checkFile('results/' .. pu.host .. "/merged/" .. p) then
749 if APT.testing("Integrity") then 751 local status, fsz = APT.execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5')
750 if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. 752 if APT.testing("Integrity") then
751 E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p .. ' should be ' .. sz .. ', but is ' .. fsz:sub(2, -2) .. '.', 'http', 'Integrity', pu.host) 753 if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end.
752 else 754 E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p .. ' should be ' .. sz .. ', but is ' .. fsz:sub(2, -2) .. '.', 'http', 'Integrity', pu.host)
753 local status, fsha = APT.execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') 755 else
754 if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end 756 local status, fsha = APT.execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1')
757 if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end
755-- TODO - maybe check the PGP key, though packages are mostly not signed. 758-- TODO - maybe check the PGP key, though packages are mostly not signed.
759 end
760 end
761 if APT.testing("Updated") then
762 if sz ~= fsz:sub(2, -2) then
763 E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host)
764 end
765 end
766 else
767 E('Failed to download - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host)
756 end 768 end
757 end 769 end
758 if APT.testing("Updated") then
759 if sz ~= fsz:sub(2, -2) then
760 E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host)
761 end
762 end
763 else
764 E('Failed to download - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host)
765 end 770 end
766 end 771 end
767 end 772 end
768 end 773 end
774 else
775 for i, n in pairs(releases) do
776 os.execute('touch results/NEW_Packages_' .. n .. '.test.txt')
777 end
769 end 778 end
770 end 779 end
771 780
@@ -781,7 +790,7 @@ if 0 < #arg then
781 end 790 end
782 791
783 local min, max, spd = 999999999999, 0 792 local min, max, spd = 999999999999, 0
784 for i, mt in pairs({'Release', 'Packages', 'META'}) do 793 for i, mt in pairs({'Release', 'Packages', 'package'}) do
785 if APT.checkFile("results/curl-" .. mt .. "-" .. pu.host .. ".log") then 794 if APT.checkFile("results/curl-" .. mt .. "-" .. pu.host .. ".log") then
786 for l in io.lines("results/curl-" .. mt .. "-" .. pu.host .. ".log") do 795 for l in io.lines("results/curl-" .. mt .. "-" .. pu.host .. ".log") do
787 local speed, crrnt = l:match('^%c *%d+ +%d+k? +%d+ +%d+k? +%d+ +%d+ +(%d+k?) +%d+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +(%d+k?)') 796 local speed, crrnt = l:match('^%c *%d+ +%d+k? +%d+ +%d+k? +%d+ +%d+ +(%d+k?) +%d+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +(%d+k?)')
@@ -827,7 +836,6 @@ else
827 os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp') 836 os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp')
828 os.execute("rm -f results/*.check 2>/dev/null") 837 os.execute("rm -f results/*.check 2>/dev/null")
829 if not APT.keep then 838 if not APT.keep then
830 os.execute("rm -f results/*.curl 2>/dev/null")
831 os.execute("rm -f results/*.log 2>/dev/null") 839 os.execute("rm -f results/*.log 2>/dev/null")
832 os.execute("rm -f results/*.html 2>/dev/null") 840 os.execute("rm -f results/*.html 2>/dev/null")
833 os.execute("rm -f results/*.txt 2>/dev/null") 841 os.execute("rm -f results/*.txt 2>/dev/null")
@@ -863,7 +871,7 @@ else
863 871
864 while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end 872 while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end
865 873
866 os.execute("rm -f results/*.check; rm -f results/*.lock 2>/dev/null") 874 os.execute("rm -f results/*.check")
867 875
868 -- Create the reports. 876 -- Create the reports.
869 for n, r in pairs(APT.options.reports.value) do 877 for n, r in pairs(APT.options.reports.value) do
generated by cgit v1.1 at 2024-07-01 15:43:20 +0000