diff options
-rwxr-xr-x | apt-panopticon.lua | 132 |
1 files changed, 70 insertions, 62 deletions
diff --git a/apt-panopticon.lua b/apt-panopticon.lua index cd3b38e..5dc2570 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua | |||
@@ -532,15 +532,15 @@ end | |||
532 | 532 | ||
533 | local downloadLock = "flock -n results/curl-" | 533 | local downloadLock = "flock -n results/curl-" |
534 | local download = "curl --connect-timeout " .. APT.options.timeout.value .. " --create-dirs -f -L --max-time " .. APT.options.maxtime.value .. " -z 'results/stamp.old' -v -R " | 534 | local download = "curl --connect-timeout " .. APT.options.timeout.value .. " --create-dirs -f -L --max-time " .. APT.options.maxtime.value .. " -z 'results/stamp.old' -v -R " |
535 | local downloads = function(host, URL, release, list) | 535 | local downloads = function(host, URL, meta, release, list) |
536 | if nil == URL then URL = "" end | 536 | if nil == URL then URL = "" end |
537 | local lock = "META-" .. host .. ".lock" | 537 | local lock = meta .. "-" .. host .. ".log" |
538 | local log = " --stderr results/curl-META-" .. host .. ".log" | 538 | local log = " --stderr results/curl-" .. meta .. "-" .. host .. ".log" |
539 | local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock .. " " .. download .. log .. " -K results/" .. host .. ".curl" | 539 | local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock .. " " .. download .. log .. " -K results/curl-" .. meta .. '-' .. host .. ".files" |
540 | if APT.testing("IPv4") and (not APT.testing("IPv6")) then cm = cm .. ' -4' end | 540 | if APT.testing("IPv4") and (not APT.testing("IPv6")) then cm = cm .. ' -4' end |
541 | if (not APT.testing("IPv4")) and APT.testing("IPv6") then cm = cm .. ' -6' end | 541 | if (not APT.testing("IPv4")) and APT.testing("IPv6") then cm = cm .. ' -6' end |
542 | f, e = io.open("results/" .. host .. ".curl", "a+") | 542 | f, e = io.open("results/curl-" .. meta .. '-' .. host .. ".files", "a+") |
543 | if nil == f then C("opening curl file - " .. e); return end | 543 | if nil == f then C("opening curl downloads list file - " .. e); return end |
544 | 544 | ||
545 | if nil ~= list then | 545 | if nil ~= list then |
546 | if "" ~= list then | 546 | if "" ~= list then |
@@ -659,7 +659,7 @@ if 0 < #arg then | |||
659 | if APT.testing("Integrity") or APT.testing("Updated") then | 659 | if APT.testing("Integrity") or APT.testing("Updated") then |
660 | if APT.origin and (APT.options.roundRobin.value ~= pu.host) then | 660 | if APT.origin and (APT.options.roundRobin.value ~= pu.host) then |
661 | I("Starting file downloads for " .. pu.host) | 661 | I("Starting file downloads for " .. pu.host) |
662 | downloads(pu.host, pu.path) | 662 | downloads(pu.host, pu.path, 'Release') |
663 | end | 663 | end |
664 | end | 664 | end |
665 | checkFiles(pu.host, pu.host, pu.path); | 665 | checkFiles(pu.host, pu.host, pu.path); |
@@ -670,9 +670,9 @@ if 0 < #arg then | |||
670 | if APT.testing("Integrity") or APT.testing("Updated") then | 670 | if APT.testing("Integrity") or APT.testing("Updated") then |
671 | if 4 > (totalTimeouts) then | 671 | if 4 > (totalTimeouts) then |
672 | if APT.origin and (APT.options.roundRobin.value ~= pu.host) then | 672 | if APT.origin and (APT.options.roundRobin.value ~= pu.host) then |
673 | while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end | 673 | while 0 < APT.checkExes(downloadLock .. "Release-" .. pu.host .. ".log") do os.execute("sleep 10") end |
674 | os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " .. | 674 | |
675 | "mv results/curl-META-" .. pu.host .. ".log results/curl-Release-" .. pu.host .. ".log") | 675 | local fcount = 0 |
676 | for i, n in pairs(releases) do | 676 | for i, n in pairs(releases) do |
677 | for l, o in pairs(releaseFiles) do | 677 | for l, o in pairs(releaseFiles) do |
678 | if repoExists(i .. o) then | 678 | if repoExists(i .. o) then |
@@ -697,75 +697,84 @@ if 0 < #arg then | |||
697 | if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else | 697 | if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else |
698 | local diff = dfile:read("*a") | 698 | local diff = dfile:read("*a") |
699 | if "" ~= diff then | 699 | if "" ~= diff then |
700 | downloads(pu.host, pu.path, n, diff) | 700 | downloads(pu.host, pu.path, 'Packages', n, diff) |
701 | end | 701 | fcount = fcount + 1 |
702 | end | ||
702 | end | 703 | end |
703 | end | 704 | end |
704 | end | 705 | end |
705 | 706 | ||
706 | end | 707 | end |
707 | 708 | ||
708 | downloads(pu.host, pu.path, "", "") | 709 | if 0 ~= fcount then |
709 | while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end | 710 | downloads(pu.host, pu.path, 'Packages', "", "") |
710 | os.execute( "rm -f results/" .. pu.host .. ".curl 2>/dev/null; rm -f results/curl-META-" .. pu.host .. ".lock 2>/dev/null; " .. | 711 | while 0 < APT.checkExes(downloadLock .. "Packages-" .. pu.host .. ".log") do os.execute("sleep 10") end |
711 | "mv results/curl-META-" .. pu.host .. ".log results/curl-Packages-" .. pu.host .. ".log") | ||
712 | 712 | ||
713 | for i, n in pairs(releases) do | 713 | for i, n in pairs(releases) do |
714 | local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") | 714 | local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") |
715 | if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else | 715 | if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else |
716 | local diff = dfile:read("*a") | 716 | local diff = dfile:read("*a") |
717 | for l in diff:gmatch("\n*([^\n]+)\n*") do | 717 | for l in diff:gmatch("\n*([^\n]+)\n*") do |
718 | postDownload(pu.host, n, "/" .. l) | 718 | postDownload(pu.host, n, "/" .. l) |
719 | end | ||
720 | end | ||
721 | if APT.options.referenceSite.value == pu.host then | ||
722 | -- In case it wasn't dealt with already. | ||
723 | os.execute('touch results/NEW_Packages_' .. n .. '.test.txt') | ||
719 | end | 724 | end |
720 | end | 725 | end |
721 | if APT.options.referenceSite.value == pu.host then | ||
722 | -- In case it wasn't dealt with already. | ||
723 | os.execute('touch results/NEW_Packages_' .. n .. '.test.txt') | ||
724 | end | ||
725 | end | ||
726 | |||
727 | 726 | ||
728 | for i, n in pairs(releases) do | 727 | fcount = 0 |
729 | local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") | 728 | for i, n in pairs(releases) do |
730 | if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else | 729 | local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") |
731 | for l in nfile:lines() do | 730 | if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else |
732 | local p = l:match('(pool/.*%.deb)') | 731 | for l in nfile:lines() do |
733 | if nil ~= p then | 732 | local p = l:match('(pool/.*%.deb)') |
734 | downloads(pu.host, pu.path, nil, p) | 733 | if nil ~= p then |
734 | downloads(pu.host, pu.path, 'package', nil, p) | ||
735 | fcount = fcount + 1 | ||
736 | end | ||
735 | end | 737 | end |
736 | end | 738 | end |
737 | end | 739 | end |
738 | end | 740 | |
739 | downloads(pu.host, pu.path, nil, "") | 741 | if 0 ~= fcount then |
740 | while 0 < APT.checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end | 742 | downloads(pu.host, pu.path, 'package', nil, "") |
741 | for i, n in pairs(releases) do | 743 | while 0 < APT.checkExes(downloadLock .. "package-" .. pu.host .. ".log") do os.execute("sleep 10") end |
742 | local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") | 744 | for i, n in pairs(releases) do |
743 | if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else | 745 | local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") |
744 | for l in nfile:lines() do | 746 | if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else |
745 | local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |') | 747 | for l in nfile:lines() do |
746 | if nil ~= p then | 748 | local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |') |
747 | if APT.checkFile('results/' .. pu.host .. "/merged/" .. p) then | 749 | if nil ~= p then |
748 | local status, fsz = APT.execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5') | 750 | if APT.checkFile('results/' .. pu.host .. "/merged/" .. p) then |
749 | if APT.testing("Integrity") then | 751 | local status, fsz = APT.execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5') |
750 | if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. | 752 | if APT.testing("Integrity") then |
751 | E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p .. ' should be ' .. sz .. ', but is ' .. fsz:sub(2, -2) .. '.', 'http', 'Integrity', pu.host) | 753 | if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. |
752 | else | 754 | E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p .. ' should be ' .. sz .. ', but is ' .. fsz:sub(2, -2) .. '.', 'http', 'Integrity', pu.host) |
753 | local status, fsha = APT.execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') | 755 | else |
754 | if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end | 756 | local status, fsha = APT.execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') |
757 | if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end | ||
755 | -- TODO - maybe check the PGP key, though packages are mostly not signed. | 758 | -- TODO - maybe check the PGP key, though packages are mostly not signed. |
759 | end | ||
760 | end | ||
761 | if APT.testing("Updated") then | ||
762 | if sz ~= fsz:sub(2, -2) then | ||
763 | E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) | ||
764 | end | ||
765 | end | ||
766 | else | ||
767 | E('Failed to download - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) | ||
756 | end | 768 | end |
757 | end | 769 | end |
758 | if APT.testing("Updated") then | ||
759 | if sz ~= fsz:sub(2, -2) then | ||
760 | E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) | ||
761 | end | ||
762 | end | ||
763 | else | ||
764 | E('Failed to download - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) | ||
765 | end | 770 | end |
766 | end | 771 | end |
767 | end | 772 | end |
768 | end | 773 | end |
774 | else | ||
775 | for i, n in pairs(releases) do | ||
776 | os.execute('touch results/NEW_Packages_' .. n .. '.test.txt') | ||
777 | end | ||
769 | end | 778 | end |
770 | end | 779 | end |
771 | 780 | ||
@@ -781,7 +790,7 @@ if 0 < #arg then | |||
781 | end | 790 | end |
782 | 791 | ||
783 | local min, max, spd = 999999999999, 0 | 792 | local min, max, spd = 999999999999, 0 |
784 | for i, mt in pairs({'Release', 'Packages', 'META'}) do | 793 | for i, mt in pairs({'Release', 'Packages', 'package'}) do |
785 | if APT.checkFile("results/curl-" .. mt .. "-" .. pu.host .. ".log") then | 794 | if APT.checkFile("results/curl-" .. mt .. "-" .. pu.host .. ".log") then |
786 | for l in io.lines("results/curl-" .. mt .. "-" .. pu.host .. ".log") do | 795 | for l in io.lines("results/curl-" .. mt .. "-" .. pu.host .. ".log") do |
787 | local speed, crrnt = l:match('^%c *%d+ +%d+k? +%d+ +%d+k? +%d+ +%d+ +(%d+k?) +%d+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +(%d+k?)') | 796 | local speed, crrnt = l:match('^%c *%d+ +%d+k? +%d+ +%d+k? +%d+ +%d+ +(%d+k?) +%d+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +(%d+k?)') |
@@ -827,7 +836,6 @@ else | |||
827 | os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp') | 836 | os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp') |
828 | os.execute("rm -f results/*.check 2>/dev/null") | 837 | os.execute("rm -f results/*.check 2>/dev/null") |
829 | if not APT.keep then | 838 | if not APT.keep then |
830 | os.execute("rm -f results/*.curl 2>/dev/null") | ||
831 | os.execute("rm -f results/*.log 2>/dev/null") | 839 | os.execute("rm -f results/*.log 2>/dev/null") |
832 | os.execute("rm -f results/*.html 2>/dev/null") | 840 | os.execute("rm -f results/*.html 2>/dev/null") |
833 | os.execute("rm -f results/*.txt 2>/dev/null") | 841 | os.execute("rm -f results/*.txt 2>/dev/null") |
@@ -863,7 +871,7 @@ else | |||
863 | 871 | ||
864 | while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end | 872 | while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end |
865 | 873 | ||
866 | os.execute("rm -f results/*.check; rm -f results/*.lock 2>/dev/null") | 874 | os.execute("rm -f results/*.check") |
867 | 875 | ||
868 | -- Create the reports. | 876 | -- Create the reports. |
869 | for n, r in pairs(APT.options.reports.value) do | 877 | for n, r in pairs(APT.options.reports.value) do |