aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/apt-panopticon.lua
diff options
context:
space:
mode:
authoronefang2019-12-02 05:50:58 +1000
committeronefang2019-12-02 05:50:58 +1000
commit0560dc5975ff29be6d63885925aa1942135f0043 (patch)
treea9ba04aadcba1f91f3b53ce860825c70fb8a61a6 /apt-panopticon.lua
parentExplain the Updated time. (diff)
downloadapt-panopticon-0560dc5975ff29be6d63885925aa1942135f0043.zip
apt-panopticon-0560dc5975ff29be6d63885925aa1942135f0043.tar.gz
apt-panopticon-0560dc5975ff29be6d63885925aa1942135f0043.tar.bz2
apt-panopticon-0560dc5975ff29be6d63885925aa1942135f0043.tar.xz
Add more report stuff.
TIMEOUT, report inactive mirrors and why it's not active, add a * for things tested but not supported.
Diffstat (limited to 'apt-panopticon.lua')
-rwxr-xr-xapt-panopticon.lua155
1 files changed, 85 insertions, 70 deletions
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index d66e32d..5d9ed82 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -756,7 +756,7 @@ local getMirrors = function ()
756 local host = "" 756 local host = ""
757 local m = {} 757 local m = {}
758 local active = true 758 local active = true
759 local URL = "https://" .. options.referenceSite.value .. "/mirror_list.txt" 759 local URL = "http://" .. options.referenceSite.value .. "/mirror_list.txt"
760 I("getting mirrors.") 760 I("getting mirrors.")
761 local p, c, h = http.request(URL) 761 local p, c, h = http.request(URL)
762 if nil == p then E(c .. " fetching " .. URL) else 762 if nil == p then E(c .. " fetching " .. URL) else
@@ -766,7 +766,8 @@ local getMirrors = function ()
766 d = string.lower(d) 766 d = string.lower(d)
767 if "FQDN" == t then 767 if "FQDN" == t then
768 if "" ~= host then 768 if "" ~= host then
769 if active then mirrors[host] = m end 769-- if active then mirrors[host] = m end
770 mirrors[host] = m
770 m = {} 771 m = {}
771 active = true 772 active = true
772 end 773 end
@@ -780,15 +781,16 @@ local getMirrors = function ()
780 prot[w] = true; 781 prot[w] = true;
781 end 782 end
782 m[t] = prot 783 m[t] = prot
783 elseif "Active" == t and nil == d:find("yes", 1, true) then 784 elseif "Active" == t and nil == d:sub(1, 3):find("yes", 1, true) then
784 W("Mirror " .. host .. " is not active - " .. d, "", "", host) 785 W("Mirror " .. host .. " is not active - " .. d, "", "", host)
785 active = false 786 active = false
787 m[t] = d
786-- TODO - Should do some input validation on BaseURL, and everything else. 788-- TODO - Should do some input validation on BaseURL, and everything else.
787 else 789 else
788 m[t] = d 790 m[t] = d
789 end 791 end
790 end 792 end
791 if "" ~= host and active then 793 if "" ~= host --[[and active]] then
792 mirrors[host] = m 794 mirrors[host] = m
793 end 795 end
794 end 796 end
@@ -933,88 +935,90 @@ if 0 < #arg then
933 end 935 end
934 936
935 if testing("Integrity") or testing("Updated") then 937 if testing("Integrity") or testing("Updated") then
936 if origin and (options.roundRobin.value ~= pu.host) then 938 if 4 > (totalTimeouts) then
937 while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end 939 if origin and (options.roundRobin.value ~= pu.host) then
938 os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock") 940 while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
939 for i, n in pairs(releases) do 941 os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock")
940 for l, o in pairs(releaseFiles) do 942 for i, n in pairs(releases) do
941 if repoExists(i .. o) then 943 for l, o in pairs(releaseFiles) do
942 postDownload(pu.host, n, o) 944 if repoExists(i .. o) then
945 postDownload(pu.host, n, o)
946 end
943 end 947 end
944 end
945 948
946 os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED') 949 os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED')
947 if options.referenceSite.value == pu.host then 950 if options.referenceSite.value == pu.host then
948 execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. 951 execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
949 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. 952 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
950 '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt') 953 '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt')
951 else 954 else
952-- TODO - compare to the pkgmaster copy. 955-- TODO - compare to the pkgmaster copy.
953 end 956 end
954 957
955 local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") 958 local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r")
956 if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else 959 if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else
957 local diff = dfile:read("*a") 960 local diff = dfile:read("*a")
958 if "" ~= diff then 961 if "" ~= diff then
959 downloads(pu.host, pu.path, n, diff) 962 downloads(pu.host, pu.path, n, diff)
963 end
960 end 964 end
961 end
962 965
963 end 966 end
964 967
965 downloads(pu.host, pu.path, "", "") 968 downloads(pu.host, pu.path, "", "")
966 while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end 969 while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
967 os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock") 970 os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock")
968 971
969 for i, n in pairs(releases) do 972 for i, n in pairs(releases) do
970 local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") 973 local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r")
971 if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else 974 if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else
972 local diff = dfile:read("*a") 975 local diff = dfile:read("*a")
973 for l in diff:gmatch("\n*([^\n]+)\n*") do 976 for l in diff:gmatch("\n*([^\n]+)\n*") do
974 postDownload(pu.host, n, "/" .. l) 977 postDownload(pu.host, n, "/" .. l)
978 end
979 end
980 if options.referenceSite.value == pu.host then
981 -- In case it wasn't dealt with already.
982 os.execute('touch results/NEW_Packages_' .. n .. '.test.txt')
975 end 983 end
976 end 984 end
977 if options.referenceSite.value == pu.host then
978 -- In case it wasn't dealt with already.
979 os.execute('touch results/NEW_Packages_' .. n .. '.test.txt')
980 end
981 end
982 985
983 986
984 for i, n in pairs(releases) do 987 for i, n in pairs(releases) do
985 local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") 988 local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
986 if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else 989 if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
987 for l in nfile:lines() do 990 for l in nfile:lines() do
988 local p = l:match('(pool/.*%.deb)') 991 local p = l:match('(pool/.*%.deb)')
989 if nil ~= p then 992 if nil ~= p then
990 downloads(pu.host, pu.path, nil, p) 993 downloads(pu.host, pu.path, nil, p)
994 end
991 end 995 end
992 end 996 end
993 end 997 end
994 end 998 downloads(pu.host, pu.path, nil, "")
995 downloads(pu.host, pu.path, nil, "") 999 while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
996 while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end 1000 for i, n in pairs(releases) do
997 for i, n in pairs(releases) do 1001 local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
998 local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") 1002 if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
999 if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else 1003 for l in nfile:lines() do
1000 for l in nfile:lines() do 1004 local v, p, sz, m, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) | (%x.+) |')
1001 local v, p, sz, m, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) | (%x.+) |') 1005 if nil ~= p then
1002 if nil ~= p then 1006 local status, fsz = execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5')
1003 local status, fsz = execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5') 1007 if testing("Integrity") then
1004 if testing("Integrity") then 1008 if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end.
1005 if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. 1009 E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host)
1006 E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) 1010 print('|' .. sz .. '~=' .. fsz:sub(2, -2) .. '|')
1007 print('|' .. sz .. '~=' .. fsz:sub(2, -2) .. '|') 1011 else
1008 else 1012 local status, fm = execute('md5sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1')
1009 local status, fm = execute('md5sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') 1013 if m ~= fm:sub(2, -2) then E('Package MD5 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end
1010 if m ~= fm:sub(2, -2) then E('Package MD5 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end 1014 local status, fsha = execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1')
1011 local status, fsha = execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') 1015 if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end
1012 if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end 1016 end
1013 end 1017 end
1014 end 1018 if testing("Updated") then
1015 if testing("Updated") then 1019 if sz ~= fsz:sub(2, -2) then
1016 if sz ~= fsz:sub(2, -2) then 1020 E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host)
1017 E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) 1021 end
1018 end 1022 end
1019 end 1023 end
1020 end 1024 end
@@ -1022,6 +1026,17 @@ if 0 < #arg then
1022 end 1026 end
1023 end 1027 end
1024 1028
1029 else
1030 for k, v in pairs{"ftp", "http", "https", "rsync"} do
1031 if testing(v) then
1032 local tests = results[v]
1033 if testing("Integrity") then tests.Integrity = {errors = -1; warnings = -1} end
1034 if testing("Protocol") then tests.Protocol = {errors = -1; warnings = -1} end
1035 if testing("Updated") then tests.Updated = {errors = -1; warnings = -1} end
1036 if testing("URLSanity") then tests.URLSanity = {errors = -1; warnings = -1} end
1037 results[v] = tests
1038 end
1039 end
1025 end 1040 end
1026 end 1041 end
1027 1042