diff options
| author | onefang | 2019-12-02 05:50:58 +1000 |
|---|---|---|
| committer | onefang | 2019-12-02 05:50:58 +1000 |
| commit | 0560dc5975ff29be6d63885925aa1942135f0043 (patch) | |
| tree | a9ba04aadcba1f91f3b53ce860825c70fb8a61a6 /apt-panopticon.lua | |
| parent | Explain the Updated time. (diff) | |
| download | apt-panopticon-0560dc5975ff29be6d63885925aa1942135f0043.zip apt-panopticon-0560dc5975ff29be6d63885925aa1942135f0043.tar.gz apt-panopticon-0560dc5975ff29be6d63885925aa1942135f0043.tar.bz2 apt-panopticon-0560dc5975ff29be6d63885925aa1942135f0043.tar.xz | |
Add more report stuff.
TIMEOUT, report inactive mirrors and why it's not active, add a * for
things tested but not supported.
Diffstat (limited to 'apt-panopticon.lua')
| -rwxr-xr-x | apt-panopticon.lua | 155 |
1 files changed, 85 insertions, 70 deletions
diff --git a/apt-panopticon.lua b/apt-panopticon.lua index d66e32d..5d9ed82 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua | |||
| @@ -756,7 +756,7 @@ local getMirrors = function () | |||
| 756 | local host = "" | 756 | local host = "" |
| 757 | local m = {} | 757 | local m = {} |
| 758 | local active = true | 758 | local active = true |
| 759 | local URL = "https://" .. options.referenceSite.value .. "/mirror_list.txt" | 759 | local URL = "http://" .. options.referenceSite.value .. "/mirror_list.txt" |
| 760 | I("getting mirrors.") | 760 | I("getting mirrors.") |
| 761 | local p, c, h = http.request(URL) | 761 | local p, c, h = http.request(URL) |
| 762 | if nil == p then E(c .. " fetching " .. URL) else | 762 | if nil == p then E(c .. " fetching " .. URL) else |
| @@ -766,7 +766,8 @@ local getMirrors = function () | |||
| 766 | d = string.lower(d) | 766 | d = string.lower(d) |
| 767 | if "FQDN" == t then | 767 | if "FQDN" == t then |
| 768 | if "" ~= host then | 768 | if "" ~= host then |
| 769 | if active then mirrors[host] = m end | 769 | -- if active then mirrors[host] = m end |
| 770 | mirrors[host] = m | ||
| 770 | m = {} | 771 | m = {} |
| 771 | active = true | 772 | active = true |
| 772 | end | 773 | end |
| @@ -780,15 +781,16 @@ local getMirrors = function () | |||
| 780 | prot[w] = true; | 781 | prot[w] = true; |
| 781 | end | 782 | end |
| 782 | m[t] = prot | 783 | m[t] = prot |
| 783 | elseif "Active" == t and nil == d:find("yes", 1, true) then | 784 | elseif "Active" == t and nil == d:sub(1, 3):find("yes", 1, true) then |
| 784 | W("Mirror " .. host .. " is not active - " .. d, "", "", host) | 785 | W("Mirror " .. host .. " is not active - " .. d, "", "", host) |
| 785 | active = false | 786 | active = false |
| 787 | m[t] = d | ||
| 786 | -- TODO - Should do some input validation on BaseURL, and everything else. | 788 | -- TODO - Should do some input validation on BaseURL, and everything else. |
| 787 | else | 789 | else |
| 788 | m[t] = d | 790 | m[t] = d |
| 789 | end | 791 | end |
| 790 | end | 792 | end |
| 791 | if "" ~= host and active then | 793 | if "" ~= host --[[and active]] then |
| 792 | mirrors[host] = m | 794 | mirrors[host] = m |
| 793 | end | 795 | end |
| 794 | end | 796 | end |
| @@ -933,88 +935,90 @@ if 0 < #arg then | |||
| 933 | end | 935 | end |
| 934 | 936 | ||
| 935 | if testing("Integrity") or testing("Updated") then | 937 | if testing("Integrity") or testing("Updated") then |
| 936 | if origin and (options.roundRobin.value ~= pu.host) then | 938 | if 4 > (totalTimeouts) then |
| 937 | while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end | 939 | if origin and (options.roundRobin.value ~= pu.host) then |
| 938 | os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock") | 940 | while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end |
| 939 | for i, n in pairs(releases) do | 941 | os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock") |
| 940 | for l, o in pairs(releaseFiles) do | 942 | for i, n in pairs(releases) do |
| 941 | if repoExists(i .. o) then | 943 | for l, o in pairs(releaseFiles) do |
| 942 | postDownload(pu.host, n, o) | 944 | if repoExists(i .. o) then |
| 945 | postDownload(pu.host, n, o) | ||
| 946 | end | ||
| 943 | end | 947 | end |
| 944 | end | ||
| 945 | 948 | ||
| 946 | os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED') | 949 | os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED') |
| 947 | if options.referenceSite.value == pu.host then | 950 | if options.referenceSite.value == pu.host then |
| 948 | execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. | 951 | execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. |
| 949 | 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. | 952 | 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. |
| 950 | '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt') | 953 | '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt') |
| 951 | else | 954 | else |
| 952 | -- TODO - compare to the pkgmaster copy. | 955 | -- TODO - compare to the pkgmaster copy. |
| 953 | end | 956 | end |
| 954 | 957 | ||
| 955 | local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") | 958 | local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") |
| 956 | if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else | 959 | if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else |
| 957 | local diff = dfile:read("*a") | 960 | local diff = dfile:read("*a") |
| 958 | if "" ~= diff then | 961 | if "" ~= diff then |
| 959 | downloads(pu.host, pu.path, n, diff) | 962 | downloads(pu.host, pu.path, n, diff) |
| 963 | end | ||
| 960 | end | 964 | end |
| 961 | end | ||
| 962 | 965 | ||
| 963 | end | 966 | end |
| 964 | 967 | ||
| 965 | downloads(pu.host, pu.path, "", "") | 968 | downloads(pu.host, pu.path, "", "") |
| 966 | while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end | 969 | while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end |
| 967 | os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock") | 970 | os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock") |
| 968 | 971 | ||
| 969 | for i, n in pairs(releases) do | 972 | for i, n in pairs(releases) do |
| 970 | local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") | 973 | local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") |
| 971 | if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else | 974 | if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else |
| 972 | local diff = dfile:read("*a") | 975 | local diff = dfile:read("*a") |
| 973 | for l in diff:gmatch("\n*([^\n]+)\n*") do | 976 | for l in diff:gmatch("\n*([^\n]+)\n*") do |
| 974 | postDownload(pu.host, n, "/" .. l) | 977 | postDownload(pu.host, n, "/" .. l) |
| 978 | end | ||
| 979 | end | ||
| 980 | if options.referenceSite.value == pu.host then | ||
| 981 | -- In case it wasn't dealt with already. | ||
| 982 | os.execute('touch results/NEW_Packages_' .. n .. '.test.txt') | ||
| 975 | end | 983 | end |
| 976 | end | 984 | end |
| 977 | if options.referenceSite.value == pu.host then | ||
| 978 | -- In case it wasn't dealt with already. | ||
| 979 | os.execute('touch results/NEW_Packages_' .. n .. '.test.txt') | ||
| 980 | end | ||
| 981 | end | ||
| 982 | 985 | ||
| 983 | 986 | ||
| 984 | for i, n in pairs(releases) do | 987 | for i, n in pairs(releases) do |
| 985 | local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") | 988 | local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") |
| 986 | if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else | 989 | if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else |
| 987 | for l in nfile:lines() do | 990 | for l in nfile:lines() do |
| 988 | local p = l:match('(pool/.*%.deb)') | 991 | local p = l:match('(pool/.*%.deb)') |
| 989 | if nil ~= p then | 992 | if nil ~= p then |
| 990 | downloads(pu.host, pu.path, nil, p) | 993 | downloads(pu.host, pu.path, nil, p) |
| 994 | end | ||
| 991 | end | 995 | end |
| 992 | end | 996 | end |
| 993 | end | 997 | end |
| 994 | end | 998 | downloads(pu.host, pu.path, nil, "") |
| 995 | downloads(pu.host, pu.path, nil, "") | 999 | while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end |
| 996 | while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end | 1000 | for i, n in pairs(releases) do |
| 997 | for i, n in pairs(releases) do | 1001 | local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") |
| 998 | local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") | 1002 | if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else |
| 999 | if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else | 1003 | for l in nfile:lines() do |
| 1000 | for l in nfile:lines() do | 1004 | local v, p, sz, m, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) | (%x.+) |') |
| 1001 | local v, p, sz, m, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) | (%x.+) |') | 1005 | if nil ~= p then |
| 1002 | if nil ~= p then | 1006 | local status, fsz = execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5') |
| 1003 | local status, fsz = execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5') | 1007 | if testing("Integrity") then |
| 1004 | if testing("Integrity") then | 1008 | if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. |
| 1005 | if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. | 1009 | E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) |
| 1006 | E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) | 1010 | print('|' .. sz .. '~=' .. fsz:sub(2, -2) .. '|') |
| 1007 | print('|' .. sz .. '~=' .. fsz:sub(2, -2) .. '|') | 1011 | else |
| 1008 | else | 1012 | local status, fm = execute('md5sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') |
| 1009 | local status, fm = execute('md5sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') | 1013 | if m ~= fm:sub(2, -2) then E('Package MD5 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end |
| 1010 | if m ~= fm:sub(2, -2) then E('Package MD5 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end | 1014 | local status, fsha = execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') |
| 1011 | local status, fsha = execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') | 1015 | if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end |
| 1012 | if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end | 1016 | end |
| 1013 | end | 1017 | end |
| 1014 | end | 1018 | if testing("Updated") then |
| 1015 | if testing("Updated") then | 1019 | if sz ~= fsz:sub(2, -2) then |
| 1016 | if sz ~= fsz:sub(2, -2) then | 1020 | E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) |
| 1017 | E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) | 1021 | end |
| 1018 | end | 1022 | end |
| 1019 | end | 1023 | end |
| 1020 | end | 1024 | end |
| @@ -1022,6 +1026,17 @@ if 0 < #arg then | |||
| 1022 | end | 1026 | end |
| 1023 | end | 1027 | end |
| 1024 | 1028 | ||
| 1029 | else | ||
| 1030 | for k, v in pairs{"ftp", "http", "https", "rsync"} do | ||
| 1031 | if testing(v) then | ||
| 1032 | local tests = results[v] | ||
| 1033 | if testing("Integrity") then tests.Integrity = {errors = -1; warnings = -1} end | ||
| 1034 | if testing("Protocol") then tests.Protocol = {errors = -1; warnings = -1} end | ||
| 1035 | if testing("Updated") then tests.Updated = {errors = -1; warnings = -1} end | ||
| 1036 | if testing("URLSanity") then tests.URLSanity = {errors = -1; warnings = -1} end | ||
| 1037 | results[v] = tests | ||
| 1038 | end | ||
| 1039 | end | ||
| 1025 | end | 1040 | end |
| 1026 | end | 1041 | end |
| 1027 | 1042 | ||
