diff options
Diffstat (limited to '')
-rwxr-xr-x | apt-panopticon.lua | 155 |
1 files changed, 85 insertions, 70 deletions
diff --git a/apt-panopticon.lua b/apt-panopticon.lua index d66e32d..5d9ed82 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua | |||
@@ -756,7 +756,7 @@ local getMirrors = function () | |||
756 | local host = "" | 756 | local host = "" |
757 | local m = {} | 757 | local m = {} |
758 | local active = true | 758 | local active = true |
759 | local URL = "https://" .. options.referenceSite.value .. "/mirror_list.txt" | 759 | local URL = "http://" .. options.referenceSite.value .. "/mirror_list.txt" |
760 | I("getting mirrors.") | 760 | I("getting mirrors.") |
761 | local p, c, h = http.request(URL) | 761 | local p, c, h = http.request(URL) |
762 | if nil == p then E(c .. " fetching " .. URL) else | 762 | if nil == p then E(c .. " fetching " .. URL) else |
@@ -766,7 +766,8 @@ local getMirrors = function () | |||
766 | d = string.lower(d) | 766 | d = string.lower(d) |
767 | if "FQDN" == t then | 767 | if "FQDN" == t then |
768 | if "" ~= host then | 768 | if "" ~= host then |
769 | if active then mirrors[host] = m end | 769 | -- if active then mirrors[host] = m end |
770 | mirrors[host] = m | ||
770 | m = {} | 771 | m = {} |
771 | active = true | 772 | active = true |
772 | end | 773 | end |
@@ -780,15 +781,16 @@ local getMirrors = function () | |||
780 | prot[w] = true; | 781 | prot[w] = true; |
781 | end | 782 | end |
782 | m[t] = prot | 783 | m[t] = prot |
783 | elseif "Active" == t and nil == d:find("yes", 1, true) then | 784 | elseif "Active" == t and nil == d:sub(1, 3):find("yes", 1, true) then |
784 | W("Mirror " .. host .. " is not active - " .. d, "", "", host) | 785 | W("Mirror " .. host .. " is not active - " .. d, "", "", host) |
785 | active = false | 786 | active = false |
787 | m[t] = d | ||
786 | -- TODO - Should do some input validation on BaseURL, and everything else. | 788 | -- TODO - Should do some input validation on BaseURL, and everything else. |
787 | else | 789 | else |
788 | m[t] = d | 790 | m[t] = d |
789 | end | 791 | end |
790 | end | 792 | end |
791 | if "" ~= host and active then | 793 | if "" ~= host --[[and active]] then |
792 | mirrors[host] = m | 794 | mirrors[host] = m |
793 | end | 795 | end |
794 | end | 796 | end |
@@ -933,88 +935,90 @@ if 0 < #arg then | |||
933 | end | 935 | end |
934 | 936 | ||
935 | if testing("Integrity") or testing("Updated") then | 937 | if testing("Integrity") or testing("Updated") then |
936 | if origin and (options.roundRobin.value ~= pu.host) then | 938 | if 4 > (totalTimeouts) then |
937 | while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end | 939 | if origin and (options.roundRobin.value ~= pu.host) then |
938 | os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock") | 940 | while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end |
939 | for i, n in pairs(releases) do | 941 | os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock") |
940 | for l, o in pairs(releaseFiles) do | 942 | for i, n in pairs(releases) do |
941 | if repoExists(i .. o) then | 943 | for l, o in pairs(releaseFiles) do |
942 | postDownload(pu.host, n, o) | 944 | if repoExists(i .. o) then |
945 | postDownload(pu.host, n, o) | ||
946 | end | ||
943 | end | 947 | end |
944 | end | ||
945 | 948 | ||
946 | os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED') | 949 | os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED') |
947 | if options.referenceSite.value == pu.host then | 950 | if options.referenceSite.value == pu.host then |
948 | execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. | 951 | execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. |
949 | 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. | 952 | 'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' .. |
950 | '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt') | 953 | '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.txt') |
951 | else | 954 | else |
952 | -- TODO - compare to the pkgmaster copy. | 955 | -- TODO - compare to the pkgmaster copy. |
953 | end | 956 | end |
954 | 957 | ||
955 | local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") | 958 | local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") |
956 | if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else | 959 | if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else |
957 | local diff = dfile:read("*a") | 960 | local diff = dfile:read("*a") |
958 | if "" ~= diff then | 961 | if "" ~= diff then |
959 | downloads(pu.host, pu.path, n, diff) | 962 | downloads(pu.host, pu.path, n, diff) |
963 | end | ||
960 | end | 964 | end |
961 | end | ||
962 | 965 | ||
963 | end | 966 | end |
964 | 967 | ||
965 | downloads(pu.host, pu.path, "", "") | 968 | downloads(pu.host, pu.path, "", "") |
966 | while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end | 969 | while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end |
967 | os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock") | 970 | os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock") |
968 | 971 | ||
969 | for i, n in pairs(releases) do | 972 | for i, n in pairs(releases) do |
970 | local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") | 973 | local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r") |
971 | if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else | 974 | if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else |
972 | local diff = dfile:read("*a") | 975 | local diff = dfile:read("*a") |
973 | for l in diff:gmatch("\n*([^\n]+)\n*") do | 976 | for l in diff:gmatch("\n*([^\n]+)\n*") do |
974 | postDownload(pu.host, n, "/" .. l) | 977 | postDownload(pu.host, n, "/" .. l) |
978 | end | ||
979 | end | ||
980 | if options.referenceSite.value == pu.host then | ||
981 | -- In case it wasn't dealt with already. | ||
982 | os.execute('touch results/NEW_Packages_' .. n .. '.test.txt') | ||
975 | end | 983 | end |
976 | end | 984 | end |
977 | if options.referenceSite.value == pu.host then | ||
978 | -- In case it wasn't dealt with already. | ||
979 | os.execute('touch results/NEW_Packages_' .. n .. '.test.txt') | ||
980 | end | ||
981 | end | ||
982 | 985 | ||
983 | 986 | ||
984 | for i, n in pairs(releases) do | 987 | for i, n in pairs(releases) do |
985 | local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") | 988 | local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") |
986 | if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else | 989 | if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else |
987 | for l in nfile:lines() do | 990 | for l in nfile:lines() do |
988 | local p = l:match('(pool/.*%.deb)') | 991 | local p = l:match('(pool/.*%.deb)') |
989 | if nil ~= p then | 992 | if nil ~= p then |
990 | downloads(pu.host, pu.path, nil, p) | 993 | downloads(pu.host, pu.path, nil, p) |
994 | end | ||
991 | end | 995 | end |
992 | end | 996 | end |
993 | end | 997 | end |
994 | end | 998 | downloads(pu.host, pu.path, nil, "") |
995 | downloads(pu.host, pu.path, nil, "") | 999 | while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end |
996 | while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end | 1000 | for i, n in pairs(releases) do |
997 | for i, n in pairs(releases) do | 1001 | local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") |
998 | local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r") | 1002 | if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else |
999 | if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else | 1003 | for l in nfile:lines() do |
1000 | for l in nfile:lines() do | 1004 | local v, p, sz, m, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) | (%x.+) |') |
1001 | local v, p, sz, m, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) | (%x.+) |') | 1005 | if nil ~= p then |
1002 | if nil ~= p then | 1006 | local status, fsz = execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5') |
1003 | local status, fsz = execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5') | 1007 | if testing("Integrity") then |
1004 | if testing("Integrity") then | 1008 | if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. |
1005 | if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end. | 1009 | E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) |
1006 | E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) | 1010 | print('|' .. sz .. '~=' .. fsz:sub(2, -2) .. '|') |
1007 | print('|' .. sz .. '~=' .. fsz:sub(2, -2) .. '|') | 1011 | else |
1008 | else | 1012 | local status, fm = execute('md5sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') |
1009 | local status, fm = execute('md5sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') | 1013 | if m ~= fm:sub(2, -2) then E('Package MD5 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end |
1010 | if m ~= fm:sub(2, -2) then E('Package MD5 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end | 1014 | local status, fsha = execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') |
1011 | local status, fsha = execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1') | 1015 | if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end |
1012 | if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end | 1016 | end |
1013 | end | 1017 | end |
1014 | end | 1018 | if testing("Updated") then |
1015 | if testing("Updated") then | 1019 | if sz ~= fsz:sub(2, -2) then |
1016 | if sz ~= fsz:sub(2, -2) then | 1020 | E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) |
1017 | E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host) | 1021 | end |
1018 | end | 1022 | end |
1019 | end | 1023 | end |
1020 | end | 1024 | end |
@@ -1022,6 +1026,17 @@ if 0 < #arg then | |||
1022 | end | 1026 | end |
1023 | end | 1027 | end |
1024 | 1028 | ||
1029 | else | ||
1030 | for k, v in pairs{"ftp", "http", "https", "rsync"} do | ||
1031 | if testing(v) then | ||
1032 | local tests = results[v] | ||
1033 | if testing("Integrity") then tests.Integrity = {errors = -1; warnings = -1} end | ||
1034 | if testing("Protocol") then tests.Protocol = {errors = -1; warnings = -1} end | ||
1035 | if testing("Updated") then tests.Updated = {errors = -1; warnings = -1} end | ||
1036 | if testing("URLSanity") then tests.URLSanity = {errors = -1; warnings = -1} end | ||
1037 | results[v] = tests | ||
1038 | end | ||
1039 | end | ||
1025 | end | 1040 | end |
1026 | end | 1041 | end |
1027 | 1042 | ||