Much work on the Updated tests.

Plus keep a history of results.

3 files changed, 206 insertions, 41 deletions

diff --git a/apt-panopticon-report-email.lua b/apt-panopticon-report-email.lua
index b9c1b30..eb2a0a7 100755
--- a/apt-panopticon-report-email.lua
+++ b/apt-panopticon-report-email.lua
@@ -240,7 +240,7 @@ if nil == file then C("opening mirrors file - " .. e) else
         local protocol  = status(k, results, "Protocol")
         local sanity    = "[skip]"
         local integrity = status(k, results, "Integrity")
-        local updated   = "[skip]"
+        local updated   = status(k, results, "Updated")
 
         -- DNS-RR test.
         if ("deb.devuan.org" ~= k) and (nil ~= mirrors["deb.devuan.org"]) then
diff --git a/apt-panopticon-report-web.lua b/apt-panopticon-report-web.lua
index 9a90860..753a23c 100755
--- a/apt-panopticon-report-web.lua
+++ b/apt-panopticon-report-web.lua
@@ -278,7 +278,7 @@ if nil == file then C("opening mirrors file - " .. e) else
         local protocol  = status(k, results, "Protocol")
         local sanity    = "[<font color='gray'><b>skip</b></font>]"
         local integrity = status(k, results, "Integrity")
-        local updated   = "[<font color='gray'><b>skip</b></font>]"
+        local updated   = status(k, results, "Updated")
 
         -- DNS-RR test.
         if ("deb.devuan.org" ~= k) and (nil ~= mirrors["deb.devuan.org"]) then
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index d40a74c..d66e32d 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -45,7 +45,7 @@ options =
                 "Protocol",
 --              "URLSanity",
                 "Integrity",
---              "Updated",
+                "Updated",
             },
         },
     timeout =
@@ -100,7 +100,7 @@ local referenceDebs =
 local referenceDevs =
 {
     -- Devuan package.  NOTE this one should not get redirected, but that's more a warning than an error.
-    "merged/pool/DEVUAN/main/d/desktop-base/desktop-base_2.0.3_all.deb",
+    "merged/pool/DEVUAN/main/d/desktop-base/desktop-base_3.0_all.deb",
     "merged/pool/DEVUAN/main/u/util-linux/util-linux_2.32.1-0.1+devuan2.1_amd64.deb",
 }
 local arg = {}
@@ -354,6 +354,11 @@ local checkExes = function (exe)
     return tonumber(count)
 end
 
+local checkFile = function(f)
+    local h, e = io.open(f, "r")
+    if nil == h then return false else h:close();  return true end
+end
+
 local repoExists = function (r)
     r = r:match("([%a-]*)")
     if nil == r then return false end
@@ -610,34 +615,106 @@ end
 
 
 local addDownload = function(host, URL, f, r, k)
+    local file = k:match(".*/([%w%.%+%-_]*)$")  -- Get the filename.
+    local o, e = io.open("results/" .. host .. "/merged/dists/" .. r .. k, "r")
+    if nil ~= o then
+        o:close()
+        -- Curls "check timestamp and overwrite file" stuff sucks.
+        -- -R means the destination file gets the timestamp of the remote file.
+        -- Can only do ONE timestamp check per command.
+        --  This doesn't work either.  All downloads get all these headers.  Pffft
+--      local status, ts = execute('TZ="GMT" ls -l --time-style="+%a, %d %b %Y %T %Z"  results/' .. host .. "/merged/dists/" .. r .. k .. ' | cut -d " " -f 6-11')
+--      f:write('header "If-Modified-Since: ' .. ts:sub(2, -2) .. '"\n')
+        -- Curl will DELETE the existing file if the timestamp fails to download a new one, unless we change directory first, 
+        -- which wont work with multiple files in multiple directories.  WTF?
+        os.execute(" mv   results/" .. host .. "/merged/dists/" .. r .. k .. 
+                        " results/" .. host .. "/merged/dists/" .. r .. k .. ".old")
+    end
+
+    D('Downloading http://' .. host .. URL .. '/merged/dists/' .. r .. k)
     f:write('url "' .. 'http://' .. host .. URL .. '/merged/dists/' .. r .. k .. '"\n')
     f:write('output "results/' .. host .. '/merged/dists/' .. r .. k .. '"\n')
-    -- Curls "check timestamp and overwrite file" stuff sucks.
-    -- Can only do ONE timestamp check per command.
-    -- Will DELETE the existing file if the timestamp fails to download a new one, unless we change directory first, which wont work here.
-    os.execute("if [ -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then mv" ..
-                    "   results/" .. host .. "/merged/dists/" .. r .. k .. 
-                    "   results/" .. host .. "/merged/dists/" .. r .. k .. ".old; fi")
 end
 
 local postDownload = function(host, r, k)
+    local file = k:match(".*/([%w%.%+%-_]*)$")  -- Get the filename.
+    local dir  = k:sub(1, 0 - (#file + 1))
     os.execute("if [ -f results/" .. host .. "/merged/dists/" .. r .. k .. ".old ]" ..
-            " && [ ! -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then cp" ..
+            " && [ ! -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then cp -a" ..
                     "   results/" .. host .. "/merged/dists/" .. r .. k .. ".old" ..
                     "   results/" .. host .. "/merged/dists/" .. r .. k .. "; fi")
+    if ".gz"  == k:sub(-3, -1) then execute("gzip -dfk results/" .. host .. "/merged/dists/" .. r .. k) end
+    if ".xz"  == k:sub(-3, -1) then execute("xz   -dfk results/" .. host .. "/merged/dists/" .. r .. k .. " 2>/dev/null") end
     if testing("Integrity") then
         if ".gpg" == k:sub(-4, -1) then
             local status, out = execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. r .. k .. 
                                         " results/" .. host .. "/merged/dists/" .. r .. k:sub(1, -5) .. " 2>/dev/null")
             if "0" ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. r .. k, "http", "Integrity", host) end
         end
+-- TODO - should check the PGP sig of InRelease as well.
+    end
+    if testing("Integrity") or testing("Updated") then
+        if options.referenceSite.value == host then
+            if "Packages." == file:sub(1, 9) then
+-- TODO - compare the SHA256 sums in pkgmaster's Release for both the packed and unpacked versions.
+                local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed', "w+")
+                if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed' .. ' file - ' .. e) else
+                    local pp = {}
+                    if checkFile('results_old/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages') then
+                        for l in io.lines('results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages') do
+                            if "Package: "      == l:sub(1, 9) then
+                                if 0 ~= #pp then
+                                    Pp:write(r .. " | ")
+                                    for i = 1, 6 do
+                                        if nil == pp[i] then print(host .. " " .. r .. " " .. dir .. " " ..  i) else Pp:write(pp[i] .. " | ") end
+                                    end
+                                    Pp:write("\n")
+                                end
+                                pp = {}
+                                pp[1] =            l:sub(10, -1)
+                            elseif "Version: "  == l:sub(1, 9) then
+                                pp[2] =            l:sub(10, -1)
+                            elseif "Filename: " == l:sub(1, 10) then
+                                pp[3] =            l:sub(11, -1)
+                            elseif "Size: "     == l:sub(1, 6) then
+                                pp[4] =            l:sub(7, -1)
+                            elseif "MD5sum: "   == l:sub(1, 8) then
+                                pp[5] =            l:sub(9, -1)
+                            elseif "SHA256: "   == l:sub(1, 8) then
+                                pp[6] =            l:sub(9, -1)
+                            end
+                        end
+                    else
+                        W("Can't find file results_old/pkgmaster.devuan.org/merged/dists/" .. r .. dir .. "Packages")
+                    end
+                    Pp:close()
+                    os.execute('sort results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed >results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed-sorted')
+                    os.execute('rm results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed')
+                    if checkFile('results_old/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages.parsed-sorted') then
+                        execute('diff -U 0   results_old/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages.parsed-sorted ' ..
+                                            'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages.parsed-sorted ' ..
+                                            ' | grep -E "^-" | grep -Ev "^\\+\\+\\+|^---" >>results/OLD_PACKAGES_' .. r .. '.txt')
+                        execute('diff -U 0   results_old/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages.parsed-sorted ' ..
+                                            'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages.parsed-sorted ' ..
+                                            ' | grep -E "^\\+" | grep -Ev "^\\+\\+\\+|^---" >>results/NEW_Packages_' .. r .. '.txt')
+                        -- Find the smallest new package for each release.
+                        os.execute('sort -b -k 9,9 -n results/NEW_Packages_' .. r .. '.txt >results/NEW_Packages_' .. r .. '.sorted.txt')
+                        os.execute('grep -s " | pool/DEBIAN/"          results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1  >results/NEW_Packages_' .. r .. '.test.txt')
+                        os.execute('grep -s " | pool/DEBIAN-SECURITY/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >>results/NEW_Packages_' .. r .. '.test.txt')
+                        os.execute('grep -s " | pool/DEVUAN/"          results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >>results/NEW_Packages_' .. r .. '.test.txt')
+                    else
+                        W("Can't find file results_old/pkgmaster.devuan.org/merged/dists/" .. r .. dir .. "Packages.parsed-sorted")
+                    end
+                end
+            end
+        end
     end
 end
 
 local downloadLock = "flock -n results/curl-"
 local download = "curl --connect-timeout " .. options.timeout.value .. " --create-dirs -L -z 'results/stamp.old' -v -R "
 local downloads = function(host, URL, release, list)
-    if nil == URL then URL = "/" end
+    if nil == URL then URL = "" end
     local lock = "%s-" .. host .. ".lock"
     local log = " --stderr results/curl-%s_" .. host .. ".log"
     local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock:format("META") .. " " .. download .. log:format("META") .. " -K results/" .. host .. ".curl"
@@ -648,24 +725,19 @@ local downloads = function(host, URL, release, list)
 
     if nil ~= list then
         if "" ~= list then
-            for l in list:gmatch("\n*([^\n]+)\n*") do
-                addDownload(host, URL, f, release, "/" .. l)
+            if nil ~= release then
+                for l in list:gmatch("\n*([^\n]+)\n*") do
+                    addDownload(host, URL, f, release, "/" .. l)
+                end
+            else
+                D('Downlaading http://' .. host .. URL .. '/merged/' .. list)
+                f:write('url "' .. 'http://' .. host .. URL .. '/merged/' .. list .. '"\n')
+                f:write('output "results/' .. host .. '/merged/' .. list .. '"\n')
             end
             f:close()
             return
         end
     else
-
---[[
-        for i, s in pairs(referenceDevs) do
-            cm = cm .. " https://" .. host .. URL .. "/" .. s
-        end
-        for i, s in pairs(referenceDebs) do
-            cm = cm .. " https://" .. host .. URL .. "/" .. s
-        end
-        execute(cm)
-]]
-
         for i, s in pairs(releases) do
             for j, k in pairs(releaseFiles) do
                 if repoExists(s .. k) then
@@ -678,6 +750,7 @@ local downloads = function(host, URL, release, list)
     fork(cm)
 end
 
+
 local getMirrors = function ()
     local mirrors = {}
     local host = ""
@@ -818,6 +891,13 @@ if 0 < #arg then
         arg[1] = arg[1]:sub(1, -2)
     end
     local pu = url.parse("http://" .. arg[1])
+
+    if testing("Integrity") or testing("Updated") then
+        if origin and options.referenceSite.value == pu.host then
+--          if not keep then execute("rm -fr results/" .. pu.host) end
+        end
+    end
+
     if nil ~= arg[2] then
         logFile, e = io.open("results/LOG_" .. pu.host .. "_" .. arg[2] .. ".html", "a+")
     else
@@ -840,25 +920,22 @@ if 0 < #arg then
             results[v] = tests
         end
     end
-    if testing("Integrity") or testing("Updated") then
-        if nil == arg[2] then
-            I("Starting file downloads for " .. pu.host)
---          if not keep then execute("rm -fr results/" .. pu.host) end
-            downloads(pu.host, pu.path)
---          checkExes("apt-panopticon.lua " .. sendArgs)
---          checkExes(downloadLock)
-        end
-    end
     if origin then
+        if testing("Integrity") or testing("Updated") then
+            if origin and (options.roundRobin.value ~= pu.host) then
+                I("Starting file downloads for " .. pu.host)
+                downloads(pu.host, pu.path)
+            end
+        end
         checkFiles(pu.host, pu.host, pu.path);
     else
         checkHost(pu.host, pu.host, pu.path, arg[2], arg[3])
     end
 
     if testing("Integrity") or testing("Updated") then
-        if nil == arg[2] then
+        if origin and (options.roundRobin.value ~= pu.host) then
             while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
-            os.execute("rm -f results/" .. pu.host .. ".curl")
+            os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock")
             for i, n in pairs(releases) do
                 for l, o in pairs(releaseFiles) do
                     if repoExists(i .. o) then
@@ -866,7 +943,85 @@ if 0 < #arg then
                     end
                 end
 
+                os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED')
+                if options.referenceSite.value == pu.host then
+                    execute('diff -U 0   results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
+                                        'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
+                                        '| grep -v "@@" | grep "^+" | grep "Packages.xz$" |  cut -c 77- >results/NEW_Release_' .. n .. '.txt')
+                else
+-- TODO - compare to the pkgmaster copy.
+                end
+
+                local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r")
+                if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else
+                    local diff = dfile:read("*a")
+                    if "" ~= diff then
+                        downloads(pu.host, pu.path, n, diff)
+                    end
+                end
+
+            end
+
+            downloads(pu.host, pu.path, "", "")
+            while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
+            os.execute("rm -f results/" .. pu.host .. ".curl; rm results/curl-" .. "META-" .. pu.host .. ".lock")
+
+            for i, n in pairs(releases) do
+                local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r")
+                if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else
+                    local diff = dfile:read("*a")
+                    for l in diff:gmatch("\n*([^\n]+)\n*") do
+                        postDownload(pu.host, n, "/" .. l)
+                    end
+                end
+                if options.referenceSite.value == pu.host then
+                    -- In case it wasn't dealt with already.
+                    os.execute('touch results/NEW_Packages_' .. n .. '.test.txt')
+                end
+            end
+
+
+            for i, n in pairs(releases) do
+                local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
+                if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
+                    for l in nfile:lines() do
+                        local p = l:match('(pool/.*%.deb)')
+                        if nil ~= p then
+                            downloads(pu.host, pu.path, nil, p)
+                        end
+                    end
+                end
             end
+            downloads(pu.host, pu.path, nil, "")
+            while 0 < checkExes(downloadLock .. "META-" .. pu.host .. ".lock") do os.execute("sleep 10") end
+            for i, n in pairs(releases) do
+                local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
+                if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
+                    for l in nfile:lines() do
+                        local v, p, sz, m, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) | (%x.+) |')
+                        if nil ~= p then
+                            local status, fsz = execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5')
+                            if testing("Integrity") then
+                                if sz ~= fsz:sub(2, -2) then    -- The sub bit is to slice off the EOLs at each end.
+                                    E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host)
+                                    print('|' .. sz .. '~=' .. fsz:sub(2, -2) .. '|')
+                                else
+                                    local status, fm = execute('md5sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1')
+                                    if m ~= fm:sub(2, -2) then E('Package MD5 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end
+                                    local status, fsha = execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1')
+                                    if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end
+                                end
+                            end
+                            if testing("Updated") then
+                                if sz ~= fsz:sub(2, -2) then
+                                    E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host)
+                                end
+                            end
+                        end
+                    end
+                end
+            end
+
         end
     end
 
@@ -880,14 +1035,21 @@ if 0 < #arg then
     logPost()
     logFile:close()
 else
-    os.execute("mkdir -p results;  if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi;  touch results/stamp")
+    local dt = os.date('!%Y-%m-%d-%H-%M')
+    local fodt = io.popen('TZ="GMT" date -r results/stamp +%Y-%m-%d-%H-%M', 'r')
+    odt = fodt:read('*l')
+    fodt:close()
+    os.execute('                               rm results_old; ln -s results_' .. odt .. ' results_old')
+    os.execute('mkdir -p results_' .. dt .. '; rm results;     ln -s results_' .. dt  .. ' results')
+    os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi;  touch results/stamp')
+    os.execute("rm -f results/*.check")
     if not keep then
         os.execute("rm -f results/*.curl")
         os.execute("rm -f results/*.log")
         os.execute("rm -f results/*.html")
         os.execute("rm -f results/*.txt")
     end
-    os.execute("rm -f results/*.check")
+
     logFile, e = io.open("results/LOG_apt-panopticon.html", "a+")
     if nil == logFile then C("opening log file - " .. e); return end
     logPre()
@@ -895,6 +1057,10 @@ else
     os.execute("mkdir -p results")
     mirrors = getMirrors()
     checkHost(options.referenceSite.value)
+    for i, n in pairs(releases) do
+        while not checkFile('results/NEW_Packages_' .. n .. '.test.txt') do os.execute("sleep 10") end
+    end
+
     for k, m in pairs(mirrors) do
         if "/" == m.BaseURL:sub(-1, -1) then
             W("slash at end of BaseURL in mirror_list.txt!  " .. m.BaseURL, "", "", m.FQDN)
@@ -911,10 +1077,9 @@ else
             if testing("Integrity") or testing("Updated") then checkExes(downloadLock) end
         end
     end
+
     while 1 <= checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end
-    if testing("Integrity") or testing("Updated") then
-        while 0 < checkExes(downloadLock) do os.execute("sleep 10") end
-    end
+
     os.execute("rm -f results/*.check; rm -f results/*.lock")
 
     -- Create the reports.