Some refactoring of the results data handling.

3 files changed, 44 insertions, 69 deletions

diff --git a/apt-panopticommon.lua b/apt-panopticommon.lua
index 5c0dfe3..aba2a2a 100644
--- a/apt-panopticommon.lua
+++ b/apt-panopticommon.lua
@@ -395,7 +395,7 @@ APT.plurals = function(e, w)
     return result
 end
 
-APT.fill = function(results)
+APT.padResults = function(results)
     for k, v in pairs{"ftp", "http", "https", "rsync"} do
         tests = results[v]
         if nil == tests then tests = {errors = 0; warnings = 0} end
@@ -410,14 +410,12 @@ APT.fill = function(results)
     return results
 end
 
-APT.collate = function(host, ip, results)
-    results = APT.fill(results)
-    local f = "results/" .. host .. "_" .. ip .. ".lua"
-    local rfile, e = io.open(f, "r")
-    if nil == rfile then I("opening " .. f .. " file - " .. e) else
-        rfile:close()
+APT.collate = function(l, host, ip, results)
+    results = APT.padResults(results)
+    local f = l .. "/" .. host .. "_" .. ip .. ".lua"
+    if APT.checkFile(f) then
         local rs = loadfile(f)()
-        rs = APT.fill(rs)
+        rs = APT.padResults(rs)
         for k, v in pairs(rs) do
             if "table" == type(v) then
                 if ("speed" == k) and (nil ~= results.speed) then
@@ -427,7 +425,11 @@ APT.collate = function(host, ip, results)
                     for i, u in pairs(v) do
                         if "table" == type(u) then
                             for h, t in pairs(u) do
-                                local a = results[k][i][h]
+                                local a = results[k]
+                                if nil == a then results[k]    = {i = {}}; a = results[k]    end
+                                a = a[i]
+                                if nil == a then results[k][i] = {h = {}}; a = results[k][i] end
+                                a = a[h]
                                 if nil == a then a = 0 end
                                 results[k][i][h] = a + t
                             end
@@ -448,8 +450,31 @@ APT.collate = function(host, ip, results)
     end
     return results
 end
-
-
+APT.collateAll = function(l, host, func)
+    results = {}
+    local f = l .. "/" .. host .. ".lua"
+    if APT.checkFile(f) then
+        results = loadfile(f)()
+        results = APT.padResults(results)
+        if nil ~= func then func(results) end
+        local v = APT.mirrors[host]
+        if nil ~= v then
+            local IPs = v.IPs
+            for i, u in pairs(IPs) do
+                if "table" == type(u) then
+                    for h, t in pairs(u) do
+                        results = APT.collate(l, host, h, results)
+                        if nil ~= func then func(results, h) end
+                    end
+                else
+                    results = APT.collate(l, host, i, results)
+                    if nil ~= func then func(results, i) end
+                end
+            end
+        end
+    end
+    return results
+end
 
 
 APT.now = 0
@@ -471,7 +496,6 @@ local DSSe = 'DS:URLSanityErrors:GAUGE:' .. hb .. ':0:U'
 local DSSw = 'DS:URLSanityWarnings:GAUGE:' .. hb .. ':0:U'
 local DSx = 'DS:max:GAUGE:' .. hb .. ':0:U'
 local DSn = 'DS:min:GAUGE:' .. hb .. ':0:U'
-
 -- What Collectd uses.
 local RRAc0  = 'RRA:AVERAGE:0.9:1:1200'
 local RRAc1  = 'RRA:MIN:0.9:1:1200'
@@ -488,7 +512,6 @@ local RRAc11 = 'RRA:MAX:0.9:223:1202'
 local RRAc12 = 'RRA:AVERAGE:0.9:2635:1201'
 local RRAc13 = 'RRA:MIN:0.9:2635:1201'
 local RRAc14 = 'RRA:MAX:0.9:2635:1201'
-
 -- Try LAST.
 local RRAl0  = 'RRA:LAST:0.9:1:1200'
 local RRAl1  = 'RRA:LAST:0.9:7:1235'
@@ -534,30 +557,12 @@ APT.updateRRD = function(results, host, ip)
     end
 end
 
-
 APT.doRRD = function(l, k, v)
-    if APT.checkFile(l .. "/" .. k .. ".lua") then
-        local results = loadfile(l .. "/" .. k .. ".lua")()
-        results = APT.fill(results)
-        APT.createRRD(k)
-        APT.updateRRD(results, k)
-        if nil ~= v then
-            local IPs = v.IPs
-            for i, u in pairs(IPs) do
-                if "table" == type(u) then
-                    for h, t in pairs(u) do
-                        APT.createRRD(k, h)
-                        results = APT.collate(k, h, results)
-                        APT.updateRRD(results, k, h)
-                    end
-                else
-                    APT.createRRD(k, i)
-                    results = APT.collate(k, i, results)
-                    APT.updateRRD(results, k, i)
-                end
-            end
-        end
-    end
+    APT.collateAll(l, k,
+        function(results, ip)
+            APT.createRRD(k, ip)
+            APT.updateRRD(results, k, ip)
+        end)
 end
 
 
diff --git a/apt-panopticon-report-email-web.lua b/apt-panopticon-report-email-web.lua
index 6319d1e..c19d211 100755
--- a/apt-panopticon-report-email-web.lua
+++ b/apt-panopticon-report-email-web.lua
@@ -160,18 +160,8 @@ if nil == email then C("opening mirrors file - " .. e) else
                 "==== package mirror status " .. os.date("!%Y-%m-%d %H:%M") .. " GMT ====\n" ..
                 "[skip] means that the test hasn't been written yet.\n\n")
     for k, v in APT.orderedPairs(APT.mirrors) do
-        local results = loadfile("results/" .. k .. ".lua")()
         email:write(k .. "....\n")
-        local IPs = v.IPs
-        for i, u in pairs(IPs) do
-            if "table" == type(u) then
-                for h, t in pairs(u) do
-                    results = APT.collate(k, h, results)
-                end
-            else
-                results = APT.collate(k, i, results)
-            end
-        end
+        local results = APT.collateAll('results', k)
         local ftp       = "[skip]"
         local http      = status(k, results, "http")
         local https     = status(k, results, "https")
@@ -325,7 +315,7 @@ if nil == web then C("opening mirrors file - " .. e) else
                 "<th>Protocol</th><th>URL sanity</th><th>Integrity</th><th>Updated</th><th colspan='2'>Speed range</th></tr>\n"
                 )
     for k, v in APT.orderedPairs(APT.mirrors) do
-        local results = loadfile("results/" .. k .. ".lua")()
+        local results = APT.collateAll('results', k)
         local active = ""
         if "yes" == v.Active then
             web:write("     <tr><th>" .. k .. "</th> ")
@@ -333,16 +323,6 @@ if nil == web then C("opening mirrors file - " .. e) else
             if nil == v.Active then active = 'nil' else active = v.Active end
             web:write("     <tr style='background-color:dimgrey'><th>" .. k .. "</th> ")
         end
-        local IPs = v.IPs
-        for i, u in pairs(IPs) do
-            if "table" == type(u) then
-                for h, t in pairs(u) do
-                    results = APT.collate(k, h, results)
-                end
-            else
-                results = APT.collate(k, i, results)
-            end
-        end
         local ftp       = "[<font color='grey'><b>skip</b></font>]"
         local http      = status(k, results, "http")
         local https     = status(k, results, "https")
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index c5b76b3..cd3b38e 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -654,17 +654,7 @@ if 0 < #arg then
     APT.mirrors = loadfile("results/mirrors.lua")()
     if nil ~= arg[2] then I("  Using IP " .. arg[2]); ip = arg[2] end
     if nil ~= arg[3] then I("  Using file " .. arg[3]); end
-
-    for k, v in pairs{"ftp", "http", "https", "rsync"} do
-        if APT.testing(v) then
-            local tests = {errors = 0; warnings = 0}
-            if APT.testing("Integrity") then tests.Integrity = {errors = 0; warnings = 0} end
-            if APT.testing("Protocol")  then tests.Protocol  = {errors = 0; warnings = 0} end
-            if APT.testing("Updated")   then tests.Updated   = {errors = 0; warnings = 0} end
-            if APT.testing("URLSanity") then tests.URLSanity = {errors = 0; warnings = 0} end
-            APT.results[v] = tests
-        end
-    end
+    APT.results = APT.padResults(APT.results)
     if APT.origin then
         if APT.testing("Integrity") or APT.testing("Updated") then
             if APT.origin and (APT.options.roundRobin.value ~= pu.host) then