From beef318bc3f330bb45541e59672f349a6cb5233b Mon Sep 17 00:00:00 2001
From: onefang
Date: Thu, 12 Dec 2019 13:53:47 +1000
Subject: Some refactoring of the results data handling.

---
 apt-panopticommon.lua               | 77 ++++++++++++++++++++-----------------
 apt-panopticon-report-email-web.lua | 24 +-----------
 apt-panopticon.lua                  | 12 +-----
 3 files changed, 44 insertions(+), 69 deletions(-)

diff --git a/apt-panopticommon.lua b/apt-panopticommon.lua
index 5c0dfe3..aba2a2a 100644
--- a/apt-panopticommon.lua
+++ b/apt-panopticommon.lua
@@ -395,7 +395,7 @@ APT.plurals = function(e, w)
     return result
 end
 
-APT.fill = function(results)
+APT.padResults = function(results)
     for k, v in pairs{"ftp", "http", "https", "rsync"} do
 	tests = results[v]
 	if nil == tests then tests = {errors = 0; warnings = 0} end
@@ -410,14 +410,12 @@ APT.fill = function(results)
     return results
 end
 
-APT.collate = function(host, ip, results)
-    results = APT.fill(results)
-    local f = "results/" .. host .. "_" .. ip .. ".lua"
-    local rfile, e = io.open(f, "r")
-    if nil == rfile then I("opening " .. f .. " file - " .. e) else
-	rfile:close()
+APT.collate = function(l, host, ip, results)
+    results = APT.padResults(results)
+    local f = l .. "/" .. host .. "_" .. ip .. ".lua"
+    if APT.checkFile(f) then
 	local rs = loadfile(f)()
-	rs = APT.fill(rs)
+	rs = APT.padResults(rs)
 	for k, v in pairs(rs) do
 	    if "table" == type(v) then
 		if ("speed" == k) and (nil ~= results.speed) then
@@ -427,7 +425,11 @@ APT.collate = function(host, ip, results)
 		    for i, u in pairs(v) do
 			if "table" == type(u) then
 			    for h, t in pairs(u) do
-				local a = results[k][i][h]
+				local a = results[k]
+				if nil == a then results[k]    = {i = {}}; a = results[k]    end
+				a = a[i]
+				if nil == a then results[k][i] = {h = {}}; a = results[k][i] end
+				a = a[h]
 				if nil == a then a = 0 end
 				results[k][i][h] = a + t
 			    end
@@ -448,8 +450,31 @@ APT.collate = function(host, ip, results)
     end
     return results
 end
-
-
+APT.collateAll = function(l, host, func)
+    results = {}
+    local f = l .. "/" .. host .. ".lua"
+    if APT.checkFile(f) then
+	results = loadfile(f)()
+	results = APT.padResults(results)
+	if nil ~= func then func(results) end
+	local v = APT.mirrors[host]
+	if nil ~= v then
+	    local IPs = v.IPs
+	    for i, u in pairs(IPs) do
+		if "table" == type(u) then
+		    for h, t in pairs(u) do
+			results = APT.collate(l, host, h, results)
+			if nil ~= func then func(results, h) end
+		    end
+		else
+		    results = APT.collate(l, host, i, results)
+		    if nil ~= func then func(results, i) end
+		end
+	    end
+	end
+    end
+    return results
+end
 
 
 APT.now = 0
@@ -471,7 +496,6 @@ local DSSe = 'DS:URLSanityErrors:GAUGE:' .. hb .. ':0:U'
 local DSSw = 'DS:URLSanityWarnings:GAUGE:' .. hb .. ':0:U'
 local DSx = 'DS:max:GAUGE:' .. hb .. ':0:U'
 local DSn = 'DS:min:GAUGE:' .. hb .. ':0:U'
-
 -- What Collectd uses.
 local RRAc0  = 'RRA:AVERAGE:0.9:1:1200'
 local RRAc1  = 'RRA:MIN:0.9:1:1200'
@@ -488,7 +512,6 @@ local RRAc11 = 'RRA:MAX:0.9:223:1202'
 local RRAc12 = 'RRA:AVERAGE:0.9:2635:1201'
 local RRAc13 = 'RRA:MIN:0.9:2635:1201'
 local RRAc14 = 'RRA:MAX:0.9:2635:1201'
-
 -- Try LAST.
 local RRAl0  = 'RRA:LAST:0.9:1:1200'
 local RRAl1  = 'RRA:LAST:0.9:7:1235'
@@ -534,30 +557,12 @@ APT.updateRRD = function(results, host, ip)
     end
 end
 
-
 APT.doRRD = function(l, k, v)
-    if APT.checkFile(l .. "/" .. k .. ".lua") then
-	local results = loadfile(l .. "/" .. k .. ".lua")()
-	results = APT.fill(results)
-	APT.createRRD(k)
-	APT.updateRRD(results, k)
-	if nil ~= v then
-	    local IPs = v.IPs
-	    for i, u in pairs(IPs) do
-		if "table" == type(u) then
-		    for h, t in pairs(u) do
-			APT.createRRD(k, h)
-			results = APT.collate(k, h, results)
-			APT.updateRRD(results, k, h)
-		    end
-		else
-		    APT.createRRD(k, i)
-		    results = APT.collate(k, i, results)
-		    APT.updateRRD(results, k, i)
-		end
-	    end
-	end
-    end
+    APT.collateAll(l, k,
+	function(results, ip)
+	    APT.createRRD(k, ip)
+	    APT.updateRRD(results, k, ip)
+	end)
 end
 
 
diff --git a/apt-panopticon-report-email-web.lua b/apt-panopticon-report-email-web.lua
index 6319d1e..c19d211 100755
--- a/apt-panopticon-report-email-web.lua
+++ b/apt-panopticon-report-email-web.lua
@@ -160,18 +160,8 @@ if nil == email then C("opening mirrors file - " .. e) else
 		"==== package mirror status " .. os.date("!%Y-%m-%d %H:%M") .. " GMT ====\n" ..
 		"[skip] means that the test hasn't been written yet.\n\n")
     for k, v in APT.orderedPairs(APT.mirrors) do
-	local results = loadfile("results/" .. k .. ".lua")()
 	email:write(k .. "....\n")
-	local IPs = v.IPs
-	for i, u in pairs(IPs) do
-	    if "table" == type(u) then
-		for h, t in pairs(u) do
-		    results = APT.collate(k, h, results)
-		end
-	    else
-		results = APT.collate(k, i, results)
-	    end
-	end
+	local results = APT.collateAll('results', k)
 	local ftp	= "[skip]"
 	local http	= status(k, results, "http")
 	local https	= status(k, results, "https")
@@ -325,7 +315,7 @@ if nil == web then C("opening mirrors file - " .. e) else
 		"<th>Protocol</th><th>URL sanity</th><th>Integrity</th><th>Updated</th><th colspan='2'>Speed range</th></tr>\n"
 		)
     for k, v in APT.orderedPairs(APT.mirrors) do
-	local results = loadfile("results/" .. k .. ".lua")()
+	local results = APT.collateAll('results', k)
 	local active = ""
 	if "yes" == v.Active then
 	    web:write("     <tr><th>" .. k .. "</th> ")
@@ -333,16 +323,6 @@ if nil == web then C("opening mirrors file - " .. e) else
 	    if nil == v.Active then active = 'nil' else active = v.Active end
 	    web:write("     <tr style='background-color:dimgrey'><th>" .. k .. "</th> ")
 	end
-	local IPs = v.IPs
-	for i, u in pairs(IPs) do
-	    if "table" == type(u) then
-		for h, t in pairs(u) do
-		    results = APT.collate(k, h, results)
-		end
-	    else
-		results = APT.collate(k, i, results)
-	    end
-	end
 	local ftp	= "[<font color='grey'><b>skip</b></font>]"
 	local http	= status(k, results, "http")
 	local https	= status(k, results, "https")
diff --git a/apt-panopticon.lua b/apt-panopticon.lua
index c5b76b3..cd3b38e 100755
--- a/apt-panopticon.lua
+++ b/apt-panopticon.lua
@@ -654,17 +654,7 @@ if 0 < #arg then
     APT.mirrors = loadfile("results/mirrors.lua")()
     if nil ~= arg[2] then I("  Using IP " .. arg[2]); ip = arg[2] end
     if nil ~= arg[3] then I("  Using file " .. arg[3]); end
-
-    for k, v in pairs{"ftp", "http", "https", "rsync"} do
-	if APT.testing(v) then
-	    local tests = {errors = 0; warnings = 0}
-	    if APT.testing("Integrity") then tests.Integrity = {errors = 0; warnings = 0} end
-	    if APT.testing("Protocol")  then tests.Protocol  = {errors = 0; warnings = 0} end
-	    if APT.testing("Updated")   then tests.Updated   = {errors = 0; warnings = 0} end
-	    if APT.testing("URLSanity") then tests.URLSanity = {errors = 0; warnings = 0} end
-	    APT.results[v] = tests
-	end
-    end
+    APT.results = APT.padResults(APT.results)
     if APT.origin then
 	if APT.testing("Integrity") or APT.testing("Updated") then
 	    if APT.origin and (APT.options.roundRobin.value ~= pu.host) then
-- 
cgit v1.1