aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/OpenSim/Framework/Monitoring/Stats
diff options
context:
space:
mode:
authorDavid Walter Seikel2016-11-03 21:44:39 +1000
committerDavid Walter Seikel2016-11-03 21:44:39 +1000
commit134f86e8d5c414409631b25b8c6f0ee45fbd8631 (patch)
tree216b89d3fb89acfb81be1e440c25c41ab09fa96d /OpenSim/Framework/Monitoring/Stats
parentMore changing to production grid. Double oops. (diff)
downloadopensim-SC-134f86e8d5c414409631b25b8c6f0ee45fbd8631.zip
opensim-SC-134f86e8d5c414409631b25b8c6f0ee45fbd8631.tar.gz
opensim-SC-134f86e8d5c414409631b25b8c6f0ee45fbd8631.tar.bz2
opensim-SC-134f86e8d5c414409631b25b8c6f0ee45fbd8631.tar.xz
Initial update to OpenSim 0.8.2.1 source code.
Diffstat (limited to '')
-rwxr-xr-xOpenSim/Framework/Monitoring/Stats/CounterStat.cs119
-rwxr-xr-xOpenSim/Framework/Monitoring/Stats/EventHistogram.cs173
-rw-r--r--OpenSim/Framework/Monitoring/Stats/PercentageStat.cs16
-rw-r--r--OpenSim/Framework/Monitoring/Stats/Stat.cs102
-rw-r--r--OpenSim/Framework/Monitoring/StatsLogger.cs151
-rw-r--r--OpenSim/Framework/Monitoring/StatsManager.cs403
6 files changed, 882 insertions, 82 deletions
diff --git a/OpenSim/Framework/Monitoring/Stats/CounterStat.cs b/OpenSim/Framework/Monitoring/Stats/CounterStat.cs
new file mode 100755
index 0000000..318cf1c
--- /dev/null
+++ b/OpenSim/Framework/Monitoring/Stats/CounterStat.cs
@@ -0,0 +1,119 @@
1/*
2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the OpenSimulator Project nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28using System;
29using System.Collections.Generic;
30using System.Linq;
31using System.Text;
32
33using OpenMetaverse.StructuredData;
34
35namespace OpenSim.Framework.Monitoring
36{
37// A statistic that wraps a counter.
38// Built this way mostly so histograms and history can be created.
39public class CounterStat : Stat
40{
41 private SortedDictionary<string, EventHistogram> m_histograms;
42 private object counterLock = new object();
43
44 public CounterStat(
45 string shortName,
46 string name,
47 string description,
48 string unitName,
49 string category,
50 string container,
51 StatVerbosity verbosity)
52 : base(shortName, name, description, unitName, category, container, StatType.Push, null, verbosity)
53 {
54 m_histograms = new SortedDictionary<string, EventHistogram>();
55 }
56
57 // Histograms are presumably added at intialization time and the list does not change thereafter.
58 // Thus no locking of the histogram list.
59 public void AddHistogram(string histoName, EventHistogram histo)
60 {
61 m_histograms.Add(histoName, histo);
62 }
63
64 public delegate void ProcessHistogram(string name, EventHistogram histo);
65 public void ForEachHistogram(ProcessHistogram process)
66 {
67 foreach (KeyValuePair<string, EventHistogram> kvp in m_histograms)
68 {
69 process(kvp.Key, kvp.Value);
70 }
71 }
72
73 public void Event()
74 {
75 this.Event(1);
76 }
77
78 // Count the underlying counter.
79 public void Event(int cnt)
80 {
81 lock (counterLock)
82 {
83 base.Value += cnt;
84
85 foreach (EventHistogram histo in m_histograms.Values)
86 {
87 histo.Event(cnt);
88 }
89 }
90 }
91
92 // CounterStat is a basic stat plus histograms
93 public override OSDMap ToOSDMap()
94 {
95 // Get the foundational instance
96 OSDMap map = base.ToOSDMap();
97
98 map["StatType"] = "CounterStat";
99
100 // If there are any histograms, add a new field that is an array of histograms as OSDMaps
101 if (m_histograms.Count > 0)
102 {
103 lock (counterLock)
104 {
105 if (m_histograms.Count > 0)
106 {
107 OSDArray histos = new OSDArray();
108 foreach (EventHistogram histo in m_histograms.Values)
109 {
110 histos.Add(histo.GetHistogramAsOSDMap());
111 }
112 map.Add("Histograms", histos);
113 }
114 }
115 }
116 return map;
117 }
118}
119}
diff --git a/OpenSim/Framework/Monitoring/Stats/EventHistogram.cs b/OpenSim/Framework/Monitoring/Stats/EventHistogram.cs
new file mode 100755
index 0000000..f51f322
--- /dev/null
+++ b/OpenSim/Framework/Monitoring/Stats/EventHistogram.cs
@@ -0,0 +1,173 @@
1/*
2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the OpenSimulator Project nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28using System;
29using System.Collections.Generic;
30using System.Linq;
31using System.Text;
32
33using OpenMetaverse.StructuredData;
34
35namespace OpenSim.Framework.Monitoring
36{
37// Create a time histogram of events. The histogram is built in a wrap-around
38// array of equally distributed buckets.
39// For instance, a minute long histogram of second sized buckets would be:
40// new EventHistogram(60, 1000)
41public class EventHistogram
42{
43 private int m_timeBase;
44 private int m_numBuckets;
45 private int m_bucketMilliseconds;
46 private int m_lastBucket;
47 private int m_totalHistogramMilliseconds;
48 private long[] m_histogram;
49 private object histoLock = new object();
50
51 public EventHistogram(int numberOfBuckets, int millisecondsPerBucket)
52 {
53 m_numBuckets = numberOfBuckets;
54 m_bucketMilliseconds = millisecondsPerBucket;
55 m_totalHistogramMilliseconds = m_numBuckets * m_bucketMilliseconds;
56
57 m_histogram = new long[m_numBuckets];
58 Zero();
59 m_lastBucket = 0;
60 m_timeBase = Util.EnvironmentTickCount();
61 }
62
63 public void Event()
64 {
65 this.Event(1);
66 }
67
68 // Record an event at time 'now' in the histogram.
69 public void Event(int cnt)
70 {
71 lock (histoLock)
72 {
73 // The time as displaced from the base of the histogram
74 int bucketTime = Util.EnvironmentTickCountSubtract(m_timeBase);
75
76 // If more than the total time of the histogram, we just start over
77 if (bucketTime > m_totalHistogramMilliseconds)
78 {
79 Zero();
80 m_lastBucket = 0;
81 m_timeBase = Util.EnvironmentTickCount();
82 }
83 else
84 {
85 // To which bucket should we add this event?
86 int bucket = bucketTime / m_bucketMilliseconds;
87
88 // Advance m_lastBucket to the new bucket. Zero any buckets skipped over.
89 while (bucket != m_lastBucket)
90 {
91 // Zero from just after the last bucket to the new bucket or the end
92 for (int jj = m_lastBucket + 1; jj <= Math.Min(bucket, m_numBuckets - 1); jj++)
93 {
94 m_histogram[jj] = 0;
95 }
96 m_lastBucket = bucket;
97 // If the new bucket is off the end, wrap around to the beginning
98 if (bucket > m_numBuckets)
99 {
100 bucket -= m_numBuckets;
101 m_lastBucket = 0;
102 m_histogram[m_lastBucket] = 0;
103 m_timeBase += m_totalHistogramMilliseconds;
104 }
105 }
106 }
107 m_histogram[m_lastBucket] += cnt;
108 }
109 }
110
111 // Get a copy of the current histogram
112 public long[] GetHistogram()
113 {
114 long[] ret = new long[m_numBuckets];
115 lock (histoLock)
116 {
117 int indx = m_lastBucket + 1;
118 for (int ii = 0; ii < m_numBuckets; ii++, indx++)
119 {
120 if (indx >= m_numBuckets)
121 indx = 0;
122 ret[ii] = m_histogram[indx];
123 }
124 }
125 return ret;
126 }
127
128 public OSDMap GetHistogramAsOSDMap()
129 {
130 OSDMap ret = new OSDMap();
131
132 ret.Add("Buckets", OSD.FromInteger(m_numBuckets));
133 ret.Add("BucketMilliseconds", OSD.FromInteger(m_bucketMilliseconds));
134 ret.Add("TotalMilliseconds", OSD.FromInteger(m_totalHistogramMilliseconds));
135
136 // Compute a number for the first bucket in the histogram.
137 // This will allow readers to know how this histogram relates to any previously read histogram.
138 int baseBucketNum = (m_timeBase / m_bucketMilliseconds) + m_lastBucket + 1;
139 ret.Add("BaseNumber", OSD.FromInteger(baseBucketNum));
140
141 ret.Add("Values", GetHistogramAsOSDArray());
142
143 return ret;
144 }
145 // Get a copy of the current histogram
146 public OSDArray GetHistogramAsOSDArray()
147 {
148 OSDArray ret = new OSDArray(m_numBuckets);
149 lock (histoLock)
150 {
151 int indx = m_lastBucket + 1;
152 for (int ii = 0; ii < m_numBuckets; ii++, indx++)
153 {
154 if (indx >= m_numBuckets)
155 indx = 0;
156 ret[ii] = OSD.FromLong(m_histogram[indx]);
157 }
158 }
159 return ret;
160 }
161
162 // Zero out the histogram
163 public void Zero()
164 {
165 lock (histoLock)
166 {
167 for (int ii = 0; ii < m_numBuckets; ii++)
168 m_histogram[ii] = 0;
169 }
170 }
171}
172
173}
diff --git a/OpenSim/Framework/Monitoring/Stats/PercentageStat.cs b/OpenSim/Framework/Monitoring/Stats/PercentageStat.cs
index 60bed55..55ddf06 100644
--- a/OpenSim/Framework/Monitoring/Stats/PercentageStat.cs
+++ b/OpenSim/Framework/Monitoring/Stats/PercentageStat.cs
@@ -29,6 +29,8 @@ using System;
29using System.Collections.Generic; 29using System.Collections.Generic;
30using System.Text; 30using System.Text;
31 31
32using OpenMetaverse.StructuredData;
33
32namespace OpenSim.Framework.Monitoring 34namespace OpenSim.Framework.Monitoring
33{ 35{
34 public class PercentageStat : Stat 36 public class PercentageStat : Stat
@@ -84,5 +86,19 @@ namespace OpenSim.Framework.Monitoring
84 86
85 return sb.ToString(); 87 return sb.ToString();
86 } 88 }
89
90 // PercentageStat is a basic stat plus percent calc
91 public override OSDMap ToOSDMap()
92 {
93 // Get the foundational instance
94 OSDMap map = base.ToOSDMap();
95
96 map["StatType"] = "PercentageStat";
97
98 map.Add("Antecedent", OSD.FromLong(Antecedent));
99 map.Add("Consequent", OSD.FromLong(Consequent));
100
101 return map;
102 }
87 } 103 }
88} \ No newline at end of file 104} \ No newline at end of file
diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs
index f91251b..a7cb2a6 100644
--- a/OpenSim/Framework/Monitoring/Stats/Stat.cs
+++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs
@@ -27,15 +27,23 @@
27 27
28using System; 28using System;
29using System.Collections.Generic; 29using System.Collections.Generic;
30using System.Linq;
31using System.Reflection;
30using System.Text; 32using System.Text;
33using log4net;
34using OpenMetaverse.StructuredData;
31 35
32namespace OpenSim.Framework.Monitoring 36namespace OpenSim.Framework.Monitoring
33{ 37{
34 /// <summary> 38 /// <summary>
35 /// Holds individual statistic details 39 /// Holds individual statistic details
36 /// </summary> 40 /// </summary>
37 public class Stat 41 public class Stat : IDisposable
38 { 42 {
43// private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
44
45 public static readonly char[] DisallowedShortNameCharacters = { '.' };
46
39 /// <summary> 47 /// <summary>
40 /// Category of this stat (e.g. cache, scene, etc). 48 /// Category of this stat (e.g. cache, scene, etc).
41 /// </summary> 49 /// </summary>
@@ -93,7 +101,7 @@ namespace OpenSim.Framework.Monitoring
93 /// <remarks> 101 /// <remarks>
94 /// Will be null if no measures of interest require samples. 102 /// Will be null if no measures of interest require samples.
95 /// </remarks> 103 /// </remarks>
96 private static Queue<double> m_samples; 104 private Queue<double> m_samples;
97 105
98 /// <summary> 106 /// <summary>
99 /// Maximum number of statistical samples. 107 /// Maximum number of statistical samples.
@@ -160,6 +168,13 @@ namespace OpenSim.Framework.Monitoring
160 throw new Exception( 168 throw new Exception(
161 string.Format("Stat cannot be in category '{0}' since this is reserved for a subcommand", category)); 169 string.Format("Stat cannot be in category '{0}' since this is reserved for a subcommand", category));
162 170
171 foreach (char c in DisallowedShortNameCharacters)
172 {
173 if (shortName.IndexOf(c) != -1)
174 shortName = shortName.Replace(c, '#');
175// throw new Exception(string.Format("Stat name {0} cannot contain character {1}", shortName, c));
176 }
177
163 ShortName = shortName; 178 ShortName = shortName;
164 Name = name; 179 Name = name;
165 Description = description; 180 Description = description;
@@ -181,6 +196,12 @@ namespace OpenSim.Framework.Monitoring
181 Verbosity = verbosity; 196 Verbosity = verbosity;
182 } 197 }
183 198
199 // IDisposable.Dispose()
200 public virtual void Dispose()
201 {
202 return;
203 }
204
184 /// <summary> 205 /// <summary>
185 /// Record a value in the sample set. 206 /// Record a value in the sample set.
186 /// </summary> 207 /// </summary>
@@ -196,6 +217,8 @@ namespace OpenSim.Framework.Monitoring
196 if (m_samples.Count >= m_maxSamples) 217 if (m_samples.Count >= m_maxSamples)
197 m_samples.Dequeue(); 218 m_samples.Dequeue();
198 219
220// m_log.DebugFormat("[STAT]: Recording value {0} for {1}", newValue, Name);
221
199 m_samples.Enqueue(newValue); 222 m_samples.Enqueue(newValue);
200 } 223 }
201 } 224 }
@@ -203,35 +226,100 @@ namespace OpenSim.Framework.Monitoring
203 public virtual string ToConsoleString() 226 public virtual string ToConsoleString()
204 { 227 {
205 StringBuilder sb = new StringBuilder(); 228 StringBuilder sb = new StringBuilder();
206 sb.AppendFormat("{0}.{1}.{2} : {3}{4}", Category, Container, ShortName, Value, UnitName); 229 sb.AppendFormat(
230 "{0}.{1}.{2} : {3}{4}",
231 Category,
232 Container,
233 ShortName,
234 Value,
235 string.IsNullOrEmpty(UnitName) ? "" : string.Format(" {0}", UnitName));
207 236
208 AppendMeasuresOfInterest(sb); 237 AppendMeasuresOfInterest(sb);
209 238
210 return sb.ToString(); 239 return sb.ToString();
211 } 240 }
212 241
213 protected void AppendMeasuresOfInterest(StringBuilder sb) 242 public virtual OSDMap ToOSDMap()
214 { 243 {
215 if ((MeasuresOfInterest & MeasuresOfInterest.AverageChangeOverTime) 244 OSDMap ret = new OSDMap();
216 == MeasuresOfInterest.AverageChangeOverTime) 245 ret.Add("StatType", "Stat"); // used by overloading classes to denote type of stat
246
247 ret.Add("Category", OSD.FromString(Category));
248 ret.Add("Container", OSD.FromString(Container));
249 ret.Add("ShortName", OSD.FromString(ShortName));
250 ret.Add("Name", OSD.FromString(Name));
251 ret.Add("Description", OSD.FromString(Description));
252 ret.Add("UnitName", OSD.FromString(UnitName));
253 ret.Add("Value", OSD.FromReal(Value));
254
255 double lastChangeOverTime, averageChangeOverTime;
256 if (ComputeMeasuresOfInterest(out lastChangeOverTime, out averageChangeOverTime))
257 {
258 ret.Add("LastChangeOverTime", OSD.FromReal(lastChangeOverTime));
259 ret.Add("AverageChangeOverTime", OSD.FromReal(averageChangeOverTime));
260 }
261
262 return ret;
263 }
264
265 // Compute the averages over time and return same.
266 // Return 'true' if averages were actually computed. 'false' if no average info.
267 public bool ComputeMeasuresOfInterest(out double lastChangeOverTime, out double averageChangeOverTime)
268 {
269 bool ret = false;
270 lastChangeOverTime = 0;
271 averageChangeOverTime = 0;
272
273 if ((MeasuresOfInterest & MeasuresOfInterest.AverageChangeOverTime) == MeasuresOfInterest.AverageChangeOverTime)
217 { 274 {
218 double totalChange = 0; 275 double totalChange = 0;
276 double? penultimateSample = null;
219 double? lastSample = null; 277 double? lastSample = null;
220 278
221 lock (m_samples) 279 lock (m_samples)
222 { 280 {
281 // m_log.DebugFormat(
282 // "[STAT]: Samples for {0} are {1}",
283 // Name, string.Join(",", m_samples.Select(s => s.ToString()).ToArray()));
284
223 foreach (double s in m_samples) 285 foreach (double s in m_samples)
224 { 286 {
225 if (lastSample != null) 287 if (lastSample != null)
226 totalChange += s - (double)lastSample; 288 totalChange += s - (double)lastSample;
227 289
290 penultimateSample = lastSample;
228 lastSample = s; 291 lastSample = s;
229 } 292 }
230 } 293 }
231 294
295 if (lastSample != null && penultimateSample != null)
296 {
297 lastChangeOverTime
298 = ((double)lastSample - (double)penultimateSample) / (Watchdog.WATCHDOG_INTERVAL_MS / 1000);
299 }
300
232 int divisor = m_samples.Count <= 1 ? 1 : m_samples.Count - 1; 301 int divisor = m_samples.Count <= 1 ? 1 : m_samples.Count - 1;
233 302
234 sb.AppendFormat(", {0:0.##}{1}/s", totalChange / divisor / (Watchdog.WATCHDOG_INTERVAL_MS / 1000), UnitName); 303 averageChangeOverTime = totalChange / divisor / (Watchdog.WATCHDOG_INTERVAL_MS / 1000);
304 ret = true;
305 }
306
307 return ret;
308 }
309
310 protected void AppendMeasuresOfInterest(StringBuilder sb)
311 {
312 double lastChangeOverTime = 0;
313 double averageChangeOverTime = 0;
314
315 if (ComputeMeasuresOfInterest(out lastChangeOverTime, out averageChangeOverTime))
316 {
317 sb.AppendFormat(
318 ", {0:0.##}{1}/s, {2:0.##}{3}/s",
319 lastChangeOverTime,
320 string.IsNullOrEmpty(UnitName) ? "" : string.Format(" {0}", UnitName),
321 averageChangeOverTime,
322 string.IsNullOrEmpty(UnitName) ? "" : string.Format(" {0}", UnitName));
235 } 323 }
236 } 324 }
237 } 325 }
diff --git a/OpenSim/Framework/Monitoring/StatsLogger.cs b/OpenSim/Framework/Monitoring/StatsLogger.cs
new file mode 100644
index 0000000..15a37aa
--- /dev/null
+++ b/OpenSim/Framework/Monitoring/StatsLogger.cs
@@ -0,0 +1,151 @@
1/*
2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the OpenSimulator Project nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28using System;
29using System.Collections.Generic;
30using System.IO;
31using System.Reflection;
32using System.Text;
33using System.Timers;
34using log4net;
35
36namespace OpenSim.Framework.Monitoring
37{
38 /// <summary>
39 /// Provides a means to continuously log stats for debugging purposes.
40 /// </summary>
41 public static class StatsLogger
42 {
43 private static readonly ILog m_statsLog = LogManager.GetLogger("special.StatsLogger");
44
45 private static Timer m_loggingTimer;
46 private static int m_statsLogIntervalMs = 5000;
47
48 public static void RegisterConsoleCommands(ICommandConsole console)
49 {
50 console.Commands.AddCommand(
51 "General",
52 false,
53 "stats record",
54 "stats record start|stop",
55 "Control whether stats are being regularly recorded to a separate file.",
56 "For debug purposes. Experimental.",
57 HandleStatsRecordCommand);
58
59 console.Commands.AddCommand(
60 "General",
61 false,
62 "stats save",
63 "stats save <path>",
64 "Save stats snapshot to a file. If the file already exists, then the report is appended.",
65 "For debug purposes. Experimental.",
66 HandleStatsSaveCommand);
67 }
68
69 public static void HandleStatsRecordCommand(string module, string[] cmd)
70 {
71 ICommandConsole con = MainConsole.Instance;
72
73 if (cmd.Length != 3)
74 {
75 con.Output("Usage: stats record start|stop");
76 return;
77 }
78
79 if (cmd[2] == "start")
80 {
81 Start();
82 con.OutputFormat("Now recording all stats to file every {0}ms", m_statsLogIntervalMs);
83 }
84 else if (cmd[2] == "stop")
85 {
86 Stop();
87 con.Output("Stopped recording stats to file.");
88 }
89 }
90
91 public static void HandleStatsSaveCommand(string module, string[] cmd)
92 {
93 ICommandConsole con = MainConsole.Instance;
94
95 if (cmd.Length != 3)
96 {
97 con.Output("Usage: stats save <path>");
98 return;
99 }
100
101 string path = cmd[2];
102
103 using (StreamWriter sw = new StreamWriter(path, true))
104 {
105 foreach (string line in GetReport())
106 sw.WriteLine(line);
107 }
108
109 MainConsole.Instance.OutputFormat("Stats saved to file {0}", path);
110 }
111
112 public static void Start()
113 {
114 if (m_loggingTimer != null)
115 Stop();
116
117 m_loggingTimer = new Timer(m_statsLogIntervalMs);
118 m_loggingTimer.AutoReset = false;
119 m_loggingTimer.Elapsed += Log;
120 m_loggingTimer.Start();
121 }
122
123 public static void Stop()
124 {
125 if (m_loggingTimer != null)
126 {
127 m_loggingTimer.Stop();
128 }
129 }
130
131 private static void Log(object sender, ElapsedEventArgs e)
132 {
133 foreach (string line in GetReport())
134 m_statsLog.Info(line);
135
136 m_loggingTimer.Start();
137 }
138
139 private static List<string> GetReport()
140 {
141 List<string> lines = new List<string>();
142
143 lines.Add(string.Format("*** STATS REPORT AT {0} ***", DateTime.Now));
144
145 foreach (string report in StatsManager.GetAllStatsReports())
146 lines.Add(report);
147
148 return lines;
149 }
150 }
151} \ No newline at end of file
diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs
index 0762b01..3136ee8 100644
--- a/OpenSim/Framework/Monitoring/StatsManager.cs
+++ b/OpenSim/Framework/Monitoring/StatsManager.cs
@@ -26,15 +26,20 @@
26 */ 26 */
27 27
28using System; 28using System;
29using System.Collections;
29using System.Collections.Generic; 30using System.Collections.Generic;
31using System.Linq;
30using System.Text; 32using System.Text;
31 33
34using OpenSim.Framework;
35using OpenMetaverse.StructuredData;
36
32namespace OpenSim.Framework.Monitoring 37namespace OpenSim.Framework.Monitoring
33{ 38{
34 /// <summary> 39 /// <summary>
35 /// Singleton used to provide access to statistics reporters 40 /// Static class used to register/deregister/fetch statistics
36 /// </summary> 41 /// </summary>
37 public class StatsManager 42 public static class StatsManager
38 { 43 {
39 // Subcommand used to list other stats. 44 // Subcommand used to list other stats.
40 public const string AllSubCommand = "all"; 45 public const string AllSubCommand = "all";
@@ -51,31 +56,43 @@ namespace OpenSim.Framework.Monitoring
51 /// <remarks> 56 /// <remarks>
52 /// Do not add or remove directly from this dictionary. 57 /// Do not add or remove directly from this dictionary.
53 /// </remarks> 58 /// </remarks>
54 public static Dictionary<string, Dictionary<string, Dictionary<string, Stat>>> RegisteredStats 59 public static SortedDictionary<string, SortedDictionary<string, SortedDictionary<string, Stat>>> RegisteredStats
55 = new Dictionary<string, Dictionary<string, Dictionary<string, Stat>>>(); 60 = new SortedDictionary<string, SortedDictionary<string, SortedDictionary<string, Stat>>>();
56 61
57 private static AssetStatsCollector assetStats; 62// private static AssetStatsCollector assetStats;
58 private static UserStatsCollector userStats; 63// private static UserStatsCollector userStats;
59 private static SimExtraStatsCollector simExtraStats = new SimExtraStatsCollector(); 64// private static SimExtraStatsCollector simExtraStats = new SimExtraStatsCollector();
60 65
61 public static AssetStatsCollector AssetStats { get { return assetStats; } } 66// public static AssetStatsCollector AssetStats { get { return assetStats; } }
62 public static UserStatsCollector UserStats { get { return userStats; } } 67// public static UserStatsCollector UserStats { get { return userStats; } }
63 public static SimExtraStatsCollector SimExtraStats { get { return simExtraStats; } } 68 public static SimExtraStatsCollector SimExtraStats { get; set; }
64 69
65 public static void RegisterConsoleCommands(ICommandConsole console) 70 public static void RegisterConsoleCommands(ICommandConsole console)
66 { 71 {
67 console.Commands.AddCommand( 72 console.Commands.AddCommand(
68 "General", 73 "General",
69 false, 74 false,
70 "show stats", 75 "stats show",
71 "show stats [list|all|<category>]", 76 "stats show [list|all|(<category>[.<container>])+",
72 "Show statistical information for this server", 77 "Show statistical information for this server",
73 "If no final argument is specified then legacy statistics information is currently shown.\n" 78 "If no final argument is specified then legacy statistics information is currently shown.\n"
74 + "If list is specified then statistic categories are shown.\n" 79 + "'list' argument will show statistic categories.\n"
75 + "If all is specified then all registered statistics are shown.\n" 80 + "'all' will show all statistics.\n"
76 + "If a category name is specified then only statistics from that category are shown.\n" 81 + "A <category> name will show statistics from that category.\n"
82 + "A <category>.<container> name will show statistics from that category in that container.\n"
83 + "More than one name can be given separated by spaces.\n"
77 + "THIS STATS FACILITY IS EXPERIMENTAL AND DOES NOT YET CONTAIN ALL STATS", 84 + "THIS STATS FACILITY IS EXPERIMENTAL AND DOES NOT YET CONTAIN ALL STATS",
78 HandleShowStatsCommand); 85 HandleShowStatsCommand);
86
87 console.Commands.AddCommand(
88 "General",
89 false,
90 "show stats",
91 "show stats [list|all|(<category>[.<container>])+",
92 "Alias for 'stats show' command",
93 HandleShowStatsCommand);
94
95 StatsLogger.RegisterConsoleCommands(console);
79 } 96 }
80 97
81 public static void HandleShowStatsCommand(string module, string[] cmd) 98 public static void HandleShowStatsCommand(string module, string[] cmd)
@@ -84,105 +101,286 @@ namespace OpenSim.Framework.Monitoring
84 101
85 if (cmd.Length > 2) 102 if (cmd.Length > 2)
86 { 103 {
87 var categoryName = cmd[2]; 104 foreach (string name in cmd.Skip(2))
88
89 if (categoryName == AllSubCommand)
90 { 105 {
91 foreach (var category in RegisteredStats.Values) 106 string[] components = name.Split('.');
107
108 string categoryName = components[0];
109 string containerName = components.Length > 1 ? components[1] : null;
110 string statName = components.Length > 2 ? components[2] : null;
111
112 if (categoryName == AllSubCommand)
92 { 113 {
93 OutputCategoryStatsToConsole(con, category); 114 OutputAllStatsToConsole(con);
94 } 115 }
95 } 116 else if (categoryName == ListSubCommand)
96 else if (categoryName == ListSubCommand)
97 {
98 con.Output("Statistic categories available are:");
99 foreach (string category in RegisteredStats.Keys)
100 con.OutputFormat(" {0}", category);
101 }
102 else
103 {
104 Dictionary<string, Dictionary<string, Stat>> category;
105 if (!RegisteredStats.TryGetValue(categoryName, out category))
106 { 117 {
107 con.OutputFormat("No such category as {0}", categoryName); 118 con.Output("Statistic categories available are:");
119 foreach (string category in RegisteredStats.Keys)
120 con.OutputFormat(" {0}", category);
108 } 121 }
109 else 122 else
110 { 123 {
111 OutputCategoryStatsToConsole(con, category); 124 SortedDictionary<string, SortedDictionary<string, Stat>> category;
125 if (!RegisteredStats.TryGetValue(categoryName, out category))
126 {
127 con.OutputFormat("No such category as {0}", categoryName);
128 }
129 else
130 {
131 if (String.IsNullOrEmpty(containerName))
132 {
133 OutputCategoryStatsToConsole(con, category);
134 }
135 else
136 {
137 SortedDictionary<string, Stat> container;
138 if (category.TryGetValue(containerName, out container))
139 {
140 if (String.IsNullOrEmpty(statName))
141 {
142 OutputContainerStatsToConsole(con, container);
143 }
144 else
145 {
146 Stat stat;
147 if (container.TryGetValue(statName, out stat))
148 {
149 OutputStatToConsole(con, stat);
150 }
151 else
152 {
153 con.OutputFormat(
154 "No such stat {0} in {1}.{2}", statName, categoryName, containerName);
155 }
156 }
157 }
158 else
159 {
160 con.OutputFormat("No such container {0} in category {1}", containerName, categoryName);
161 }
162 }
163 }
112 } 164 }
113 } 165 }
114 } 166 }
115 else 167 else
116 { 168 {
117 // Legacy 169 // Legacy
118 con.Output(SimExtraStats.Report()); 170 if (SimExtraStats != null)
171 con.Output(SimExtraStats.Report());
172 else
173 OutputAllStatsToConsole(con);
119 } 174 }
120 } 175 }
121 176
122 private static void OutputCategoryStatsToConsole( 177 public static List<string> GetAllStatsReports()
123 ICommandConsole con, Dictionary<string, Dictionary<string, Stat>> category) 178 {
179 List<string> reports = new List<string>();
180
181 foreach (var category in RegisteredStats.Values)
182 reports.AddRange(GetCategoryStatsReports(category));
183
184 return reports;
185 }
186
187 private static void OutputAllStatsToConsole(ICommandConsole con)
124 { 188 {
189 foreach (string report in GetAllStatsReports())
190 con.Output(report);
191 }
192
193 private static List<string> GetCategoryStatsReports(
194 SortedDictionary<string, SortedDictionary<string, Stat>> category)
195 {
196 List<string> reports = new List<string>();
197
125 foreach (var container in category.Values) 198 foreach (var container in category.Values)
199 reports.AddRange(GetContainerStatsReports(container));
200
201 return reports;
202 }
203
204 private static void OutputCategoryStatsToConsole(
205 ICommandConsole con, SortedDictionary<string, SortedDictionary<string, Stat>> category)
206 {
207 foreach (string report in GetCategoryStatsReports(category))
208 con.Output(report);
209 }
210
211 private static List<string> GetContainerStatsReports(SortedDictionary<string, Stat> container)
212 {
213 List<string> reports = new List<string>();
214
215 foreach (Stat stat in container.Values)
216 reports.Add(stat.ToConsoleString());
217
218 return reports;
219 }
220
221 private static void OutputContainerStatsToConsole(
222 ICommandConsole con, SortedDictionary<string, Stat> container)
223 {
224 foreach (string report in GetContainerStatsReports(container))
225 con.Output(report);
226 }
227
228 private static void OutputStatToConsole(ICommandConsole con, Stat stat)
229 {
230 con.Output(stat.ToConsoleString());
231 }
232
233 // Creates an OSDMap of the format:
234 // { categoryName: {
235 // containerName: {
236 // statName: {
237 // "Name": name,
238 // "ShortName": shortName,
239 // ...
240 // },
241 // statName: {
242 // "Name": name,
243 // "ShortName": shortName,
244 // ...
245 // },
246 // ...
247 // },
248 // containerName: {
249 // ...
250 // },
251 // ...
252 // },
253 // categoryName: {
254 // ...
255 // },
256 // ...
257 // }
258 // The passed in parameters will filter the categories, containers and stats returned. If any of the
259 // parameters are either EmptyOrNull or the AllSubCommand value, all of that type will be returned.
260 // Case matters.
261 public static OSDMap GetStatsAsOSDMap(string pCategoryName, string pContainerName, string pStatName)
262 {
263 OSDMap map = new OSDMap();
264
265 foreach (string catName in RegisteredStats.Keys)
126 { 266 {
127 foreach (Stat stat in container.Values) 267 // Do this category if null spec, "all" subcommand or category name matches passed parameter.
268 // Skip category if none of the above.
269 if (!(String.IsNullOrEmpty(pCategoryName) || pCategoryName == AllSubCommand || pCategoryName == catName))
270 continue;
271
272 OSDMap contMap = new OSDMap();
273 foreach (string contName in RegisteredStats[catName].Keys)
128 { 274 {
129 con.Output(stat.ToConsoleString()); 275 if (!(string.IsNullOrEmpty(pContainerName) || pContainerName == AllSubCommand || pContainerName == contName))
276 continue;
277
278 OSDMap statMap = new OSDMap();
279
280 SortedDictionary<string, Stat> theStats = RegisteredStats[catName][contName];
281 foreach (string statName in theStats.Keys)
282 {
283 if (!(String.IsNullOrEmpty(pStatName) || pStatName == AllSubCommand || pStatName == statName))
284 continue;
285
286 statMap.Add(statName, theStats[statName].ToOSDMap());
287 }
288
289 contMap.Add(contName, statMap);
130 } 290 }
291 map.Add(catName, contMap);
131 } 292 }
293
294 return map;
132 } 295 }
133 296
134 /// <summary> 297 public static Hashtable HandleStatsRequest(Hashtable request)
135 /// Start collecting statistics related to assets.
136 /// Should only be called once.
137 /// </summary>
138 public static AssetStatsCollector StartCollectingAssetStats()
139 { 298 {
140 assetStats = new AssetStatsCollector(); 299 Hashtable responsedata = new Hashtable();
300// string regpath = request["uri"].ToString();
301 int response_code = 200;
302 string contenttype = "text/json";
141 303
142 return assetStats; 304 string pCategoryName = StatsManager.AllSubCommand;
143 } 305 string pContainerName = StatsManager.AllSubCommand;
306 string pStatName = StatsManager.AllSubCommand;
144 307
145 /// <summary> 308 if (request.ContainsKey("cat")) pCategoryName = request["cat"].ToString();
146 /// Start collecting statistics related to users. 309 if (request.ContainsKey("cont")) pContainerName = request["cat"].ToString();
147 /// Should only be called once. 310 if (request.ContainsKey("stat")) pStatName = request["stat"].ToString();
148 /// </summary>
149 public static UserStatsCollector StartCollectingUserStats()
150 {
151 userStats = new UserStatsCollector();
152 311
153 return userStats; 312 string strOut = StatsManager.GetStatsAsOSDMap(pCategoryName, pContainerName, pStatName).ToString();
313
314 // If requestor wants it as a callback function, build response as a function rather than just the JSON string.
315 if (request.ContainsKey("callback"))
316 {
317 strOut = request["callback"].ToString() + "(" + strOut + ");";
318 }
319
320 // m_log.DebugFormat("{0} StatFetch: uri={1}, cat={2}, cont={3}, stat={4}, resp={5}",
321 // LogHeader, regpath, pCategoryName, pContainerName, pStatName, strOut);
322
323 responsedata["int_response_code"] = response_code;
324 responsedata["content_type"] = contenttype;
325 responsedata["keepalive"] = false;
326 responsedata["str_response_string"] = strOut;
327 responsedata["access_control_allow_origin"] = "*";
328
329 return responsedata;
154 } 330 }
155 331
332// /// <summary>
333// /// Start collecting statistics related to assets.
334// /// Should only be called once.
335// /// </summary>
336// public static AssetStatsCollector StartCollectingAssetStats()
337// {
338// assetStats = new AssetStatsCollector();
339//
340// return assetStats;
341// }
342//
343// /// <summary>
344// /// Start collecting statistics related to users.
345// /// Should only be called once.
346// /// </summary>
347// public static UserStatsCollector StartCollectingUserStats()
348// {
349// userStats = new UserStatsCollector();
350//
351// return userStats;
352// }
353
156 /// <summary> 354 /// <summary>
157 /// Registers a statistic. 355 /// Register a statistic.
158 /// </summary> 356 /// </summary>
159 /// <param name='stat'></param> 357 /// <param name='stat'></param>
160 /// <returns></returns> 358 /// <returns></returns>
161 public static bool RegisterStat(Stat stat) 359 public static bool RegisterStat(Stat stat)
162 { 360 {
163 Dictionary<string, Dictionary<string, Stat>> category = null, newCategory; 361 SortedDictionary<string, SortedDictionary<string, Stat>> category = null, newCategory;
164 Dictionary<string, Stat> container = null, newContainer; 362 SortedDictionary<string, Stat> container = null, newContainer;
165 363
166 lock (RegisteredStats) 364 lock (RegisteredStats)
167 { 365 {
168 // Stat name is not unique across category/container/shortname key. 366 // Stat name is not unique across category/container/shortname key.
169 // XXX: For now just return false. This is to avoid problems in regression tests where all tests 367 // XXX: For now just return false. This is to avoid problems in regression tests where all tests
170 // in a class are run in the same instance of the VM. 368 // in a class are run in the same instance of the VM.
171 if (TryGetStat(stat, out category, out container)) 369 if (TryGetStatParents(stat, out category, out container))
172 return false; 370 return false;
173 371
174 // We take a copy-on-write approach here of replacing dictionaries when keys are added or removed. 372 // We take a copy-on-write approach here of replacing dictionaries when keys are added or removed.
175 // This means that we don't need to lock or copy them on iteration, which will be a much more 373 // This means that we don't need to lock or copy them on iteration, which will be a much more
176 // common operation after startup. 374 // common operation after startup.
177 if (container != null) 375 if (container != null)
178 newContainer = new Dictionary<string, Stat>(container); 376 newContainer = new SortedDictionary<string, Stat>(container);
179 else 377 else
180 newContainer = new Dictionary<string, Stat>(); 378 newContainer = new SortedDictionary<string, Stat>();
181 379
182 if (category != null) 380 if (category != null)
183 newCategory = new Dictionary<string, Dictionary<string, Stat>>(category); 381 newCategory = new SortedDictionary<string, SortedDictionary<string, Stat>>(category);
184 else 382 else
185 newCategory = new Dictionary<string, Dictionary<string, Stat>>(); 383 newCategory = new SortedDictionary<string, SortedDictionary<string, Stat>>();
186 384
187 newContainer[stat.ShortName] = stat; 385 newContainer[stat.ShortName] = stat;
188 newCategory[stat.Container] = newContainer; 386 newCategory[stat.Container] = newContainer;
@@ -196,21 +394,21 @@ namespace OpenSim.Framework.Monitoring
196 /// Deregister a statistic 394 /// Deregister a statistic
197 /// </summary>> 395 /// </summary>>
198 /// <param name='stat'></param> 396 /// <param name='stat'></param>
199 /// <returns></returns 397 /// <returns></returns>
200 public static bool DeregisterStat(Stat stat) 398 public static bool DeregisterStat(Stat stat)
201 { 399 {
202 Dictionary<string, Dictionary<string, Stat>> category = null, newCategory; 400 SortedDictionary<string, SortedDictionary<string, Stat>> category = null, newCategory;
203 Dictionary<string, Stat> container = null, newContainer; 401 SortedDictionary<string, Stat> container = null, newContainer;
204 402
205 lock (RegisteredStats) 403 lock (RegisteredStats)
206 { 404 {
207 if (!TryGetStat(stat, out category, out container)) 405 if (!TryGetStatParents(stat, out category, out container))
208 return false; 406 return false;
209 407
210 newContainer = new Dictionary<string, Stat>(container); 408 newContainer = new SortedDictionary<string, Stat>(container);
211 newContainer.Remove(stat.ShortName); 409 newContainer.Remove(stat.ShortName);
212 410
213 newCategory = new Dictionary<string, Dictionary<string, Stat>>(category); 411 newCategory = new SortedDictionary<string, SortedDictionary<string, Stat>>(category);
214 newCategory.Remove(stat.Container); 412 newCategory.Remove(stat.Container);
215 413
216 newCategory[stat.Container] = newContainer; 414 newCategory[stat.Container] = newContainer;
@@ -220,15 +418,70 @@ namespace OpenSim.Framework.Monitoring
220 } 418 }
221 } 419 }
222 420
223 public static bool TryGetStats(string category, out Dictionary<string, Dictionary<string, Stat>> stats) 421 public static bool TryGetStat(string category, string container, string statShortName, out Stat stat)
224 { 422 {
225 return RegisteredStats.TryGetValue(category, out stats); 423 stat = null;
424 SortedDictionary<string, SortedDictionary<string, Stat>> categoryStats;
425
426 lock (RegisteredStats)
427 {
428 if (!TryGetStatsForCategory(category, out categoryStats))
429 return false;
430
431 SortedDictionary<string, Stat> containerStats;
432
433 if (!categoryStats.TryGetValue(container, out containerStats))
434 return false;
435
436 return containerStats.TryGetValue(statShortName, out stat);
437 }
438 }
439
440 public static bool TryGetStatsForCategory(
441 string category, out SortedDictionary<string, SortedDictionary<string, Stat>> stats)
442 {
443 lock (RegisteredStats)
444 return RegisteredStats.TryGetValue(category, out stats);
445 }
446
447 /// <summary>
448 /// Get the same stat for each container in a given category.
449 /// </summary>
450 /// <returns>
451 /// The stats if there were any to fetch. Otherwise null.
452 /// </returns>
453 /// <param name='category'></param>
454 /// <param name='statShortName'></param>
455 public static List<Stat> GetStatsFromEachContainer(string category, string statShortName)
456 {
457 SortedDictionary<string, SortedDictionary<string, Stat>> categoryStats;
458
459 lock (RegisteredStats)
460 {
461 if (!RegisteredStats.TryGetValue(category, out categoryStats))
462 return null;
463
464 List<Stat> stats = null;
465
466 foreach (SortedDictionary<string, Stat> containerStats in categoryStats.Values)
467 {
468 if (containerStats.ContainsKey(statShortName))
469 {
470 if (stats == null)
471 stats = new List<Stat>();
472
473 stats.Add(containerStats[statShortName]);
474 }
475 }
476
477 return stats;
478 }
226 } 479 }
227 480
228 public static bool TryGetStat( 481 public static bool TryGetStatParents(
229 Stat stat, 482 Stat stat,
230 out Dictionary<string, Dictionary<string, Stat>> category, 483 out SortedDictionary<string, SortedDictionary<string, Stat>> category,
231 out Dictionary<string, Stat> container) 484 out SortedDictionary<string, Stat> container)
232 { 485 {
233 category = null; 486 category = null;
234 container = null; 487 container = null;
@@ -252,9 +505,9 @@ namespace OpenSim.Framework.Monitoring
252 { 505 {
253 lock (RegisteredStats) 506 lock (RegisteredStats)
254 { 507 {
255 foreach (Dictionary<string, Dictionary<string, Stat>> category in RegisteredStats.Values) 508 foreach (SortedDictionary<string, SortedDictionary<string, Stat>> category in RegisteredStats.Values)
256 { 509 {
257 foreach (Dictionary<string, Stat> container in category.Values) 510 foreach (SortedDictionary<string, Stat> container in category.Values)
258 { 511 {
259 foreach (Stat stat in container.Values) 512 foreach (Stat stat in container.Values)
260 { 513 {
generated by cgit v1.1 at 2025-03-11 15:55:09 +0000