aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/OpenSim/Framework/Monitoring
diff options
context:
space:
mode:
authorDavid Walter Seikel2016-11-03 21:44:39 +1000
committerDavid Walter Seikel2016-11-03 21:44:39 +1000
commit134f86e8d5c414409631b25b8c6f0ee45fbd8631 (patch)
tree216b89d3fb89acfb81be1e440c25c41ab09fa96d /OpenSim/Framework/Monitoring
parentMore changing to production grid. Double oops. (diff)
downloadopensim-SC_OLD-134f86e8d5c414409631b25b8c6f0ee45fbd8631.zip
opensim-SC_OLD-134f86e8d5c414409631b25b8c6f0ee45fbd8631.tar.gz
opensim-SC_OLD-134f86e8d5c414409631b25b8c6f0ee45fbd8631.tar.bz2
opensim-SC_OLD-134f86e8d5c414409631b25b8c6f0ee45fbd8631.tar.xz
Initial update to OpenSim 0.8.2.1 source code.
Diffstat (limited to 'OpenSim/Framework/Monitoring')
-rw-r--r--OpenSim/Framework/Monitoring/AssetStatsCollector.cs26
-rw-r--r--OpenSim/Framework/Monitoring/BaseStatsCollector.cs19
-rw-r--r--OpenSim/Framework/Monitoring/Checks/Check.cs118
-rw-r--r--OpenSim/Framework/Monitoring/ChecksManager.cs262
-rw-r--r--OpenSim/Framework/Monitoring/Interfaces/IStatsCollector.cs9
-rw-r--r--OpenSim/Framework/Monitoring/JobEngine.cs341
-rw-r--r--OpenSim/Framework/Monitoring/MemoryWatchdog.cs8
-rw-r--r--OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs4
-rw-r--r--OpenSim/Framework/Monitoring/ServerStatsCollector.cs346
-rwxr-xr-x[-rw-r--r--]OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs106
-rwxr-xr-xOpenSim/Framework/Monitoring/Stats/CounterStat.cs119
-rwxr-xr-xOpenSim/Framework/Monitoring/Stats/EventHistogram.cs173
-rw-r--r--OpenSim/Framework/Monitoring/Stats/PercentageStat.cs16
-rw-r--r--OpenSim/Framework/Monitoring/Stats/Stat.cs102
-rw-r--r--OpenSim/Framework/Monitoring/StatsLogger.cs151
-rw-r--r--OpenSim/Framework/Monitoring/StatsManager.cs403
-rw-r--r--OpenSim/Framework/Monitoring/UserStatsCollector.cs18
-rw-r--r--OpenSim/Framework/Monitoring/Watchdog.cs106
-rw-r--r--OpenSim/Framework/Monitoring/WorkManager.cs290
19 files changed, 2442 insertions, 175 deletions
diff --git a/OpenSim/Framework/Monitoring/AssetStatsCollector.cs b/OpenSim/Framework/Monitoring/AssetStatsCollector.cs
index 2a4d45b..6a0f676 100644
--- a/OpenSim/Framework/Monitoring/AssetStatsCollector.cs
+++ b/OpenSim/Framework/Monitoring/AssetStatsCollector.cs
@@ -28,6 +28,8 @@
28using System; 28using System;
29using System.Timers; 29using System.Timers;
30 30
31using OpenMetaverse.StructuredData;
32
31namespace OpenSim.Framework.Monitoring 33namespace OpenSim.Framework.Monitoring
32{ 34{
33 /// <summary> 35 /// <summary>
@@ -100,5 +102,29 @@ Asset requests yesterday : {3} ({4} per hour) of which {5} were not found",
100 AssetRequestsToday, assetRequestsTodayPerHour, AssetRequestsNotFoundToday, 102 AssetRequestsToday, assetRequestsTodayPerHour, AssetRequestsNotFoundToday,
101 AssetRequestsYesterday, assetRequestsYesterdayPerHour, AssetRequestsNotFoundYesterday); 103 AssetRequestsYesterday, assetRequestsYesterdayPerHour, AssetRequestsNotFoundYesterday);
102 } 104 }
105
106 public override string XReport(string uptime, string version)
107 {
108 return OSDParser.SerializeJsonString(OReport(uptime, version));
109 }
110
111 public override OSDMap OReport(string uptime, string version)
112 {
113 double elapsedHours = (DateTime.Now - startTime).TotalHours;
114 if (elapsedHours <= 0) { elapsedHours = 1; } // prevent divide by zero
115
116 long assetRequestsTodayPerHour = (long)Math.Round(AssetRequestsToday / elapsedHours);
117 long assetRequestsYesterdayPerHour = (long)Math.Round(AssetRequestsYesterday / 24.0);
118
119 OSDMap ret = new OSDMap();
120 ret.Add("AssetRequestsToday", OSD.FromLong(AssetRequestsToday));
121 ret.Add("AssetRequestsTodayPerHour", OSD.FromLong(assetRequestsTodayPerHour));
122 ret.Add("AssetRequestsNotFoundToday", OSD.FromLong(AssetRequestsNotFoundToday));
123 ret.Add("AssetRequestsYesterday", OSD.FromLong(AssetRequestsYesterday));
124 ret.Add("AssetRequestsYesterdayPerHour", OSD.FromLong(assetRequestsYesterdayPerHour));
125 ret.Add("AssetRequestsNotFoundYesterday", OSD.FromLong(assetRequestsNotFoundYesterday));
126
127 return ret;
128 }
103 } 129 }
104} 130}
diff --git a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs
index 2903b6e..20495f6 100644
--- a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs
+++ b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs
@@ -1,4 +1,4 @@
1/* 1/*
2 * Copyright (c) Contributors, http://opensimulator.org/ 2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders. 3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 * 4 *
@@ -45,16 +45,16 @@ namespace OpenSim.Framework.Monitoring
45 sb.Append(Environment.NewLine); 45 sb.Append(Environment.NewLine);
46 46
47 sb.AppendFormat( 47 sb.AppendFormat(
48 "Allocated to OpenSim objects: {0} MB\n", 48 "Heap allocated to OpenSim : {0} MB\n",
49 Math.Round(GC.GetTotalMemory(false) / 1024.0 / 1024.0)); 49 Math.Round(GC.GetTotalMemory(false) / 1024.0 / 1024.0));
50 50
51 sb.AppendFormat( 51 sb.AppendFormat(
52 "OpenSim last object memory churn : {0} MB/s\n", 52 "Last heap allocation rate : {0} MB/s\n",
53 Math.Round((MemoryWatchdog.LastMemoryChurn * 1000) / 1024.0 / 1024, 3)); 53 Math.Round((MemoryWatchdog.LastHeapAllocationRate * 1000) / 1024.0 / 1024, 3));
54 54
55 sb.AppendFormat( 55 sb.AppendFormat(
56 "OpenSim average object memory churn : {0} MB/s\n", 56 "Average heap allocation rate: {0} MB/s\n",
57 Math.Round((MemoryWatchdog.AverageMemoryChurn * 1000) / 1024.0 / 1024, 3)); 57 Math.Round((MemoryWatchdog.AverageHeapAllocationRate * 1000) / 1024.0 / 1024, 3));
58 58
59 sb.AppendFormat( 59 sb.AppendFormat(
60 "Process memory : {0} MB\n", 60 "Process memory : {0} MB\n",
@@ -67,5 +67,12 @@ namespace OpenSim.Framework.Monitoring
67 { 67 {
68 return (string) Math.Round(GC.GetTotalMemory(false) / 1024.0 / 1024.0).ToString() ; 68 return (string) Math.Round(GC.GetTotalMemory(false) / 1024.0 / 1024.0).ToString() ;
69 } 69 }
70
71 public virtual OSDMap OReport(string uptime, string version)
72 {
73 OSDMap ret = new OSDMap();
74 ret.Add("TotalMemory", new OSDReal(Math.Round(GC.GetTotalMemory(false) / 1024.0 / 1024.0)));
75 return ret;
76 }
70 } 77 }
71} 78}
diff --git a/OpenSim/Framework/Monitoring/Checks/Check.cs b/OpenSim/Framework/Monitoring/Checks/Check.cs
new file mode 100644
index 0000000..594386a
--- /dev/null
+++ b/OpenSim/Framework/Monitoring/Checks/Check.cs
@@ -0,0 +1,118 @@
1/*
2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the OpenSimulator Project nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28using System;
29using System.Text;
30
31namespace OpenSim.Framework.Monitoring
32{
33 public class Check
34 {
35// private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
36
37 public static readonly char[] DisallowedShortNameCharacters = { '.' };
38
39 /// <summary>
40 /// Category of this stat (e.g. cache, scene, etc).
41 /// </summary>
42 public string Category { get; private set; }
43
44 /// <summary>
45 /// Containing name for this stat.
46 /// FIXME: In the case of a scene, this is currently the scene name (though this leaves
47 /// us with a to-be-resolved problem of non-unique region names).
48 /// </summary>
49 /// <value>
50 /// The container.
51 /// </value>
52 public string Container { get; private set; }
53
54 /// <summary>
55 /// Action used to check whether alert should go off.
56 /// </summary>
57 /// <remarks>
58 /// Should return true if check passes. False otherwise.
59 /// </remarks>
60 public Func<Check, bool> CheckFunc { get; private set; }
61
62 /// <summary>
63 /// Message from the last failure, if any. If there is no message or no failure then will be null.
64 /// </summary>
65 /// <remarks>
66 /// Should be set by the CheckFunc when applicable.
67 /// </remarks>
68 public string LastFailureMessage { get; set; }
69
70 public StatVerbosity Verbosity { get; private set; }
71 public string ShortName { get; private set; }
72 public string Name { get; private set; }
73 public string Description { get; private set; }
74
75 public Check(
76 string shortName,
77 string name,
78 string description,
79 string category,
80 string container,
81 Func<Check, bool> checkFunc,
82 StatVerbosity verbosity)
83 {
84 if (ChecksManager.SubCommands.Contains(category))
85 throw new Exception(
86 string.Format("Alert cannot be in category '{0}' since this is reserved for a subcommand", category));
87
88 foreach (char c in DisallowedShortNameCharacters)
89 {
90 if (shortName.IndexOf(c) != -1)
91 throw new Exception(string.Format("Alert name {0} cannot contain character {1}", shortName, c));
92 }
93
94 ShortName = shortName;
95 Name = name;
96 Description = description;
97 Category = category;
98 Container = container;
99 CheckFunc = checkFunc;
100 Verbosity = verbosity;
101 }
102
103 public bool CheckIt()
104 {
105 return CheckFunc(this);
106 }
107
108 public virtual string ToConsoleString()
109 {
110 return string.Format(
111 "{0}.{1}.{2} - {3}",
112 Category,
113 Container,
114 ShortName,
115 Description);
116 }
117 }
118} \ No newline at end of file
diff --git a/OpenSim/Framework/Monitoring/ChecksManager.cs b/OpenSim/Framework/Monitoring/ChecksManager.cs
new file mode 100644
index 0000000..e4a7f8c
--- /dev/null
+++ b/OpenSim/Framework/Monitoring/ChecksManager.cs
@@ -0,0 +1,262 @@
1/*
2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the OpenSimulator Project nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28using System;
29using System.Collections.Generic;
30using System.Linq;
31using System.Reflection;
32using System.Text;
33using log4net;
34
35namespace OpenSim.Framework.Monitoring
36{
37 /// <summary>
38 /// Static class used to register/deregister checks on runtime conditions.
39 /// </summary>
40 public static class ChecksManager
41 {
42 private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
43
44 // Subcommand used to list other stats.
45 public const string ListSubCommand = "list";
46
47 // All subcommands
48 public static HashSet<string> SubCommands = new HashSet<string> { ListSubCommand };
49
50 /// <summary>
51 /// Checks categorized by category/container/shortname
52 /// </summary>
53 /// <remarks>
54 /// Do not add or remove directly from this dictionary.
55 /// </remarks>
56 public static SortedDictionary<string, SortedDictionary<string, SortedDictionary<string, Check>>> RegisteredChecks
57 = new SortedDictionary<string, SortedDictionary<string, SortedDictionary<string, Check>>>();
58
59 public static void RegisterConsoleCommands(ICommandConsole console)
60 {
61 console.Commands.AddCommand(
62 "General",
63 false,
64 "show checks",
65 "show checks",
66 "Show checks configured for this server",
67 "If no argument is specified then info on all checks will be shown.\n"
68 + "'list' argument will show check categories.\n"
69 + "THIS FACILITY IS EXPERIMENTAL",
70 HandleShowchecksCommand);
71 }
72
73 public static void HandleShowchecksCommand(string module, string[] cmd)
74 {
75 ICommandConsole con = MainConsole.Instance;
76
77 if (cmd.Length > 2)
78 {
79 foreach (string name in cmd.Skip(2))
80 {
81 string[] components = name.Split('.');
82
83 string categoryName = components[0];
84// string containerName = components.Length > 1 ? components[1] : null;
85
86 if (categoryName == ListSubCommand)
87 {
88 con.Output("check categories available are:");
89
90 foreach (string category in RegisteredChecks.Keys)
91 con.OutputFormat(" {0}", category);
92 }
93// else
94// {
95// SortedDictionary<string, SortedDictionary<string, Check>> category;
96// if (!Registeredchecks.TryGetValue(categoryName, out category))
97// {
98// con.OutputFormat("No such category as {0}", categoryName);
99// }
100// else
101// {
102// if (String.IsNullOrEmpty(containerName))
103// {
104// OutputConfiguredToConsole(con, category);
105// }
106// else
107// {
108// SortedDictionary<string, Check> container;
109// if (category.TryGetValue(containerName, out container))
110// {
111// OutputContainerChecksToConsole(con, container);
112// }
113// else
114// {
115// con.OutputFormat("No such container {0} in category {1}", containerName, categoryName);
116// }
117// }
118// }
119// }
120 }
121 }
122 else
123 {
124 OutputAllChecksToConsole(con);
125 }
126 }
127
128 /// <summary>
129 /// Registers a statistic.
130 /// </summary>
131 /// <param name='stat'></param>
132 /// <returns></returns>
133 public static bool RegisterCheck(Check check)
134 {
135 SortedDictionary<string, SortedDictionary<string, Check>> category = null, newCategory;
136 SortedDictionary<string, Check> container = null, newContainer;
137
138 lock (RegisteredChecks)
139 {
140 // Check name is not unique across category/container/shortname key.
141 // XXX: For now just return false. This is to avoid problems in regression tests where all tests
142 // in a class are run in the same instance of the VM.
143 if (TryGetCheckParents(check, out category, out container))
144 return false;
145
146 // We take a copy-on-write approach here of replacing dictionaries when keys are added or removed.
147 // This means that we don't need to lock or copy them on iteration, which will be a much more
148 // common operation after startup.
149 if (container != null)
150 newContainer = new SortedDictionary<string, Check>(container);
151 else
152 newContainer = new SortedDictionary<string, Check>();
153
154 if (category != null)
155 newCategory = new SortedDictionary<string, SortedDictionary<string, Check>>(category);
156 else
157 newCategory = new SortedDictionary<string, SortedDictionary<string, Check>>();
158
159 newContainer[check.ShortName] = check;
160 newCategory[check.Container] = newContainer;
161 RegisteredChecks[check.Category] = newCategory;
162 }
163
164 return true;
165 }
166
167 /// <summary>
168 /// Deregister an check
169 /// </summary>>
170 /// <param name='stat'></param>
171 /// <returns></returns>
172 public static bool DeregisterCheck(Check check)
173 {
174 SortedDictionary<string, SortedDictionary<string, Check>> category = null, newCategory;
175 SortedDictionary<string, Check> container = null, newContainer;
176
177 lock (RegisteredChecks)
178 {
179 if (!TryGetCheckParents(check, out category, out container))
180 return false;
181
182 newContainer = new SortedDictionary<string, Check>(container);
183 newContainer.Remove(check.ShortName);
184
185 newCategory = new SortedDictionary<string, SortedDictionary<string, Check>>(category);
186 newCategory.Remove(check.Container);
187
188 newCategory[check.Container] = newContainer;
189 RegisteredChecks[check.Category] = newCategory;
190
191 return true;
192 }
193 }
194
195 public static bool TryGetCheckParents(
196 Check check,
197 out SortedDictionary<string, SortedDictionary<string, Check>> category,
198 out SortedDictionary<string, Check> container)
199 {
200 category = null;
201 container = null;
202
203 lock (RegisteredChecks)
204 {
205 if (RegisteredChecks.TryGetValue(check.Category, out category))
206 {
207 if (category.TryGetValue(check.Container, out container))
208 {
209 if (container.ContainsKey(check.ShortName))
210 return true;
211 }
212 }
213 }
214
215 return false;
216 }
217
218 public static void CheckChecks()
219 {
220 lock (RegisteredChecks)
221 {
222 foreach (SortedDictionary<string, SortedDictionary<string, Check>> category in RegisteredChecks.Values)
223 {
224 foreach (SortedDictionary<string, Check> container in category.Values)
225 {
226 foreach (Check check in container.Values)
227 {
228 if (!check.CheckIt())
229 m_log.WarnFormat(
230 "[CHECKS MANAGER]: Check {0}.{1}.{2} failed with message {3}", check.Category, check.Container, check.ShortName, check.LastFailureMessage);
231 }
232 }
233 }
234 }
235 }
236
237 private static void OutputAllChecksToConsole(ICommandConsole con)
238 {
239 foreach (var category in RegisteredChecks.Values)
240 {
241 OutputCategoryChecksToConsole(con, category);
242 }
243 }
244
245 private static void OutputCategoryChecksToConsole(
246 ICommandConsole con, SortedDictionary<string, SortedDictionary<string, Check>> category)
247 {
248 foreach (var container in category.Values)
249 {
250 OutputContainerChecksToConsole(con, container);
251 }
252 }
253
254 private static void OutputContainerChecksToConsole(ICommandConsole con, SortedDictionary<string, Check> container)
255 {
256 foreach (Check check in container.Values)
257 {
258 con.Output(check.ToConsoleString());
259 }
260 }
261 }
262} \ No newline at end of file
diff --git a/OpenSim/Framework/Monitoring/Interfaces/IStatsCollector.cs b/OpenSim/Framework/Monitoring/Interfaces/IStatsCollector.cs
index 99f75e3..40df562 100644
--- a/OpenSim/Framework/Monitoring/Interfaces/IStatsCollector.cs
+++ b/OpenSim/Framework/Monitoring/Interfaces/IStatsCollector.cs
@@ -25,6 +25,8 @@
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */ 26 */
27 27
28using OpenMetaverse.StructuredData;
29
28namespace OpenSim.Framework.Monitoring 30namespace OpenSim.Framework.Monitoring
29{ 31{
30 /// <summary> 32 /// <summary>
@@ -45,5 +47,12 @@ namespace OpenSim.Framework.Monitoring
45 /// A <see cref="System.String"/> 47 /// A <see cref="System.String"/>
46 /// </returns> 48 /// </returns>
47 string XReport(string uptime, string version); 49 string XReport(string uptime, string version);
50
51 /// <summary>
52 /// Report back collected statistical information as an OSDMap of key/values
53 /// </summary>
54 /// <returns>
55 /// </returns>
56 OSDMap OReport(string uptime, string version);
48 } 57 }
49} 58}
diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs
new file mode 100644
index 0000000..6db9a67
--- /dev/null
+++ b/OpenSim/Framework/Monitoring/JobEngine.cs
@@ -0,0 +1,341 @@
1/*
2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the OpenSimulator Project nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28using System;
29using System.Collections.Concurrent;
30using System.Reflection;
31using System.Threading;
32using log4net;
33using OpenSim.Framework;
34
35namespace OpenSim.Framework.Monitoring
36{
37 public class JobEngine
38 {
39 private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
40
41 public int LogLevel { get; set; }
42
43 public string Name { get; private set; }
44
45 public string LoggingName { get; private set; }
46
47 /// <summary>
48 /// Is this engine running?
49 /// </summary>
50 public bool IsRunning { get; private set; }
51
52 /// <summary>
53 /// The current job that the engine is running.
54 /// </summary>
55 /// <remarks>
56 /// Will be null if no job is currently running.
57 /// </remarks>
58 public Job CurrentJob { get; private set; }
59
60 /// <summary>
61 /// Number of jobs waiting to be processed.
62 /// </summary>
63 public int JobsWaiting { get { return m_jobQueue.Count; } }
64
65 /// <summary>
66 /// The timeout in milliseconds to wait for at least one event to be written when the recorder is stopping.
67 /// </summary>
68 public int RequestProcessTimeoutOnStop { get; set; }
69
70 /// <summary>
71 /// Controls whether we need to warn in the log about exceeding the max queue size.
72 /// </summary>
73 /// <remarks>
74 /// This is flipped to false once queue max has been exceeded and back to true when it falls below max, in
75 /// order to avoid spamming the log with lots of warnings.
76 /// </remarks>
77 private bool m_warnOverMaxQueue = true;
78
79 private BlockingCollection<Job> m_jobQueue;
80
81 private CancellationTokenSource m_cancelSource;
82
83 /// <summary>
84 /// Used to signal that we are ready to complete stop.
85 /// </summary>
86 private ManualResetEvent m_finishedProcessingAfterStop = new ManualResetEvent(false);
87
88 public JobEngine(string name, string loggingName)
89 {
90 Name = name;
91 LoggingName = loggingName;
92
93 RequestProcessTimeoutOnStop = 5000;
94 }
95
96 public void Start()
97 {
98 lock (this)
99 {
100 if (IsRunning)
101 return;
102
103 IsRunning = true;
104
105 m_finishedProcessingAfterStop.Reset();
106
107 m_jobQueue = new BlockingCollection<Job>(new ConcurrentQueue<Job>(), 5000);
108 m_cancelSource = new CancellationTokenSource();
109
110 WorkManager.StartThread(
111 ProcessRequests,
112 Name,
113 ThreadPriority.Normal,
114 false,
115 true,
116 null,
117 int.MaxValue);
118 }
119 }
120
121 public void Stop()
122 {
123 lock (this)
124 {
125 try
126 {
127 if (!IsRunning)
128 return;
129
130 IsRunning = false;
131
132 int requestsLeft = m_jobQueue.Count;
133
134 if (requestsLeft <= 0)
135 {
136 m_cancelSource.Cancel();
137 }
138 else
139 {
140 m_log.InfoFormat("[{0}]: Waiting to write {1} events after stop.", LoggingName, requestsLeft);
141
142 while (requestsLeft > 0)
143 {
144 if (!m_finishedProcessingAfterStop.WaitOne(RequestProcessTimeoutOnStop))
145 {
146 // After timeout no events have been written
147 if (requestsLeft == m_jobQueue.Count)
148 {
149 m_log.WarnFormat(
150 "[{0}]: No requests processed after {1} ms wait. Discarding remaining {2} requests",
151 LoggingName, RequestProcessTimeoutOnStop, requestsLeft);
152
153 break;
154 }
155 }
156
157 requestsLeft = m_jobQueue.Count;
158 }
159 }
160 }
161 finally
162 {
163 m_cancelSource.Dispose();
164 }
165 }
166 }
167
168 /// <summary>
169 /// Make a job.
170 /// </summary>
171 /// <remarks>
172 /// We provide this method to replace the constructor so that we can later pool job objects if necessary to
173 /// reduce memory churn. Normally one would directly call QueueJob() with parameters anyway.
174 /// </remarks>
175 /// <returns></returns>
176 /// <param name="name">Name.</param>
177 /// <param name="action">Action.</param>
178 /// <param name="commonId">Common identifier.</param>
179 public static Job MakeJob(string name, Action action, string commonId = null)
180 {
181 return Job.MakeJob(name, action, commonId);
182 }
183
184 /// <summary>
185 /// Remove the next job queued for processing.
186 /// </summary>
187 /// <remarks>
188 /// Returns null if there is no next job.
189 /// Will not remove a job currently being performed.
190 /// </remarks>
191 public Job RemoveNextJob()
192 {
193 Job nextJob;
194 m_jobQueue.TryTake(out nextJob);
195
196 return nextJob;
197 }
198
199 /// <summary>
200 /// Queue the job for processing.
201 /// </summary>
202 /// <returns><c>true</c>, if job was queued, <c>false</c> otherwise.</returns>
203 /// <param name="name">Name of job. This appears on the console and in logging.</param>
204 /// <param name="action">Action to perform.</param>
205 /// <param name="commonId">
206 /// Common identifier for a set of jobs. This is allows a set of jobs to be removed
207 /// if required (e.g. all jobs for a given agent. Optional.
208 /// </param>
209 public bool QueueJob(string name, Action action, string commonId = null)
210 {
211 return QueueJob(MakeJob(name, action, commonId));
212 }
213
214 /// <summary>
215 /// Queue the job for processing.
216 /// </summary>
217 /// <returns><c>true</c>, if job was queued, <c>false</c> otherwise.</returns>
218 /// <param name="job">The job</param>
219 /// </param>
220 public bool QueueJob(Job job)
221 {
222 if (m_jobQueue.Count < m_jobQueue.BoundedCapacity)
223 {
224 m_jobQueue.Add(job);
225
226 if (!m_warnOverMaxQueue)
227 m_warnOverMaxQueue = true;
228
229 return true;
230 }
231 else
232 {
233 if (m_warnOverMaxQueue)
234 {
235 m_log.WarnFormat(
236 "[{0}]: Job queue at maximum capacity, not recording job from {1} in {2}",
237 LoggingName, job.Name, Name);
238
239 m_warnOverMaxQueue = false;
240 }
241
242 return false;
243 }
244 }
245
246 private void ProcessRequests()
247 {
248 try
249 {
250 while (IsRunning || m_jobQueue.Count > 0)
251 {
252 try
253 {
254 CurrentJob = m_jobQueue.Take(m_cancelSource.Token);
255 }
256 catch (ObjectDisposedException e)
257 {
258 // If we see this whilst not running then it may be due to a race where this thread checks
259 // IsRunning after the stopping thread sets it to false and disposes of the cancellation source.
260 if (IsRunning)
261 throw e;
262 else
263 break;
264 }
265
266 if (LogLevel >= 1)
267 m_log.DebugFormat("[{0}]: Processing job {1}", LoggingName, CurrentJob.Name);
268
269 try
270 {
271 CurrentJob.Action();
272 }
273 catch (Exception e)
274 {
275 m_log.Error(
276 string.Format(
277 "[{0}]: Job {1} failed, continuing. Exception ", LoggingName, CurrentJob.Name), e);
278 }
279
280 if (LogLevel >= 1)
281 m_log.DebugFormat("[{0}]: Processed job {1}", LoggingName, CurrentJob.Name);
282
283 CurrentJob = null;
284 }
285 }
286 catch (OperationCanceledException)
287 {
288 }
289
290 m_finishedProcessingAfterStop.Set();
291 }
292
293 public class Job
294 {
295 /// <summary>
296 /// Name of the job.
297 /// </summary>
298 /// <remarks>
299 /// This appears on console and debug output.
300 /// </remarks>
301 public string Name { get; private set; }
302
303 /// <summary>
304 /// Common ID for this job.
305 /// </summary>
306 /// <remarks>
307 /// This allows all jobs with a certain common ID (e.g. a client UUID) to be removed en-masse if required.
308 /// Can be null if this is not required.
309 /// </remarks>
310 public string CommonId { get; private set; }
311
312 /// <summary>
313 /// Action to perform when this job is processed.
314 /// </summary>
315 public Action Action { get; private set; }
316
317 private Job(string name, string commonId, Action action)
318 {
319 Name = name;
320 CommonId = commonId;
321 Action = action;
322 }
323
324 /// <summary>
325 /// Make a job. It needs to be separately queued.
326 /// </summary>
327 /// <remarks>
328 /// We provide this method to replace the constructor so that we can pool job objects if necessary to
329 /// to reduce memory churn. Normally one would directly call JobEngine.QueueJob() with parameters anyway.
330 /// </remarks>
331 /// <returns></returns>
332 /// <param name="name">Name.</param>
333 /// <param name="action">Action.</param>
334 /// <param name="commonId">Common identifier.</param>
335 public static Job MakeJob(string name, Action action, string commonId = null)
336 {
337 return new Job(name, commonId, action);
338 }
339 }
340 }
341} \ No newline at end of file
diff --git a/OpenSim/Framework/Monitoring/MemoryWatchdog.cs b/OpenSim/Framework/Monitoring/MemoryWatchdog.cs
index c6010cd..c474622 100644
--- a/OpenSim/Framework/Monitoring/MemoryWatchdog.cs
+++ b/OpenSim/Framework/Monitoring/MemoryWatchdog.cs
@@ -60,17 +60,17 @@ namespace OpenSim.Framework.Monitoring
60 private static bool m_enabled; 60 private static bool m_enabled;
61 61
62 /// <summary> 62 /// <summary>
63 /// Last memory churn in bytes per millisecond. 63 /// Average heap allocation rate in bytes per millisecond.
64 /// </summary> 64 /// </summary>
65 public static double AverageMemoryChurn 65 public static double AverageHeapAllocationRate
66 { 66 {
67 get { if (m_samples.Count > 0) return m_samples.Average(); else return 0; } 67 get { if (m_samples.Count > 0) return m_samples.Average(); else return 0; }
68 } 68 }
69 69
70 /// <summary> 70 /// <summary>
71 /// Average memory churn in bytes per millisecond. 71 /// Last heap allocation in bytes
72 /// </summary> 72 /// </summary>
73 public static double LastMemoryChurn 73 public static double LastHeapAllocationRate
74 { 74 {
75 get { if (m_samples.Count > 0) return m_samples.Last(); else return 0; } 75 get { if (m_samples.Count > 0) return m_samples.Last(); else return 0; }
76 } 76 }
diff --git a/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs
index 1f2bb40..a617b93 100644
--- a/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs
+++ b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs
@@ -29,5 +29,5 @@ using System.Runtime.InteropServices;
29// Build Number 29// Build Number
30// Revision 30// Revision
31// 31//
32[assembly: AssemblyVersion("0.7.5.*")] 32[assembly: AssemblyVersion("0.8.3.*")]
33[assembly: AssemblyFileVersion("1.0.0.0")] 33
diff --git a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs
new file mode 100644
index 0000000..77315bb
--- /dev/null
+++ b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs
@@ -0,0 +1,346 @@
1/*
2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the OpenSimulator Project nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28using System;
29using System.Collections.Generic;
30using System.Diagnostics;
31using System.Linq;
32using System.Net.NetworkInformation;
33using System.Text;
34using System.Threading;
35using log4net;
36using Nini.Config;
37using OpenMetaverse.StructuredData;
38using OpenSim.Framework;
39
40namespace OpenSim.Framework.Monitoring
41{
42 public class ServerStatsCollector
43 {
44 private readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
45 private readonly string LogHeader = "[SERVER STATS]";
46
47 public bool Enabled = false;
48 private static Dictionary<string, Stat> RegisteredStats = new Dictionary<string, Stat>();
49
50 public readonly string CategoryServer = "server";
51
52 public readonly string ContainerThreadpool = "threadpool";
53 public readonly string ContainerProcessor = "processor";
54 public readonly string ContainerMemory = "memory";
55 public readonly string ContainerNetwork = "network";
56 public readonly string ContainerProcess = "process";
57
58 public string NetworkInterfaceTypes = "Ethernet";
59
60 readonly int performanceCounterSampleInterval = 500;
61// int lastperformanceCounterSampleTime = 0;
62
63 private class PerfCounterControl
64 {
65 public PerformanceCounter perfCounter;
66 public int lastFetch;
67 public string name;
68 public PerfCounterControl(PerformanceCounter pPc)
69 : this(pPc, String.Empty)
70 {
71 }
72 public PerfCounterControl(PerformanceCounter pPc, string pName)
73 {
74 perfCounter = pPc;
75 lastFetch = 0;
76 name = pName;
77 }
78 }
79
80 PerfCounterControl processorPercentPerfCounter = null;
81
82 // IRegionModuleBase.Initialize
83 public void Initialise(IConfigSource source)
84 {
85 if (source == null)
86 return;
87
88 IConfig cfg = source.Configs["Monitoring"];
89
90 if (cfg != null)
91 Enabled = cfg.GetBoolean("ServerStatsEnabled", true);
92
93 if (Enabled)
94 {
95 NetworkInterfaceTypes = cfg.GetString("NetworkInterfaceTypes", "Ethernet");
96 }
97 }
98
99 public void Start()
100 {
101 if (RegisteredStats.Count == 0)
102 RegisterServerStats();
103 }
104
105 public void Close()
106 {
107 if (RegisteredStats.Count > 0)
108 {
109 foreach (Stat stat in RegisteredStats.Values)
110 {
111 StatsManager.DeregisterStat(stat);
112 stat.Dispose();
113 }
114 RegisteredStats.Clear();
115 }
116 }
117
118 private void MakeStat(string pName, string pDesc, string pUnit, string pContainer, Action<Stat> act)
119 {
120 MakeStat(pName, pDesc, pUnit, pContainer, act, MeasuresOfInterest.None);
121 }
122
123 private void MakeStat(string pName, string pDesc, string pUnit, string pContainer, Action<Stat> act, MeasuresOfInterest moi)
124 {
125 string desc = pDesc;
126 if (desc == null)
127 desc = pName;
128 Stat stat = new Stat(pName, pName, desc, pUnit, CategoryServer, pContainer, StatType.Pull, moi, act, StatVerbosity.Debug);
129 StatsManager.RegisterStat(stat);
130 RegisteredStats.Add(pName, stat);
131 }
132
133 public void RegisterServerStats()
134 {
135// lastperformanceCounterSampleTime = Util.EnvironmentTickCount();
136 PerformanceCounter tempPC;
137 Stat tempStat;
138 string tempName;
139
140 try
141 {
142 tempName = "CPUPercent";
143 tempPC = new PerformanceCounter("Processor", "% Processor Time", "_Total");
144 processorPercentPerfCounter = new PerfCounterControl(tempPC);
145 // A long time bug in mono is that CPU percent is reported as CPU percent idle. Windows reports CPU percent busy.
146 tempStat = new Stat(tempName, tempName, "", "percent", CategoryServer, ContainerProcessor,
147 StatType.Pull, (s) => { GetNextValue(s, processorPercentPerfCounter); },
148 StatVerbosity.Info);
149 StatsManager.RegisterStat(tempStat);
150 RegisteredStats.Add(tempName, tempStat);
151
152 MakeStat("TotalProcessorTime", null, "sec", ContainerProcessor,
153 (s) => { s.Value = Math.Round(Process.GetCurrentProcess().TotalProcessorTime.TotalSeconds, 3); });
154
155 MakeStat("UserProcessorTime", null, "sec", ContainerProcessor,
156 (s) => { s.Value = Math.Round(Process.GetCurrentProcess().UserProcessorTime.TotalSeconds, 3); });
157
158 MakeStat("PrivilegedProcessorTime", null, "sec", ContainerProcessor,
159 (s) => { s.Value = Math.Round(Process.GetCurrentProcess().PrivilegedProcessorTime.TotalSeconds, 3); });
160
161 MakeStat("Threads", null, "threads", ContainerProcessor,
162 (s) => { s.Value = Process.GetCurrentProcess().Threads.Count; });
163 }
164 catch (Exception e)
165 {
166 m_log.ErrorFormat("{0} Exception creating 'Process': {1}", LogHeader, e);
167 }
168
169 MakeStat("BuiltinThreadpoolWorkerThreadsAvailable", null, "threads", ContainerThreadpool,
170 s =>
171 {
172 int workerThreads, iocpThreads;
173 ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads);
174 s.Value = workerThreads;
175 });
176
177 MakeStat("BuiltinThreadpoolIOCPThreadsAvailable", null, "threads", ContainerThreadpool,
178 s =>
179 {
180 int workerThreads, iocpThreads;
181 ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads);
182 s.Value = iocpThreads;
183 });
184
185 if (Util.FireAndForgetMethod == FireAndForgetMethod.SmartThreadPool && Util.GetSmartThreadPoolInfo() != null)
186 {
187 MakeStat("STPMaxThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().MaxThreads);
188 MakeStat("STPMinThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().MinThreads);
189 MakeStat("STPConcurrency", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().MaxConcurrentWorkItems);
190 MakeStat("STPActiveThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().ActiveThreads);
191 MakeStat("STPInUseThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().InUseThreads);
192 MakeStat("STPWorkItemsWaiting", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().WaitingCallbacks);
193 }
194
195 MakeStat(
196 "HTTPRequestsMade",
197 "Number of outbound HTTP requests made",
198 "requests",
199 ContainerNetwork,
200 s => s.Value = WebUtil.RequestNumber,
201 MeasuresOfInterest.AverageChangeOverTime);
202
203 try
204 {
205 List<string> okInterfaceTypes = new List<string>(NetworkInterfaceTypes.Split(','));
206
207 IEnumerable<NetworkInterface> nics = NetworkInterface.GetAllNetworkInterfaces();
208 foreach (NetworkInterface nic in nics)
209 {
210 if (nic.OperationalStatus != OperationalStatus.Up)
211 continue;
212
213 string nicInterfaceType = nic.NetworkInterfaceType.ToString();
214 if (!okInterfaceTypes.Contains(nicInterfaceType))
215 {
216 m_log.DebugFormat("{0} Not including stats for network interface '{1}' of type '{2}'.",
217 LogHeader, nic.Name, nicInterfaceType);
218 m_log.DebugFormat("{0} To include, add to comma separated list in [Monitoring]NetworkInterfaceTypes={1}",
219 LogHeader, NetworkInterfaceTypes);
220 continue;
221 }
222
223 if (nic.Supports(NetworkInterfaceComponent.IPv4))
224 {
225 IPv4InterfaceStatistics nicStats = nic.GetIPv4Statistics();
226 if (nicStats != null)
227 {
228 MakeStat("BytesRcvd/" + nic.Name, nic.Name, "KB", ContainerNetwork,
229 (s) => { LookupNic(s, (ns) => { return ns.BytesReceived; }, 1024.0); });
230 MakeStat("BytesSent/" + nic.Name, nic.Name, "KB", ContainerNetwork,
231 (s) => { LookupNic(s, (ns) => { return ns.BytesSent; }, 1024.0); });
232 MakeStat("TotalBytes/" + nic.Name, nic.Name, "KB", ContainerNetwork,
233 (s) => { LookupNic(s, (ns) => { return ns.BytesSent + ns.BytesReceived; }, 1024.0); });
234 }
235 }
236 // TODO: add IPv6 (it may actually happen someday)
237 }
238 }
239 catch (Exception e)
240 {
241 m_log.ErrorFormat("{0} Exception creating 'Network Interface': {1}", LogHeader, e);
242 }
243
244 MakeStat("ProcessMemory", null, "MB", ContainerMemory,
245 (s) => { s.Value = Math.Round(Process.GetCurrentProcess().WorkingSet64 / 1024d / 1024d, 3); });
246 MakeStat("HeapMemory", null, "MB", ContainerMemory,
247 (s) => { s.Value = Math.Round(GC.GetTotalMemory(false) / 1024d / 1024d, 3); });
248 MakeStat("LastHeapAllocationRate", null, "MB/sec", ContainerMemory,
249 (s) => { s.Value = Math.Round(MemoryWatchdog.LastHeapAllocationRate * 1000d / 1024d / 1024d, 3); });
250 MakeStat("AverageHeapAllocationRate", null, "MB/sec", ContainerMemory,
251 (s) => { s.Value = Math.Round(MemoryWatchdog.AverageHeapAllocationRate * 1000d / 1024d / 1024d, 3); });
252 }
253
254 // Notes on performance counters:
255 // "How To Read Performance Counters": http://blogs.msdn.com/b/bclteam/archive/2006/06/02/618156.aspx
256 // "How to get the CPU Usage in C#": http://stackoverflow.com/questions/278071/how-to-get-the-cpu-usage-in-c
257 // "Mono Performance Counters": http://www.mono-project.com/Mono_Performance_Counters
258 private delegate double PerfCounterNextValue();
259
260 private void GetNextValue(Stat stat, PerfCounterControl perfControl)
261 {
262 if (Util.EnvironmentTickCountSubtract(perfControl.lastFetch) > performanceCounterSampleInterval)
263 {
264 if (perfControl != null && perfControl.perfCounter != null)
265 {
266 try
267 {
268 stat.Value = Math.Round(perfControl.perfCounter.NextValue(), 3);
269 }
270 catch (Exception e)
271 {
272 m_log.ErrorFormat("{0} Exception on NextValue fetching {1}: {2}", LogHeader, stat.Name, e);
273 }
274
275 perfControl.lastFetch = Util.EnvironmentTickCount();
276 }
277 }
278 }
279
280 // Lookup the nic that goes with this stat and set the value by using a fetch action.
281 // Not sure about closure with delegates inside delegates.
282 private delegate double GetIPv4StatValue(IPv4InterfaceStatistics interfaceStat);
283 private void LookupNic(Stat stat, GetIPv4StatValue getter, double factor)
284 {
285 // Get the one nic that has the name of this stat
286 IEnumerable<NetworkInterface> nics = NetworkInterface.GetAllNetworkInterfaces().Where(
287 (network) => network.Name == stat.Description);
288 try
289 {
290 foreach (NetworkInterface nic in nics)
291 {
292 IPv4InterfaceStatistics intrStats = nic.GetIPv4Statistics();
293 if (intrStats != null)
294 {
295 double newVal = Math.Round(getter(intrStats) / factor, 3);
296 stat.Value = newVal;
297 }
298 break;
299 }
300 }
301 catch
302 {
303 // There are times interfaces go away so we just won't update the stat for this
304 m_log.ErrorFormat("{0} Exception fetching stat on interface '{1}'", LogHeader, stat.Description);
305 }
306 }
307 }
308
309 public class ServerStatsAggregator : Stat
310 {
311 public ServerStatsAggregator(
312 string shortName,
313 string name,
314 string description,
315 string unitName,
316 string category,
317 string container
318 )
319 : base(
320 shortName,
321 name,
322 description,
323 unitName,
324 category,
325 container,
326 StatType.Push,
327 MeasuresOfInterest.None,
328 null,
329 StatVerbosity.Info)
330 {
331 }
332 public override string ToConsoleString()
333 {
334 StringBuilder sb = new StringBuilder();
335
336 return sb.ToString();
337 }
338
339 public override OSDMap ToOSDMap()
340 {
341 OSDMap ret = new OSDMap();
342
343 return ret;
344 }
345 }
346}
diff --git a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs
index aa86202..e4df7ee 100644..100755
--- a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs
+++ b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs
@@ -27,6 +27,8 @@
27 27
28using System; 28using System;
29using System.Collections.Generic; 29using System.Collections.Generic;
30using System.Diagnostics;
31using System.Linq;
30using System.Text; 32using System.Text;
31using OpenMetaverse; 33using OpenMetaverse;
32using OpenMetaverse.StructuredData; 34using OpenMetaverse.StructuredData;
@@ -39,8 +41,6 @@ namespace OpenSim.Framework.Monitoring
39 /// </summary> 41 /// </summary>
40 public class SimExtraStatsCollector : BaseStatsCollector 42 public class SimExtraStatsCollector : BaseStatsCollector
41 { 43 {
42 private long abnormalClientThreadTerminations;
43
44// private long assetsInCache; 44// private long assetsInCache;
45// private long texturesInCache; 45// private long texturesInCache;
46// private long assetCacheMemoryUsage; 46// private long assetCacheMemoryUsage;
@@ -72,11 +72,11 @@ namespace OpenSim.Framework.Monitoring
72 private volatile float pendingUploads; 72 private volatile float pendingUploads;
73 private volatile float activeScripts; 73 private volatile float activeScripts;
74 private volatile float scriptLinesPerSecond; 74 private volatile float scriptLinesPerSecond;
75 75 private volatile float m_frameDilation;
76 /// <summary> 76 private volatile float m_usersLoggingIn;
77 /// Number of times that a client thread terminated because of an exception 77 private volatile float m_totalGeoPrims;
78 /// </summary> 78 private volatile float m_totalMeshes;
79 public long AbnormalClientThreadTerminations { get { return abnormalClientThreadTerminations; } } 79 private volatile float m_inUseThreads;
80 80
81// /// <summary> 81// /// <summary>
82// /// These statistics are being collected by push rather than pull. Pull would be simpler, but I had the 82// /// These statistics are being collected by push rather than pull. Pull would be simpler, but I had the
@@ -166,11 +166,6 @@ namespace OpenSim.Framework.Monitoring
166 private IDictionary<UUID, PacketQueueStatsCollector> packetQueueStatsCollectors 166 private IDictionary<UUID, PacketQueueStatsCollector> packetQueueStatsCollectors
167 = new Dictionary<UUID, PacketQueueStatsCollector>(); 167 = new Dictionary<UUID, PacketQueueStatsCollector>();
168 168
169 public void AddAbnormalClientThreadTermination()
170 {
171 abnormalClientThreadTerminations++;
172 }
173
174// public void AddAsset(AssetBase asset) 169// public void AddAsset(AssetBase asset)
175// { 170// {
176// assetsInCache++; 171// assetsInCache++;
@@ -260,6 +255,10 @@ namespace OpenSim.Framework.Monitoring
260 { 255 {
261 // FIXME: SimStats shouldn't allow an arbitrary stat packing order (which is inherited from the original 256 // FIXME: SimStats shouldn't allow an arbitrary stat packing order (which is inherited from the original
262 // SimStatsPacket that was being used). 257 // SimStatsPacket that was being used).
258
259 // For an unknown reason the original designers decided not to
260 // include the spare MS statistic inside of this class, this is
261 // located inside the StatsBlock at location 21, thus it is skipped
263 timeDilation = stats.StatsBlock[0].StatValue; 262 timeDilation = stats.StatsBlock[0].StatValue;
264 simFps = stats.StatsBlock[1].StatValue; 263 simFps = stats.StatsBlock[1].StatValue;
265 physicsFps = stats.StatsBlock[2].StatValue; 264 physicsFps = stats.StatsBlock[2].StatValue;
@@ -281,6 +280,11 @@ namespace OpenSim.Framework.Monitoring
281 pendingUploads = stats.StatsBlock[18].StatValue; 280 pendingUploads = stats.StatsBlock[18].StatValue;
282 activeScripts = stats.StatsBlock[19].StatValue; 281 activeScripts = stats.StatsBlock[19].StatValue;
283 scriptLinesPerSecond = stats.StatsBlock[20].StatValue; 282 scriptLinesPerSecond = stats.StatsBlock[20].StatValue;
283 m_frameDilation = stats.StatsBlock[22].StatValue;
284 m_usersLoggingIn = stats.StatsBlock[23].StatValue;
285 m_totalGeoPrims = stats.StatsBlock[24].StatValue;
286 m_totalMeshes = stats.StatsBlock[25].StatValue;
287 m_inUseThreads = stats.StatsBlock[26].StatValue;
284 } 288 }
285 289
286 /// <summary> 290 /// <summary>
@@ -324,10 +328,12 @@ Asset service request failures: {3}" + Environment.NewLine,
324 sb.Append(Environment.NewLine); 328 sb.Append(Environment.NewLine);
325 sb.Append("CONNECTION STATISTICS"); 329 sb.Append("CONNECTION STATISTICS");
326 sb.Append(Environment.NewLine); 330 sb.Append(Environment.NewLine);
327 sb.Append( 331
328 string.Format( 332 List<Stat> stats = StatsManager.GetStatsFromEachContainer("clientstack", "ClientLogoutsDueToNoReceives");
329 "Abnormal client thread terminations: {0}" + Environment.NewLine, 333
330 abnormalClientThreadTerminations)); 334 sb.AppendFormat(
335 "Client logouts due to no data receive timeout: {0}\n\n",
336 stats != null ? stats.Sum(s => s.Value).ToString() : "unknown");
331 337
332// sb.Append(Environment.NewLine); 338// sb.Append(Environment.NewLine);
333// sb.Append("INVENTORY STATISTICS"); 339// sb.Append("INVENTORY STATISTICS");
@@ -338,7 +344,7 @@ Asset service request failures: {3}" + Environment.NewLine,
338// InventoryServiceRetrievalFailures)); 344// InventoryServiceRetrievalFailures));
339 345
340 sb.Append(Environment.NewLine); 346 sb.Append(Environment.NewLine);
341 sb.Append("FRAME STATISTICS"); 347 sb.Append("SAMPLE FRAME STATISTICS");
342 sb.Append(Environment.NewLine); 348 sb.Append(Environment.NewLine);
343 sb.Append("Dilatn SimFPS PhyFPS AgntUp RootAg ChldAg Prims AtvPrm AtvScr ScrLPS"); 349 sb.Append("Dilatn SimFPS PhyFPS AgntUp RootAg ChldAg Prims AtvPrm AtvScr ScrLPS");
344 sb.Append(Environment.NewLine); 350 sb.Append(Environment.NewLine);
@@ -359,11 +365,12 @@ Asset service request failures: {3}" + Environment.NewLine,
359 inPacketsPerSecond, outPacketsPerSecond, pendingDownloads, pendingUploads, unackedBytes, totalFrameTime, 365 inPacketsPerSecond, outPacketsPerSecond, pendingDownloads, pendingUploads, unackedBytes, totalFrameTime,
360 netFrameTime, physicsFrameTime, otherFrameTime, agentFrameTime, imageFrameTime)); 366 netFrameTime, physicsFrameTime, otherFrameTime, agentFrameTime, imageFrameTime));
361 367
362 Dictionary<string, Dictionary<string, Stat>> sceneStats; 368 /* 20130319 RA: For the moment, disable the dump of 'scene' catagory as they are mostly output by
363 369 * the two formatted printouts above.
370 SortedDictionary<string, SortedDictionary<string, Stat>> sceneStats;
364 if (StatsManager.TryGetStats("scene", out sceneStats)) 371 if (StatsManager.TryGetStats("scene", out sceneStats))
365 { 372 {
366 foreach (KeyValuePair<string, Dictionary<string, Stat>> kvp in sceneStats) 373 foreach (KeyValuePair<string, SortedDictionary<string, Stat>> kvp in sceneStats)
367 { 374 {
368 foreach (Stat stat in kvp.Value.Values) 375 foreach (Stat stat in kvp.Value.Values)
369 { 376 {
@@ -374,6 +381,7 @@ Asset service request failures: {3}" + Environment.NewLine,
374 } 381 }
375 } 382 }
376 } 383 }
384 */
377 385
378 /* 386 /*
379 sb.Append(Environment.NewLine); 387 sb.Append(Environment.NewLine);
@@ -405,6 +413,36 @@ Asset service request failures: {3}" + Environment.NewLine,
405 /// <returns></returns> 413 /// <returns></returns>
406 public override string XReport(string uptime, string version) 414 public override string XReport(string uptime, string version)
407 { 415 {
416 return OSDParser.SerializeJsonString(OReport(uptime, version));
417 }
418
419 /// <summary>
420 /// Report back collected statistical information as an OSDMap
421 /// </summary>
422 /// <returns></returns>
423 public override OSDMap OReport(string uptime, string version)
424 {
425 // Get the amount of physical memory, allocated with the instance of this program, in kilobytes;
426 // the working set is the set of memory pages currently visible to this program in physical RAM
427 // memory and includes both shared (e.g. system libraries) and private data
428 double memUsage = Process.GetCurrentProcess().WorkingSet64 / 1024.0;
429
430 // Get the number of threads from the system that are currently
431 // running
432 int numberThreadsRunning = 0;
433 foreach (ProcessThread currentThread in
434 Process.GetCurrentProcess().Threads)
435 {
436 // A known issue with the current process .Threads property is
437 // that it can return null threads, thus don't count those as
438 // running threads and prevent the program function from failing
439 if (currentThread != null &&
440 currentThread.ThreadState == ThreadState.Running)
441 {
442 numberThreadsRunning++;
443 }
444 }
445
408 OSDMap args = new OSDMap(30); 446 OSDMap args = new OSDMap(30);
409// args["AssetsInCache"] = OSD.FromString (String.Format ("{0:0.##}", AssetsInCache)); 447// args["AssetsInCache"] = OSD.FromString (String.Format ("{0:0.##}", AssetsInCache));
410// args["TimeAfterCacheMiss"] = OSD.FromString (String.Format ("{0:0.##}", 448// args["TimeAfterCacheMiss"] = OSD.FromString (String.Format ("{0:0.##}",
@@ -441,14 +479,28 @@ Asset service request failures: {3}" + Environment.NewLine,
441 args["Memory"] = OSD.FromString (base.XReport (uptime, version)); 479 args["Memory"] = OSD.FromString (base.XReport (uptime, version));
442 args["Uptime"] = OSD.FromString (uptime); 480 args["Uptime"] = OSD.FromString (uptime);
443 args["Version"] = OSD.FromString (version); 481 args["Version"] = OSD.FromString (version);
444
445 string strBuffer = "";
446 strBuffer = OSDParser.SerializeJsonString(args);
447 482
448 return strBuffer; 483 args["FrameDilatn"] = OSD.FromString(String.Format("{0:0.##}", m_frameDilation));
484 args["Logging in Users"] = OSD.FromString(String.Format("{0:0.##}",
485 m_usersLoggingIn));
486 args["GeoPrims"] = OSD.FromString(String.Format("{0:0.##}",
487 m_totalGeoPrims));
488 args["Mesh Objects"] = OSD.FromString(String.Format("{0:0.##}",
489 m_totalMeshes));
490 args["XEngine Thread Count"] = OSD.FromString(String.Format("{0:0.##}",
491 m_inUseThreads));
492 args["Util Thread Count"] = OSD.FromString(String.Format("{0:0.##}",
493 Util.GetSmartThreadPoolInfo().InUseThreads));
494 args["System Thread Count"] = OSD.FromString(String.Format(
495 "{0:0.##}", numberThreadsRunning));
496 args["ProcMem"] = OSD.FromString(String.Format("{0:#,###,###.##}",
497 memUsage));
498
499 return args;
449 } 500 }
450 } 501 }
451 502
503
452 /// <summary> 504 /// <summary>
453 /// Pull packet queue stats from packet queues and report 505 /// Pull packet queue stats from packet queues and report
454 /// </summary> 506 /// </summary>
@@ -474,5 +526,11 @@ Asset service request failures: {3}" + Environment.NewLine,
474 { 526 {
475 return ""; 527 return "";
476 } 528 }
529
530 public OSDMap OReport(string uptime, string version)
531 {
532 OSDMap ret = new OSDMap();
533 return ret;
534 }
477 } 535 }
478} 536}
diff --git a/OpenSim/Framework/Monitoring/Stats/CounterStat.cs b/OpenSim/Framework/Monitoring/Stats/CounterStat.cs
new file mode 100755
index 0000000..318cf1c
--- /dev/null
+++ b/OpenSim/Framework/Monitoring/Stats/CounterStat.cs
@@ -0,0 +1,119 @@
1/*
2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the OpenSimulator Project nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28using System;
29using System.Collections.Generic;
30using System.Linq;
31using System.Text;
32
33using OpenMetaverse.StructuredData;
34
35namespace OpenSim.Framework.Monitoring
36{
37// A statistic that wraps a counter.
38// Built this way mostly so histograms and history can be created.
39public class CounterStat : Stat
40{
41 private SortedDictionary<string, EventHistogram> m_histograms;
42 private object counterLock = new object();
43
44 public CounterStat(
45 string shortName,
46 string name,
47 string description,
48 string unitName,
49 string category,
50 string container,
51 StatVerbosity verbosity)
52 : base(shortName, name, description, unitName, category, container, StatType.Push, null, verbosity)
53 {
54 m_histograms = new SortedDictionary<string, EventHistogram>();
55 }
56
57 // Histograms are presumably added at intialization time and the list does not change thereafter.
58 // Thus no locking of the histogram list.
59 public void AddHistogram(string histoName, EventHistogram histo)
60 {
61 m_histograms.Add(histoName, histo);
62 }
63
64 public delegate void ProcessHistogram(string name, EventHistogram histo);
65 public void ForEachHistogram(ProcessHistogram process)
66 {
67 foreach (KeyValuePair<string, EventHistogram> kvp in m_histograms)
68 {
69 process(kvp.Key, kvp.Value);
70 }
71 }
72
73 public void Event()
74 {
75 this.Event(1);
76 }
77
78 // Count the underlying counter.
79 public void Event(int cnt)
80 {
81 lock (counterLock)
82 {
83 base.Value += cnt;
84
85 foreach (EventHistogram histo in m_histograms.Values)
86 {
87 histo.Event(cnt);
88 }
89 }
90 }
91
92 // CounterStat is a basic stat plus histograms
93 public override OSDMap ToOSDMap()
94 {
95 // Get the foundational instance
96 OSDMap map = base.ToOSDMap();
97
98 map["StatType"] = "CounterStat";
99
100 // If there are any histograms, add a new field that is an array of histograms as OSDMaps
101 if (m_histograms.Count > 0)
102 {
103 lock (counterLock)
104 {
105 if (m_histograms.Count > 0)
106 {
107 OSDArray histos = new OSDArray();
108 foreach (EventHistogram histo in m_histograms.Values)
109 {
110 histos.Add(histo.GetHistogramAsOSDMap());
111 }
112 map.Add("Histograms", histos);
113 }
114 }
115 }
116 return map;
117 }
118}
119}
diff --git a/OpenSim/Framework/Monitoring/Stats/EventHistogram.cs b/OpenSim/Framework/Monitoring/Stats/EventHistogram.cs
new file mode 100755
index 0000000..f51f322
--- /dev/null
+++ b/OpenSim/Framework/Monitoring/Stats/EventHistogram.cs
@@ -0,0 +1,173 @@
1/*
2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the OpenSimulator Project nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28using System;
29using System.Collections.Generic;
30using System.Linq;
31using System.Text;
32
33using OpenMetaverse.StructuredData;
34
35namespace OpenSim.Framework.Monitoring
36{
37// Create a time histogram of events. The histogram is built in a wrap-around
38// array of equally distributed buckets.
39// For instance, a minute long histogram of second sized buckets would be:
40// new EventHistogram(60, 1000)
41public class EventHistogram
42{
43 private int m_timeBase;
44 private int m_numBuckets;
45 private int m_bucketMilliseconds;
46 private int m_lastBucket;
47 private int m_totalHistogramMilliseconds;
48 private long[] m_histogram;
49 private object histoLock = new object();
50
51 public EventHistogram(int numberOfBuckets, int millisecondsPerBucket)
52 {
53 m_numBuckets = numberOfBuckets;
54 m_bucketMilliseconds = millisecondsPerBucket;
55 m_totalHistogramMilliseconds = m_numBuckets * m_bucketMilliseconds;
56
57 m_histogram = new long[m_numBuckets];
58 Zero();
59 m_lastBucket = 0;
60 m_timeBase = Util.EnvironmentTickCount();
61 }
62
63 public void Event()
64 {
65 this.Event(1);
66 }
67
68 // Record an event at time 'now' in the histogram.
69 public void Event(int cnt)
70 {
71 lock (histoLock)
72 {
73 // The time as displaced from the base of the histogram
74 int bucketTime = Util.EnvironmentTickCountSubtract(m_timeBase);
75
76 // If more than the total time of the histogram, we just start over
77 if (bucketTime > m_totalHistogramMilliseconds)
78 {
79 Zero();
80 m_lastBucket = 0;
81 m_timeBase = Util.EnvironmentTickCount();
82 }
83 else
84 {
85 // To which bucket should we add this event?
86 int bucket = bucketTime / m_bucketMilliseconds;
87
88 // Advance m_lastBucket to the new bucket. Zero any buckets skipped over.
89 while (bucket != m_lastBucket)
90 {
91 // Zero from just after the last bucket to the new bucket or the end
92 for (int jj = m_lastBucket + 1; jj <= Math.Min(bucket, m_numBuckets - 1); jj++)
93 {
94 m_histogram[jj] = 0;
95 }
96 m_lastBucket = bucket;
97 // If the new bucket is off the end, wrap around to the beginning
98 if (bucket > m_numBuckets)
99 {
100 bucket -= m_numBuckets;
101 m_lastBucket = 0;
102 m_histogram[m_lastBucket] = 0;
103 m_timeBase += m_totalHistogramMilliseconds;
104 }
105 }
106 }
107 m_histogram[m_lastBucket] += cnt;
108 }
109 }
110
111 // Get a copy of the current histogram
112 public long[] GetHistogram()
113 {
114 long[] ret = new long[m_numBuckets];
115 lock (histoLock)
116 {
117 int indx = m_lastBucket + 1;
118 for (int ii = 0; ii < m_numBuckets; ii++, indx++)
119 {
120 if (indx >= m_numBuckets)
121 indx = 0;
122 ret[ii] = m_histogram[indx];
123 }
124 }
125 return ret;
126 }
127
128 public OSDMap GetHistogramAsOSDMap()
129 {
130 OSDMap ret = new OSDMap();
131
132 ret.Add("Buckets", OSD.FromInteger(m_numBuckets));
133 ret.Add("BucketMilliseconds", OSD.FromInteger(m_bucketMilliseconds));
134 ret.Add("TotalMilliseconds", OSD.FromInteger(m_totalHistogramMilliseconds));
135
136 // Compute a number for the first bucket in the histogram.
137 // This will allow readers to know how this histogram relates to any previously read histogram.
138 int baseBucketNum = (m_timeBase / m_bucketMilliseconds) + m_lastBucket + 1;
139 ret.Add("BaseNumber", OSD.FromInteger(baseBucketNum));
140
141 ret.Add("Values", GetHistogramAsOSDArray());
142
143 return ret;
144 }
145 // Get a copy of the current histogram
146 public OSDArray GetHistogramAsOSDArray()
147 {
148 OSDArray ret = new OSDArray(m_numBuckets);
149 lock (histoLock)
150 {
151 int indx = m_lastBucket + 1;
152 for (int ii = 0; ii < m_numBuckets; ii++, indx++)
153 {
154 if (indx >= m_numBuckets)
155 indx = 0;
156 ret[ii] = OSD.FromLong(m_histogram[indx]);
157 }
158 }
159 return ret;
160 }
161
162 // Zero out the histogram
163 public void Zero()
164 {
165 lock (histoLock)
166 {
167 for (int ii = 0; ii < m_numBuckets; ii++)
168 m_histogram[ii] = 0;
169 }
170 }
171}
172
173}
diff --git a/OpenSim/Framework/Monitoring/Stats/PercentageStat.cs b/OpenSim/Framework/Monitoring/Stats/PercentageStat.cs
index 60bed55..55ddf06 100644
--- a/OpenSim/Framework/Monitoring/Stats/PercentageStat.cs
+++ b/OpenSim/Framework/Monitoring/Stats/PercentageStat.cs
@@ -29,6 +29,8 @@ using System;
29using System.Collections.Generic; 29using System.Collections.Generic;
30using System.Text; 30using System.Text;
31 31
32using OpenMetaverse.StructuredData;
33
32namespace OpenSim.Framework.Monitoring 34namespace OpenSim.Framework.Monitoring
33{ 35{
34 public class PercentageStat : Stat 36 public class PercentageStat : Stat
@@ -84,5 +86,19 @@ namespace OpenSim.Framework.Monitoring
84 86
85 return sb.ToString(); 87 return sb.ToString();
86 } 88 }
89
90 // PercentageStat is a basic stat plus percent calc
91 public override OSDMap ToOSDMap()
92 {
93 // Get the foundational instance
94 OSDMap map = base.ToOSDMap();
95
96 map["StatType"] = "PercentageStat";
97
98 map.Add("Antecedent", OSD.FromLong(Antecedent));
99 map.Add("Consequent", OSD.FromLong(Consequent));
100
101 return map;
102 }
87 } 103 }
88} \ No newline at end of file 104} \ No newline at end of file
diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs
index f91251b..a7cb2a6 100644
--- a/OpenSim/Framework/Monitoring/Stats/Stat.cs
+++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs
@@ -27,15 +27,23 @@
27 27
28using System; 28using System;
29using System.Collections.Generic; 29using System.Collections.Generic;
30using System.Linq;
31using System.Reflection;
30using System.Text; 32using System.Text;
33using log4net;
34using OpenMetaverse.StructuredData;
31 35
32namespace OpenSim.Framework.Monitoring 36namespace OpenSim.Framework.Monitoring
33{ 37{
34 /// <summary> 38 /// <summary>
35 /// Holds individual statistic details 39 /// Holds individual statistic details
36 /// </summary> 40 /// </summary>
37 public class Stat 41 public class Stat : IDisposable
38 { 42 {
43// private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
44
45 public static readonly char[] DisallowedShortNameCharacters = { '.' };
46
39 /// <summary> 47 /// <summary>
40 /// Category of this stat (e.g. cache, scene, etc). 48 /// Category of this stat (e.g. cache, scene, etc).
41 /// </summary> 49 /// </summary>
@@ -93,7 +101,7 @@ namespace OpenSim.Framework.Monitoring
93 /// <remarks> 101 /// <remarks>
94 /// Will be null if no measures of interest require samples. 102 /// Will be null if no measures of interest require samples.
95 /// </remarks> 103 /// </remarks>
96 private static Queue<double> m_samples; 104 private Queue<double> m_samples;
97 105
98 /// <summary> 106 /// <summary>
99 /// Maximum number of statistical samples. 107 /// Maximum number of statistical samples.
@@ -160,6 +168,13 @@ namespace OpenSim.Framework.Monitoring
160 throw new Exception( 168 throw new Exception(
161 string.Format("Stat cannot be in category '{0}' since this is reserved for a subcommand", category)); 169 string.Format("Stat cannot be in category '{0}' since this is reserved for a subcommand", category));
162 170
171 foreach (char c in DisallowedShortNameCharacters)
172 {
173 if (shortName.IndexOf(c) != -1)
174 shortName = shortName.Replace(c, '#');
175// throw new Exception(string.Format("Stat name {0} cannot contain character {1}", shortName, c));
176 }
177
163 ShortName = shortName; 178 ShortName = shortName;
164 Name = name; 179 Name = name;
165 Description = description; 180 Description = description;
@@ -181,6 +196,12 @@ namespace OpenSim.Framework.Monitoring
181 Verbosity = verbosity; 196 Verbosity = verbosity;
182 } 197 }
183 198
199 // IDisposable.Dispose()
200 public virtual void Dispose()
201 {
202 return;
203 }
204
184 /// <summary> 205 /// <summary>
185 /// Record a value in the sample set. 206 /// Record a value in the sample set.
186 /// </summary> 207 /// </summary>
@@ -196,6 +217,8 @@ namespace OpenSim.Framework.Monitoring
196 if (m_samples.Count >= m_maxSamples) 217 if (m_samples.Count >= m_maxSamples)
197 m_samples.Dequeue(); 218 m_samples.Dequeue();
198 219
220// m_log.DebugFormat("[STAT]: Recording value {0} for {1}", newValue, Name);
221
199 m_samples.Enqueue(newValue); 222 m_samples.Enqueue(newValue);
200 } 223 }
201 } 224 }
@@ -203,35 +226,100 @@ namespace OpenSim.Framework.Monitoring
203 public virtual string ToConsoleString() 226 public virtual string ToConsoleString()
204 { 227 {
205 StringBuilder sb = new StringBuilder(); 228 StringBuilder sb = new StringBuilder();
206 sb.AppendFormat("{0}.{1}.{2} : {3}{4}", Category, Container, ShortName, Value, UnitName); 229 sb.AppendFormat(
230 "{0}.{1}.{2} : {3}{4}",
231 Category,
232 Container,
233 ShortName,
234 Value,
235 string.IsNullOrEmpty(UnitName) ? "" : string.Format(" {0}", UnitName));
207 236
208 AppendMeasuresOfInterest(sb); 237 AppendMeasuresOfInterest(sb);
209 238
210 return sb.ToString(); 239 return sb.ToString();
211 } 240 }
212 241
213 protected void AppendMeasuresOfInterest(StringBuilder sb) 242 public virtual OSDMap ToOSDMap()
214 { 243 {
215 if ((MeasuresOfInterest & MeasuresOfInterest.AverageChangeOverTime) 244 OSDMap ret = new OSDMap();
216 == MeasuresOfInterest.AverageChangeOverTime) 245 ret.Add("StatType", "Stat"); // used by overloading classes to denote type of stat
246
247 ret.Add("Category", OSD.FromString(Category));
248 ret.Add("Container", OSD.FromString(Container));
249 ret.Add("ShortName", OSD.FromString(ShortName));
250 ret.Add("Name", OSD.FromString(Name));
251 ret.Add("Description", OSD.FromString(Description));
252 ret.Add("UnitName", OSD.FromString(UnitName));
253 ret.Add("Value", OSD.FromReal(Value));
254
255 double lastChangeOverTime, averageChangeOverTime;
256 if (ComputeMeasuresOfInterest(out lastChangeOverTime, out averageChangeOverTime))
257 {
258 ret.Add("LastChangeOverTime", OSD.FromReal(lastChangeOverTime));
259 ret.Add("AverageChangeOverTime", OSD.FromReal(averageChangeOverTime));
260 }
261
262 return ret;
263 }
264
265 // Compute the averages over time and return same.
266 // Return 'true' if averages were actually computed. 'false' if no average info.
267 public bool ComputeMeasuresOfInterest(out double lastChangeOverTime, out double averageChangeOverTime)
268 {
269 bool ret = false;
270 lastChangeOverTime = 0;
271 averageChangeOverTime = 0;
272
273 if ((MeasuresOfInterest & MeasuresOfInterest.AverageChangeOverTime) == MeasuresOfInterest.AverageChangeOverTime)
217 { 274 {
218 double totalChange = 0; 275 double totalChange = 0;
276 double? penultimateSample = null;
219 double? lastSample = null; 277 double? lastSample = null;
220 278
221 lock (m_samples) 279 lock (m_samples)
222 { 280 {
281 // m_log.DebugFormat(
282 // "[STAT]: Samples for {0} are {1}",
283 // Name, string.Join(",", m_samples.Select(s => s.ToString()).ToArray()));
284
223 foreach (double s in m_samples) 285 foreach (double s in m_samples)
224 { 286 {
225 if (lastSample != null) 287 if (lastSample != null)
226 totalChange += s - (double)lastSample; 288 totalChange += s - (double)lastSample;
227 289
290 penultimateSample = lastSample;
228 lastSample = s; 291 lastSample = s;
229 } 292 }
230 } 293 }
231 294
295 if (lastSample != null && penultimateSample != null)
296 {
297 lastChangeOverTime
298 = ((double)lastSample - (double)penultimateSample) / (Watchdog.WATCHDOG_INTERVAL_MS / 1000);
299 }
300
232 int divisor = m_samples.Count <= 1 ? 1 : m_samples.Count - 1; 301 int divisor = m_samples.Count <= 1 ? 1 : m_samples.Count - 1;
233 302
234 sb.AppendFormat(", {0:0.##}{1}/s", totalChange / divisor / (Watchdog.WATCHDOG_INTERVAL_MS / 1000), UnitName); 303 averageChangeOverTime = totalChange / divisor / (Watchdog.WATCHDOG_INTERVAL_MS / 1000);
304 ret = true;
305 }
306
307 return ret;
308 }
309
310 protected void AppendMeasuresOfInterest(StringBuilder sb)
311 {
312 double lastChangeOverTime = 0;
313 double averageChangeOverTime = 0;
314
315 if (ComputeMeasuresOfInterest(out lastChangeOverTime, out averageChangeOverTime))
316 {
317 sb.AppendFormat(
318 ", {0:0.##}{1}/s, {2:0.##}{3}/s",
319 lastChangeOverTime,
320 string.IsNullOrEmpty(UnitName) ? "" : string.Format(" {0}", UnitName),
321 averageChangeOverTime,
322 string.IsNullOrEmpty(UnitName) ? "" : string.Format(" {0}", UnitName));
235 } 323 }
236 } 324 }
237 } 325 }
diff --git a/OpenSim/Framework/Monitoring/StatsLogger.cs b/OpenSim/Framework/Monitoring/StatsLogger.cs
new file mode 100644
index 0000000..15a37aa
--- /dev/null
+++ b/OpenSim/Framework/Monitoring/StatsLogger.cs
@@ -0,0 +1,151 @@
1/*
2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the OpenSimulator Project nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28using System;
29using System.Collections.Generic;
30using System.IO;
31using System.Reflection;
32using System.Text;
33using System.Timers;
34using log4net;
35
36namespace OpenSim.Framework.Monitoring
37{
38 /// <summary>
39 /// Provides a means to continuously log stats for debugging purposes.
40 /// </summary>
41 public static class StatsLogger
42 {
43 private static readonly ILog m_statsLog = LogManager.GetLogger("special.StatsLogger");
44
45 private static Timer m_loggingTimer;
46 private static int m_statsLogIntervalMs = 5000;
47
48 public static void RegisterConsoleCommands(ICommandConsole console)
49 {
50 console.Commands.AddCommand(
51 "General",
52 false,
53 "stats record",
54 "stats record start|stop",
55 "Control whether stats are being regularly recorded to a separate file.",
56 "For debug purposes. Experimental.",
57 HandleStatsRecordCommand);
58
59 console.Commands.AddCommand(
60 "General",
61 false,
62 "stats save",
63 "stats save <path>",
64 "Save stats snapshot to a file. If the file already exists, then the report is appended.",
65 "For debug purposes. Experimental.",
66 HandleStatsSaveCommand);
67 }
68
69 public static void HandleStatsRecordCommand(string module, string[] cmd)
70 {
71 ICommandConsole con = MainConsole.Instance;
72
73 if (cmd.Length != 3)
74 {
75 con.Output("Usage: stats record start|stop");
76 return;
77 }
78
79 if (cmd[2] == "start")
80 {
81 Start();
82 con.OutputFormat("Now recording all stats to file every {0}ms", m_statsLogIntervalMs);
83 }
84 else if (cmd[2] == "stop")
85 {
86 Stop();
87 con.Output("Stopped recording stats to file.");
88 }
89 }
90
91 public static void HandleStatsSaveCommand(string module, string[] cmd)
92 {
93 ICommandConsole con = MainConsole.Instance;
94
95 if (cmd.Length != 3)
96 {
97 con.Output("Usage: stats save <path>");
98 return;
99 }
100
101 string path = cmd[2];
102
103 using (StreamWriter sw = new StreamWriter(path, true))
104 {
105 foreach (string line in GetReport())
106 sw.WriteLine(line);
107 }
108
109 MainConsole.Instance.OutputFormat("Stats saved to file {0}", path);
110 }
111
112 public static void Start()
113 {
114 if (m_loggingTimer != null)
115 Stop();
116
117 m_loggingTimer = new Timer(m_statsLogIntervalMs);
118 m_loggingTimer.AutoReset = false;
119 m_loggingTimer.Elapsed += Log;
120 m_loggingTimer.Start();
121 }
122
123 public static void Stop()
124 {
125 if (m_loggingTimer != null)
126 {
127 m_loggingTimer.Stop();
128 }
129 }
130
131 private static void Log(object sender, ElapsedEventArgs e)
132 {
133 foreach (string line in GetReport())
134 m_statsLog.Info(line);
135
136 m_loggingTimer.Start();
137 }
138
139 private static List<string> GetReport()
140 {
141 List<string> lines = new List<string>();
142
143 lines.Add(string.Format("*** STATS REPORT AT {0} ***", DateTime.Now));
144
145 foreach (string report in StatsManager.GetAllStatsReports())
146 lines.Add(report);
147
148 return lines;
149 }
150 }
151} \ No newline at end of file
diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs
index 0762b01..3136ee8 100644
--- a/OpenSim/Framework/Monitoring/StatsManager.cs
+++ b/OpenSim/Framework/Monitoring/StatsManager.cs
@@ -26,15 +26,20 @@
26 */ 26 */
27 27
28using System; 28using System;
29using System.Collections;
29using System.Collections.Generic; 30using System.Collections.Generic;
31using System.Linq;
30using System.Text; 32using System.Text;
31 33
34using OpenSim.Framework;
35using OpenMetaverse.StructuredData;
36
32namespace OpenSim.Framework.Monitoring 37namespace OpenSim.Framework.Monitoring
33{ 38{
34 /// <summary> 39 /// <summary>
35 /// Singleton used to provide access to statistics reporters 40 /// Static class used to register/deregister/fetch statistics
36 /// </summary> 41 /// </summary>
37 public class StatsManager 42 public static class StatsManager
38 { 43 {
39 // Subcommand used to list other stats. 44 // Subcommand used to list other stats.
40 public const string AllSubCommand = "all"; 45 public const string AllSubCommand = "all";
@@ -51,31 +56,43 @@ namespace OpenSim.Framework.Monitoring
51 /// <remarks> 56 /// <remarks>
52 /// Do not add or remove directly from this dictionary. 57 /// Do not add or remove directly from this dictionary.
53 /// </remarks> 58 /// </remarks>
54 public static Dictionary<string, Dictionary<string, Dictionary<string, Stat>>> RegisteredStats 59 public static SortedDictionary<string, SortedDictionary<string, SortedDictionary<string, Stat>>> RegisteredStats
55 = new Dictionary<string, Dictionary<string, Dictionary<string, Stat>>>(); 60 = new SortedDictionary<string, SortedDictionary<string, SortedDictionary<string, Stat>>>();
56 61
57 private static AssetStatsCollector assetStats; 62// private static AssetStatsCollector assetStats;
58 private static UserStatsCollector userStats; 63// private static UserStatsCollector userStats;
59 private static SimExtraStatsCollector simExtraStats = new SimExtraStatsCollector(); 64// private static SimExtraStatsCollector simExtraStats = new SimExtraStatsCollector();
60 65
61 public static AssetStatsCollector AssetStats { get { return assetStats; } } 66// public static AssetStatsCollector AssetStats { get { return assetStats; } }
62 public static UserStatsCollector UserStats { get { return userStats; } } 67// public static UserStatsCollector UserStats { get { return userStats; } }
63 public static SimExtraStatsCollector SimExtraStats { get { return simExtraStats; } } 68 public static SimExtraStatsCollector SimExtraStats { get; set; }
64 69
65 public static void RegisterConsoleCommands(ICommandConsole console) 70 public static void RegisterConsoleCommands(ICommandConsole console)
66 { 71 {
67 console.Commands.AddCommand( 72 console.Commands.AddCommand(
68 "General", 73 "General",
69 false, 74 false,
70 "show stats", 75 "stats show",
71 "show stats [list|all|<category>]", 76 "stats show [list|all|(<category>[.<container>])+",
72 "Show statistical information for this server", 77 "Show statistical information for this server",
73 "If no final argument is specified then legacy statistics information is currently shown.\n" 78 "If no final argument is specified then legacy statistics information is currently shown.\n"
74 + "If list is specified then statistic categories are shown.\n" 79 + "'list' argument will show statistic categories.\n"
75 + "If all is specified then all registered statistics are shown.\n" 80 + "'all' will show all statistics.\n"
76 + "If a category name is specified then only statistics from that category are shown.\n" 81 + "A <category> name will show statistics from that category.\n"
82 + "A <category>.<container> name will show statistics from that category in that container.\n"
83 + "More than one name can be given separated by spaces.\n"
77 + "THIS STATS FACILITY IS EXPERIMENTAL AND DOES NOT YET CONTAIN ALL STATS", 84 + "THIS STATS FACILITY IS EXPERIMENTAL AND DOES NOT YET CONTAIN ALL STATS",
78 HandleShowStatsCommand); 85 HandleShowStatsCommand);
86
87 console.Commands.AddCommand(
88 "General",
89 false,
90 "show stats",
91 "show stats [list|all|(<category>[.<container>])+",
92 "Alias for 'stats show' command",
93 HandleShowStatsCommand);
94
95 StatsLogger.RegisterConsoleCommands(console);
79 } 96 }
80 97
81 public static void HandleShowStatsCommand(string module, string[] cmd) 98 public static void HandleShowStatsCommand(string module, string[] cmd)
@@ -84,105 +101,286 @@ namespace OpenSim.Framework.Monitoring
84 101
85 if (cmd.Length > 2) 102 if (cmd.Length > 2)
86 { 103 {
87 var categoryName = cmd[2]; 104 foreach (string name in cmd.Skip(2))
88
89 if (categoryName == AllSubCommand)
90 { 105 {
91 foreach (var category in RegisteredStats.Values) 106 string[] components = name.Split('.');
107
108 string categoryName = components[0];
109 string containerName = components.Length > 1 ? components[1] : null;
110 string statName = components.Length > 2 ? components[2] : null;
111
112 if (categoryName == AllSubCommand)
92 { 113 {
93 OutputCategoryStatsToConsole(con, category); 114 OutputAllStatsToConsole(con);
94 } 115 }
95 } 116 else if (categoryName == ListSubCommand)
96 else if (categoryName == ListSubCommand)
97 {
98 con.Output("Statistic categories available are:");
99 foreach (string category in RegisteredStats.Keys)
100 con.OutputFormat(" {0}", category);
101 }
102 else
103 {
104 Dictionary<string, Dictionary<string, Stat>> category;
105 if (!RegisteredStats.TryGetValue(categoryName, out category))
106 { 117 {
107 con.OutputFormat("No such category as {0}", categoryName); 118 con.Output("Statistic categories available are:");
119 foreach (string category in RegisteredStats.Keys)
120 con.OutputFormat(" {0}", category);
108 } 121 }
109 else 122 else
110 { 123 {
111 OutputCategoryStatsToConsole(con, category); 124 SortedDictionary<string, SortedDictionary<string, Stat>> category;
125 if (!RegisteredStats.TryGetValue(categoryName, out category))
126 {
127 con.OutputFormat("No such category as {0}", categoryName);
128 }
129 else
130 {
131 if (String.IsNullOrEmpty(containerName))
132 {
133 OutputCategoryStatsToConsole(con, category);
134 }
135 else
136 {
137 SortedDictionary<string, Stat> container;
138 if (category.TryGetValue(containerName, out container))
139 {
140 if (String.IsNullOrEmpty(statName))
141 {
142 OutputContainerStatsToConsole(con, container);
143 }
144 else
145 {
146 Stat stat;
147 if (container.TryGetValue(statName, out stat))
148 {
149 OutputStatToConsole(con, stat);
150 }
151 else
152 {
153 con.OutputFormat(
154 "No such stat {0} in {1}.{2}", statName, categoryName, containerName);
155 }
156 }
157 }
158 else
159 {
160 con.OutputFormat("No such container {0} in category {1}", containerName, categoryName);
161 }
162 }
163 }
112 } 164 }
113 } 165 }
114 } 166 }
115 else 167 else
116 { 168 {
117 // Legacy 169 // Legacy
118 con.Output(SimExtraStats.Report()); 170 if (SimExtraStats != null)
171 con.Output(SimExtraStats.Report());
172 else
173 OutputAllStatsToConsole(con);
119 } 174 }
120 } 175 }
121 176
122 private static void OutputCategoryStatsToConsole( 177 public static List<string> GetAllStatsReports()
123 ICommandConsole con, Dictionary<string, Dictionary<string, Stat>> category) 178 {
179 List<string> reports = new List<string>();
180
181 foreach (var category in RegisteredStats.Values)
182 reports.AddRange(GetCategoryStatsReports(category));
183
184 return reports;
185 }
186
187 private static void OutputAllStatsToConsole(ICommandConsole con)
124 { 188 {
189 foreach (string report in GetAllStatsReports())
190 con.Output(report);
191 }
192
193 private static List<string> GetCategoryStatsReports(
194 SortedDictionary<string, SortedDictionary<string, Stat>> category)
195 {
196 List<string> reports = new List<string>();
197
125 foreach (var container in category.Values) 198 foreach (var container in category.Values)
199 reports.AddRange(GetContainerStatsReports(container));
200
201 return reports;
202 }
203
204 private static void OutputCategoryStatsToConsole(
205 ICommandConsole con, SortedDictionary<string, SortedDictionary<string, Stat>> category)
206 {
207 foreach (string report in GetCategoryStatsReports(category))
208 con.Output(report);
209 }
210
211 private static List<string> GetContainerStatsReports(SortedDictionary<string, Stat> container)
212 {
213 List<string> reports = new List<string>();
214
215 foreach (Stat stat in container.Values)
216 reports.Add(stat.ToConsoleString());
217
218 return reports;
219 }
220
221 private static void OutputContainerStatsToConsole(
222 ICommandConsole con, SortedDictionary<string, Stat> container)
223 {
224 foreach (string report in GetContainerStatsReports(container))
225 con.Output(report);
226 }
227
228 private static void OutputStatToConsole(ICommandConsole con, Stat stat)
229 {
230 con.Output(stat.ToConsoleString());
231 }
232
233 // Creates an OSDMap of the format:
234 // { categoryName: {
235 // containerName: {
236 // statName: {
237 // "Name": name,
238 // "ShortName": shortName,
239 // ...
240 // },
241 // statName: {
242 // "Name": name,
243 // "ShortName": shortName,
244 // ...
245 // },
246 // ...
247 // },
248 // containerName: {
249 // ...
250 // },
251 // ...
252 // },
253 // categoryName: {
254 // ...
255 // },
256 // ...
257 // }
258 // The passed in parameters will filter the categories, containers and stats returned. If any of the
259 // parameters are either EmptyOrNull or the AllSubCommand value, all of that type will be returned.
260 // Case matters.
261 public static OSDMap GetStatsAsOSDMap(string pCategoryName, string pContainerName, string pStatName)
262 {
263 OSDMap map = new OSDMap();
264
265 foreach (string catName in RegisteredStats.Keys)
126 { 266 {
127 foreach (Stat stat in container.Values) 267 // Do this category if null spec, "all" subcommand or category name matches passed parameter.
268 // Skip category if none of the above.
269 if (!(String.IsNullOrEmpty(pCategoryName) || pCategoryName == AllSubCommand || pCategoryName == catName))
270 continue;
271
272 OSDMap contMap = new OSDMap();
273 foreach (string contName in RegisteredStats[catName].Keys)
128 { 274 {
129 con.Output(stat.ToConsoleString()); 275 if (!(string.IsNullOrEmpty(pContainerName) || pContainerName == AllSubCommand || pContainerName == contName))
276 continue;
277
278 OSDMap statMap = new OSDMap();
279
280 SortedDictionary<string, Stat> theStats = RegisteredStats[catName][contName];
281 foreach (string statName in theStats.Keys)
282 {
283 if (!(String.IsNullOrEmpty(pStatName) || pStatName == AllSubCommand || pStatName == statName))
284 continue;
285
286 statMap.Add(statName, theStats[statName].ToOSDMap());
287 }
288
289 contMap.Add(contName, statMap);
130 } 290 }
291 map.Add(catName, contMap);
131 } 292 }
293
294 return map;
132 } 295 }
133 296
134 /// <summary> 297 public static Hashtable HandleStatsRequest(Hashtable request)
135 /// Start collecting statistics related to assets.
136 /// Should only be called once.
137 /// </summary>
138 public static AssetStatsCollector StartCollectingAssetStats()
139 { 298 {
140 assetStats = new AssetStatsCollector(); 299 Hashtable responsedata = new Hashtable();
300// string regpath = request["uri"].ToString();
301 int response_code = 200;
302 string contenttype = "text/json";
141 303
142 return assetStats; 304 string pCategoryName = StatsManager.AllSubCommand;
143 } 305 string pContainerName = StatsManager.AllSubCommand;
306 string pStatName = StatsManager.AllSubCommand;
144 307
145 /// <summary> 308 if (request.ContainsKey("cat")) pCategoryName = request["cat"].ToString();
146 /// Start collecting statistics related to users. 309 if (request.ContainsKey("cont")) pContainerName = request["cat"].ToString();
147 /// Should only be called once. 310 if (request.ContainsKey("stat")) pStatName = request["stat"].ToString();
148 /// </summary>
149 public static UserStatsCollector StartCollectingUserStats()
150 {
151 userStats = new UserStatsCollector();
152 311
153 return userStats; 312 string strOut = StatsManager.GetStatsAsOSDMap(pCategoryName, pContainerName, pStatName).ToString();
313
314 // If requestor wants it as a callback function, build response as a function rather than just the JSON string.
315 if (request.ContainsKey("callback"))
316 {
317 strOut = request["callback"].ToString() + "(" + strOut + ");";
318 }
319
320 // m_log.DebugFormat("{0} StatFetch: uri={1}, cat={2}, cont={3}, stat={4}, resp={5}",
321 // LogHeader, regpath, pCategoryName, pContainerName, pStatName, strOut);
322
323 responsedata["int_response_code"] = response_code;
324 responsedata["content_type"] = contenttype;
325 responsedata["keepalive"] = false;
326 responsedata["str_response_string"] = strOut;
327 responsedata["access_control_allow_origin"] = "*";
328
329 return responsedata;
154 } 330 }
155 331
332// /// <summary>
333// /// Start collecting statistics related to assets.
334// /// Should only be called once.
335// /// </summary>
336// public static AssetStatsCollector StartCollectingAssetStats()
337// {
338// assetStats = new AssetStatsCollector();
339//
340// return assetStats;
341// }
342//
343// /// <summary>
344// /// Start collecting statistics related to users.
345// /// Should only be called once.
346// /// </summary>
347// public static UserStatsCollector StartCollectingUserStats()
348// {
349// userStats = new UserStatsCollector();
350//
351// return userStats;
352// }
353
156 /// <summary> 354 /// <summary>
157 /// Registers a statistic. 355 /// Register a statistic.
158 /// </summary> 356 /// </summary>
159 /// <param name='stat'></param> 357 /// <param name='stat'></param>
160 /// <returns></returns> 358 /// <returns></returns>
161 public static bool RegisterStat(Stat stat) 359 public static bool RegisterStat(Stat stat)
162 { 360 {
163 Dictionary<string, Dictionary<string, Stat>> category = null, newCategory; 361 SortedDictionary<string, SortedDictionary<string, Stat>> category = null, newCategory;
164 Dictionary<string, Stat> container = null, newContainer; 362 SortedDictionary<string, Stat> container = null, newContainer;
165 363
166 lock (RegisteredStats) 364 lock (RegisteredStats)
167 { 365 {
168 // Stat name is not unique across category/container/shortname key. 366 // Stat name is not unique across category/container/shortname key.
169 // XXX: For now just return false. This is to avoid problems in regression tests where all tests 367 // XXX: For now just return false. This is to avoid problems in regression tests where all tests
170 // in a class are run in the same instance of the VM. 368 // in a class are run in the same instance of the VM.
171 if (TryGetStat(stat, out category, out container)) 369 if (TryGetStatParents(stat, out category, out container))
172 return false; 370 return false;
173 371
174 // We take a copy-on-write approach here of replacing dictionaries when keys are added or removed. 372 // We take a copy-on-write approach here of replacing dictionaries when keys are added or removed.
175 // This means that we don't need to lock or copy them on iteration, which will be a much more 373 // This means that we don't need to lock or copy them on iteration, which will be a much more
176 // common operation after startup. 374 // common operation after startup.
177 if (container != null) 375 if (container != null)
178 newContainer = new Dictionary<string, Stat>(container); 376 newContainer = new SortedDictionary<string, Stat>(container);
179 else 377 else
180 newContainer = new Dictionary<string, Stat>(); 378 newContainer = new SortedDictionary<string, Stat>();
181 379
182 if (category != null) 380 if (category != null)
183 newCategory = new Dictionary<string, Dictionary<string, Stat>>(category); 381 newCategory = new SortedDictionary<string, SortedDictionary<string, Stat>>(category);
184 else 382 else
185 newCategory = new Dictionary<string, Dictionary<string, Stat>>(); 383 newCategory = new SortedDictionary<string, SortedDictionary<string, Stat>>();
186 384
187 newContainer[stat.ShortName] = stat; 385 newContainer[stat.ShortName] = stat;
188 newCategory[stat.Container] = newContainer; 386 newCategory[stat.Container] = newContainer;
@@ -196,21 +394,21 @@ namespace OpenSim.Framework.Monitoring
196 /// Deregister a statistic 394 /// Deregister a statistic
197 /// </summary>> 395 /// </summary>>
198 /// <param name='stat'></param> 396 /// <param name='stat'></param>
199 /// <returns></returns 397 /// <returns></returns>
200 public static bool DeregisterStat(Stat stat) 398 public static bool DeregisterStat(Stat stat)
201 { 399 {
202 Dictionary<string, Dictionary<string, Stat>> category = null, newCategory; 400 SortedDictionary<string, SortedDictionary<string, Stat>> category = null, newCategory;
203 Dictionary<string, Stat> container = null, newContainer; 401 SortedDictionary<string, Stat> container = null, newContainer;
204 402
205 lock (RegisteredStats) 403 lock (RegisteredStats)
206 { 404 {
207 if (!TryGetStat(stat, out category, out container)) 405 if (!TryGetStatParents(stat, out category, out container))
208 return false; 406 return false;
209 407
210 newContainer = new Dictionary<string, Stat>(container); 408 newContainer = new SortedDictionary<string, Stat>(container);
211 newContainer.Remove(stat.ShortName); 409 newContainer.Remove(stat.ShortName);
212 410
213 newCategory = new Dictionary<string, Dictionary<string, Stat>>(category); 411 newCategory = new SortedDictionary<string, SortedDictionary<string, Stat>>(category);
214 newCategory.Remove(stat.Container); 412 newCategory.Remove(stat.Container);
215 413
216 newCategory[stat.Container] = newContainer; 414 newCategory[stat.Container] = newContainer;
@@ -220,15 +418,70 @@ namespace OpenSim.Framework.Monitoring
220 } 418 }
221 } 419 }
222 420
223 public static bool TryGetStats(string category, out Dictionary<string, Dictionary<string, Stat>> stats) 421 public static bool TryGetStat(string category, string container, string statShortName, out Stat stat)
224 { 422 {
225 return RegisteredStats.TryGetValue(category, out stats); 423 stat = null;
424 SortedDictionary<string, SortedDictionary<string, Stat>> categoryStats;
425
426 lock (RegisteredStats)
427 {
428 if (!TryGetStatsForCategory(category, out categoryStats))
429 return false;
430
431 SortedDictionary<string, Stat> containerStats;
432
433 if (!categoryStats.TryGetValue(container, out containerStats))
434 return false;
435
436 return containerStats.TryGetValue(statShortName, out stat);
437 }
438 }
439
440 public static bool TryGetStatsForCategory(
441 string category, out SortedDictionary<string, SortedDictionary<string, Stat>> stats)
442 {
443 lock (RegisteredStats)
444 return RegisteredStats.TryGetValue(category, out stats);
445 }
446
447 /// <summary>
448 /// Get the same stat for each container in a given category.
449 /// </summary>
450 /// <returns>
451 /// The stats if there were any to fetch. Otherwise null.
452 /// </returns>
453 /// <param name='category'></param>
454 /// <param name='statShortName'></param>
455 public static List<Stat> GetStatsFromEachContainer(string category, string statShortName)
456 {
457 SortedDictionary<string, SortedDictionary<string, Stat>> categoryStats;
458
459 lock (RegisteredStats)
460 {
461 if (!RegisteredStats.TryGetValue(category, out categoryStats))
462 return null;
463
464 List<Stat> stats = null;
465
466 foreach (SortedDictionary<string, Stat> containerStats in categoryStats.Values)
467 {
468 if (containerStats.ContainsKey(statShortName))
469 {
470 if (stats == null)
471 stats = new List<Stat>();
472
473 stats.Add(containerStats[statShortName]);
474 }
475 }
476
477 return stats;
478 }
226 } 479 }
227 480
228 public static bool TryGetStat( 481 public static bool TryGetStatParents(
229 Stat stat, 482 Stat stat,
230 out Dictionary<string, Dictionary<string, Stat>> category, 483 out SortedDictionary<string, SortedDictionary<string, Stat>> category,
231 out Dictionary<string, Stat> container) 484 out SortedDictionary<string, Stat> container)
232 { 485 {
233 category = null; 486 category = null;
234 container = null; 487 container = null;
@@ -252,9 +505,9 @@ namespace OpenSim.Framework.Monitoring
252 { 505 {
253 lock (RegisteredStats) 506 lock (RegisteredStats)
254 { 507 {
255 foreach (Dictionary<string, Dictionary<string, Stat>> category in RegisteredStats.Values) 508 foreach (SortedDictionary<string, SortedDictionary<string, Stat>> category in RegisteredStats.Values)
256 { 509 {
257 foreach (Dictionary<string, Stat> container in category.Values) 510 foreach (SortedDictionary<string, Stat> container in category.Values)
258 { 511 {
259 foreach (Stat stat in container.Values) 512 foreach (Stat stat in container.Values)
260 { 513 {
diff --git a/OpenSim/Framework/Monitoring/UserStatsCollector.cs b/OpenSim/Framework/Monitoring/UserStatsCollector.cs
index e89c8e6..81e0fa4 100644
--- a/OpenSim/Framework/Monitoring/UserStatsCollector.cs
+++ b/OpenSim/Framework/Monitoring/UserStatsCollector.cs
@@ -27,6 +27,8 @@
27 27
28using System.Timers; 28using System.Timers;
29 29
30using OpenMetaverse.StructuredData;
31
30namespace OpenSim.Framework.Monitoring 32namespace OpenSim.Framework.Monitoring
31{ 33{
32 /// <summary> 34 /// <summary>
@@ -88,5 +90,21 @@ namespace OpenSim.Framework.Monitoring
88 Logouts total : {3}", 90 Logouts total : {3}",
89 SuccessfulLogins, SuccessfulLoginsToday, SuccessfulLoginsYesterday, Logouts); 91 SuccessfulLogins, SuccessfulLoginsToday, SuccessfulLoginsYesterday, Logouts);
90 } 92 }
93
94 public override string XReport(string uptime, string version)
95 {
96 return OSDParser.SerializeJsonString(OReport(uptime, version));
97 }
98
99 public override OSDMap OReport(string uptime, string version)
100 {
101 OSDMap ret = new OSDMap();
102 ret.Add("SuccessfulLogins", OSD.FromInteger(SuccessfulLogins));
103 ret.Add("SuccessfulLoginsToday", OSD.FromInteger(SuccessfulLoginsToday));
104 ret.Add("SuccessfulLoginsYesterday", OSD.FromInteger(SuccessfulLoginsYesterday));
105 ret.Add("Logouts", OSD.FromInteger(Logouts));
106
107 return ret;
108 }
91 } 109 }
92} 110}
diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs
index 3f992b1..a644fa5 100644
--- a/OpenSim/Framework/Monitoring/Watchdog.cs
+++ b/OpenSim/Framework/Monitoring/Watchdog.cs
@@ -38,6 +38,8 @@ namespace OpenSim.Framework.Monitoring
38 /// </summary> 38 /// </summary>
39 public static class Watchdog 39 public static class Watchdog
40 { 40 {
41 private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
42
41 /// <summary>Timer interval in milliseconds for the watchdog timer</summary> 43 /// <summary>Timer interval in milliseconds for the watchdog timer</summary>
42 public const double WATCHDOG_INTERVAL_MS = 2500.0d; 44 public const double WATCHDOG_INTERVAL_MS = 2500.0d;
43 45
@@ -82,12 +84,32 @@ namespace OpenSim.Framework.Monitoring
82 /// </summary> 84 /// </summary>
83 public Func<string> AlarmMethod { get; set; } 85 public Func<string> AlarmMethod { get; set; }
84 86
85 public ThreadWatchdogInfo(Thread thread, int timeout) 87 /// <summary>
88 /// Stat structure associated with this thread.
89 /// </summary>
90 public Stat Stat { get; set; }
91
92 public ThreadWatchdogInfo(Thread thread, int timeout, string name)
86 { 93 {
87 Thread = thread; 94 Thread = thread;
88 Timeout = timeout; 95 Timeout = timeout;
89 FirstTick = Environment.TickCount & Int32.MaxValue; 96 FirstTick = Environment.TickCount & Int32.MaxValue;
90 LastTick = FirstTick; 97 LastTick = FirstTick;
98
99 Stat
100 = new Stat(
101 name,
102 string.Format("Last update of thread {0}", name),
103 "",
104 "ms",
105 "server",
106 "thread",
107 StatType.Pull,
108 MeasuresOfInterest.None,
109 stat => stat.Value = Environment.TickCount & Int32.MaxValue - LastTick,
110 StatVerbosity.Debug);
111
112 StatsManager.RegisterStat(Stat);
91 } 113 }
92 114
93 public ThreadWatchdogInfo(ThreadWatchdogInfo previousTwi) 115 public ThreadWatchdogInfo(ThreadWatchdogInfo previousTwi)
@@ -100,6 +122,11 @@ namespace OpenSim.Framework.Monitoring
100 AlarmIfTimeout = previousTwi.AlarmIfTimeout; 122 AlarmIfTimeout = previousTwi.AlarmIfTimeout;
101 AlarmMethod = previousTwi.AlarmMethod; 123 AlarmMethod = previousTwi.AlarmMethod;
102 } 124 }
125
126 public void Cleanup()
127 {
128 StatsManager.DeregisterStat(Stat);
129 }
103 } 130 }
104 131
105 /// <summary> 132 /// <summary>
@@ -116,7 +143,7 @@ namespace OpenSim.Framework.Monitoring
116 get { return m_enabled; } 143 get { return m_enabled; }
117 set 144 set
118 { 145 {
119// m_log.DebugFormat("[MEMORY WATCHDOG]: Setting MemoryWatchdog.Enabled to {0}", value); 146 // m_log.DebugFormat("[MEMORY WATCHDOG]: Setting MemoryWatchdog.Enabled to {0}", value);
120 147
121 if (value == m_enabled) 148 if (value == m_enabled)
122 return; 149 return;
@@ -132,9 +159,8 @@ namespace OpenSim.Framework.Monitoring
132 m_watchdogTimer.Enabled = m_enabled; 159 m_watchdogTimer.Enabled = m_enabled;
133 } 160 }
134 } 161 }
135 private static bool m_enabled;
136 162
137 private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); 163 private static bool m_enabled;
138 private static Dictionary<int, ThreadWatchdogInfo> m_threads; 164 private static Dictionary<int, ThreadWatchdogInfo> m_threads;
139 private static System.Timers.Timer m_watchdogTimer; 165 private static System.Timers.Timer m_watchdogTimer;
140 166
@@ -155,57 +181,19 @@ namespace OpenSim.Framework.Monitoring
155 } 181 }
156 182
157 /// <summary> 183 /// <summary>
158 /// Start a new thread that is tracked by the watchdog timer. 184 /// Add a thread to the watchdog tracker.
159 /// </summary>
160 /// <param name="start">The method that will be executed in a new thread</param>
161 /// <param name="name">A name to give to the new thread</param>
162 /// <param name="priority">Priority to run the thread at</param>
163 /// <param name="isBackground">True to run this thread as a background thread, otherwise false</param>
164 /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
165 /// <returns>The newly created Thread object</returns>
166 public static Thread StartThread(
167 ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout)
168 {
169 return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS);
170 }
171
172 /// <summary>
173 /// Start a new thread that is tracked by the watchdog timer
174 /// </summary> 185 /// </summary>
175 /// <param name="start">The method that will be executed in a new thread</param> 186 /// <param name="info">Information about the thread.</info>
176 /// <param name="name">A name to give to the new thread</param> 187 /// <param name="info">Name of the thread.</info>
177 /// <param name="priority">Priority to run the thread at</param> 188 /// <param name="log">If true then creation of thread is logged.</param>
178 /// <param name="isBackground">True to run this thread as a background 189 public static void AddThread(ThreadWatchdogInfo info, string name, bool log = true)
179 /// thread, otherwise false</param>
180 /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
181 /// <param name="alarmMethod">
182 /// Alarm method to call if alarmIfTimeout is true and there is a timeout.
183 /// Normally, this will just return some useful debugging information.
184 /// </param>
185 /// <param name="timeout">Number of milliseconds to wait until we issue a warning about timeout.</param>
186 /// <returns>The newly created Thread object</returns>
187 public static Thread StartThread(
188 ThreadStart start, string name, ThreadPriority priority, bool isBackground,
189 bool alarmIfTimeout, Func<string> alarmMethod, int timeout)
190 { 190 {
191 Thread thread = new Thread(start); 191 if (log)
192 thread.Name = name; 192 m_log.DebugFormat(
193 thread.Priority = priority; 193 "[WATCHDOG]: Started tracking thread {0}, ID {1}", name, info.Thread.ManagedThreadId);
194 thread.IsBackground = isBackground;
195
196 ThreadWatchdogInfo twi
197 = new ThreadWatchdogInfo(thread, timeout)
198 { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod };
199
200 m_log.DebugFormat(
201 "[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
202 194
203 lock (m_threads) 195 lock (m_threads)
204 m_threads.Add(twi.Thread.ManagedThreadId, twi); 196 m_threads.Add(info.Thread.ManagedThreadId, info);
205
206 thread.Start();
207
208 return thread;
209 } 197 }
210 198
211 /// <summary> 199 /// <summary>
@@ -219,25 +207,28 @@ namespace OpenSim.Framework.Monitoring
219 /// <summary> 207 /// <summary>
220 /// Stops watchdog tracking on the current thread 208 /// Stops watchdog tracking on the current thread
221 /// </summary> 209 /// </summary>
210 /// <param name="log">If true then normal events in thread removal are not logged.</param>
222 /// <returns> 211 /// <returns>
223 /// True if the thread was removed from the list of tracked 212 /// True if the thread was removed from the list of tracked
224 /// threads, otherwise false 213 /// threads, otherwise false
225 /// </returns> 214 /// </returns>
226 public static bool RemoveThread() 215 public static bool RemoveThread(bool log = true)
227 { 216 {
228 return RemoveThread(Thread.CurrentThread.ManagedThreadId); 217 return RemoveThread(Thread.CurrentThread.ManagedThreadId, log);
229 } 218 }
230 219
231 private static bool RemoveThread(int threadID) 220 private static bool RemoveThread(int threadID, bool log = true)
232 { 221 {
233 lock (m_threads) 222 lock (m_threads)
234 { 223 {
235 ThreadWatchdogInfo twi; 224 ThreadWatchdogInfo twi;
236 if (m_threads.TryGetValue(threadID, out twi)) 225 if (m_threads.TryGetValue(threadID, out twi))
237 { 226 {
238 m_log.DebugFormat( 227 if (log)
239 "[WATCHDOG]: Removing thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId); 228 m_log.DebugFormat(
229 "[WATCHDOG]: Removing thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
240 230
231 twi.Cleanup();
241 m_threads.Remove(threadID); 232 m_threads.Remove(threadID);
242 233
243 return true; 234 return true;
@@ -293,7 +284,7 @@ namespace OpenSim.Framework.Monitoring
293 } 284 }
294 catch { } 285 catch { }
295 } 286 }
296 287
297 /// <summary> 288 /// <summary>
298 /// Get currently watched threads for diagnostic purposes 289 /// Get currently watched threads for diagnostic purposes
299 /// </summary> 290 /// </summary>
@@ -380,6 +371,7 @@ namespace OpenSim.Framework.Monitoring
380 if (MemoryWatchdog.Enabled) 371 if (MemoryWatchdog.Enabled)
381 MemoryWatchdog.Update(); 372 MemoryWatchdog.Update();
382 373
374 ChecksManager.CheckChecks();
383 StatsManager.RecordStats(); 375 StatsManager.RecordStats();
384 376
385 m_watchdogTimer.Start(); 377 m_watchdogTimer.Start();
diff --git a/OpenSim/Framework/Monitoring/WorkManager.cs b/OpenSim/Framework/Monitoring/WorkManager.cs
new file mode 100644
index 0000000..d1a74ce
--- /dev/null
+++ b/OpenSim/Framework/Monitoring/WorkManager.cs
@@ -0,0 +1,290 @@
1/*
2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the OpenSimulator Project nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28using System;
29using System.Reflection;
30using System.Threading;
31using log4net;
32
33namespace OpenSim.Framework.Monitoring
34{
35 /// <summary>
36 /// Manages various work items in the simulator.
37 /// </summary>
38 /// <remarks>
39 /// Currently, here work can be started
40 /// * As a long-running and monitored thread.
41 /// * In a thread that will never timeout but where the job is expected to eventually complete.
42 /// * In a threadpool thread that will timeout if it takes a very long time to complete (> 10 mins).
43 /// * As a job which will be run in a single-threaded job engine. Such jobs must not incorporate delays (sleeps,
44 /// network waits, etc.).
45 ///
46 /// This is an evolving approach to better manage the work that OpenSimulator is asked to do from a very diverse
47 /// range of sources (client actions, incoming network, outgoing network calls, etc.).
48 ///
49 /// Util.FireAndForget is still available to insert jobs in the threadpool, though this is equivalent to
50 /// WorkManager.RunInThreadPool().
51 /// </remarks>
52 public static class WorkManager
53 {
54 private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
55
56 public static JobEngine JobEngine { get; private set; }
57
58 static WorkManager()
59 {
60 JobEngine = new JobEngine("Non-blocking non-critical job engine", "JOB ENGINE");
61
62 StatsManager.RegisterStat(
63 new Stat(
64 "JobsWaiting",
65 "Number of jobs waiting for processing.",
66 "",
67 "",
68 "server",
69 "jobengine",
70 StatType.Pull,
71 MeasuresOfInterest.None,
72 stat => stat.Value = JobEngine.JobsWaiting,
73 StatVerbosity.Debug));
74
75 MainConsole.Instance.Commands.AddCommand(
76 "Debug",
77 false,
78 "debug jobengine",
79 "debug jobengine <start|stop|status|log>",
80 "Start, stop, get status or set logging level of the job engine.",
81 "If stopped then all outstanding jobs are processed immediately.",
82 HandleControlCommand);
83 }
84
85 /// <summary>
86 /// Start a new long-lived thread.
87 /// </summary>
88 /// <param name="start">The method that will be executed in a new thread</param>
89 /// <param name="name">A name to give to the new thread</param>
90 /// <param name="priority">Priority to run the thread at</param>
91 /// <param name="isBackground">True to run this thread as a background thread, otherwise false</param>
92 /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
93 /// <param name="log">If true then creation of thread is logged.</param>
94 /// <returns>The newly created Thread object</returns>
95 public static Thread StartThread(
96 ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, bool log = true)
97 {
98 return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, Watchdog.DEFAULT_WATCHDOG_TIMEOUT_MS, log);
99 }
100
101 /// <summary>
102 /// Start a new thread that is tracked by the watchdog
103 /// </summary>
104 /// <param name="start">The method that will be executed in a new thread</param>
105 /// <param name="name">A name to give to the new thread</param>
106 /// <param name="priority">Priority to run the thread at</param>
107 /// <param name="isBackground">True to run this thread as a background
108 /// thread, otherwise false</param>
109 /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
110 /// <param name="alarmMethod">
111 /// Alarm method to call if alarmIfTimeout is true and there is a timeout.
112 /// Normally, this will just return some useful debugging information.
113 /// </param>
114 /// <param name="timeout">Number of milliseconds to wait until we issue a warning about timeout.</param>
115 /// <param name="log">If true then creation of thread is logged.</param>
116 /// <returns>The newly created Thread object</returns>
117 public static Thread StartThread(
118 ThreadStart start, string name, ThreadPriority priority, bool isBackground,
119 bool alarmIfTimeout, Func<string> alarmMethod, int timeout, bool log = true)
120 {
121 Thread thread = new Thread(start);
122 thread.Priority = priority;
123 thread.IsBackground = isBackground;
124
125 Watchdog.ThreadWatchdogInfo twi
126 = new Watchdog.ThreadWatchdogInfo(thread, timeout, name)
127 { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod };
128
129 Watchdog.AddThread(twi, name, log:log);
130
131 thread.Start();
132 thread.Name = name;
133
134 return thread;
135 }
136
137 /// <summary>
138 /// Run the callback in a new thread immediately. If the thread exits with an exception log it but do
139 /// not propogate it.
140 /// </summary>
141 /// <param name="callback">Code for the thread to execute.</param>
142 /// <param name="obj">Object to pass to the thread.</param>
143 /// <param name="name">Name of the thread</param>
144 public static void RunInThread(WaitCallback callback, object obj, string name, bool log = false)
145 {
146 if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
147 {
148 Culture.SetCurrentCulture();
149 callback(obj);
150 return;
151 }
152
153 ThreadStart ts = new ThreadStart(delegate()
154 {
155 try
156 {
157 Culture.SetCurrentCulture();
158 callback(obj);
159 Watchdog.RemoveThread(log:false);
160 }
161 catch (Exception e)
162 {
163 m_log.Error(string.Format("[WATCHDOG]: Exception in thread {0}.", name), e);
164 }
165 });
166
167 StartThread(ts, name, ThreadPriority.Normal, true, false, log:log);
168 }
169
170 /// <summary>
171 /// Run the callback via a threadpool thread.
172 /// </summary>
173 /// <remarks>
174 /// Such jobs may run after some delay but must always complete.
175 /// </remarks>
176 /// <param name="callback"></param>
177 /// <param name="obj"></param>
178 /// <param name="name">The name of the job. This is used in monitoring and debugging.</param>
179 public static void RunInThreadPool(System.Threading.WaitCallback callback, object obj, string name)
180 {
181 Util.FireAndForget(callback, obj, name);
182 }
183
184 /// <summary>
185 /// Run a job.
186 /// </summary>
187 /// <remarks>
188 /// This differs from direct scheduling (e.g. Util.FireAndForget) in that a job can be run in the job
189 /// engine if it is running, where all jobs are currently performed in sequence on a single thread. This is
190 /// to prevent observed overload and server freeze problems when there are hundreds of connections which all attempt to
191 /// perform work at once (e.g. in conference situations). With lower numbers of connections, the small
192 /// delay in performing jobs in sequence rather than concurrently has not been notiecable in testing, though a future more
193 /// sophisticated implementation could perform jobs concurrently when the server is under low load.
194 ///
195 /// However, be advised that some callers of this function rely on all jobs being performed in sequence if any
196 /// jobs are performed in sequence (i.e. if jobengine is active or not). Therefore, expanding the jobengine
197 /// beyond a single thread will require considerable thought.
198 ///
199 /// Also, any jobs submitted must be guaranteed to complete within a reasonable timeframe (e.g. they cannot
200 /// incorporate a network delay with a long timeout). At the moment, work that could suffer such issues
201 /// should still be run directly with RunInThread(), Util.FireAndForget(), etc. This is another area where
202 /// the job engine could be improved and so CPU utilization improved by better management of concurrency within
203 /// OpenSimulator.
204 /// </remarks>
205 /// <param name="jobType">General classification for the job (e.g. "RezAttachments").</param>
206 /// <param name="callback">Callback for job.</param>
207 /// <param name="obj">Object to pass to callback when run</param>
208 /// <param name="name">Specific name of job (e.g. "RezAttachments for Joe Bloggs"</param>
209 /// <param name="canRunInThisThread">If set to true then the job may be run in ths calling thread.</param>
210 /// <param name="mustNotTimeout">If the true then the job must never timeout.</param>
211 /// <param name="log">If set to true then extra logging is performed.</param>
212 public static void RunJob(
213 string jobType, WaitCallback callback, object obj, string name,
214 bool canRunInThisThread = false, bool mustNotTimeout = false,
215 bool log = false)
216 {
217 if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
218 {
219 Culture.SetCurrentCulture();
220 callback(obj);
221 return;
222 }
223
224 if (JobEngine.IsRunning)
225 JobEngine.QueueJob(name, () => callback(obj));
226 else if (canRunInThisThread)
227 callback(obj);
228 else if (mustNotTimeout)
229 RunInThread(callback, obj, name, log);
230 else
231 Util.FireAndForget(callback, obj, name);
232 }
233
234 private static void HandleControlCommand(string module, string[] args)
235 {
236 // if (SceneManager.Instance.CurrentScene != null && SceneManager.Instance.CurrentScene != m_udpServer.Scene)
237 // return;
238
239 if (args.Length < 3)
240 {
241 MainConsole.Instance.Output("Usage: debug jobengine <stop|start|status|log>");
242 return;
243 }
244
245 string subCommand = args[2];
246
247 if (subCommand == "stop")
248 {
249 JobEngine.Stop();
250 MainConsole.Instance.OutputFormat("Stopped job engine.");
251 }
252 else if (subCommand == "start")
253 {
254 JobEngine.Start();
255 MainConsole.Instance.OutputFormat("Started job engine.");
256 }
257 else if (subCommand == "status")
258 {
259 MainConsole.Instance.OutputFormat("Job engine running: {0}", JobEngine.IsRunning);
260
261 JobEngine.Job job = JobEngine.CurrentJob;
262 MainConsole.Instance.OutputFormat("Current job {0}", job != null ? job.Name : "none");
263
264 MainConsole.Instance.OutputFormat(
265 "Jobs waiting: {0}", JobEngine.IsRunning ? JobEngine.JobsWaiting.ToString() : "n/a");
266 MainConsole.Instance.OutputFormat("Log Level: {0}", JobEngine.LogLevel);
267 }
268 else if (subCommand == "log")
269 {
270 if (args.Length < 4)
271 {
272 MainConsole.Instance.Output("Usage: debug jobengine log <level>");
273 return;
274 }
275
276 // int logLevel;
277 int logLevel = int.Parse(args[3]);
278 // if (ConsoleUtil.TryParseConsoleInt(MainConsole.Instance, args[4], out logLevel))
279 // {
280 JobEngine.LogLevel = logLevel;
281 MainConsole.Instance.OutputFormat("Set debug log level to {0}", JobEngine.LogLevel);
282 // }
283 }
284 else
285 {
286 MainConsole.Instance.OutputFormat("Unrecognized job engine subcommand {0}", subCommand);
287 }
288 }
289 }
290} \ No newline at end of file
generated by cgit v1.1 at 2024-07-01 02:05:38 +0000