aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/OpenSim/Framework/Monitoring
diff options
context:
space:
mode:
Diffstat (limited to 'OpenSim/Framework/Monitoring')
-rw-r--r--OpenSim/Framework/Monitoring/JobEngine.cs2
-rw-r--r--OpenSim/Framework/Monitoring/Watchdog.cs149
-rw-r--r--OpenSim/Framework/Monitoring/WorkManager.cs212
3 files changed, 224 insertions, 139 deletions
diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs
index 4a46328..5925867 100644
--- a/OpenSim/Framework/Monitoring/JobEngine.cs
+++ b/OpenSim/Framework/Monitoring/JobEngine.cs
@@ -125,7 +125,7 @@ namespace OpenSim.Framework.Monitoring
125 125
126 StatsManager.RegisterStat(m_requestsWaitingStat); 126 StatsManager.RegisterStat(m_requestsWaitingStat);
127 127
128 Watchdog.StartThread( 128 WorkManager.StartThread(
129 ProcessRequests, 129 ProcessRequests,
130 "JobEngineThread", 130 "JobEngineThread",
131 ThreadPriority.Normal, 131 ThreadPriority.Normal,
diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs
index 0feec7c..a644fa5 100644
--- a/OpenSim/Framework/Monitoring/Watchdog.cs
+++ b/OpenSim/Framework/Monitoring/Watchdog.cs
@@ -38,6 +38,8 @@ namespace OpenSim.Framework.Monitoring
38 /// </summary> 38 /// </summary>
39 public static class Watchdog 39 public static class Watchdog
40 { 40 {
41 private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
42
41 /// <summary>Timer interval in milliseconds for the watchdog timer</summary> 43 /// <summary>Timer interval in milliseconds for the watchdog timer</summary>
42 public const double WATCHDOG_INTERVAL_MS = 2500.0d; 44 public const double WATCHDOG_INTERVAL_MS = 2500.0d;
43 45
@@ -133,8 +135,6 @@ namespace OpenSim.Framework.Monitoring
133 /// /summary> 135 /// /summary>
134 public static event Action<ThreadWatchdogInfo> OnWatchdogTimeout; 136 public static event Action<ThreadWatchdogInfo> OnWatchdogTimeout;
135 137
136 public static JobEngine JobEngine { get; private set; }
137
138 /// <summary> 138 /// <summary>
139 /// Is this watchdog active? 139 /// Is this watchdog active?
140 /// </summary> 140 /// </summary>
@@ -143,7 +143,7 @@ namespace OpenSim.Framework.Monitoring
143 get { return m_enabled; } 143 get { return m_enabled; }
144 set 144 set
145 { 145 {
146// m_log.DebugFormat("[MEMORY WATCHDOG]: Setting MemoryWatchdog.Enabled to {0}", value); 146 // m_log.DebugFormat("[MEMORY WATCHDOG]: Setting MemoryWatchdog.Enabled to {0}", value);
147 147
148 if (value == m_enabled) 148 if (value == m_enabled)
149 return; 149 return;
@@ -159,9 +159,8 @@ namespace OpenSim.Framework.Monitoring
159 m_watchdogTimer.Enabled = m_enabled; 159 m_watchdogTimer.Enabled = m_enabled;
160 } 160 }
161 } 161 }
162 private static bool m_enabled;
163 162
164 private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); 163 private static bool m_enabled;
165 private static Dictionary<int, ThreadWatchdogInfo> m_threads; 164 private static Dictionary<int, ThreadWatchdogInfo> m_threads;
166 private static System.Timers.Timer m_watchdogTimer; 165 private static System.Timers.Timer m_watchdogTimer;
167 166
@@ -175,7 +174,6 @@ namespace OpenSim.Framework.Monitoring
175 174
176 static Watchdog() 175 static Watchdog()
177 { 176 {
178 JobEngine = new JobEngine();
179 m_threads = new Dictionary<int, ThreadWatchdogInfo>(); 177 m_threads = new Dictionary<int, ThreadWatchdogInfo>();
180 m_watchdogTimer = new System.Timers.Timer(WATCHDOG_INTERVAL_MS); 178 m_watchdogTimer = new System.Timers.Timer(WATCHDOG_INTERVAL_MS);
181 m_watchdogTimer.AutoReset = false; 179 m_watchdogTimer.AutoReset = false;
@@ -183,94 +181,19 @@ namespace OpenSim.Framework.Monitoring
183 } 181 }
184 182
185 /// <summary> 183 /// <summary>
186 /// Start a new thread that is tracked by the watchdog timer. 184 /// Add a thread to the watchdog tracker.
187 /// </summary>
188 /// <param name="start">The method that will be executed in a new thread</param>
189 /// <param name="name">A name to give to the new thread</param>
190 /// <param name="priority">Priority to run the thread at</param>
191 /// <param name="isBackground">True to run this thread as a background thread, otherwise false</param>
192 /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
193 /// <param name="log">If true then creation of thread is logged.</param>
194 /// <returns>The newly created Thread object</returns>
195 public static Thread StartThread(
196 ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, bool log = true)
197 {
198 return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS, log);
199 }
200
201 /// <summary>
202 /// Start a new thread that is tracked by the watchdog
203 /// </summary> 185 /// </summary>
204 /// <param name="start">The method that will be executed in a new thread</param> 186 /// <param name="info">Information about the thread.</info>
205 /// <param name="name">A name to give to the new thread</param> 187 /// <param name="info">Name of the thread.</info>
206 /// <param name="priority">Priority to run the thread at</param>
207 /// <param name="isBackground">True to run this thread as a background
208 /// thread, otherwise false</param>
209 /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
210 /// <param name="alarmMethod">
211 /// Alarm method to call if alarmIfTimeout is true and there is a timeout.
212 /// Normally, this will just return some useful debugging information.
213 /// </param>
214 /// <param name="timeout">Number of milliseconds to wait until we issue a warning about timeout.</param>
215 /// <param name="log">If true then creation of thread is logged.</param> 188 /// <param name="log">If true then creation of thread is logged.</param>
216 /// <returns>The newly created Thread object</returns> 189 public static void AddThread(ThreadWatchdogInfo info, string name, bool log = true)
217 public static Thread StartThread(
218 ThreadStart start, string name, ThreadPriority priority, bool isBackground,
219 bool alarmIfTimeout, Func<string> alarmMethod, int timeout, bool log = true)
220 { 190 {
221 Thread thread = new Thread(start);
222 thread.Priority = priority;
223 thread.IsBackground = isBackground;
224
225 ThreadWatchdogInfo twi
226 = new ThreadWatchdogInfo(thread, timeout, name)
227 { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod };
228
229 if (log) 191 if (log)
230 m_log.DebugFormat( 192 m_log.DebugFormat(
231 "[WATCHDOG]: Started tracking thread {0}, ID {1}", name, twi.Thread.ManagedThreadId); 193 "[WATCHDOG]: Started tracking thread {0}, ID {1}", name, info.Thread.ManagedThreadId);
232 194
233 lock (m_threads) 195 lock (m_threads)
234 m_threads.Add(twi.Thread.ManagedThreadId, twi); 196 m_threads.Add(info.Thread.ManagedThreadId, info);
235
236 thread.Start();
237 thread.Name = name;
238
239
240 return thread;
241 }
242
243 /// <summary>
244 /// Run the callback in a new thread immediately. If the thread exits with an exception log it but do
245 /// not propogate it.
246 /// </summary>
247 /// <param name="callback">Code for the thread to execute.</param>
248 /// <param name="name">Name of the thread</param>
249 /// <param name="obj">Object to pass to the thread.</param>
250 public static void RunInThread(WaitCallback callback, string name, object obj, bool log = false)
251 {
252 if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
253 {
254 Culture.SetCurrentCulture();
255 callback(obj);
256 return;
257 }
258
259 ThreadStart ts = new ThreadStart(delegate()
260 {
261 try
262 {
263 Culture.SetCurrentCulture();
264 callback(obj);
265 Watchdog.RemoveThread(log:false);
266 }
267 catch (Exception e)
268 {
269 m_log.Error(string.Format("[WATCHDOG]: Exception in thread {0}.", name), e);
270 }
271 });
272
273 StartThread(ts, name, ThreadPriority.Normal, true, false, log:log);
274 } 197 }
275 198
276 /// <summary> 199 /// <summary>
@@ -361,7 +284,7 @@ namespace OpenSim.Framework.Monitoring
361 } 284 }
362 catch { } 285 catch { }
363 } 286 }
364 287
365 /// <summary> 288 /// <summary>
366 /// Get currently watched threads for diagnostic purposes 289 /// Get currently watched threads for diagnostic purposes
367 /// </summary> 290 /// </summary>
@@ -453,55 +376,5 @@ namespace OpenSim.Framework.Monitoring
453 376
454 m_watchdogTimer.Start(); 377 m_watchdogTimer.Start();
455 } 378 }
456
457 /// <summary>
458 /// Run a job.
459 /// </summary>
460 /// <remarks>
461 /// This differs from direct scheduling (e.g. Util.FireAndForget) in that a job can be run in the job
462 /// engine if it is running, where all jobs are currently performed in sequence on a single thread. This is
463 /// to prevent observed overload and server freeze problems when there are hundreds of connections which all attempt to
464 /// perform work at once (e.g. in conference situations). With lower numbers of connections, the small
465 /// delay in performing jobs in sequence rather than concurrently has not been notiecable in testing, though a future more
466 /// sophisticated implementation could perform jobs concurrently when the server is under low load.
467 ///
468 /// However, be advised that some callers of this function rely on all jobs being performed in sequence if any
469 /// jobs are performed in sequence (i.e. if jobengine is active or not). Therefore, expanding the jobengine
470 /// beyond a single thread will require considerable thought.
471 ///
472 /// Also, any jobs submitted must be guaranteed to complete within a reasonable timeframe (e.g. they cannot
473 /// incorporate a network delay with a long timeout). At the moment, work that could suffer such issues
474 /// should still be run directly with RunInThread(), Util.FireAndForget(), etc. This is another area where
475 /// the job engine could be improved and so CPU utilization improved by better management of concurrency within
476 /// OpenSimulator.
477 /// </remarks>
478 /// <param name="jobType">General classification for the job (e.g. "RezAttachments").</param>
479 /// <param name="callback">Callback for job.</param>
480 /// <param name="name">Specific name of job (e.g. "RezAttachments for Joe Bloggs"</param>
481 /// <param name="obj">Object to pass to callback when run</param>
482 /// <param name="canRunInThisThread">If set to true then the job may be run in ths calling thread.</param>
483 /// <param name="mustNotTimeout">If the true then the job must never timeout.</param>
484 /// <param name="log">If set to true then extra logging is performed.</param>
485 public static void RunJob(
486 string jobType, WaitCallback callback, string name, object obj,
487 bool canRunInThisThread = false, bool mustNotTimeout = false,
488 bool log = false)
489 {
490 if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
491 {
492 Culture.SetCurrentCulture();
493 callback(obj);
494 return;
495 }
496
497 if (JobEngine.IsRunning)
498 JobEngine.QueueRequest(name, callback, obj);
499 else if (canRunInThisThread)
500 callback(obj);
501 else if (mustNotTimeout)
502 RunInThread(callback, name, obj, log);
503 else
504 Util.FireAndForget(callback, obj, name);
505 }
506 } 379 }
507} \ No newline at end of file 380} \ No newline at end of file
diff --git a/OpenSim/Framework/Monitoring/WorkManager.cs b/OpenSim/Framework/Monitoring/WorkManager.cs
new file mode 100644
index 0000000..9d0eefc
--- /dev/null
+++ b/OpenSim/Framework/Monitoring/WorkManager.cs
@@ -0,0 +1,212 @@
1/*
2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the OpenSimulator Project nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28using System;
29using System.Reflection;
30using System.Threading;
31using log4net;
32
33namespace OpenSim.Framework.Monitoring
34{
35 /// <summary>
36 /// Manages various work items in the simulator.
37 /// </summary>
38 /// <remarks>
39 /// Currently, here work can be started
40 /// * As a long-running and monitored thread.
41 /// * In a thread that will never timeout but where the job is expected to eventually complete.
42 /// * In a threadpool thread that will timeout if it takes a very long time to complete (> 10 mins).
43 /// * As a job which will be run in a single-threaded job engine. Such jobs must not incorporate delays (sleeps,
44 /// network waits, etc.).
45 ///
46 /// This is an evolving approach to better manage the work that OpenSimulator is asked to do from a very diverse
47 /// range of sources (client actions, incoming network, outgoing network calls, etc.).
48 ///
49 /// Util.FireAndForget is still available to insert jobs in the threadpool, though this is equivalent to
50 /// WorkManager.RunInThreadPool().
51 /// </remarks>
52 public static class WorkManager
53 {
54 private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
55
56 public static JobEngine JobEngine { get; private set; }
57
58 static WorkManager()
59 {
60 JobEngine = new JobEngine();
61 }
62
63 /// <summary>
64 /// Start a new long-lived thread.
65 /// </summary>
66 /// <param name="start">The method that will be executed in a new thread</param>
67 /// <param name="name">A name to give to the new thread</param>
68 /// <param name="priority">Priority to run the thread at</param>
69 /// <param name="isBackground">True to run this thread as a background thread, otherwise false</param>
70 /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
71 /// <param name="log">If true then creation of thread is logged.</param>
72 /// <returns>The newly created Thread object</returns>
73 public static Thread StartThread(
74 ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, bool log = true)
75 {
76 return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, Watchdog.DEFAULT_WATCHDOG_TIMEOUT_MS, log);
77 }
78
79 /// <summary>
80 /// Start a new thread that is tracked by the watchdog
81 /// </summary>
82 /// <param name="start">The method that will be executed in a new thread</param>
83 /// <param name="name">A name to give to the new thread</param>
84 /// <param name="priority">Priority to run the thread at</param>
85 /// <param name="isBackground">True to run this thread as a background
86 /// thread, otherwise false</param>
87 /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
88 /// <param name="alarmMethod">
89 /// Alarm method to call if alarmIfTimeout is true and there is a timeout.
90 /// Normally, this will just return some useful debugging information.
91 /// </param>
92 /// <param name="timeout">Number of milliseconds to wait until we issue a warning about timeout.</param>
93 /// <param name="log">If true then creation of thread is logged.</param>
94 /// <returns>The newly created Thread object</returns>
95 public static Thread StartThread(
96 ThreadStart start, string name, ThreadPriority priority, bool isBackground,
97 bool alarmIfTimeout, Func<string> alarmMethod, int timeout, bool log = true)
98 {
99 Thread thread = new Thread(start);
100 thread.Priority = priority;
101 thread.IsBackground = isBackground;
102
103 Watchdog.ThreadWatchdogInfo twi
104 = new Watchdog.ThreadWatchdogInfo(thread, timeout, name)
105 { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod };
106
107 Watchdog.AddThread(twi, name, log:log);
108
109 thread.Start();
110 thread.Name = name;
111
112 return thread;
113 }
114
115 /// <summary>
116 /// Run the callback in a new thread immediately. If the thread exits with an exception log it but do
117 /// not propogate it.
118 /// </summary>
119 /// <param name="callback">Code for the thread to execute.</param>
120 /// <param name="obj">Object to pass to the thread.</param>
121 /// <param name="name">Name of the thread</param>
122 public static void RunInThread(WaitCallback callback, object obj, string name, bool log = false)
123 {
124 if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
125 {
126 Culture.SetCurrentCulture();
127 callback(obj);
128 return;
129 }
130
131 ThreadStart ts = new ThreadStart(delegate()
132 {
133 try
134 {
135 Culture.SetCurrentCulture();
136 callback(obj);
137 Watchdog.RemoveThread(log:false);
138 }
139 catch (Exception e)
140 {
141 m_log.Error(string.Format("[WATCHDOG]: Exception in thread {0}.", name), e);
142 }
143 });
144
145 StartThread(ts, name, ThreadPriority.Normal, true, false, log:log);
146 }
147
148 /// <summary>
149 /// Run the callback via a threadpool thread.
150 /// </summary>
151 /// <remarks>
152 /// Such jobs may run after some delay but must always complete.
153 /// </remarks>
154 /// <param name="callback"></param>
155 /// <param name="obj"></param>
156 /// <param name="name">The name of the job. This is used in monitoring and debugging.</param>
157 public static void RunInThreadPool(System.Threading.WaitCallback callback, object obj, string name)
158 {
159 Util.FireAndForget(callback, obj, name);
160 }
161
162 /// <summary>
163 /// Run a job.
164 /// </summary>
165 /// <remarks>
166 /// This differs from direct scheduling (e.g. Util.FireAndForget) in that a job can be run in the job
167 /// engine if it is running, where all jobs are currently performed in sequence on a single thread. This is
168 /// to prevent observed overload and server freeze problems when there are hundreds of connections which all attempt to
169 /// perform work at once (e.g. in conference situations). With lower numbers of connections, the small
170 /// delay in performing jobs in sequence rather than concurrently has not been notiecable in testing, though a future more
171 /// sophisticated implementation could perform jobs concurrently when the server is under low load.
172 ///
173 /// However, be advised that some callers of this function rely on all jobs being performed in sequence if any
174 /// jobs are performed in sequence (i.e. if jobengine is active or not). Therefore, expanding the jobengine
175 /// beyond a single thread will require considerable thought.
176 ///
177 /// Also, any jobs submitted must be guaranteed to complete within a reasonable timeframe (e.g. they cannot
178 /// incorporate a network delay with a long timeout). At the moment, work that could suffer such issues
179 /// should still be run directly with RunInThread(), Util.FireAndForget(), etc. This is another area where
180 /// the job engine could be improved and so CPU utilization improved by better management of concurrency within
181 /// OpenSimulator.
182 /// </remarks>
183 /// <param name="jobType">General classification for the job (e.g. "RezAttachments").</param>
184 /// <param name="callback">Callback for job.</param>
185 /// <param name="obj">Object to pass to callback when run</param>
186 /// <param name="name">Specific name of job (e.g. "RezAttachments for Joe Bloggs"</param>
187 /// <param name="canRunInThisThread">If set to true then the job may be run in ths calling thread.</param>
188 /// <param name="mustNotTimeout">If the true then the job must never timeout.</param>
189 /// <param name="log">If set to true then extra logging is performed.</param>
190 public static void RunJob(
191 string jobType, WaitCallback callback, object obj, string name,
192 bool canRunInThisThread = false, bool mustNotTimeout = false,
193 bool log = false)
194 {
195 if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
196 {
197 Culture.SetCurrentCulture();
198 callback(obj);
199 return;
200 }
201
202 if (JobEngine.IsRunning)
203 JobEngine.QueueRequest(name, callback, obj);
204 else if (canRunInThisThread)
205 callback(obj);
206 else if (mustNotTimeout)
207 RunInThread(callback, obj, name, log);
208 else
209 Util.FireAndForget(callback, obj, name);
210 }
211 }
212} \ No newline at end of file