aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/OpenSim/Framework/Monitoring
diff options
context:
space:
mode:
authorJustin Clark-Casey (justincc)2014-11-25 23:56:32 +0000
committerJustin Clark-Casey (justincc)2014-11-25 23:56:32 +0000
commit86367d7219b3bd52f63045b2b17bcbde328844ed (patch)
tree789227d95cb152a437481e741b55d86dfeba9f74 /OpenSim/Framework/Monitoring
parentFix ghosts copy of SimulatorFeatureModule to enable logging. (diff)
downloadopensim-SC-86367d7219b3bd52f63045b2b17bcbde328844ed.zip
opensim-SC-86367d7219b3bd52f63045b2b17bcbde328844ed.tar.gz
opensim-SC-86367d7219b3bd52f63045b2b17bcbde328844ed.tar.bz2
opensim-SC-86367d7219b3bd52f63045b2b17bcbde328844ed.tar.xz
refactor: Move methods to start a monitored thread, start work in its own thread and run work in the jobengine from Watchdog to a WorkManager class.
This is to achieve a clean separation of concerns - the watchdog is an inappropriate place for work management. Also adds a WorkManager.RunInThreadPool() class which feeds through to Util.FireAndForget. Also switches around the name and obj arguments to the new RunInThread() and RunJob() methods so that the callback obj comes after the callback as seen in the SDK and elsewhere
Diffstat (limited to 'OpenSim/Framework/Monitoring')
-rw-r--r--OpenSim/Framework/Monitoring/JobEngine.cs2
-rw-r--r--OpenSim/Framework/Monitoring/Watchdog.cs149
-rw-r--r--OpenSim/Framework/Monitoring/WorkManager.cs212
3 files changed, 224 insertions, 139 deletions
diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs
index 4a46328..5925867 100644
--- a/OpenSim/Framework/Monitoring/JobEngine.cs
+++ b/OpenSim/Framework/Monitoring/JobEngine.cs
@@ -125,7 +125,7 @@ namespace OpenSim.Framework.Monitoring
125 125
126 StatsManager.RegisterStat(m_requestsWaitingStat); 126 StatsManager.RegisterStat(m_requestsWaitingStat);
127 127
128 Watchdog.StartThread( 128 WorkManager.StartThread(
129 ProcessRequests, 129 ProcessRequests,
130 "JobEngineThread", 130 "JobEngineThread",
131 ThreadPriority.Normal, 131 ThreadPriority.Normal,
diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs
index 0feec7c..a644fa5 100644
--- a/OpenSim/Framework/Monitoring/Watchdog.cs
+++ b/OpenSim/Framework/Monitoring/Watchdog.cs
@@ -38,6 +38,8 @@ namespace OpenSim.Framework.Monitoring
38 /// </summary> 38 /// </summary>
39 public static class Watchdog 39 public static class Watchdog
40 { 40 {
41 private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
42
41 /// <summary>Timer interval in milliseconds for the watchdog timer</summary> 43 /// <summary>Timer interval in milliseconds for the watchdog timer</summary>
42 public const double WATCHDOG_INTERVAL_MS = 2500.0d; 44 public const double WATCHDOG_INTERVAL_MS = 2500.0d;
43 45
@@ -133,8 +135,6 @@ namespace OpenSim.Framework.Monitoring
133 /// /summary> 135 /// /summary>
134 public static event Action<ThreadWatchdogInfo> OnWatchdogTimeout; 136 public static event Action<ThreadWatchdogInfo> OnWatchdogTimeout;
135 137
136 public static JobEngine JobEngine { get; private set; }
137
138 /// <summary> 138 /// <summary>
139 /// Is this watchdog active? 139 /// Is this watchdog active?
140 /// </summary> 140 /// </summary>
@@ -143,7 +143,7 @@ namespace OpenSim.Framework.Monitoring
143 get { return m_enabled; } 143 get { return m_enabled; }
144 set 144 set
145 { 145 {
146// m_log.DebugFormat("[MEMORY WATCHDOG]: Setting MemoryWatchdog.Enabled to {0}", value); 146 // m_log.DebugFormat("[MEMORY WATCHDOG]: Setting MemoryWatchdog.Enabled to {0}", value);
147 147
148 if (value == m_enabled) 148 if (value == m_enabled)
149 return; 149 return;
@@ -159,9 +159,8 @@ namespace OpenSim.Framework.Monitoring
159 m_watchdogTimer.Enabled = m_enabled; 159 m_watchdogTimer.Enabled = m_enabled;
160 } 160 }
161 } 161 }
162 private static bool m_enabled;
163 162
164 private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); 163 private static bool m_enabled;
165 private static Dictionary<int, ThreadWatchdogInfo> m_threads; 164 private static Dictionary<int, ThreadWatchdogInfo> m_threads;
166 private static System.Timers.Timer m_watchdogTimer; 165 private static System.Timers.Timer m_watchdogTimer;
167 166
@@ -175,7 +174,6 @@ namespace OpenSim.Framework.Monitoring
175 174
176 static Watchdog() 175 static Watchdog()
177 { 176 {
178 JobEngine = new JobEngine();
179 m_threads = new Dictionary<int, ThreadWatchdogInfo>(); 177 m_threads = new Dictionary<int, ThreadWatchdogInfo>();
180 m_watchdogTimer = new System.Timers.Timer(WATCHDOG_INTERVAL_MS); 178 m_watchdogTimer = new System.Timers.Timer(WATCHDOG_INTERVAL_MS);
181 m_watchdogTimer.AutoReset = false; 179 m_watchdogTimer.AutoReset = false;
@@ -183,94 +181,19 @@ namespace OpenSim.Framework.Monitoring
183 } 181 }
184 182
185 /// <summary> 183 /// <summary>
186 /// Start a new thread that is tracked by the watchdog timer. 184 /// Add a thread to the watchdog tracker.
187 /// </summary>
188 /// <param name="start">The method that will be executed in a new thread</param>
189 /// <param name="name">A name to give to the new thread</param>
190 /// <param name="priority">Priority to run the thread at</param>
191 /// <param name="isBackground">True to run this thread as a background thread, otherwise false</param>
192 /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
193 /// <param name="log">If true then creation of thread is logged.</param>
194 /// <returns>The newly created Thread object</returns>
195 public static Thread StartThread(
196 ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, bool log = true)
197 {
198 return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS, log);
199 }
200
201 /// <summary>
202 /// Start a new thread that is tracked by the watchdog
203 /// </summary> 185 /// </summary>
204 /// <param name="start">The method that will be executed in a new thread</param> 186 /// <param name="info">Information about the thread.</info>
205 /// <param name="name">A name to give to the new thread</param> 187 /// <param name="info">Name of the thread.</info>
206 /// <param name="priority">Priority to run the thread at</param>
207 /// <param name="isBackground">True to run this thread as a background
208 /// thread, otherwise false</param>
209 /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
210 /// <param name="alarmMethod">
211 /// Alarm method to call if alarmIfTimeout is true and there is a timeout.
212 /// Normally, this will just return some useful debugging information.
213 /// </param>
214 /// <param name="timeout">Number of milliseconds to wait until we issue a warning about timeout.</param>
215 /// <param name="log">If true then creation of thread is logged.</param> 188 /// <param name="log">If true then creation of thread is logged.</param>
216 /// <returns>The newly created Thread object</returns> 189 public static void AddThread(ThreadWatchdogInfo info, string name, bool log = true)
217 public static Thread StartThread(
218 ThreadStart start, string name, ThreadPriority priority, bool isBackground,
219 bool alarmIfTimeout, Func<string> alarmMethod, int timeout, bool log = true)
220 { 190 {
221 Thread thread = new Thread(start);
222 thread.Priority = priority;
223 thread.IsBackground = isBackground;
224
225 ThreadWatchdogInfo twi
226 = new ThreadWatchdogInfo(thread, timeout, name)
227 { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod };
228
229 if (log) 191 if (log)
230 m_log.DebugFormat( 192 m_log.DebugFormat(
231 "[WATCHDOG]: Started tracking thread {0}, ID {1}", name, twi.Thread.ManagedThreadId); 193 "[WATCHDOG]: Started tracking thread {0}, ID {1}", name, info.Thread.ManagedThreadId);
232 194
233 lock (m_threads) 195 lock (m_threads)
234 m_threads.Add(twi.Thread.ManagedThreadId, twi); 196 m_threads.Add(info.Thread.ManagedThreadId, info);
235
236 thread.Start();
237 thread.Name = name;
238
239
240 return thread;
241 }
242
243 /// <summary>
244 /// Run the callback in a new thread immediately. If the thread exits with an exception log it but do
245 /// not propogate it.
246 /// </summary>
247 /// <param name="callback">Code for the thread to execute.</param>
248 /// <param name="name">Name of the thread</param>
249 /// <param name="obj">Object to pass to the thread.</param>
250 public static void RunInThread(WaitCallback callback, string name, object obj, bool log = false)
251 {
252 if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
253 {
254 Culture.SetCurrentCulture();
255 callback(obj);
256 return;
257 }
258
259 ThreadStart ts = new ThreadStart(delegate()
260 {
261 try
262 {
263 Culture.SetCurrentCulture();
264 callback(obj);
265 Watchdog.RemoveThread(log:false);
266 }
267 catch (Exception e)
268 {
269 m_log.Error(string.Format("[WATCHDOG]: Exception in thread {0}.", name), e);
270 }
271 });
272
273 StartThread(ts, name, ThreadPriority.Normal, true, false, log:log);
274 } 197 }
275 198
276 /// <summary> 199 /// <summary>
@@ -361,7 +284,7 @@ namespace OpenSim.Framework.Monitoring
361 } 284 }
362 catch { } 285 catch { }
363 } 286 }
364 287
365 /// <summary> 288 /// <summary>
366 /// Get currently watched threads for diagnostic purposes 289 /// Get currently watched threads for diagnostic purposes
367 /// </summary> 290 /// </summary>
@@ -453,55 +376,5 @@ namespace OpenSim.Framework.Monitoring
453 376
454 m_watchdogTimer.Start(); 377 m_watchdogTimer.Start();
455 } 378 }
456
457 /// <summary>
458 /// Run a job.
459 /// </summary>
460 /// <remarks>
461 /// This differs from direct scheduling (e.g. Util.FireAndForget) in that a job can be run in the job
462 /// engine if it is running, where all jobs are currently performed in sequence on a single thread. This is
463 /// to prevent observed overload and server freeze problems when there are hundreds of connections which all attempt to
464 /// perform work at once (e.g. in conference situations). With lower numbers of connections, the small
465 /// delay in performing jobs in sequence rather than concurrently has not been notiecable in testing, though a future more
466 /// sophisticated implementation could perform jobs concurrently when the server is under low load.
467 ///
468 /// However, be advised that some callers of this function rely on all jobs being performed in sequence if any
469 /// jobs are performed in sequence (i.e. if jobengine is active or not). Therefore, expanding the jobengine
470 /// beyond a single thread will require considerable thought.
471 ///
472 /// Also, any jobs submitted must be guaranteed to complete within a reasonable timeframe (e.g. they cannot
473 /// incorporate a network delay with a long timeout). At the moment, work that could suffer such issues
474 /// should still be run directly with RunInThread(), Util.FireAndForget(), etc. This is another area where
475 /// the job engine could be improved and so CPU utilization improved by better management of concurrency within
476 /// OpenSimulator.
477 /// </remarks>
478 /// <param name="jobType">General classification for the job (e.g. "RezAttachments").</param>
479 /// <param name="callback">Callback for job.</param>
480 /// <param name="name">Specific name of job (e.g. "RezAttachments for Joe Bloggs"</param>
481 /// <param name="obj">Object to pass to callback when run</param>
482 /// <param name="canRunInThisThread">If set to true then the job may be run in ths calling thread.</param>
483 /// <param name="mustNotTimeout">If the true then the job must never timeout.</param>
484 /// <param name="log">If set to true then extra logging is performed.</param>
485 public static void RunJob(
486 string jobType, WaitCallback callback, string name, object obj,
487 bool canRunInThisThread = false, bool mustNotTimeout = false,
488 bool log = false)
489 {
490 if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
491 {
492 Culture.SetCurrentCulture();
493 callback(obj);
494 return;
495 }
496
497 if (JobEngine.IsRunning)
498 JobEngine.QueueRequest(name, callback, obj);
499 else if (canRunInThisThread)
500 callback(obj);
501 else if (mustNotTimeout)
502 RunInThread(callback, name, obj, log);
503 else
504 Util.FireAndForget(callback, obj, name);
505 }
506 } 379 }
507} \ No newline at end of file 380} \ No newline at end of file
diff --git a/OpenSim/Framework/Monitoring/WorkManager.cs b/OpenSim/Framework/Monitoring/WorkManager.cs
new file mode 100644
index 0000000..9d0eefc
--- /dev/null
+++ b/OpenSim/Framework/Monitoring/WorkManager.cs
@@ -0,0 +1,212 @@
1/*
2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the OpenSimulator Project nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28using System;
29using System.Reflection;
30using System.Threading;
31using log4net;
32
33namespace OpenSim.Framework.Monitoring
34{
35 /// <summary>
36 /// Manages various work items in the simulator.
37 /// </summary>
38 /// <remarks>
39 /// Currently, here work can be started
40 /// * As a long-running and monitored thread.
41 /// * In a thread that will never timeout but where the job is expected to eventually complete.
42 /// * In a threadpool thread that will timeout if it takes a very long time to complete (> 10 mins).
43 /// * As a job which will be run in a single-threaded job engine. Such jobs must not incorporate delays (sleeps,
44 /// network waits, etc.).
45 ///
46 /// This is an evolving approach to better manage the work that OpenSimulator is asked to do from a very diverse
47 /// range of sources (client actions, incoming network, outgoing network calls, etc.).
48 ///
49 /// Util.FireAndForget is still available to insert jobs in the threadpool, though this is equivalent to
50 /// WorkManager.RunInThreadPool().
51 /// </remarks>
52 public static class WorkManager
53 {
54 private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
55
56 public static JobEngine JobEngine { get; private set; }
57
58 static WorkManager()
59 {
60 JobEngine = new JobEngine();
61 }
62
63 /// <summary>
64 /// Start a new long-lived thread.
65 /// </summary>
66 /// <param name="start">The method that will be executed in a new thread</param>
67 /// <param name="name">A name to give to the new thread</param>
68 /// <param name="priority">Priority to run the thread at</param>
69 /// <param name="isBackground">True to run this thread as a background thread, otherwise false</param>
70 /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
71 /// <param name="log">If true then creation of thread is logged.</param>
72 /// <returns>The newly created Thread object</returns>
73 public static Thread StartThread(
74 ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, bool log = true)
75 {
76 return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, Watchdog.DEFAULT_WATCHDOG_TIMEOUT_MS, log);
77 }
78
79 /// <summary>
80 /// Start a new thread that is tracked by the watchdog
81 /// </summary>
82 /// <param name="start">The method that will be executed in a new thread</param>
83 /// <param name="name">A name to give to the new thread</param>
84 /// <param name="priority">Priority to run the thread at</param>
85 /// <param name="isBackground">True to run this thread as a background
86 /// thread, otherwise false</param>
87 /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
88 /// <param name="alarmMethod">
89 /// Alarm method to call if alarmIfTimeout is true and there is a timeout.
90 /// Normally, this will just return some useful debugging information.
91 /// </param>
92 /// <param name="timeout">Number of milliseconds to wait until we issue a warning about timeout.</param>
93 /// <param name="log">If true then creation of thread is logged.</param>
94 /// <returns>The newly created Thread object</returns>
95 public static Thread StartThread(
96 ThreadStart start, string name, ThreadPriority priority, bool isBackground,
97 bool alarmIfTimeout, Func<string> alarmMethod, int timeout, bool log = true)
98 {
99 Thread thread = new Thread(start);
100 thread.Priority = priority;
101 thread.IsBackground = isBackground;
102
103 Watchdog.ThreadWatchdogInfo twi
104 = new Watchdog.ThreadWatchdogInfo(thread, timeout, name)
105 { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod };
106
107 Watchdog.AddThread(twi, name, log:log);
108
109 thread.Start();
110 thread.Name = name;
111
112 return thread;
113 }
114
115 /// <summary>
116 /// Run the callback in a new thread immediately. If the thread exits with an exception log it but do
117 /// not propogate it.
118 /// </summary>
119 /// <param name="callback">Code for the thread to execute.</param>
120 /// <param name="obj">Object to pass to the thread.</param>
121 /// <param name="name">Name of the thread</param>
122 public static void RunInThread(WaitCallback callback, object obj, string name, bool log = false)
123 {
124 if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
125 {
126 Culture.SetCurrentCulture();
127 callback(obj);
128 return;
129 }
130
131 ThreadStart ts = new ThreadStart(delegate()
132 {
133 try
134 {
135 Culture.SetCurrentCulture();
136 callback(obj);
137 Watchdog.RemoveThread(log:false);
138 }
139 catch (Exception e)
140 {
141 m_log.Error(string.Format("[WATCHDOG]: Exception in thread {0}.", name), e);
142 }
143 });
144
145 StartThread(ts, name, ThreadPriority.Normal, true, false, log:log);
146 }
147
148 /// <summary>
149 /// Run the callback via a threadpool thread.
150 /// </summary>
151 /// <remarks>
152 /// Such jobs may run after some delay but must always complete.
153 /// </remarks>
154 /// <param name="callback"></param>
155 /// <param name="obj"></param>
156 /// <param name="name">The name of the job. This is used in monitoring and debugging.</param>
157 public static void RunInThreadPool(System.Threading.WaitCallback callback, object obj, string name)
158 {
159 Util.FireAndForget(callback, obj, name);
160 }
161
162 /// <summary>
163 /// Run a job.
164 /// </summary>
165 /// <remarks>
166 /// This differs from direct scheduling (e.g. Util.FireAndForget) in that a job can be run in the job
167 /// engine if it is running, where all jobs are currently performed in sequence on a single thread. This is
168 /// to prevent observed overload and server freeze problems when there are hundreds of connections which all attempt to
169 /// perform work at once (e.g. in conference situations). With lower numbers of connections, the small
170 /// delay in performing jobs in sequence rather than concurrently has not been notiecable in testing, though a future more
171 /// sophisticated implementation could perform jobs concurrently when the server is under low load.
172 ///
173 /// However, be advised that some callers of this function rely on all jobs being performed in sequence if any
174 /// jobs are performed in sequence (i.e. if jobengine is active or not). Therefore, expanding the jobengine
175 /// beyond a single thread will require considerable thought.
176 ///
177 /// Also, any jobs submitted must be guaranteed to complete within a reasonable timeframe (e.g. they cannot
178 /// incorporate a network delay with a long timeout). At the moment, work that could suffer such issues
179 /// should still be run directly with RunInThread(), Util.FireAndForget(), etc. This is another area where
180 /// the job engine could be improved and so CPU utilization improved by better management of concurrency within
181 /// OpenSimulator.
182 /// </remarks>
183 /// <param name="jobType">General classification for the job (e.g. "RezAttachments").</param>
184 /// <param name="callback">Callback for job.</param>
185 /// <param name="obj">Object to pass to callback when run</param>
186 /// <param name="name">Specific name of job (e.g. "RezAttachments for Joe Bloggs"</param>
187 /// <param name="canRunInThisThread">If set to true then the job may be run in ths calling thread.</param>
188 /// <param name="mustNotTimeout">If the true then the job must never timeout.</param>
189 /// <param name="log">If set to true then extra logging is performed.</param>
190 public static void RunJob(
191 string jobType, WaitCallback callback, object obj, string name,
192 bool canRunInThisThread = false, bool mustNotTimeout = false,
193 bool log = false)
194 {
195 if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
196 {
197 Culture.SetCurrentCulture();
198 callback(obj);
199 return;
200 }
201
202 if (JobEngine.IsRunning)
203 JobEngine.QueueRequest(name, callback, obj);
204 else if (canRunInThisThread)
205 callback(obj);
206 else if (mustNotTimeout)
207 RunInThread(callback, obj, name, log);
208 else
209 Util.FireAndForget(callback, obj, name);
210 }
211 }
212} \ No newline at end of file