aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/OpenSim/Framework/Monitoring/WorkManager.cs
diff options
context:
space:
mode:
Diffstat (limited to 'OpenSim/Framework/Monitoring/WorkManager.cs')
-rw-r--r--OpenSim/Framework/Monitoring/WorkManager.cs290
1 files changed, 290 insertions, 0 deletions
diff --git a/OpenSim/Framework/Monitoring/WorkManager.cs b/OpenSim/Framework/Monitoring/WorkManager.cs
new file mode 100644
index 0000000..d1a74ce
--- /dev/null
+++ b/OpenSim/Framework/Monitoring/WorkManager.cs
@@ -0,0 +1,290 @@
1/*
2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the OpenSimulator Project nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28using System;
29using System.Reflection;
30using System.Threading;
31using log4net;
32
33namespace OpenSim.Framework.Monitoring
34{
35 /// <summary>
36 /// Manages various work items in the simulator.
37 /// </summary>
38 /// <remarks>
39 /// Currently, here work can be started
40 /// * As a long-running and monitored thread.
41 /// * In a thread that will never timeout but where the job is expected to eventually complete.
42 /// * In a threadpool thread that will timeout if it takes a very long time to complete (> 10 mins).
43 /// * As a job which will be run in a single-threaded job engine. Such jobs must not incorporate delays (sleeps,
44 /// network waits, etc.).
45 ///
46 /// This is an evolving approach to better manage the work that OpenSimulator is asked to do from a very diverse
47 /// range of sources (client actions, incoming network, outgoing network calls, etc.).
48 ///
49 /// Util.FireAndForget is still available to insert jobs in the threadpool, though this is equivalent to
50 /// WorkManager.RunInThreadPool().
51 /// </remarks>
52 public static class WorkManager
53 {
54 private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
55
56 public static JobEngine JobEngine { get; private set; }
57
58 static WorkManager()
59 {
60 JobEngine = new JobEngine("Non-blocking non-critical job engine", "JOB ENGINE");
61
62 StatsManager.RegisterStat(
63 new Stat(
64 "JobsWaiting",
65 "Number of jobs waiting for processing.",
66 "",
67 "",
68 "server",
69 "jobengine",
70 StatType.Pull,
71 MeasuresOfInterest.None,
72 stat => stat.Value = JobEngine.JobsWaiting,
73 StatVerbosity.Debug));
74
75 MainConsole.Instance.Commands.AddCommand(
76 "Debug",
77 false,
78 "debug jobengine",
79 "debug jobengine <start|stop|status|log>",
80 "Start, stop, get status or set logging level of the job engine.",
81 "If stopped then all outstanding jobs are processed immediately.",
82 HandleControlCommand);
83 }
84
85 /// <summary>
86 /// Start a new long-lived thread.
87 /// </summary>
88 /// <param name="start">The method that will be executed in a new thread</param>
89 /// <param name="name">A name to give to the new thread</param>
90 /// <param name="priority">Priority to run the thread at</param>
91 /// <param name="isBackground">True to run this thread as a background thread, otherwise false</param>
92 /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
93 /// <param name="log">If true then creation of thread is logged.</param>
94 /// <returns>The newly created Thread object</returns>
95 public static Thread StartThread(
96 ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, bool log = true)
97 {
98 return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, Watchdog.DEFAULT_WATCHDOG_TIMEOUT_MS, log);
99 }
100
101 /// <summary>
102 /// Start a new thread that is tracked by the watchdog
103 /// </summary>
104 /// <param name="start">The method that will be executed in a new thread</param>
105 /// <param name="name">A name to give to the new thread</param>
106 /// <param name="priority">Priority to run the thread at</param>
107 /// <param name="isBackground">True to run this thread as a background
108 /// thread, otherwise false</param>
109 /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
110 /// <param name="alarmMethod">
111 /// Alarm method to call if alarmIfTimeout is true and there is a timeout.
112 /// Normally, this will just return some useful debugging information.
113 /// </param>
114 /// <param name="timeout">Number of milliseconds to wait until we issue a warning about timeout.</param>
115 /// <param name="log">If true then creation of thread is logged.</param>
116 /// <returns>The newly created Thread object</returns>
117 public static Thread StartThread(
118 ThreadStart start, string name, ThreadPriority priority, bool isBackground,
119 bool alarmIfTimeout, Func<string> alarmMethod, int timeout, bool log = true)
120 {
121 Thread thread = new Thread(start);
122 thread.Priority = priority;
123 thread.IsBackground = isBackground;
124
125 Watchdog.ThreadWatchdogInfo twi
126 = new Watchdog.ThreadWatchdogInfo(thread, timeout, name)
127 { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod };
128
129 Watchdog.AddThread(twi, name, log:log);
130
131 thread.Start();
132 thread.Name = name;
133
134 return thread;
135 }
136
137 /// <summary>
138 /// Run the callback in a new thread immediately. If the thread exits with an exception log it but do
139 /// not propogate it.
140 /// </summary>
141 /// <param name="callback">Code for the thread to execute.</param>
142 /// <param name="obj">Object to pass to the thread.</param>
143 /// <param name="name">Name of the thread</param>
144 public static void RunInThread(WaitCallback callback, object obj, string name, bool log = false)
145 {
146 if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
147 {
148 Culture.SetCurrentCulture();
149 callback(obj);
150 return;
151 }
152
153 ThreadStart ts = new ThreadStart(delegate()
154 {
155 try
156 {
157 Culture.SetCurrentCulture();
158 callback(obj);
159 Watchdog.RemoveThread(log:false);
160 }
161 catch (Exception e)
162 {
163 m_log.Error(string.Format("[WATCHDOG]: Exception in thread {0}.", name), e);
164 }
165 });
166
167 StartThread(ts, name, ThreadPriority.Normal, true, false, log:log);
168 }
169
170 /// <summary>
171 /// Run the callback via a threadpool thread.
172 /// </summary>
173 /// <remarks>
174 /// Such jobs may run after some delay but must always complete.
175 /// </remarks>
176 /// <param name="callback"></param>
177 /// <param name="obj"></param>
178 /// <param name="name">The name of the job. This is used in monitoring and debugging.</param>
179 public static void RunInThreadPool(System.Threading.WaitCallback callback, object obj, string name)
180 {
181 Util.FireAndForget(callback, obj, name);
182 }
183
184 /// <summary>
185 /// Run a job.
186 /// </summary>
187 /// <remarks>
188 /// This differs from direct scheduling (e.g. Util.FireAndForget) in that a job can be run in the job
189 /// engine if it is running, where all jobs are currently performed in sequence on a single thread. This is
190 /// to prevent observed overload and server freeze problems when there are hundreds of connections which all attempt to
191 /// perform work at once (e.g. in conference situations). With lower numbers of connections, the small
192 /// delay in performing jobs in sequence rather than concurrently has not been notiecable in testing, though a future more
193 /// sophisticated implementation could perform jobs concurrently when the server is under low load.
194 ///
195 /// However, be advised that some callers of this function rely on all jobs being performed in sequence if any
196 /// jobs are performed in sequence (i.e. if jobengine is active or not). Therefore, expanding the jobengine
197 /// beyond a single thread will require considerable thought.
198 ///
199 /// Also, any jobs submitted must be guaranteed to complete within a reasonable timeframe (e.g. they cannot
200 /// incorporate a network delay with a long timeout). At the moment, work that could suffer such issues
201 /// should still be run directly with RunInThread(), Util.FireAndForget(), etc. This is another area where
202 /// the job engine could be improved and so CPU utilization improved by better management of concurrency within
203 /// OpenSimulator.
204 /// </remarks>
205 /// <param name="jobType">General classification for the job (e.g. "RezAttachments").</param>
206 /// <param name="callback">Callback for job.</param>
207 /// <param name="obj">Object to pass to callback when run</param>
208 /// <param name="name">Specific name of job (e.g. "RezAttachments for Joe Bloggs"</param>
209 /// <param name="canRunInThisThread">If set to true then the job may be run in ths calling thread.</param>
210 /// <param name="mustNotTimeout">If the true then the job must never timeout.</param>
211 /// <param name="log">If set to true then extra logging is performed.</param>
212 public static void RunJob(
213 string jobType, WaitCallback callback, object obj, string name,
214 bool canRunInThisThread = false, bool mustNotTimeout = false,
215 bool log = false)
216 {
217 if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
218 {
219 Culture.SetCurrentCulture();
220 callback(obj);
221 return;
222 }
223
224 if (JobEngine.IsRunning)
225 JobEngine.QueueJob(name, () => callback(obj));
226 else if (canRunInThisThread)
227 callback(obj);
228 else if (mustNotTimeout)
229 RunInThread(callback, obj, name, log);
230 else
231 Util.FireAndForget(callback, obj, name);
232 }
233
234 private static void HandleControlCommand(string module, string[] args)
235 {
236 // if (SceneManager.Instance.CurrentScene != null && SceneManager.Instance.CurrentScene != m_udpServer.Scene)
237 // return;
238
239 if (args.Length < 3)
240 {
241 MainConsole.Instance.Output("Usage: debug jobengine <stop|start|status|log>");
242 return;
243 }
244
245 string subCommand = args[2];
246
247 if (subCommand == "stop")
248 {
249 JobEngine.Stop();
250 MainConsole.Instance.OutputFormat("Stopped job engine.");
251 }
252 else if (subCommand == "start")
253 {
254 JobEngine.Start();
255 MainConsole.Instance.OutputFormat("Started job engine.");
256 }
257 else if (subCommand == "status")
258 {
259 MainConsole.Instance.OutputFormat("Job engine running: {0}", JobEngine.IsRunning);
260
261 JobEngine.Job job = JobEngine.CurrentJob;
262 MainConsole.Instance.OutputFormat("Current job {0}", job != null ? job.Name : "none");
263
264 MainConsole.Instance.OutputFormat(
265 "Jobs waiting: {0}", JobEngine.IsRunning ? JobEngine.JobsWaiting.ToString() : "n/a");
266 MainConsole.Instance.OutputFormat("Log Level: {0}", JobEngine.LogLevel);
267 }
268 else if (subCommand == "log")
269 {
270 if (args.Length < 4)
271 {
272 MainConsole.Instance.Output("Usage: debug jobengine log <level>");
273 return;
274 }
275
276 // int logLevel;
277 int logLevel = int.Parse(args[3]);
278 // if (ConsoleUtil.TryParseConsoleInt(MainConsole.Instance, args[4], out logLevel))
279 // {
280 JobEngine.LogLevel = logLevel;
281 MainConsole.Instance.OutputFormat("Set debug log level to {0}", JobEngine.LogLevel);
282 // }
283 }
284 else
285 {
286 MainConsole.Instance.OutputFormat("Unrecognized job engine subcommand {0}", subCommand);
287 }
288 }
289 }
290} \ No newline at end of file