1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
|
/*
* Copyright (c) Contributors, http://opensimulator.org/
* See CONTRIBUTORS.TXT for a full list of copyright holders.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of the OpenSimulator Project nor the
* names of its contributors may be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
using System;
using System.Reflection;
using System.Threading;
using log4net;
namespace OpenSim.Framework.Monitoring
{
/// <summary>
/// Manages various work items in the simulator.
/// </summary>
/// <remarks>
/// Currently, here work can be started
/// * As a long-running and monitored thread.
/// * In a thread that will never timeout but where the job is expected to eventually complete.
/// * In a threadpool thread that will timeout if it takes a very long time to complete (> 10 mins).
/// * As a job which will be run in a single-threaded job engine. Such jobs must not incorporate delays (sleeps,
/// network waits, etc.).
///
/// This is an evolving approach to better manage the work that OpenSimulator is asked to do from a very diverse
/// range of sources (client actions, incoming network, outgoing network calls, etc.).
///
/// Util.FireAndForget is still available to insert jobs in the threadpool, though this is equivalent to
/// WorkManager.RunInThreadPool().
/// </remarks>
public static class WorkManager
{
private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
public static JobEngine JobEngine { get; private set; }
static WorkManager()
{
JobEngine = new JobEngine();
}
/// <summary>
/// Start a new long-lived thread.
/// </summary>
/// <param name="start">The method that will be executed in a new thread</param>
/// <param name="name">A name to give to the new thread</param>
/// <param name="priority">Priority to run the thread at</param>
/// <param name="isBackground">True to run this thread as a background thread, otherwise false</param>
/// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
/// <param name="log">If true then creation of thread is logged.</param>
/// <returns>The newly created Thread object</returns>
public static Thread StartThread(
ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, bool log = true)
{
return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, Watchdog.DEFAULT_WATCHDOG_TIMEOUT_MS, log);
}
/// <summary>
/// Start a new thread that is tracked by the watchdog
/// </summary>
/// <param name="start">The method that will be executed in a new thread</param>
/// <param name="name">A name to give to the new thread</param>
/// <param name="priority">Priority to run the thread at</param>
/// <param name="isBackground">True to run this thread as a background
/// thread, otherwise false</param>
/// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
/// <param name="alarmMethod">
/// Alarm method to call if alarmIfTimeout is true and there is a timeout.
/// Normally, this will just return some useful debugging information.
/// </param>
/// <param name="timeout">Number of milliseconds to wait until we issue a warning about timeout.</param>
/// <param name="log">If true then creation of thread is logged.</param>
/// <returns>The newly created Thread object</returns>
public static Thread StartThread(
ThreadStart start, string name, ThreadPriority priority, bool isBackground,
bool alarmIfTimeout, Func<string> alarmMethod, int timeout, bool log = true)
{
Thread thread = new Thread(start);
thread.Priority = priority;
thread.IsBackground = isBackground;
Watchdog.ThreadWatchdogInfo twi
= new Watchdog.ThreadWatchdogInfo(thread, timeout, name)
{ AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod };
Watchdog.AddThread(twi, name, log:log);
thread.Start();
thread.Name = name;
return thread;
}
/// <summary>
/// Run the callback in a new thread immediately. If the thread exits with an exception log it but do
/// not propogate it.
/// </summary>
/// <param name="callback">Code for the thread to execute.</param>
/// <param name="obj">Object to pass to the thread.</param>
/// <param name="name">Name of the thread</param>
public static void RunInThread(WaitCallback callback, object obj, string name, bool log = false)
{
if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
{
Culture.SetCurrentCulture();
callback(obj);
return;
}
ThreadStart ts = new ThreadStart(delegate()
{
try
{
Culture.SetCurrentCulture();
callback(obj);
Watchdog.RemoveThread(log:false);
}
catch (Exception e)
{
m_log.Error(string.Format("[WATCHDOG]: Exception in thread {0}.", name), e);
}
});
StartThread(ts, name, ThreadPriority.Normal, true, false, log:log);
}
/// <summary>
/// Run the callback via a threadpool thread.
/// </summary>
/// <remarks>
/// Such jobs may run after some delay but must always complete.
/// </remarks>
/// <param name="callback"></param>
/// <param name="obj"></param>
/// <param name="name">The name of the job. This is used in monitoring and debugging.</param>
public static void RunInThreadPool(System.Threading.WaitCallback callback, object obj, string name)
{
Util.FireAndForget(callback, obj, name);
}
/// <summary>
/// Run a job.
/// </summary>
/// <remarks>
/// This differs from direct scheduling (e.g. Util.FireAndForget) in that a job can be run in the job
/// engine if it is running, where all jobs are currently performed in sequence on a single thread. This is
/// to prevent observed overload and server freeze problems when there are hundreds of connections which all attempt to
/// perform work at once (e.g. in conference situations). With lower numbers of connections, the small
/// delay in performing jobs in sequence rather than concurrently has not been notiecable in testing, though a future more
/// sophisticated implementation could perform jobs concurrently when the server is under low load.
///
/// However, be advised that some callers of this function rely on all jobs being performed in sequence if any
/// jobs are performed in sequence (i.e. if jobengine is active or not). Therefore, expanding the jobengine
/// beyond a single thread will require considerable thought.
///
/// Also, any jobs submitted must be guaranteed to complete within a reasonable timeframe (e.g. they cannot
/// incorporate a network delay with a long timeout). At the moment, work that could suffer such issues
/// should still be run directly with RunInThread(), Util.FireAndForget(), etc. This is another area where
/// the job engine could be improved and so CPU utilization improved by better management of concurrency within
/// OpenSimulator.
/// </remarks>
/// <param name="jobType">General classification for the job (e.g. "RezAttachments").</param>
/// <param name="callback">Callback for job.</param>
/// <param name="obj">Object to pass to callback when run</param>
/// <param name="name">Specific name of job (e.g. "RezAttachments for Joe Bloggs"</param>
/// <param name="canRunInThisThread">If set to true then the job may be run in ths calling thread.</param>
/// <param name="mustNotTimeout">If the true then the job must never timeout.</param>
/// <param name="log">If set to true then extra logging is performed.</param>
public static void RunJob(
string jobType, WaitCallback callback, object obj, string name,
bool canRunInThisThread = false, bool mustNotTimeout = false,
bool log = false)
{
if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
{
Culture.SetCurrentCulture();
callback(obj);
return;
}
if (JobEngine.IsRunning)
JobEngine.QueueRequest(name, callback, obj);
else if (canRunInThisThread)
callback(obj);
else if (mustNotTimeout)
RunInThread(callback, obj, name, log);
else
Util.FireAndForget(callback, obj, name);
}
}
}
|
