refactor: Move methods to start a monitored thread, start work in its own thread and run work in the jobengine from Watchdog to a WorkManager class.

This is to achieve a clean separation of concerns - the watchdog is an inappropriate place for work management.
Also adds a WorkManager.RunInThreadPool() class which feeds through to Util.FireAndForget.
Also switches around the name and obj arguments to the new RunInThread() and RunJob() methods so that the callback obj comes after the callback as seen in the SDK and elsewhere

27 files changed, 270 insertions, 185 deletions

diff --git a/OpenSim/Addons/Groups/Hypergrid/GroupsServiceHGConnectorModule.cs b/OpenSim/Addons/Groups/Hypergrid/GroupsServiceHGConnectorModule.cs
index 9fbc1b3..7d57de1 100644
--- a/OpenSim/Addons/Groups/Hypergrid/GroupsServiceHGConnectorModule.cs
+++ b/OpenSim/Addons/Groups/Hypergrid/GroupsServiceHGConnectorModule.cs
@@ -561,7 +561,7 @@ namespace OpenSim.Groups
 
                     // so we have the list of urls to send the notice to
                     // this may take a long time...
-                    Watchdog.RunInThread(delegate
+                    WorkManager.RunInThread(delegate
                     {
                         foreach (string u in urls)
                         {
@@ -572,7 +572,7 @@ namespace OpenSim.Groups
                                     hasAttachment, attType, attName, attItemID, AgentUUIForOutside(attOwnerID));
                             }
                         }
-                    }, string.Format("AddGroupNotice (agent {0}, group {1})", RequestingAgentID, groupID) , null);
+                    }, null, string.Format("AddGroupNotice (agent {0}, group {1})", RequestingAgentID, groupID));
 
                     return true;
                 }
diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs
index 4a46328..5925867 100644
--- a/OpenSim/Framework/Monitoring/JobEngine.cs
+++ b/OpenSim/Framework/Monitoring/JobEngine.cs
@@ -125,7 +125,7 @@ namespace OpenSim.Framework.Monitoring
 
                 StatsManager.RegisterStat(m_requestsWaitingStat);
 
-                Watchdog.StartThread(
+                WorkManager.StartThread(
                     ProcessRequests,
                     "JobEngineThread",
                     ThreadPriority.Normal,
diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs
index 0feec7c..a644fa5 100644
--- a/OpenSim/Framework/Monitoring/Watchdog.cs
+++ b/OpenSim/Framework/Monitoring/Watchdog.cs
@@ -38,6 +38,8 @@ namespace OpenSim.Framework.Monitoring
     /// </summary>
     public static class Watchdog
     {
+        private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
+
         /// <summary>Timer interval in milliseconds for the watchdog timer</summary>
         public const double WATCHDOG_INTERVAL_MS = 2500.0d;
 
@@ -133,8 +135,6 @@ namespace OpenSim.Framework.Monitoring
         /// /summary>
         public static event Action<ThreadWatchdogInfo> OnWatchdogTimeout;
 
-        public static JobEngine JobEngine { get; private set; }
-
         /// <summary>
         /// Is this watchdog active?
         /// </summary>
@@ -143,7 +143,7 @@ namespace OpenSim.Framework.Monitoring
             get { return m_enabled; }
             set
             {
-//                m_log.DebugFormat("[MEMORY WATCHDOG]: Setting MemoryWatchdog.Enabled to {0}", value);
+                //                m_log.DebugFormat("[MEMORY WATCHDOG]: Setting MemoryWatchdog.Enabled to {0}", value);
 
                 if (value == m_enabled)
                     return;
@@ -159,9 +159,8 @@ namespace OpenSim.Framework.Monitoring
                 m_watchdogTimer.Enabled = m_enabled;
             }
         }
-        private static bool m_enabled;
 
-        private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
+        private static bool m_enabled;
         private static Dictionary<int, ThreadWatchdogInfo> m_threads;
         private static System.Timers.Timer m_watchdogTimer;
 
@@ -175,7 +174,6 @@ namespace OpenSim.Framework.Monitoring
 
         static Watchdog()
         {
-            JobEngine = new JobEngine();
             m_threads = new Dictionary<int, ThreadWatchdogInfo>();
             m_watchdogTimer = new System.Timers.Timer(WATCHDOG_INTERVAL_MS);
             m_watchdogTimer.AutoReset = false;
@@ -183,94 +181,19 @@ namespace OpenSim.Framework.Monitoring
         }
 
         /// <summary>
-        /// Start a new thread that is tracked by the watchdog timer.
-        /// </summary>
-        /// <param name="start">The method that will be executed in a new thread</param>
-        /// <param name="name">A name to give to the new thread</param>
-        /// <param name="priority">Priority to run the thread at</param>
-        /// <param name="isBackground">True to run this thread as a background thread, otherwise false</param>
-        /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
-        /// <param name="log">If true then creation of thread is logged.</param>
-        /// <returns>The newly created Thread object</returns>
-        public static Thread StartThread(
-            ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, bool log = true)
-        {
-            return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS, log);
-        }
-
-        /// <summary>
-        /// Start a new thread that is tracked by the watchdog
+        /// Add a thread to the watchdog tracker.
         /// </summary>
-        /// <param name="start">The method that will be executed in a new thread</param>
-        /// <param name="name">A name to give to the new thread</param>
-        /// <param name="priority">Priority to run the thread at</param>
-        /// <param name="isBackground">True to run this thread as a background
-        /// thread, otherwise false</param>
-        /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
-        /// <param name="alarmMethod">
-        /// Alarm method to call if alarmIfTimeout is true and there is a timeout.
-        /// Normally, this will just return some useful debugging information.
-        /// </param>
-        /// <param name="timeout">Number of milliseconds to wait until we issue a warning about timeout.</param>
+        /// <param name="info">Information about the thread.</info>
+        /// <param name="info">Name of the thread.</info>
         /// <param name="log">If true then creation of thread is logged.</param>
-        /// <returns>The newly created Thread object</returns>
-        public static Thread StartThread(
-            ThreadStart start, string name, ThreadPriority priority, bool isBackground,
-            bool alarmIfTimeout, Func<string> alarmMethod, int timeout, bool log = true)
+        public static void AddThread(ThreadWatchdogInfo info, string name, bool log = true)
         {
-            Thread thread = new Thread(start);
-            thread.Priority = priority;
-            thread.IsBackground = isBackground;
-            
-            ThreadWatchdogInfo twi
-                = new ThreadWatchdogInfo(thread, timeout, name)
-                    { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod };
-
             if (log)
                 m_log.DebugFormat(
-                    "[WATCHDOG]: Started tracking thread {0}, ID {1}", name, twi.Thread.ManagedThreadId);
+                    "[WATCHDOG]: Started tracking thread {0}, ID {1}", name, info.Thread.ManagedThreadId);
 
             lock (m_threads)
-                m_threads.Add(twi.Thread.ManagedThreadId, twi);
-            
-            thread.Start();
-            thread.Name = name;
-
-
-            return thread;
-        }
-
-        /// <summary>
-        /// Run the callback in a new thread immediately.  If the thread exits with an exception log it but do
-        /// not propogate it.
-        /// </summary>
-        /// <param name="callback">Code for the thread to execute.</param>
-        /// <param name="name">Name of the thread</param>
-        /// <param name="obj">Object to pass to the thread.</param>
-        public static void RunInThread(WaitCallback callback, string name, object obj, bool log = false)
-        {
-            if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)           
-            {
-                Culture.SetCurrentCulture();
-                callback(obj);
-                return;
-            }
-
-            ThreadStart ts = new ThreadStart(delegate()
-            {
-                try
-                {
-                    Culture.SetCurrentCulture();
-                    callback(obj);
-                    Watchdog.RemoveThread(log:false);
-                }
-                catch (Exception e)
-                {
-                    m_log.Error(string.Format("[WATCHDOG]: Exception in thread {0}.", name), e);
-                }
-            });
-
-            StartThread(ts, name, ThreadPriority.Normal, true, false, log:log);
+                m_threads.Add(info.Thread.ManagedThreadId, info);
         }
 
         /// <summary>
@@ -361,7 +284,7 @@ namespace OpenSim.Framework.Monitoring
             }
             catch { }
         }
-        
+
         /// <summary>
         /// Get currently watched threads for diagnostic purposes
         /// </summary>
@@ -453,55 +376,5 @@ namespace OpenSim.Framework.Monitoring
 
             m_watchdogTimer.Start();
         }
-
-        /// <summary>
-        /// Run a job.
-        /// </summary>
-        /// <remarks>
-        /// This differs from direct scheduling (e.g. Util.FireAndForget) in that a job can be run in the job
-        /// engine if it is running, where all jobs are currently performed in sequence on a single thread.  This is
-        /// to prevent observed overload and server freeze problems when there are hundreds of connections which all attempt to 
-        /// perform work at once (e.g. in conference situations).  With lower numbers of connections, the small
-        /// delay in performing jobs in sequence rather than concurrently has not been notiecable in testing, though a future more 
-        /// sophisticated implementation could perform jobs concurrently when the server is under low load.
-        /// 
-        /// However, be advised that some callers of this function rely on all jobs being performed in sequence if any
-        /// jobs are performed in sequence (i.e. if jobengine is active or not).  Therefore, expanding the jobengine
-        /// beyond a single thread will require considerable thought.
-        /// 
-        /// Also, any jobs submitted must be guaranteed to complete within a reasonable timeframe (e.g. they cannot
-        /// incorporate a network delay with a long timeout).  At the moment, work that could suffer such issues 
-        /// should still be run directly with RunInThread(), Util.FireAndForget(), etc.  This is another area where
-        /// the job engine could be improved and so CPU utilization improved by better management of concurrency within
-        /// OpenSimulator.
-        /// </remarks>
-        /// <param name="jobType">General classification for the job (e.g. "RezAttachments").</param>
-        /// <param name="callback">Callback for job.</param>
-        /// <param name="name">Specific name of job (e.g. "RezAttachments for Joe Bloggs"</param>
-        /// <param name="obj">Object to pass to callback when run</param>
-        /// <param name="canRunInThisThread">If set to true then the job may be run in ths calling thread.</param>
-        /// <param name="mustNotTimeout">If the true then the job must never timeout.</param>
-        /// <param name="log">If set to true then extra logging is performed.</param>
-        public static void RunJob(
-            string jobType, WaitCallback callback, string name, object obj, 
-            bool canRunInThisThread = false, bool mustNotTimeout = false, 
-            bool log = false)
-        {
-            if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)           
-            {
-                Culture.SetCurrentCulture();
-                callback(obj);
-                return;
-            }
-
-            if (JobEngine.IsRunning)
-                JobEngine.QueueRequest(name, callback, obj);
-            else if (canRunInThisThread)
-                callback(obj);
-            else if (mustNotTimeout)
-                RunInThread(callback, name, obj, log);
-            else
-                Util.FireAndForget(callback, obj, name);
-        }
     }
 }
\ No newline at end of file
diff --git a/OpenSim/Framework/Monitoring/WorkManager.cs b/OpenSim/Framework/Monitoring/WorkManager.cs
new file mode 100644
index 0000000..9d0eefc
--- /dev/null
+++ b/OpenSim/Framework/Monitoring/WorkManager.cs
@@ -0,0 +1,212 @@
+/*
+ * Copyright (c) Contributors, http://opensimulator.org/
+ * See CONTRIBUTORS.TXT for a full list of copyright holders.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the OpenSimulator Project nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+using System;
+using System.Reflection;
+using System.Threading;
+using log4net;
+
+namespace OpenSim.Framework.Monitoring
+{
+    /// <summary>
+    /// Manages various work items in the simulator.
+    /// </summary>
+    /// <remarks>
+    /// Currently, here work can be started 
+    ///  * As a long-running and monitored thread.
+    ///  * In a thread that will never timeout but where the job is expected to eventually complete.
+    ///  * In a threadpool thread that will timeout if it takes a very long time to complete (> 10 mins).
+    ///  * As a job which will be run in a single-threaded job engine.  Such jobs must not incorporate delays (sleeps,
+    /// network waits, etc.).
+    /// 
+    /// This is an evolving approach to better manage the work that OpenSimulator is asked to do from a very diverse
+    /// range of sources (client actions, incoming network, outgoing network calls, etc.).
+    /// 
+    /// Util.FireAndForget is still available to insert jobs in the threadpool, though this is equivalent to
+    /// WorkManager.RunInThreadPool().
+    /// </remarks>
+    public static class WorkManager
+    {
+        private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
+
+        public static JobEngine JobEngine { get; private set; }
+
+        static WorkManager()
+        {
+            JobEngine = new JobEngine();
+        }
+
+        /// <summary>
+        /// Start a new long-lived thread.
+        /// </summary>
+        /// <param name="start">The method that will be executed in a new thread</param>
+        /// <param name="name">A name to give to the new thread</param>
+        /// <param name="priority">Priority to run the thread at</param>
+        /// <param name="isBackground">True to run this thread as a background thread, otherwise false</param>
+        /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
+        /// <param name="log">If true then creation of thread is logged.</param>
+        /// <returns>The newly created Thread object</returns>
+        public static Thread StartThread(
+            ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, bool log = true)
+        {
+            return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, Watchdog.DEFAULT_WATCHDOG_TIMEOUT_MS, log);
+        }
+
+        /// <summary>
+        /// Start a new thread that is tracked by the watchdog
+        /// </summary>
+        /// <param name="start">The method that will be executed in a new thread</param>
+        /// <param name="name">A name to give to the new thread</param>
+        /// <param name="priority">Priority to run the thread at</param>
+        /// <param name="isBackground">True to run this thread as a background
+        /// thread, otherwise false</param>
+        /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
+        /// <param name="alarmMethod">
+        /// Alarm method to call if alarmIfTimeout is true and there is a timeout.
+        /// Normally, this will just return some useful debugging information.
+        /// </param>
+        /// <param name="timeout">Number of milliseconds to wait until we issue a warning about timeout.</param>
+        /// <param name="log">If true then creation of thread is logged.</param>
+        /// <returns>The newly created Thread object</returns>
+        public static Thread StartThread(
+            ThreadStart start, string name, ThreadPriority priority, bool isBackground,
+            bool alarmIfTimeout, Func<string> alarmMethod, int timeout, bool log = true)
+        {
+            Thread thread = new Thread(start);
+            thread.Priority = priority;
+            thread.IsBackground = isBackground;
+
+            Watchdog.ThreadWatchdogInfo twi
+                = new Watchdog.ThreadWatchdogInfo(thread, timeout, name)
+            { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod };
+
+            Watchdog.AddThread(twi, name, log:log);
+
+            thread.Start();
+            thread.Name = name;
+
+            return thread;
+        }
+
+        /// <summary>
+        /// Run the callback in a new thread immediately.  If the thread exits with an exception log it but do
+        /// not propogate it.
+        /// </summary>
+        /// <param name="callback">Code for the thread to execute.</param>
+        /// <param name="obj">Object to pass to the thread.</param>
+        /// <param name="name">Name of the thread</param>
+        public static void RunInThread(WaitCallback callback, object obj, string name, bool log = false)
+        {
+            if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)           
+            {
+                Culture.SetCurrentCulture();
+                callback(obj);
+                return;
+            }
+
+            ThreadStart ts = new ThreadStart(delegate()
+            {
+                try
+                {
+                    Culture.SetCurrentCulture();
+                    callback(obj);
+                    Watchdog.RemoveThread(log:false);
+                }
+                catch (Exception e)
+                {
+                    m_log.Error(string.Format("[WATCHDOG]: Exception in thread {0}.", name), e);
+                }
+            });
+
+            StartThread(ts, name, ThreadPriority.Normal, true, false, log:log);
+        }
+
+        /// <summary>
+        /// Run the callback via a threadpool thread. 
+        /// </summary>
+        /// <remarks>
+        /// Such jobs may run after some delay but must always complete.
+        /// </remarks>
+        /// <param name="callback"></param>
+        /// <param name="obj"></param>
+        /// <param name="name">The name of the job.  This is used in monitoring and debugging.</param>
+        public static void RunInThreadPool(System.Threading.WaitCallback callback, object obj, string name)
+        {
+            Util.FireAndForget(callback, obj, name);
+        }
+
+        /// <summary>
+        /// Run a job.
+        /// </summary>
+        /// <remarks>
+        /// This differs from direct scheduling (e.g. Util.FireAndForget) in that a job can be run in the job
+        /// engine if it is running, where all jobs are currently performed in sequence on a single thread.  This is
+        /// to prevent observed overload and server freeze problems when there are hundreds of connections which all attempt to 
+        /// perform work at once (e.g. in conference situations).  With lower numbers of connections, the small
+        /// delay in performing jobs in sequence rather than concurrently has not been notiecable in testing, though a future more 
+        /// sophisticated implementation could perform jobs concurrently when the server is under low load.
+        /// 
+        /// However, be advised that some callers of this function rely on all jobs being performed in sequence if any
+        /// jobs are performed in sequence (i.e. if jobengine is active or not).  Therefore, expanding the jobengine
+        /// beyond a single thread will require considerable thought.
+        /// 
+        /// Also, any jobs submitted must be guaranteed to complete within a reasonable timeframe (e.g. they cannot
+        /// incorporate a network delay with a long timeout).  At the moment, work that could suffer such issues 
+        /// should still be run directly with RunInThread(), Util.FireAndForget(), etc.  This is another area where
+        /// the job engine could be improved and so CPU utilization improved by better management of concurrency within
+        /// OpenSimulator.
+        /// </remarks>
+        /// <param name="jobType">General classification for the job (e.g. "RezAttachments").</param>
+        /// <param name="callback">Callback for job.</param>
+        /// <param name="obj">Object to pass to callback when run</param>
+        /// <param name="name">Specific name of job (e.g. "RezAttachments for Joe Bloggs"</param>
+        /// <param name="canRunInThisThread">If set to true then the job may be run in ths calling thread.</param>
+        /// <param name="mustNotTimeout">If the true then the job must never timeout.</param>
+        /// <param name="log">If set to true then extra logging is performed.</param>
+        public static void RunJob(
+            string jobType, WaitCallback callback, object obj, string name,
+            bool canRunInThisThread = false, bool mustNotTimeout = false, 
+            bool log = false)
+        {
+            if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)           
+            {
+                Culture.SetCurrentCulture();
+                callback(obj);
+                return;
+            }
+
+            if (JobEngine.IsRunning)
+                JobEngine.QueueRequest(name, callback, obj);
+            else if (canRunInThisThread)
+                callback(obj);
+            else if (mustNotTimeout)
+                RunInThread(callback, obj, name, log);
+            else
+                Util.FireAndForget(callback, obj, name);
+        }
+    }
+}
\ No newline at end of file
diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs
index 4561d23..28bba70 100644
--- a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs
+++ b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs
@@ -120,7 +120,7 @@ namespace OpenSim.Framework.Servers.HttpServer
                 for (uint i = 0; i < m_WorkerThreadCount; i++)
                 {
                     m_workerThreads[i]
-                        = Watchdog.StartThread(
+                        = WorkManager.StartThread(
                             PoolWorkerJob,
                             string.Format("PollServiceWorkerThread{0}:{1}", i, m_server.Port),
                             ThreadPriority.Normal,
@@ -130,7 +130,7 @@ namespace OpenSim.Framework.Servers.HttpServer
                             int.MaxValue);
                 }
 
-                Watchdog.StartThread(
+                WorkManager.StartThread(
                     this.CheckLongPollThreads,
                     string.Format("LongPollServiceWatcherThread:{0}", m_server.Port),
                     ThreadPriority.Normal,
diff --git a/OpenSim/Region/Application/OpenSimBase.cs b/OpenSim/Region/Application/OpenSimBase.cs
index 0f61faf..3be411a 100644
--- a/OpenSim/Region/Application/OpenSimBase.cs
+++ b/OpenSim/Region/Application/OpenSimBase.cs
@@ -339,7 +339,7 @@ namespace OpenSim
 
             IConfig startupConfig = Config.Configs["Startup"];
             if (startupConfig == null || startupConfig.GetBoolean("JobEngineEnabled", true))
-                Watchdog.JobEngine.Start();
+                WorkManager.JobEngine.Start();
 
             m_httpServerPort = m_networkServersInfo.HttpListenerPort;
             SceneManager.OnRestartSim += HandleRestartRegion;
diff --git a/OpenSim/Region/ClientStack/Linden/Caps/WebFetchInvDescModule.cs b/OpenSim/Region/ClientStack/Linden/Caps/WebFetchInvDescModule.cs
index 87192a0..84ca4bb 100644
--- a/OpenSim/Region/ClientStack/Linden/Caps/WebFetchInvDescModule.cs
+++ b/OpenSim/Region/ClientStack/Linden/Caps/WebFetchInvDescModule.cs
@@ -207,7 +207,7 @@ namespace OpenSim.Region.ClientStack.Linden
 
                 for (uint i = 0; i < 2; i++)
                 {
-                    m_workerThreads[i] = Watchdog.StartThread(DoInventoryRequests,
+                    m_workerThreads[i] = WorkManager.StartThread(DoInventoryRequests,
                             String.Format("InventoryWorkerThread{0}", i),
                             ThreadPriority.Normal,
                             false,
diff --git a/OpenSim/Region/ClientStack/Linden/UDP/IncomingPacketAsyncHandlingEngine.cs b/OpenSim/Region/ClientStack/Linden/UDP/IncomingPacketAsyncHandlingEngine.cs
index 874ddae..6f40b24 100644
--- a/OpenSim/Region/ClientStack/Linden/UDP/IncomingPacketAsyncHandlingEngine.cs
+++ b/OpenSim/Region/ClientStack/Linden/UDP/IncomingPacketAsyncHandlingEngine.cs
@@ -132,7 +132,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
                 StatsManager.RegisterStat(m_requestsWaitingStat);
 
-                Watchdog.StartThread(
+                WorkManager.StartThread(
                     ProcessRequests,
                     string.Format("Incoming Packet Async Handling Engine Thread ({0})", m_udpServer.Scene.Name),
                     ThreadPriority.Normal,
diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs
index d8cf7a5..2f97516 100644
--- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs
+++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs
@@ -483,7 +483,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             base.StartInbound(m_recvBufferSize, m_asyncPacketHandling);
 
             // This thread will process the packets received that are placed on the packetInbox
-            Watchdog.StartThread(
+            WorkManager.StartThread(
                 IncomingPacketHandler,
                 string.Format("Incoming Packets ({0})", Scene.Name),
                 ThreadPriority.Normal,
@@ -499,7 +499,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
             base.StartOutbound();
 
-            Watchdog.StartThread(
+            WorkManager.StartThread(
                 OutgoingPacketHandler,
                 string.Format("Outgoing Packets ({0})", Scene.Name),
                 ThreadPriority.Normal,
diff --git a/OpenSim/Region/ClientStack/Linden/UDP/OutgoingQueueRefillEngine.cs b/OpenSim/Region/ClientStack/Linden/UDP/OutgoingQueueRefillEngine.cs
index 2ec1733..1e915c3 100644
--- a/OpenSim/Region/ClientStack/Linden/UDP/OutgoingQueueRefillEngine.cs
+++ b/OpenSim/Region/ClientStack/Linden/UDP/OutgoingQueueRefillEngine.cs
@@ -124,7 +124,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
                 StatsManager.RegisterStat(m_oqreRequestsWaitingStat);
 
-                Watchdog.StartThread(
+                WorkManager.StartThread(
                     ProcessRequests,
                     String.Format("OutgoingQueueRefillEngineThread ({0})", m_udpServer.Scene.Name),
                     ThreadPriority.Normal,
diff --git a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs
index 5eca025..fe9a17d 100644
--- a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs
+++ b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs
@@ -964,11 +964,11 @@ namespace OpenSim.Region.CoreModules.Asset
                     case "assets":
                         con.Output("Ensuring assets are cached for all scenes.");
 
-                        Watchdog.RunInThread(delegate 
+                        WorkManager.RunInThread(delegate 
                         {
                             int assetReferenceTotal = TouchAllSceneAssets(true);
                             con.OutputFormat("Completed check with {0} assets.", assetReferenceTotal);
-                        }, "TouchAllSceneAssets", null);
+                        }, null, "TouchAllSceneAssets");
 
                         break;
 
diff --git a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs
index 9fb8aa5..6dab227 100644
--- a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs
+++ b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs
@@ -431,7 +431,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
                             m_scene.UserAccountService, m_scene.RegionInfo.ScopeID,
                             options, ReceivedAllAssets);
 
-                    Watchdog.RunInThread(o => ar.Execute(), string.Format("AssetsRequest ({0})", m_scene.Name), null);
+                    WorkManager.RunInThread(o => ar.Execute(), null, string.Format("AssetsRequest ({0})", m_scene.Name));
                 }
                 else
                 {
diff --git a/OpenSim/Region/CoreModules/Framework/EntityTransfer/HGIncomingSceneObjectEngine.cs b/OpenSim/Region/CoreModules/Framework/EntityTransfer/HGIncomingSceneObjectEngine.cs
index ce6cdc9..f62e7f4 100644
--- a/OpenSim/Region/CoreModules/Framework/EntityTransfer/HGIncomingSceneObjectEngine.cs
+++ b/OpenSim/Region/CoreModules/Framework/EntityTransfer/HGIncomingSceneObjectEngine.cs
@@ -134,7 +134,7 @@ namespace OpenSim.Region.CoreModules.Framework.EntityTransfer
 
                 StatsManager.RegisterStat(m_requestsWaitingStat);
 
-                Watchdog.StartThread(
+                WorkManager.StartThread(
                     ProcessRequests,
                     string.Format("HG Incoming Scene Object Engine Thread ({0})", Name),
                     ThreadPriority.Normal,
diff --git a/OpenSim/Region/CoreModules/Framework/ServiceThrottle/ServiceThrottleModule.cs b/OpenSim/Region/CoreModules/Framework/ServiceThrottle/ServiceThrottleModule.cs
index a70261e..3abacbd 100644
--- a/OpenSim/Region/CoreModules/Framework/ServiceThrottle/ServiceThrottleModule.cs
+++ b/OpenSim/Region/CoreModules/Framework/ServiceThrottle/ServiceThrottleModule.cs
@@ -67,7 +67,7 @@ namespace OpenSim.Region.CoreModules.Framework
             m_timer.Elapsed += ProcessQueue;
             m_timer.Start();
 
-            //Watchdog.StartThread(
+            //WorkManager.StartThread(
             //    ProcessQueue,
             //    "GridServiceRequestThread",
             //    ThreadPriority.BelowNormal,
diff --git a/OpenSim/Region/CoreModules/Scripting/XMLRPC/XMLRPCModule.cs b/OpenSim/Region/CoreModules/Scripting/XMLRPC/XMLRPCModule.cs
index d7ea906..af3700b 100644
--- a/OpenSim/Region/CoreModules/Scripting/XMLRPC/XMLRPCModule.cs
+++ b/OpenSim/Region/CoreModules/Scripting/XMLRPC/XMLRPCModule.cs
@@ -658,7 +658,7 @@ namespace OpenSim.Region.CoreModules.Scripting.XMLRPC
         public void Process()
         {
             _finished = false;
-            Watchdog.StartThread(SendRequest, "HttpRequestThread", ThreadPriority.BelowNormal, true, false);
+            WorkManager.StartThread(SendRequest, "HttpRequestThread", ThreadPriority.BelowNormal, true, false);
         }
 
         /*
diff --git a/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs b/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs
index 470ef02..cbe0e37 100644
--- a/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs
+++ b/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs
@@ -184,12 +184,12 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsOut.Inventory
                 // Protect ourselves against the caller subsequently modifying the items list
                 List<InventoryItemBase> items = new List<InventoryItemBase>(invCol.Items);
 
-                Watchdog.RunInThread(delegate
+                WorkManager.RunInThread(delegate
                 {
                     foreach (InventoryItemBase item in items)
                         if (!string.IsNullOrEmpty(item.CreatorData))
                             UserManager.AddUser(item.CreatorIdAsUuid, item.CreatorData);
-                }, string.Format("GetFolderContent (user {0}, folder {1})", userID, folderID), null);
+                }, null, string.Format("GetFolderContent (user {0}, folder {1})", userID, folderID));
             }
 
             return invCol;
diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs
index 3e0c9f3..9c6706f 100644
--- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs
+++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs
@@ -372,7 +372,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
             // Start the scripts. We delayed this because we want the OAR to finish loading ASAP, so
             // that users can enter the scene. If we allow the scripts to start in the loop above
             // then they significantly increase the time until the OAR finishes loading.
-            Watchdog.RunInThread(o =>
+            WorkManager.RunInThread(o =>
             {
                 Thread.Sleep(15000);
                 m_log.Info("[ARCHIVER]: Starting scripts in scene objects");
@@ -387,7 +387,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
 
                     sceneContext.SceneObjects.Clear();
                 }
-            }, string.Format("ReadArchiveStartScripts (request {0})", m_requestId), null);
+            }, null, string.Format("ReadArchiveStartScripts (request {0})", m_requestId));
 
             m_log.InfoFormat("[ARCHIVER]: Successfully loaded archive");
 
diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs
index 924b999..b7d7c26 100644
--- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs
+++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs
@@ -200,7 +200,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
                             m_rootScene.AssetService, m_rootScene.UserAccountService,
                             m_rootScene.RegionInfo.ScopeID, options, ReceivedAllAssets);
 
-                    Watchdog.RunInThread(o => ar.Execute(), "Archive Assets Request", null);
+                    WorkManager.RunInThread(o => ar.Execute(), null, "Archive Assets Request");
 
                     // CloseArchive() will be called from ReceivedAllAssets()
                 }
diff --git a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
index ec39bc0..4d99a6e 100644
--- a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
+++ b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
@@ -227,7 +227,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
             finally
             {
                 if (timedOut)
-                    Watchdog.RunInThread(PerformAssetsRequestCallback, "Archive Assets Request Callback", true);
+                    WorkManager.RunInThread(PerformAssetsRequestCallback, true, "Archive Assets Request Callback");
             }
         }
 
@@ -296,7 +296,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
                         
                         // We want to stop using the asset cache thread asap 
                         // as we now need to do the work of producing the rest of the archive
-                        Watchdog.RunInThread(PerformAssetsRequestCallback, "Archive Assets Request Callback", false);
+                        WorkManager.RunInThread(PerformAssetsRequestCallback, false, "Archive Assets Request Callback");
                     }
                     else
                     {
diff --git a/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs b/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs
index 78fbefe..767f75f 100644
--- a/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs
+++ b/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs
@@ -381,7 +381,7 @@ namespace OpenSim.Region.CoreModules.World.WorldMap
 
 //            m_log.Debug("[WORLD MAP]: Starting remote MapItem request thread");
 
-            Watchdog.StartThread(
+            WorkManager.StartThread(
                 process,
                 string.Format("MapItemRequestThread ({0})", m_scene.RegionInfo.RegionName),
                 ThreadPriority.BelowNormal,
diff --git a/OpenSim/Region/Framework/Scenes/Scene.cs b/OpenSim/Region/Framework/Scenes/Scene.cs
index 171f066..f7c12d6 100644
--- a/OpenSim/Region/Framework/Scenes/Scene.cs
+++ b/OpenSim/Region/Framework/Scenes/Scene.cs
@@ -1428,7 +1428,7 @@ namespace OpenSim.Region.Framework.Scenes
             }
 
             m_heartbeatThread
-                = Watchdog.StartThread(
+                = WorkManager.StartThread(
                     Heartbeat, string.Format("Heartbeat-({0})", RegionInfo.RegionName.Replace(" ", "_")), ThreadPriority.Normal, false, false);
 
             StartScripts();
@@ -1469,7 +1469,7 @@ namespace OpenSim.Region.Framework.Scenes
             // alarms for scenes with many objects.
             Update(1);
 
-            Watchdog.StartThread(
+            WorkManager.StartThread(
                 Maintenance, string.Format("Maintenance ({0})", RegionInfo.RegionName), ThreadPriority.Normal, false, true);
 
             Watchdog.GetCurrentThreadInfo().AlarmIfTimeout = true;
@@ -1568,10 +1568,10 @@ namespace OpenSim.Region.Framework.Scenes
                     tmpMS = Util.EnvironmentTickCount();
                     m_cleaningTemps = true;
 
-                    Watchdog.RunInThread(
+                    WorkManager.RunInThread(
                         delegate { CleanTempObjects(); m_cleaningTemps = false;  }, 
-                        string.Format("CleanTempObjects ({0})", Name), 
-                        null);
+                        null,
+                        string.Format("CleanTempObjects ({0})", Name));
 
                     tempOnRezMS = Util.EnvironmentTickCountSubtract(tmpMS);
                 }
@@ -1843,7 +1843,7 @@ namespace OpenSim.Region.Framework.Scenes
             if (!m_backingup)
             {
                 m_backingup = true;
-                Watchdog.RunInThread(o => Backup(false), string.Format("BackupWaitCallback ({0})", Name), null);
+                WorkManager.RunInThread(o => Backup(false), null, string.Format("BackupWaitCallback ({0})", Name));
             }
         }
 
diff --git a/OpenSim/Region/Framework/Scenes/ScenePresence.cs b/OpenSim/Region/Framework/Scenes/ScenePresence.cs
index 757ec43..5a35aff 100644
--- a/OpenSim/Region/Framework/Scenes/ScenePresence.cs
+++ b/OpenSim/Region/Framework/Scenes/ScenePresence.cs
@@ -1228,11 +1228,11 @@ namespace OpenSim.Region.Framework.Scenes
                 // viewers without (e.g. v1 viewers) will not, so we still need to make this call.
                 if (Scene.AttachmentsModule != null)
                 {
-                    Watchdog.RunJob(
+                    WorkManager.RunJob(
                         "RezAttachments", 
                         o => Scene.AttachmentsModule.RezAttachments(this), 
-                        string.Format("Rez attachments for {0} in {1}", Name, Scene.Name), 
-                        null);
+                        null,
+                        string.Format("Rez attachments for {0} in {1}", Name, Scene.Name));
                 }
             }
             else
@@ -1254,11 +1254,11 @@ namespace OpenSim.Region.Framework.Scenes
 
                 if (attachments.Count > 0)
                 {
-                    Watchdog.RunJob(
+                    WorkManager.RunJob(
                         "StartAttachmentScripts", 
                         o => RestartAttachmentScripts(attachments),
-                        string.Format("Start attachment scripts for {0} in {1}", Name, Scene.Name),
                         null,
+                        string.Format("Start attachment scripts for {0} in {1}", Name, Scene.Name),
                         true);
                 }
             }
@@ -1815,11 +1815,11 @@ namespace OpenSim.Region.Framework.Scenes
                 // XXX: If we force an update after activity has completed, then multiple attachments do appear correctly on a destination region
                 // If we do it a little bit earlier (e.g. when converting the child to a root agent) then this does not work.
                 // This may be due to viewer code or it may be something we're not doing properly simulator side.
-                Watchdog.RunJob(
+                WorkManager.RunJob(
                     "ScheduleAttachmentsForFullUpdate", 
                     o => ScheduleAttachmentsForFullUpdate(),
-                    string.Format("Schedule attachments for full update for {0} in {1}", Name, Scene.Name),
                     null,
+                    string.Format("Schedule attachments for full update for {0} in {1}", Name, Scene.Name),
                     true);
 
     //            m_log.DebugFormat(
@@ -3375,7 +3375,7 @@ namespace OpenSim.Region.Framework.Scenes
             SentInitialDataToClient = true;
 
             // Send all scene object to the new client
-            Watchdog.RunJob("SendInitialDataToClient", delegate
+            WorkManager.RunJob("SendInitialDataToClient", delegate
             {
 //                m_log.DebugFormat(
 //                    "[SCENE PRESENCE]: Sending initial data to {0} agent {1} in {2}, tp flags {3}", 
@@ -3393,7 +3393,7 @@ namespace OpenSim.Region.Framework.Scenes
                     if (e != null && e is SceneObjectGroup)
                         ((SceneObjectGroup)e).SendFullUpdateToClient(ControllingClient);
                 }
-            }, string.Format("SendInitialDataToClient ({0} in {1})", Name, Scene.Name),null,  false, true);
+            }, null, string.Format("SendInitialDataToClient ({0} in {1})", Name, Scene.Name), false, true);
         }
 
         /// <summary>
@@ -4057,11 +4057,11 @@ namespace OpenSim.Region.Framework.Scenes
                 // We don't need to worry about a race condition as the job to later start the scripts is also 
                 // JobEngine scheduled and so will always occur after this task.
                 // XXX: This will not be true if JobEngine ever gets more than one thread.
-                Watchdog.RunJob(
+                WorkManager.RunJob(
                     "CopyAttachments", 
                     o => Scene.AttachmentsModule.CopyAttachments(cAgent, this), 
-                    string.Format("Copy attachments for {0} entering {1}", Name, Scene.Name),
                     null,
+                    string.Format("Copy attachments for {0} entering {1}", Name, Scene.Name),
                     true);
             }
 
diff --git a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs
index e1aaf18..6fe86b2 100644
--- a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs
+++ b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs
@@ -71,7 +71,7 @@ namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server
             m_client = client;
             m_scene = scene;
 
-            Watchdog.StartThread(InternalLoop, "IRCClientView", ThreadPriority.Normal, false, true);
+            WorkManager.StartThread(InternalLoop, "IRCClientView", ThreadPriority.Normal, false, true);
         }
 
         private void SendServerCommand(string command)
diff --git a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs
index 9d27386..a1682d2 100644
--- a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs
+++ b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs
@@ -58,7 +58,7 @@ namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server
 
             m_listener.Start(50);
 
-            Watchdog.StartThread(ListenLoop, "IRCServer", ThreadPriority.Normal, false, true);
+            WorkManager.StartThread(ListenLoop, "IRCServer", ThreadPriority.Normal, false, true);
             m_baseScene = baseScene;
         }
 
diff --git a/OpenSim/Region/OptionalModules/Avatar/Chat/IRCConnector.cs b/OpenSim/Region/OptionalModules/Avatar/Chat/IRCConnector.cs
index bdd07e0..6985371 100644
--- a/OpenSim/Region/OptionalModules/Avatar/Chat/IRCConnector.cs
+++ b/OpenSim/Region/OptionalModules/Avatar/Chat/IRCConnector.cs
@@ -359,7 +359,7 @@ namespace OpenSim.Region.OptionalModules.Avatar.Chat
 
                     m_log.InfoFormat("[IRC-Connector-{0}]: Connected to {1}:{2}", idn, m_server, m_port);
 
-                    Watchdog.StartThread(ListenerRun, "IRCConnectionListenerThread", ThreadPriority.Normal, true, false);
+                    WorkManager.StartThread(ListenerRun, "IRCConnectionListenerThread", ThreadPriority.Normal, true, false);
 
                     // This is the message order recommended by RFC 2812
                     if (m_password != null)
diff --git a/OpenSim/Region/Physics/BulletSPlugin/BSScene.cs b/OpenSim/Region/Physics/BulletSPlugin/BSScene.cs
index 46a13ab..0f79a10 100644
--- a/OpenSim/Region/Physics/BulletSPlugin/BSScene.cs
+++ b/OpenSim/Region/Physics/BulletSPlugin/BSScene.cs
@@ -293,7 +293,7 @@ public sealed class BSScene : PhysicsScene, IPhysicsParameters
         {
             // The physics simulation should happen independently of the heartbeat loop
             m_physicsThread 
-                = Watchdog.StartThread(
+                = WorkManager.StartThread(
                     BulletSPluginPhysicsThread, 
                     string.Format("{0} ({1})", BulletEngineName, RegionName), 
                     ThreadPriority.Normal, 
diff --git a/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs b/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs
index 40a05cf..036cb5d 100644
--- a/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs
+++ b/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs
@@ -182,7 +182,7 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api
             {
                 // Start the thread that will be doing the work
                 cmdHandlerThread
-                    = Watchdog.StartThread(
+                    = WorkManager.StartThread(
                         CmdHandlerThreadLoop, "AsyncLSLCmdHandlerThread", ThreadPriority.Normal, true, true);
             }
         }