10 files changed, 669 insertions, 101 deletions

diff --git a/OpenSim/Framework/ClientInfo.cs b/OpenSim/Framework/ClientInfo.cs
index d68078e..98e4465 100644
--- a/OpenSim/Framework/ClientInfo.cs
+++ b/OpenSim/Framework/ClientInfo.cs
@@ -54,6 +54,10 @@ namespace OpenSim.Framework
         public int assetThrottle;
         public int textureThrottle;
         public int totalThrottle;
+
+        // Used by adaptive only
+        public int targetThrottle;
+
         public int maxThrottle;
 
         public Dictionary<string, int> SyncRequests = new Dictionary<string,int>();
diff --git a/OpenSim/Framework/Communications/RestClient.cs b/OpenSim/Framework/Communications/RestClient.cs
index de6fe30..72018e4 100644
--- a/OpenSim/Framework/Communications/RestClient.cs
+++ b/OpenSim/Framework/Communications/RestClient.cs
@@ -483,7 +483,7 @@ namespace OpenSim.Framework.Communications
             /// In case, we are invoked asynchroneously this object will keep track of the state
             /// </summary>
             AsyncResult<Stream> ar = new AsyncResult<Stream>(callback, state);
-            Util.FireAndForget(RequestHelper, ar);
+            Util.FireAndForget(RequestHelper, ar, "RestClient.BeginRequest");
             return ar;
         }
 
diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs
new file mode 100644
index 0000000..5925867
--- /dev/null
+++ b/OpenSim/Framework/Monitoring/JobEngine.cs
@@ -0,0 +1,320 @@
+/*
+ * Copyright (c) Contributors, http://opensimulator.org/
+ * See CONTRIBUTORS.TXT for a full list of copyright holders.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the OpenSimulator Project nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+using System;
+using System.Collections.Concurrent;
+using System.Reflection;
+using System.Threading;
+using log4net;
+using OpenSim.Framework;
+
+namespace OpenSim.Framework.Monitoring
+{
+    public class Job
+    {
+        public string Name;
+        public WaitCallback Callback;
+        public object O;
+
+        public Job(string name, WaitCallback callback, object o)
+        {
+            Name = name;
+            Callback = callback;
+            O = o;
+        }
+    }
+
+    public class JobEngine
+    {
+        private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
+
+        public int LogLevel { get; set; }
+
+        public bool IsRunning { get; private set; } 
+
+        /// <summary>
+        /// The timeout in milliseconds to wait for at least one event to be written when the recorder is stopping.
+        /// </summary>
+        public int RequestProcessTimeoutOnStop { get; set; }
+
+        /// <summary>
+        /// Controls whether we need to warn in the log about exceeding the max queue size.
+        /// </summary>
+        /// <remarks>
+        /// This is flipped to false once queue max has been exceeded and back to true when it falls below max, in 
+        /// order to avoid spamming the log with lots of warnings.
+        /// </remarks>
+        private bool m_warnOverMaxQueue = true;
+
+        private BlockingCollection<Job> m_requestQueue;
+
+        private CancellationTokenSource m_cancelSource = new CancellationTokenSource();
+
+        private Stat m_requestsWaitingStat;
+
+        private Job m_currentJob;
+
+        /// <summary>
+        /// Used to signal that we are ready to complete stop.
+        /// </summary>
+        private ManualResetEvent m_finishedProcessingAfterStop = new ManualResetEvent(false);
+
+        public JobEngine()
+        {
+            RequestProcessTimeoutOnStop = 5000;
+
+            MainConsole.Instance.Commands.AddCommand(
+                "Debug",
+                false,
+                "debug jobengine",
+                "debug jobengine <start|stop|status|log>",
+                "Start, stop, get status or set logging level of the job engine.",
+                "If stopped then all outstanding jobs are processed immediately.",
+                HandleControlCommand);
+        }
+
+        public void Start()
+        {
+            lock (this)
+            {
+                if (IsRunning)
+                    return;
+
+                IsRunning = true;
+
+                m_finishedProcessingAfterStop.Reset();
+
+                m_requestQueue = new BlockingCollection<Job>(new ConcurrentQueue<Job>(), 5000);
+
+                m_requestsWaitingStat = 
+                    new Stat(
+                        "JobsWaiting",
+                        "Number of jobs waiting for processing.",
+                        "",
+                        "",
+                        "server",
+                        "jobengine",
+                        StatType.Pull,
+                        MeasuresOfInterest.None,
+                        stat => stat.Value = m_requestQueue.Count,
+                        StatVerbosity.Debug);
+
+                StatsManager.RegisterStat(m_requestsWaitingStat);
+
+                WorkManager.StartThread(
+                    ProcessRequests,
+                    "JobEngineThread",
+                    ThreadPriority.Normal,
+                    false,
+                    true,
+                    null,
+                    int.MaxValue);
+            }
+        }
+
+        public void Stop()
+        {   
+            lock (this)
+            {
+                try
+                {
+                    if (!IsRunning)
+                        return;
+
+                    IsRunning = false;
+
+                    int requestsLeft = m_requestQueue.Count;
+
+                    if (requestsLeft <= 0)
+                    {
+                        m_cancelSource.Cancel();
+                    }
+                    else 
+                    {
+                        m_log.InfoFormat("[JOB ENGINE]: Waiting to write {0} events after stop.", requestsLeft);
+
+                        while (requestsLeft > 0)
+                        {
+                            if (!m_finishedProcessingAfterStop.WaitOne(RequestProcessTimeoutOnStop))
+                            {
+                                // After timeout no events have been written
+                                if (requestsLeft == m_requestQueue.Count)
+                                {
+                                    m_log.WarnFormat(
+                                        "[JOB ENGINE]: No requests processed after {0} ms wait.  Discarding remaining {1} requests", 
+                                        RequestProcessTimeoutOnStop, requestsLeft);
+
+                                    break;
+                                }
+                            }
+
+                            requestsLeft = m_requestQueue.Count;
+                        }
+                    }
+                }
+                finally
+                {
+                    m_cancelSource.Dispose();
+                    StatsManager.DeregisterStat(m_requestsWaitingStat);
+                    m_requestsWaitingStat = null;
+                    m_requestQueue = null;
+                }
+            }
+        }
+
+        public bool QueueRequest(string name, WaitCallback req, object o)
+        {
+            if (LogLevel >= 1)
+                m_log.DebugFormat("[JOB ENGINE]: Queued job {0}", name);
+
+            if (m_requestQueue.Count < m_requestQueue.BoundedCapacity)
+            {
+                //                m_log.DebugFormat(
+                //                    "[OUTGOING QUEUE REFILL ENGINE]: Adding request for categories {0} for {1} in {2}", 
+                //                    categories, client.AgentID, m_udpServer.Scene.Name);
+
+                m_requestQueue.Add(new Job(name, req, o));
+
+                if (!m_warnOverMaxQueue)
+                    m_warnOverMaxQueue = true;
+
+                return true;
+            }
+            else
+            {
+                if (m_warnOverMaxQueue)
+                {
+//                    m_log.WarnFormat(
+//                        "[JOB ENGINE]: Request queue at maximum capacity, not recording request from {0} in {1}", 
+//                        client.AgentID, m_udpServer.Scene.Name);
+
+                    m_log.WarnFormat("[JOB ENGINE]: Request queue at maximum capacity, not recording job");
+
+                    m_warnOverMaxQueue = false;
+                }
+
+                return false;
+            }
+        }
+
+        private void ProcessRequests()
+        {
+            try
+            {
+                while (IsRunning || m_requestQueue.Count > 0)
+                {
+                    m_currentJob = m_requestQueue.Take(m_cancelSource.Token);
+
+                    //                QueueEmpty callback = req.Client.OnQueueEmpty;                      
+                    //
+                    //                if (callback != null)
+                    //                {               
+                    //                    try 
+                    //                    { 
+                    //                        callback(req.Categories); 
+                    //                    }
+                    //                    catch (Exception e) 
+                    //                    { 
+                    //                        m_log.Error("[OUTGOING QUEUE REFILL ENGINE]: ProcessRequests(" + req.Categories + ") threw an exception: " + e.Message, e); 
+                    //                    }
+                    //                }
+
+                    if (LogLevel >= 1)
+                        m_log.DebugFormat("[JOB ENGINE]: Processing job {0}", m_currentJob.Name);
+
+                    try
+                    {
+                        m_currentJob.Callback.Invoke(m_currentJob.O);
+                    }
+                    catch (Exception e)
+                    {
+                        m_log.Error(
+                            string.Format(
+                                "[JOB ENGINE]: Job {0} failed, continuing.  Exception  ", m_currentJob.Name), e);
+                    }
+
+                    if (LogLevel >= 1)
+                        m_log.DebugFormat("[JOB ENGINE]: Processed job {0}", m_currentJob.Name);
+
+                    m_currentJob = null;
+                }
+            }
+            catch (OperationCanceledException)
+            {
+            }
+
+            m_finishedProcessingAfterStop.Set();
+        }
+
+        private void HandleControlCommand(string module, string[] args)
+        {
+//            if (SceneManager.Instance.CurrentScene != null && SceneManager.Instance.CurrentScene != m_udpServer.Scene)
+//                return;
+
+            if (args.Length < 3)
+            {
+                MainConsole.Instance.Output("Usage: debug jobengine <stop|start|status|log>");
+                return;
+            }
+
+            string subCommand = args[2];
+
+            if (subCommand == "stop")
+            {
+                Stop();
+                MainConsole.Instance.OutputFormat("Stopped job engine.");
+            }
+            else if (subCommand == "start")
+            {
+                Start();
+                MainConsole.Instance.OutputFormat("Started job engine.");
+            }
+            else if (subCommand == "status")
+            {
+                MainConsole.Instance.OutputFormat("Job engine running: {0}", IsRunning);
+                MainConsole.Instance.OutputFormat("Current job {0}", m_currentJob != null ? m_currentJob.Name : "none");
+                MainConsole.Instance.OutputFormat(
+                    "Jobs waiting: {0}", IsRunning ? m_requestQueue.Count.ToString() : "n/a");
+                MainConsole.Instance.OutputFormat("Log Level: {0}", LogLevel);
+            }
+            else if (subCommand == "log")
+            {
+//                int logLevel;
+                int logLevel = int.Parse(args[3]);
+//                if (ConsoleUtil.TryParseConsoleInt(MainConsole.Instance, args[4], out logLevel))
+//                {                 
+                    LogLevel = logLevel;
+                    MainConsole.Instance.OutputFormat("Set debug log level to {0}", LogLevel);
+//                }
+            }
+            else 
+            {
+                MainConsole.Instance.OutputFormat("Unrecognized job engine subcommand {0}", subCommand);
+            }
+        }
+    }
+}
diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs
index 0fcb195..a644fa5 100644
--- a/OpenSim/Framework/Monitoring/Watchdog.cs
+++ b/OpenSim/Framework/Monitoring/Watchdog.cs
@@ -38,6 +38,8 @@ namespace OpenSim.Framework.Monitoring
     /// </summary>
     public static class Watchdog
     {
+        private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
+
         /// <summary>Timer interval in milliseconds for the watchdog timer</summary>
         public const double WATCHDOG_INTERVAL_MS = 2500.0d;
 
@@ -141,7 +143,7 @@ namespace OpenSim.Framework.Monitoring
             get { return m_enabled; }
             set
             {
-//                m_log.DebugFormat("[MEMORY WATCHDOG]: Setting MemoryWatchdog.Enabled to {0}", value);
+                //                m_log.DebugFormat("[MEMORY WATCHDOG]: Setting MemoryWatchdog.Enabled to {0}", value);
 
                 if (value == m_enabled)
                     return;
@@ -157,9 +159,8 @@ namespace OpenSim.Framework.Monitoring
                 m_watchdogTimer.Enabled = m_enabled;
             }
         }
-        private static bool m_enabled;
 
-        private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
+        private static bool m_enabled;
         private static Dictionary<int, ThreadWatchdogInfo> m_threads;
         private static System.Timers.Timer m_watchdogTimer;
 
@@ -180,94 +181,19 @@ namespace OpenSim.Framework.Monitoring
         }
 
         /// <summary>
-        /// Start a new thread that is tracked by the watchdog timer.
+        /// Add a thread to the watchdog tracker.
         /// </summary>
-        /// <param name="start">The method that will be executed in a new thread</param>
-        /// <param name="name">A name to give to the new thread</param>
-        /// <param name="priority">Priority to run the thread at</param>
-        /// <param name="isBackground">True to run this thread as a background thread, otherwise false</param>
-        /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
+        /// <param name="info">Information about the thread.</info>
+        /// <param name="info">Name of the thread.</info>
         /// <param name="log">If true then creation of thread is logged.</param>
-        /// <returns>The newly created Thread object</returns>
-        public static Thread StartThread(
-            ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, bool log = true)
+        public static void AddThread(ThreadWatchdogInfo info, string name, bool log = true)
         {
-            return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS, log);
-        }
-
-        /// <summary>
-        /// Start a new thread that is tracked by the watchdog
-        /// </summary>
-        /// <param name="start">The method that will be executed in a new thread</param>
-        /// <param name="name">A name to give to the new thread</param>
-        /// <param name="priority">Priority to run the thread at</param>
-        /// <param name="isBackground">True to run this thread as a background
-        /// thread, otherwise false</param>
-        /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
-        /// <param name="alarmMethod">
-        /// Alarm method to call if alarmIfTimeout is true and there is a timeout.
-        /// Normally, this will just return some useful debugging information.
-        /// </param>
-        /// <param name="timeout">Number of milliseconds to wait until we issue a warning about timeout.</param>
-        /// <param name="log">If true then creation of thread is logged.</param>
-        /// <returns>The newly created Thread object</returns>
-        public static Thread StartThread(
-            ThreadStart start, string name, ThreadPriority priority, bool isBackground,
-            bool alarmIfTimeout, Func<string> alarmMethod, int timeout, bool log = true)
-        {
-            Thread thread = new Thread(start);
-            thread.Priority = priority;
-            thread.IsBackground = isBackground;
-            
-            ThreadWatchdogInfo twi
-                = new ThreadWatchdogInfo(thread, timeout, name)
-                    { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod };
-
             if (log)
                 m_log.DebugFormat(
-                    "[WATCHDOG]: Started tracking thread {0}, ID {1}", name, twi.Thread.ManagedThreadId);
+                    "[WATCHDOG]: Started tracking thread {0}, ID {1}", name, info.Thread.ManagedThreadId);
 
             lock (m_threads)
-                m_threads.Add(twi.Thread.ManagedThreadId, twi);
-            
-            thread.Start();
-            thread.Name = name;
-
-
-            return thread;
-        }
-
-        /// <summary>
-        /// Run the callback in a new thread immediately.  If the thread exits with an exception log it but do
-        /// not propogate it.
-        /// </summary>
-        /// <param name="callback">Code for the thread to execute.</param>
-        /// <param name="name">Name of the thread</param>
-        /// <param name="obj">Object to pass to the thread.</param>
-        public static void RunInThread(WaitCallback callback, string name, object obj, bool log = false)
-        {
-            if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)           
-            {
-                Culture.SetCurrentCulture();
-                callback(obj);
-                return;
-            }
-
-            ThreadStart ts = new ThreadStart(delegate()
-            {
-                try
-                {
-                    Culture.SetCurrentCulture();
-                    callback(obj);
-                    Watchdog.RemoveThread(log:false);
-                }
-                catch (Exception e)
-                {
-                    m_log.Error(string.Format("[WATCHDOG]: Exception in thread {0}.", name), e);
-                }
-            });
-
-            StartThread(ts, name, ThreadPriority.Normal, true, false, log:log);
+                m_threads.Add(info.Thread.ManagedThreadId, info);
         }
 
         /// <summary>
@@ -358,7 +284,7 @@ namespace OpenSim.Framework.Monitoring
             }
             catch { }
         }
-        
+
         /// <summary>
         /// Get currently watched threads for diagnostic purposes
         /// </summary>
diff --git a/OpenSim/Framework/Monitoring/WorkManager.cs b/OpenSim/Framework/Monitoring/WorkManager.cs
new file mode 100644
index 0000000..9d0eefc
--- /dev/null
+++ b/OpenSim/Framework/Monitoring/WorkManager.cs
@@ -0,0 +1,212 @@
+/*
+ * Copyright (c) Contributors, http://opensimulator.org/
+ * See CONTRIBUTORS.TXT for a full list of copyright holders.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the OpenSimulator Project nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+using System;
+using System.Reflection;
+using System.Threading;
+using log4net;
+
+namespace OpenSim.Framework.Monitoring
+{
+    /// <summary>
+    /// Manages various work items in the simulator.
+    /// </summary>
+    /// <remarks>
+    /// Currently, here work can be started 
+    ///  * As a long-running and monitored thread.
+    ///  * In a thread that will never timeout but where the job is expected to eventually complete.
+    ///  * In a threadpool thread that will timeout if it takes a very long time to complete (> 10 mins).
+    ///  * As a job which will be run in a single-threaded job engine.  Such jobs must not incorporate delays (sleeps,
+    /// network waits, etc.).
+    /// 
+    /// This is an evolving approach to better manage the work that OpenSimulator is asked to do from a very diverse
+    /// range of sources (client actions, incoming network, outgoing network calls, etc.).
+    /// 
+    /// Util.FireAndForget is still available to insert jobs in the threadpool, though this is equivalent to
+    /// WorkManager.RunInThreadPool().
+    /// </remarks>
+    public static class WorkManager
+    {
+        private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
+
+        public static JobEngine JobEngine { get; private set; }
+
+        static WorkManager()
+        {
+            JobEngine = new JobEngine();
+        }
+
+        /// <summary>
+        /// Start a new long-lived thread.
+        /// </summary>
+        /// <param name="start">The method that will be executed in a new thread</param>
+        /// <param name="name">A name to give to the new thread</param>
+        /// <param name="priority">Priority to run the thread at</param>
+        /// <param name="isBackground">True to run this thread as a background thread, otherwise false</param>
+        /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
+        /// <param name="log">If true then creation of thread is logged.</param>
+        /// <returns>The newly created Thread object</returns>
+        public static Thread StartThread(
+            ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, bool log = true)
+        {
+            return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, Watchdog.DEFAULT_WATCHDOG_TIMEOUT_MS, log);
+        }
+
+        /// <summary>
+        /// Start a new thread that is tracked by the watchdog
+        /// </summary>
+        /// <param name="start">The method that will be executed in a new thread</param>
+        /// <param name="name">A name to give to the new thread</param>
+        /// <param name="priority">Priority to run the thread at</param>
+        /// <param name="isBackground">True to run this thread as a background
+        /// thread, otherwise false</param>
+        /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
+        /// <param name="alarmMethod">
+        /// Alarm method to call if alarmIfTimeout is true and there is a timeout.
+        /// Normally, this will just return some useful debugging information.
+        /// </param>
+        /// <param name="timeout">Number of milliseconds to wait until we issue a warning about timeout.</param>
+        /// <param name="log">If true then creation of thread is logged.</param>
+        /// <returns>The newly created Thread object</returns>
+        public static Thread StartThread(
+            ThreadStart start, string name, ThreadPriority priority, bool isBackground,
+            bool alarmIfTimeout, Func<string> alarmMethod, int timeout, bool log = true)
+        {
+            Thread thread = new Thread(start);
+            thread.Priority = priority;
+            thread.IsBackground = isBackground;
+
+            Watchdog.ThreadWatchdogInfo twi
+                = new Watchdog.ThreadWatchdogInfo(thread, timeout, name)
+            { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod };
+
+            Watchdog.AddThread(twi, name, log:log);
+
+            thread.Start();
+            thread.Name = name;
+
+            return thread;
+        }
+
+        /// <summary>
+        /// Run the callback in a new thread immediately.  If the thread exits with an exception log it but do
+        /// not propogate it.
+        /// </summary>
+        /// <param name="callback">Code for the thread to execute.</param>
+        /// <param name="obj">Object to pass to the thread.</param>
+        /// <param name="name">Name of the thread</param>
+        public static void RunInThread(WaitCallback callback, object obj, string name, bool log = false)
+        {
+            if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)           
+            {
+                Culture.SetCurrentCulture();
+                callback(obj);
+                return;
+            }
+
+            ThreadStart ts = new ThreadStart(delegate()
+            {
+                try
+                {
+                    Culture.SetCurrentCulture();
+                    callback(obj);
+                    Watchdog.RemoveThread(log:false);
+                }
+                catch (Exception e)
+                {
+                    m_log.Error(string.Format("[WATCHDOG]: Exception in thread {0}.", name), e);
+                }
+            });
+
+            StartThread(ts, name, ThreadPriority.Normal, true, false, log:log);
+        }
+
+        /// <summary>
+        /// Run the callback via a threadpool thread. 
+        /// </summary>
+        /// <remarks>
+        /// Such jobs may run after some delay but must always complete.
+        /// </remarks>
+        /// <param name="callback"></param>
+        /// <param name="obj"></param>
+        /// <param name="name">The name of the job.  This is used in monitoring and debugging.</param>
+        public static void RunInThreadPool(System.Threading.WaitCallback callback, object obj, string name)
+        {
+            Util.FireAndForget(callback, obj, name);
+        }
+
+        /// <summary>
+        /// Run a job.
+        /// </summary>
+        /// <remarks>
+        /// This differs from direct scheduling (e.g. Util.FireAndForget) in that a job can be run in the job
+        /// engine if it is running, where all jobs are currently performed in sequence on a single thread.  This is
+        /// to prevent observed overload and server freeze problems when there are hundreds of connections which all attempt to 
+        /// perform work at once (e.g. in conference situations).  With lower numbers of connections, the small
+        /// delay in performing jobs in sequence rather than concurrently has not been notiecable in testing, though a future more 
+        /// sophisticated implementation could perform jobs concurrently when the server is under low load.
+        /// 
+        /// However, be advised that some callers of this function rely on all jobs being performed in sequence if any
+        /// jobs are performed in sequence (i.e. if jobengine is active or not).  Therefore, expanding the jobengine
+        /// beyond a single thread will require considerable thought.
+        /// 
+        /// Also, any jobs submitted must be guaranteed to complete within a reasonable timeframe (e.g. they cannot
+        /// incorporate a network delay with a long timeout).  At the moment, work that could suffer such issues 
+        /// should still be run directly with RunInThread(), Util.FireAndForget(), etc.  This is another area where
+        /// the job engine could be improved and so CPU utilization improved by better management of concurrency within
+        /// OpenSimulator.
+        /// </remarks>
+        /// <param name="jobType">General classification for the job (e.g. "RezAttachments").</param>
+        /// <param name="callback">Callback for job.</param>
+        /// <param name="obj">Object to pass to callback when run</param>
+        /// <param name="name">Specific name of job (e.g. "RezAttachments for Joe Bloggs"</param>
+        /// <param name="canRunInThisThread">If set to true then the job may be run in ths calling thread.</param>
+        /// <param name="mustNotTimeout">If the true then the job must never timeout.</param>
+        /// <param name="log">If set to true then extra logging is performed.</param>
+        public static void RunJob(
+            string jobType, WaitCallback callback, object obj, string name,
+            bool canRunInThisThread = false, bool mustNotTimeout = false, 
+            bool log = false)
+        {
+            if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)           
+            {
+                Culture.SetCurrentCulture();
+                callback(obj);
+                return;
+            }
+
+            if (JobEngine.IsRunning)
+                JobEngine.QueueRequest(name, callback, obj);
+            else if (canRunInThisThread)
+                callback(obj);
+            else if (mustNotTimeout)
+                RunInThread(callback, obj, name, log);
+            else
+                Util.FireAndForget(callback, obj, name);
+        }
+    }
+}
\ No newline at end of file
diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs
index 4561d23..28bba70 100644
--- a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs
+++ b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs
@@ -120,7 +120,7 @@ namespace OpenSim.Framework.Servers.HttpServer
                 for (uint i = 0; i < m_WorkerThreadCount; i++)
                 {
                     m_workerThreads[i]
-                        = Watchdog.StartThread(
+                        = WorkManager.StartThread(
                             PoolWorkerJob,
                             string.Format("PollServiceWorkerThread{0}:{1}", i, m_server.Port),
                             ThreadPriority.Normal,
@@ -130,7 +130,7 @@ namespace OpenSim.Framework.Servers.HttpServer
                             int.MaxValue);
                 }
 
-                Watchdog.StartThread(
+                WorkManager.StartThread(
                     this.CheckLongPollThreads,
                     string.Format("LongPollServiceWatcherThread:{0}", m_server.Port),
                     ThreadPriority.Normal,
diff --git a/OpenSim/Framework/Servers/ServerBase.cs b/OpenSim/Framework/Servers/ServerBase.cs
index fd56587..c22c119 100644
--- a/OpenSim/Framework/Servers/ServerBase.cs
+++ b/OpenSim/Framework/Servers/ServerBase.cs
@@ -29,6 +29,7 @@ using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
+using System.Linq;
 using System.Reflection;
 using System.Text;
 using System.Text.RegularExpressions;
@@ -292,6 +293,18 @@ namespace OpenSim.Framework.Servers
                 HandleDebugThreadpoolLevel);
 
             m_console.Commands.AddCommand(
+                "Debug", false, "show threadpool calls active",
+                "show threadpool calls active",
+                "Show details about threadpool calls that are still active (currently waiting or in progress)",
+                HandleShowThreadpoolCallsActive);
+
+            m_console.Commands.AddCommand(
+                "Debug", false, "show threadpool calls complete",
+                "show threadpool calls complete",
+                "Show details about threadpool calls that have been completed.",
+                HandleShowThreadpoolCallsComplete);
+
+            m_console.Commands.AddCommand(
                 "Debug", false, "force gc",
                 "force gc",
                 "Manually invoke runtime garbage collection.  For debugging purposes",
@@ -354,6 +367,57 @@ namespace OpenSim.Framework.Servers
             Notice("serialosdreq is now {0}", setSerializeOsdRequests);
         }
 
+        private void HandleShowThreadpoolCallsActive(string module, string[] args)
+        {
+            List<KeyValuePair<string, int>> calls = Util.GetFireAndForgetCallsInProgress().ToList();
+            calls.Sort((kvp1, kvp2) => kvp2.Value.CompareTo(kvp1.Value));
+            int namedCalls = 0;
+
+            ConsoleDisplayList cdl = new ConsoleDisplayList();
+            foreach (KeyValuePair<string, int> kvp in calls)
+            {
+                if (kvp.Value > 0)
+                {
+                    cdl.AddRow(kvp.Key, kvp.Value);
+                    namedCalls += kvp.Value;
+                }
+            }
+
+            cdl.AddRow("TOTAL NAMED", namedCalls);
+
+            long allQueuedCalls = Util.TotalQueuedFireAndForgetCalls;
+            long allRunningCalls = Util.TotalRunningFireAndForgetCalls;
+
+            cdl.AddRow("TOTAL QUEUED", allQueuedCalls);
+            cdl.AddRow("TOTAL RUNNING", allRunningCalls);
+            cdl.AddRow("TOTAL ANONYMOUS", allQueuedCalls + allRunningCalls - namedCalls);
+            cdl.AddRow("TOTAL ALL", allQueuedCalls + allRunningCalls);
+
+            MainConsole.Instance.Output(cdl.ToString());
+        }
+
+        private void HandleShowThreadpoolCallsComplete(string module, string[] args)
+        {
+            List<KeyValuePair<string, int>> calls = Util.GetFireAndForgetCallsMade().ToList();
+            calls.Sort((kvp1, kvp2) => kvp2.Value.CompareTo(kvp1.Value));
+            int namedCallsMade = 0;
+
+            ConsoleDisplayList cdl = new ConsoleDisplayList();
+            foreach (KeyValuePair<string, int> kvp in calls)
+            {
+                cdl.AddRow(kvp.Key, kvp.Value);
+                namedCallsMade += kvp.Value;
+            }
+
+            cdl.AddRow("TOTAL NAMED", namedCallsMade);
+
+            long allCallsMade = Util.TotalFireAndForgetCallsMade;
+            cdl.AddRow("TOTAL ANONYMOUS", allCallsMade - namedCallsMade);
+            cdl.AddRow("TOTAL ALL", allCallsMade);
+
+            MainConsole.Instance.Output(cdl.ToString());
+        }
+
         private void HandleDebugThreadpoolStatus(string module, string[] args)
         {
             int workerThreads, iocpThreads;
diff --git a/OpenSim/Framework/ServiceAuth/BasicHttpAuthentication.cs b/OpenSim/Framework/ServiceAuth/BasicHttpAuthentication.cs
index f33a045..d182a71 100644
--- a/OpenSim/Framework/ServiceAuth/BasicHttpAuthentication.cs
+++ b/OpenSim/Framework/ServiceAuth/BasicHttpAuthentication.cs
@@ -15,7 +15,7 @@ namespace OpenSim.Framework.ServiceAuth
         private string m_Username, m_Password;
         private string m_CredentialsB64;
 
-        private string remove_me;
+//        private string remove_me;
 
         public string Credentials
         {
@@ -24,7 +24,7 @@ namespace OpenSim.Framework.ServiceAuth
 
         public BasicHttpAuthentication(IConfigSource config, string section)
         {
-            remove_me = section;
+//            remove_me = section;
             m_Username = Util.GetConfigVarFromSections<string>(config, "HttpAuthUsername", new string[] { "Network", section }, string.Empty);
             m_Password = Util.GetConfigVarFromSections<string>(config, "HttpAuthPassword", new string[] { "Network", section }, string.Empty); 
             string str = m_Username + ":" + m_Password;
diff --git a/OpenSim/Framework/Tests/AnimationTests.cs b/OpenSim/Framework/Tests/AnimationTests.cs
index f3be81b..d8f17d0 100644
--- a/OpenSim/Framework/Tests/AnimationTests.cs
+++ b/OpenSim/Framework/Tests/AnimationTests.cs
@@ -32,7 +32,6 @@ using OpenMetaverse;
 using OpenMetaverse.StructuredData;
 using OpenSim.Framework;
 using OpenSim.Tests.Common;
-using OpenSim.Tests.Common.Mock;
 using Animation = OpenSim.Framework.Animation;
 
 namespace OpenSim.Framework.Tests
diff --git a/OpenSim/Framework/Util.cs b/OpenSim/Framework/Util.cs
index fefa050..97c958a 100644
--- a/OpenSim/Framework/Util.cs
+++ b/OpenSim/Framework/Util.cs
@@ -1928,11 +1928,6 @@ namespace OpenSim.Framework
             }
         }
 
-        public static void FireAndForget(System.Threading.WaitCallback callback)
-        {
-            FireAndForget(callback, null, null);
-        }
-
         public static void InitThreadPool(int minThreads, int maxThreads)
         {
             if (maxThreads < 2)
@@ -1977,8 +1972,7 @@ namespace OpenSim.Framework
                     throw new NotImplementedException();
             }
         }
-
-        
+                
         /// <summary>
         /// Additional information about threads in the main thread pool. Used to time how long the
         /// thread has been running, and abort it if it has timed-out.
@@ -2052,12 +2046,15 @@ namespace OpenSim.Framework
             }
         }
 
-
         private static long nextThreadFuncNum = 0;
         private static long numQueuedThreadFuncs = 0;
         private static long numRunningThreadFuncs = 0;
+        private static long numTotalThreadFuncsCalled = 0;
         private static Int32 threadFuncOverloadMode = 0;
 
+        public static long TotalQueuedFireAndForgetCalls { get { return numQueuedThreadFuncs; } }
+        public static long TotalRunningFireAndForgetCalls { get { return numRunningThreadFuncs; } }
+
         // Maps (ThreadFunc number -> Thread)
         private static ConcurrentDictionary<long, ThreadInfo> activeThreads = new ConcurrentDictionary<long, ThreadInfo>();
 
@@ -2086,14 +2083,49 @@ namespace OpenSim.Framework
             }
         }
 
+        public static long TotalFireAndForgetCallsMade { get { return numTotalThreadFuncsCalled; } }
+
+        public static Dictionary<string, int> GetFireAndForgetCallsMade()
+        {
+            return new Dictionary<string, int>(m_fireAndForgetCallsMade);
+        }       
+
+        private static Dictionary<string, int> m_fireAndForgetCallsMade = new Dictionary<string, int>();
+
+        public static Dictionary<string, int> GetFireAndForgetCallsInProgress()
+        {
+            return new Dictionary<string, int>(m_fireAndForgetCallsInProgress);
+        }
+
+        private static Dictionary<string, int> m_fireAndForgetCallsInProgress = new Dictionary<string, int>();
+
+        public static void FireAndForget(System.Threading.WaitCallback callback)
+        {
+            FireAndForget(callback, null, null);
+        }
 
         public static void FireAndForget(System.Threading.WaitCallback callback, object obj)
         {
             FireAndForget(callback, obj, null);
         }
-
+     
         public static void FireAndForget(System.Threading.WaitCallback callback, object obj, string context)
         {
+            Interlocked.Increment(ref numTotalThreadFuncsCalled);
+
+            if (context != null)
+            {
+                if (!m_fireAndForgetCallsMade.ContainsKey(context))
+                    m_fireAndForgetCallsMade[context] = 1;
+                else
+                    m_fireAndForgetCallsMade[context]++;
+
+                if (!m_fireAndForgetCallsInProgress.ContainsKey(context))
+                    m_fireAndForgetCallsInProgress[context] = 1;
+                else
+                    m_fireAndForgetCallsInProgress[context]++;
+            }
+
             WaitCallback realCallback;
 
             bool loggingEnabled = LogThreadPool > 0;
@@ -2104,7 +2136,15 @@ namespace OpenSim.Framework
             if (FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
             {
                 // If we're running regression tests, then we want any exceptions to rise up to the test code.
-                realCallback = o => { Culture.SetCurrentCulture(); callback(o); };
+                realCallback = 
+                    o => 
+                    { 
+                        Culture.SetCurrentCulture(); 
+                        callback(o); 
+                        
+                        if (context != null)
+                            m_fireAndForgetCallsInProgress[context]--;
+                    };
             }
             else
             {
@@ -2143,6 +2183,9 @@ namespace OpenSim.Framework
                         activeThreads.TryRemove(threadFuncNum, out dummy);
                         if ((loggingEnabled || (threadFuncOverloadMode == 1)) && threadInfo.LogThread)
                             m_log.DebugFormat("Exit threadfunc {0} ({1})", threadFuncNum, FormatDuration(threadInfo.Elapsed()));
+
+                        if (context != null)
+                            m_fireAndForgetCallsInProgress[context]--;
                     }
                 };
             }