4 files changed, 64 insertions, 65 deletions

diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs
index a6a059d..115871e 100644
--- a/OpenSim/Framework/Monitoring/JobEngine.cs
+++ b/OpenSim/Framework/Monitoring/JobEngine.cs
@@ -57,7 +57,8 @@ namespace OpenSim.Framework.Monitoring
         /// <remarks>
         /// Will be null if no job is currently running.
         /// </remarks>
-        public Job CurrentJob { get; private set; }
+        private Job m_currentJob;
+        public Job CurrentJob { get { return m_currentJob;} }
 
         /// <summary>
         /// Number of jobs waiting to be processed.
@@ -82,16 +83,15 @@ namespace OpenSim.Framework.Monitoring
 
         private CancellationTokenSource m_cancelSource;
 
-        /// <summary>
-        /// Used to signal that we are ready to complete stop.
-        /// </summary>
-        private ManualResetEvent m_finishedProcessingAfterStop = new ManualResetEvent(false);
+        private int m_timeout = -1;
+
+        private bool m_threadRunnig = false;
 
-        public JobEngine(string name, string loggingName)
+        public JobEngine(string name, string loggingName, int timeout = -1)
         {
             Name = name;
             LoggingName = loggingName;
-
+            m_timeout = timeout;
             RequestProcessTimeoutOnStop = 5000;
         }
 
@@ -104,18 +104,9 @@ namespace OpenSim.Framework.Monitoring
 
                 IsRunning = true;
 
-                m_finishedProcessingAfterStop.Reset();
-
                 m_cancelSource = new CancellationTokenSource();
-
-                WorkManager.StartThread(
-                    ProcessRequests,
-                    Name,
-                    ThreadPriority.Normal,
-                    false,
-                    true,
-                    null,
-                    int.MaxValue);
+                WorkManager.RunInThreadPool(ProcessRequests, null, Name, false);
+                m_threadRunnig = true;
             }
         }
 
@@ -131,20 +122,16 @@ namespace OpenSim.Framework.Monitoring
                     m_log.DebugFormat("[JobEngine] Stopping {0}", Name);
 
                     IsRunning = false;
-
-                    m_finishedProcessingAfterStop.Reset();
-                    if(m_jobQueue.Count <= 0)
+                    if(m_threadRunnig)
+                    {
                         m_cancelSource.Cancel();
-
-                    m_finishedProcessingAfterStop.WaitOne(RequestProcessTimeoutOnStop);
-                        m_finishedProcessingAfterStop.Close();
+                        m_threadRunnig = false;
+                    }
                 }
                 finally
                 {
                     if(m_cancelSource != null)
                         m_cancelSource.Dispose();
-                    if(m_finishedProcessingAfterStop != null)
-                        m_finishedProcessingAfterStop.Dispose();
                 }
             }
         }
@@ -203,6 +190,18 @@ namespace OpenSim.Framework.Monitoring
         /// </param>
         public bool QueueJob(Job job)
         {
+            lock(JobLock)
+            {
+                if(!IsRunning)
+                    return false;
+                
+                if(!m_threadRunnig)
+                {
+                     WorkManager.RunInThreadPool(ProcessRequests, null, Name, false);
+                     m_threadRunnig = true;
+                }
+            }
+
             if (m_jobQueue.Count < m_jobQueue.BoundedCapacity)
             {
                 m_jobQueue.Add(job);
@@ -222,59 +221,53 @@ namespace OpenSim.Framework.Monitoring
 
                     m_warnOverMaxQueue = false;
                 }
-
                 return false;
             }
         }
 
-        private void ProcessRequests()
+        private void ProcessRequests(Object o)
         {
-            while(IsRunning || m_jobQueue.Count > 0)
+            while(IsRunning)
             {
                 try
                 {
-                    CurrentJob = m_jobQueue.Take(m_cancelSource.Token);
-                }
-                catch(ObjectDisposedException e)
-                {
-                    // If we see this whilst not running then it may be due to a race where this thread checks
-                    // IsRunning after the stopping thread sets it to false and disposes of the cancellation source.
-                    if(IsRunning)
-                        throw e;
-                    else
+                    if(!m_jobQueue.TryTake(out m_currentJob, m_timeout, m_cancelSource.Token))
                     {
-                        m_log.DebugFormat("[JobEngine] {0} stopping ignoring {1} jobs in queue",
-                                Name,m_jobQueue.Count);
+                        lock(JobLock)
+                            m_threadRunnig = false;
                         break;
                     }
                 }
+                catch(ObjectDisposedException e)
+                {
+                    m_log.DebugFormat("[JobEngine] {0} stopping ignoring {1} jobs in queue",
+                        Name,m_jobQueue.Count);
+                    break;
+                }
                 catch(OperationCanceledException)
                 {
                     break;
                 }
 
                 if(LogLevel >= 1)
-                    m_log.DebugFormat("[{0}]: Processing job {1}",LoggingName,CurrentJob.Name);
+                    m_log.DebugFormat("[{0}]: Processing job {1}",LoggingName,m_currentJob.Name);
 
                 try
                 {
-                    CurrentJob.Action();
+                    m_currentJob.Action();
                 }
                 catch(Exception e)
                 {
                     m_log.Error(
                         string.Format(
-                        "[{0}]: Job {1} failed, continuing.  Exception  ",LoggingName,CurrentJob.Name),e);
+                        "[{0}]: Job {1} failed, continuing.  Exception  ",LoggingName,m_currentJob.Name),e);
                 }
 
                 if(LogLevel >= 1)
-                    m_log.DebugFormat("[{0}]: Processed job {1}",LoggingName,CurrentJob.Name);
+                    m_log.DebugFormat("[{0}]: Processed job {1}",LoggingName,m_currentJob.Name);
 
-                CurrentJob = null;
+                m_currentJob = null;
             }
-
-            Watchdog.RemoveThread(false);
-            m_finishedProcessingAfterStop.Set();
         }
 
         public class Job
diff --git a/OpenSim/Framework/Monitoring/WorkManager.cs b/OpenSim/Framework/Monitoring/WorkManager.cs
index 9d52f71..5d9b185 100644
--- a/OpenSim/Framework/Monitoring/WorkManager.cs
+++ b/OpenSim/Framework/Monitoring/WorkManager.cs
@@ -57,7 +57,7 @@ namespace OpenSim.Framework.Monitoring
 
         static WorkManager()
         {
-            JobEngine = new JobEngine("Non-blocking non-critical job engine", "JOB ENGINE");
+            JobEngine = new JobEngine("Non-blocking non-critical job engine", "JOB ENGINE", 30000);
 
             StatsManager.RegisterStat(
                 new Stat(
@@ -182,9 +182,9 @@ namespace OpenSim.Framework.Monitoring
         /// <param name="callback"></param>
         /// <param name="obj"></param>
         /// <param name="name">The name of the job.  This is used in monitoring and debugging.</param>
-        public static void RunInThreadPool(System.Threading.WaitCallback callback, object obj, string name)
+        public static void RunInThreadPool(System.Threading.WaitCallback callback, object obj, string name, bool timeout = true)
         {
-            Util.FireAndForget(callback, obj, name);
+            Util.FireAndForget(callback, obj, name, timeout);
         }
 
         /// <summary>
@@ -231,10 +231,8 @@ namespace OpenSim.Framework.Monitoring
                 JobEngine.QueueJob(name, () => callback(obj));
             else if (canRunInThisThread)
                 callback(obj);
-            else if (mustNotTimeout)
-                RunInThread(callback, obj, name, log);
             else
-                Util.FireAndForget(callback, obj, name);
+                Util.FireAndForget(callback, obj, name, !mustNotTimeout);
         }
 
         private static void HandleControlCommand(string module, string[] args)
diff --git a/OpenSim/Framework/Util.cs b/OpenSim/Framework/Util.cs
index f52a84c..9a1e348 100644
--- a/OpenSim/Framework/Util.cs
+++ b/OpenSim/Framework/Util.cs
@@ -2492,8 +2492,9 @@ namespace OpenSim.Framework
             public bool Running { get; set; }
             public bool Aborted { get; set; }
             private int started;
+            public bool DoTimeout;
 
-            public ThreadInfo(long threadFuncNum, string context)
+            public ThreadInfo(long threadFuncNum, string context, bool dotimeout = true)
             {
                 ThreadFuncNum = threadFuncNum;
                 this.context = context;
@@ -2501,6 +2502,7 @@ namespace OpenSim.Framework
                 Thread = null;
                 Running = false;
                 Aborted = false;
+                DoTimeout = dotimeout;
             }
 
             public void Started()
@@ -2571,7 +2573,7 @@ namespace OpenSim.Framework
             foreach (KeyValuePair<long, ThreadInfo> entry in activeThreads)
             {
                 ThreadInfo t = entry.Value;
-                if (t.Running && !t.Aborted && (t.Elapsed() >= THREAD_TIMEOUT))
+                if (t.DoTimeout && t.Running && !t.Aborted && (t.Elapsed() >= THREAD_TIMEOUT))
                 {
                     m_log.WarnFormat("Timeout in threadfunc {0} ({1}) {2}", t.ThreadFuncNum, t.Thread.Name, t.GetStackTrace());
                     t.Abort();
@@ -2612,7 +2614,7 @@ namespace OpenSim.Framework
             FireAndForget(callback, obj, null);
         }
 
-        public static void FireAndForget(System.Threading.WaitCallback callback, object obj, string context)
+        public static void FireAndForget(System.Threading.WaitCallback callback, object obj, string context, bool dotimeout = true)
         {
             Interlocked.Increment(ref numTotalThreadFuncsCalled);
 
@@ -2634,7 +2636,7 @@ namespace OpenSim.Framework
             bool loggingEnabled = LogThreadPool > 0;
 
             long threadFuncNum = Interlocked.Increment(ref nextThreadFuncNum);
-            ThreadInfo threadInfo = new ThreadInfo(threadFuncNum, context);
+            ThreadInfo threadInfo = new ThreadInfo(threadFuncNum, context, dotimeout);
 
             if (FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
             {
diff --git a/OpenSim/Framework/WebUtil.cs b/OpenSim/Framework/WebUtil.cs
index 7b085d0..48078ad 100644
--- a/OpenSim/Framework/WebUtil.cs
+++ b/OpenSim/Framework/WebUtil.cs
@@ -1262,18 +1262,24 @@ namespace OpenSim.Framework
                         {
                             if (hwr.StatusCode == HttpStatusCode.NotFound)
                                 return deserial;
+
                             if (hwr.StatusCode == HttpStatusCode.Unauthorized)
                             {
-                                m_log.Error(string.Format(
-                                    "[SynchronousRestObjectRequester]: Web request {0} requires authentication ",
-                                    requestUrl));
-                                return deserial;
+                                m_log.ErrorFormat("[SynchronousRestObjectRequester]: Web request {0} requires authentication",
+                                    requestUrl);
+                            }
+                            else
+                            {
+                                m_log.WarnFormat("[SynchronousRestObjectRequester]: Web request {0} returned error: {1}",
+                                    requestUrl, hwr.StatusCode);
                             }
                         }
                         else
-                            m_log.Error(string.Format(
-                                "[SynchronousRestObjectRequester]: WebException for {0} {1} {2} ",
-                                verb, requestUrl, typeof(TResponse).ToString()), e);
+                            m_log.ErrorFormat(
+                                "[SynchronousRestObjectRequester]: WebException for {0} {1} {2} {3}",
+                                verb, requestUrl, typeof(TResponse).ToString(), e.Message);
+
+                       return deserial;
                     }
                 }
                 catch (System.InvalidOperationException)