11 files changed, 78 insertions, 65 deletions

diff --git a/OpenSim/Addons/Groups/Hypergrid/GroupsServiceHGConnectorModule.cs b/OpenSim/Addons/Groups/Hypergrid/GroupsServiceHGConnectorModule.cs
index a750d8d..9fbc1b3 100644
--- a/OpenSim/Addons/Groups/Hypergrid/GroupsServiceHGConnectorModule.cs
+++ b/OpenSim/Addons/Groups/Hypergrid/GroupsServiceHGConnectorModule.cs
@@ -32,6 +32,7 @@ using System.Reflection;
 using System.Text;
 
 using OpenSim.Framework;
+using OpenSim.Framework.Monitoring;
 using OpenSim.Framework.Servers;
 using OpenSim.Region.Framework.Scenes;
 using OpenSim.Region.Framework.Interfaces;
@@ -560,7 +561,7 @@ namespace OpenSim.Groups
 
                     // so we have the list of urls to send the notice to
                     // this may take a long time...
-                    Util.RunThreadNoTimeout(delegate
+                    Watchdog.RunInThread(delegate
                     {
                         foreach (string u in urls)
                         {
@@ -571,7 +572,7 @@ namespace OpenSim.Groups
                                     hasAttachment, attType, attName, attItemID, AgentUUIForOutside(attOwnerID));
                             }
                         }
-                    }, "AddGroupNotice", null);
+                    }, string.Format("AddGroupNotice (agent {0}, group {1})", RequestingAgentID, groupID) , null);
 
                     return true;
                 }
diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs
index 427ed7b..e9e7bd2 100644
--- a/OpenSim/Framework/Monitoring/Watchdog.cs
+++ b/OpenSim/Framework/Monitoring/Watchdog.cs
@@ -187,15 +187,16 @@ namespace OpenSim.Framework.Monitoring
         /// <param name="priority">Priority to run the thread at</param>
         /// <param name="isBackground">True to run this thread as a background thread, otherwise false</param>
         /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
+        /// <param name="log">If true then creation of thread is logged.</param>
         /// <returns>The newly created Thread object</returns>
         public static Thread StartThread(
-            ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout)
+            ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, bool log = true)
         {
-            return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS);
+            return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS, log);
         }
 
         /// <summary>
-        /// Start a new thread that is tracked by the watchdog timer
+        /// Start a new thread that is tracked by the watchdog
         /// </summary>
         /// <param name="start">The method that will be executed in a new thread</param>
         /// <param name="name">A name to give to the new thread</param>
@@ -208,10 +209,11 @@ namespace OpenSim.Framework.Monitoring
         /// Normally, this will just return some useful debugging information.
         /// </param>
         /// <param name="timeout">Number of milliseconds to wait until we issue a warning about timeout.</param>
+        /// <param name="log">If true then creation of thread is logged.</param>
         /// <returns>The newly created Thread object</returns>
         public static Thread StartThread(
             ThreadStart start, string name, ThreadPriority priority, bool isBackground,
-            bool alarmIfTimeout, Func<string> alarmMethod, int timeout)
+            bool alarmIfTimeout, Func<string> alarmMethod, int timeout, bool log = true)
         {
             Thread thread = new Thread(start);
             thread.Name = name;
@@ -222,8 +224,9 @@ namespace OpenSim.Framework.Monitoring
                 = new ThreadWatchdogInfo(thread, timeout)
                     { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod };
 
-            m_log.DebugFormat(
-                "[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
+            if (log)
+                m_log.DebugFormat(
+                    "[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
 
             lock (m_threads)
                 m_threads.Add(twi.Thread.ManagedThreadId, twi);
@@ -234,6 +237,39 @@ namespace OpenSim.Framework.Monitoring
         }
 
         /// <summary>
+        /// Run the callback in a new thread immediately.  If the thread exits with an exception log it but do
+        /// not propogate it.
+        /// </summary>
+        /// <param name="callback">Code for the thread to execute.</param>
+        /// <param name="name">Name of the thread</param>
+        /// <param name="obj">Object to pass to the thread.</param>
+        public static void RunInThread(WaitCallback callback, string name, object obj, bool log = false)
+        {
+            if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)           
+            {
+                Culture.SetCurrentCulture();
+                callback(obj);
+                return;
+            }
+
+            ThreadStart ts = new ThreadStart(delegate()
+            {
+                try
+                {
+                    Culture.SetCurrentCulture();
+                    callback(obj);
+                    Watchdog.RemoveThread(log:false);
+                }
+                catch (Exception e)
+                {
+                    m_log.Error(string.Format("[WATCHDOG]: Exception in thread {0}.", name), e);
+                }
+            });
+
+            StartThread(ts, name, ThreadPriority.Normal, true, false, log:log);
+        }
+
+        /// <summary>
         /// Marks the current thread as alive
         /// </summary>
         public static void UpdateThread()
@@ -244,24 +280,26 @@ namespace OpenSim.Framework.Monitoring
         /// <summary>
         /// Stops watchdog tracking on the current thread
         /// </summary>
+        /// <param name="log">If true then normal events in thread removal are not logged.</param>
         /// <returns>
         /// True if the thread was removed from the list of tracked
         /// threads, otherwise false
         /// </returns>
-        public static bool RemoveThread()
+        public static bool RemoveThread(bool log = true)
         {
-            return RemoveThread(Thread.CurrentThread.ManagedThreadId);
+            return RemoveThread(Thread.CurrentThread.ManagedThreadId, log);
         }
 
-        private static bool RemoveThread(int threadID)
+        private static bool RemoveThread(int threadID, bool log = true)
         {
             lock (m_threads)
             {
                 ThreadWatchdogInfo twi;
                 if (m_threads.TryGetValue(threadID, out twi))
                 {
-                    m_log.DebugFormat(
-                        "[WATCHDOG]: Removing thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
+                    if (log)
+                        m_log.DebugFormat(
+                            "[WATCHDOG]: Removing thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
 
                     twi.Cleanup();
                     m_threads.Remove(threadID);
diff --git a/OpenSim/Framework/Util.cs b/OpenSim/Framework/Util.cs
index 2c38571..fefa050 100644
--- a/OpenSim/Framework/Util.cs
+++ b/OpenSim/Framework/Util.cs
@@ -2400,36 +2400,6 @@ namespace OpenSim.Framework
         #endregion FireAndForget Threading Pattern
 
         /// <summary>
-        /// Run the callback on a different thread, outside the thread pool. This is used for tasks
-        /// that may take a long time.
-        /// </summary>
-        public static void RunThreadNoTimeout(WaitCallback callback, string name, object obj)
-        {
-            if (FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
-            {
-                Culture.SetCurrentCulture();
-                callback(obj);
-                return;
-            }
-
-            Thread t = new Thread(delegate()
-            {
-                try
-                {
-                    Culture.SetCurrentCulture();
-                    callback(obj);
-                }
-                catch (Exception e)
-                {
-                    m_log.Error("Exception in thread " + name, e);
-                }
-            });
-            
-            t.Name = name;
-            t.Start();
-        }
-
-        /// <summary>
         /// Environment.TickCount is an int but it counts all 32 bits so it goes positive
         /// and negative every 24.9 days. This trims down TickCount so it doesn't wrap
         /// for the callers. 
diff --git a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs
index f06d70d..9d6870f 100644
--- a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs
+++ b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs
@@ -43,6 +43,7 @@ using Mono.Addins;
 using OpenMetaverse;
 using OpenSim.Framework;
 using OpenSim.Framework.Console;
+using OpenSim.Framework.Monitoring;
 using OpenSim.Region.Framework.Interfaces;
 using OpenSim.Region.Framework.Scenes;
 using OpenSim.Services.Interfaces;
@@ -963,7 +964,8 @@ namespace OpenSim.Region.CoreModules.Asset
                     case "assets":
                         con.Output("Ensuring assets are cached for all scenes.");
 
-                        Util.RunThreadNoTimeout(delegate {
+                        Watchdog.RunInThread(delegate 
+                        {
                             int assetReferenceTotal = TouchAllSceneAssets(true);
                             con.OutputFormat("Completed check with {0} assets.", assetReferenceTotal);
                         }, "TouchAllSceneAssets", null);
diff --git a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs
index d15703e..a5b5aa2 100644
--- a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs
+++ b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs
@@ -34,6 +34,7 @@ using System.Xml;
 using log4net;
 using OpenMetaverse;
 using OpenSim.Framework;
+using OpenSim.Framework.Monitoring;
 using OpenSim.Framework.Serialization;
 using OpenSim.Framework.Serialization.External;
 using OpenSim.Region.CoreModules.World.Archiver;
@@ -356,7 +357,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
                             m_scene.UserAccountService, m_scene.RegionInfo.ScopeID,
                             options, ReceivedAllAssets);
 
-                    Util.RunThreadNoTimeout(o => ar.Execute(), "AssetsRequest", null);
+                    Watchdog.RunInThread(o => ar.Execute(), string.Format("AssetsRequest ({0})", m_scene.Name), null);
                 }
                 else
                 {
diff --git a/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs b/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs
index f80fab3..470ef02 100644
--- a/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs
+++ b/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs
@@ -33,6 +33,7 @@ using System;
 using System.Collections.Generic;
 using System.Reflection;
 using OpenSim.Framework;
+using OpenSim.Framework.Monitoring;
 using OpenSim.Data;
 using OpenSim.Server.Base;
 using OpenSim.Region.Framework.Interfaces;
@@ -183,12 +184,12 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsOut.Inventory
                 // Protect ourselves against the caller subsequently modifying the items list
                 List<InventoryItemBase> items = new List<InventoryItemBase>(invCol.Items);
 
-                Util.RunThreadNoTimeout(delegate
+                Watchdog.RunInThread(delegate
                 {
                     foreach (InventoryItemBase item in items)
                         if (!string.IsNullOrEmpty(item.CreatorData))
                             UserManager.AddUser(item.CreatorIdAsUuid, item.CreatorData);
-                }, "GetFolderContent", null);
+                }, string.Format("GetFolderContent (user {0}, folder {1})", userID, folderID), null);
             }
 
             return invCol;
diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs
index c8056d4..3e0c9f3 100644
--- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs
+++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs
@@ -36,6 +36,7 @@ using System.Xml;
 using log4net;
 using OpenMetaverse;
 using OpenSim.Framework;
+using OpenSim.Framework.Monitoring;
 using OpenSim.Framework.Serialization;
 using OpenSim.Framework.Serialization.External;
 using OpenSim.Region.CoreModules.World.Terrain;
@@ -371,7 +372,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
             // Start the scripts. We delayed this because we want the OAR to finish loading ASAP, so
             // that users can enter the scene. If we allow the scripts to start in the loop above
             // then they significantly increase the time until the OAR finishes loading.
-            Util.RunThreadNoTimeout(delegate(object o)
+            Watchdog.RunInThread(o =>
             {
                 Thread.Sleep(15000);
                 m_log.Info("[ARCHIVER]: Starting scripts in scene objects");
@@ -386,7 +387,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
 
                     sceneContext.SceneObjects.Clear();
                 }
-            }, "ReadArchiveStartScripts", null);
+            }, string.Format("ReadArchiveStartScripts (request {0})", m_requestId), null);
 
             m_log.InfoFormat("[ARCHIVER]: Successfully loaded archive");
 
diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs
index 448147a..803f24e 100644
--- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs
+++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs
@@ -36,6 +36,7 @@ using System.Xml;
 using log4net;
 using OpenMetaverse;
 using OpenSim.Framework;
+using OpenSim.Framework.Monitoring;
 using OpenSim.Framework.Serialization;
 using OpenSim.Region.CoreModules.World.Terrain;
 using OpenSim.Region.Framework.Interfaces;
@@ -199,7 +200,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
                             m_rootScene.AssetService, m_rootScene.UserAccountService,
                             m_rootScene.RegionInfo.ScopeID, options, ReceivedAllAssets);
 
-                    Util.RunThreadNoTimeout(o => ar.Execute(), "AssetsRequest", null);
+                    Watchdog.RunInThread(o => ar.Execute(), "Archive Assets Request", null);
 
                     // CloseArchive() will be called from ReceivedAllAssets()
                 }
diff --git a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
index 4f428c3..ec39bc0 100644
--- a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
+++ b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
@@ -33,6 +33,7 @@ using System.Timers;
 using log4net;
 using OpenMetaverse;
 using OpenSim.Framework;
+using OpenSim.Framework.Monitoring;
 using OpenSim.Framework.Serialization;
 using OpenSim.Framework.Serialization.External;
 using OpenSim.Services.Interfaces;
@@ -226,7 +227,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
             finally
             {
                 if (timedOut)
-                    Util.RunThreadNoTimeout(PerformAssetsRequestCallback, "AssetsRequestCallback", true);
+                    Watchdog.RunInThread(PerformAssetsRequestCallback, "Archive Assets Request Callback", true);
             }
         }
 
@@ -295,7 +296,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
                         
                         // We want to stop using the asset cache thread asap 
                         // as we now need to do the work of producing the rest of the archive
-                        Util.RunThreadNoTimeout(PerformAssetsRequestCallback, "AssetsRequestCallback", false);
+                        Watchdog.RunInThread(PerformAssetsRequestCallback, "Archive Assets Request Callback", false);
                     }
                     else
                     {
diff --git a/OpenSim/Region/Framework/Scenes/Scene.cs b/OpenSim/Region/Framework/Scenes/Scene.cs
index 5f0dbd7..28dbccb 100644
--- a/OpenSim/Region/Framework/Scenes/Scene.cs
+++ b/OpenSim/Region/Framework/Scenes/Scene.cs
@@ -1623,7 +1623,12 @@ namespace OpenSim.Region.Framework.Scenes
                     {
                         tmpMS = Util.EnvironmentTickCount();
                         m_cleaningTemps = true;
-                        Util.RunThreadNoTimeout(delegate { CleanTempObjects(); m_cleaningTemps = false;  }, "CleanTempObjects", null);
+
+                        Watchdog.RunInThread(
+                            delegate { CleanTempObjects(); m_cleaningTemps = false;  }, 
+                            string.Format("CleanTempObjects ({0})", Name), 
+                            null);
+
                         tempOnRezMS = Util.EnvironmentTickCountSubtract(tmpMS);
                     }
     
@@ -1803,7 +1808,7 @@ namespace OpenSim.Region.Framework.Scenes
             if (!m_backingup)
             {
                 m_backingup = true;
-                Util.RunThreadNoTimeout(BackupWaitCallback, "BackupWaitCallback", null);
+                Watchdog.RunInThread(o => Backup(false), string.Format("BackupWaitCallback ({0})", Name), null);
             }
         }
 
@@ -1814,14 +1819,6 @@ namespace OpenSim.Region.Framework.Scenes
         {
             m_eventManager.TriggerOnFrame();
         }
-
-        /// <summary>
-        /// Wrapper for Backup() that can be called with Util.RunThreadNoTimeout()
-        /// </summary>
-        private void BackupWaitCallback(object o)
-        {
-            Backup(false);
-        }
         
         /// <summary>
         /// Backup the scene.
diff --git a/OpenSim/Region/Framework/Scenes/ScenePresence.cs b/OpenSim/Region/Framework/Scenes/ScenePresence.cs
index 3c37de8..59b521d 100644
--- a/OpenSim/Region/Framework/Scenes/ScenePresence.cs
+++ b/OpenSim/Region/Framework/Scenes/ScenePresence.cs
@@ -37,6 +37,7 @@ using log4net;
 using Nini.Config;
 using OpenSim.Framework;
 using OpenSim.Framework.Client;
+using OpenSim.Framework.Monitoring;
 using OpenSim.Region.Framework.Interfaces;
 using OpenSim.Region.Framework.Scenes.Animation;
 using OpenSim.Region.Framework.Scenes.Types;
@@ -3362,7 +3363,7 @@ namespace OpenSim.Region.Framework.Scenes
             SentInitialDataToClient = true;
 
             // Send all scene object to the new client
-            Util.RunThreadNoTimeout(delegate
+            Watchdog.RunInThread(delegate
             {
 //                m_log.DebugFormat(
 //                    "[SCENE PRESENCE]: Sending initial data to {0} agent {1} in {2}, tp flags {3}", 
@@ -3380,8 +3381,7 @@ namespace OpenSim.Region.Framework.Scenes
                     if (e != null && e is SceneObjectGroup)
                         ((SceneObjectGroup)e).SendFullUpdateToClient(ControllingClient);
                 }
-
-            }, "SendInitialDataToClient", null);
+            }, string.Format("SendInitialDataToClient ({0} in {1})", Name, Scene.Name), null);
         }
 
         /// <summary>