For monitoring purposes, start non-timeout tasks (which do not currently use a threadpool) via Watchdog.RunInThread() rather than Util.RunThreadNoTimeout()

The functionality is the same but this allow us to monitor such tasks via "show threads" and abort them for test purposes, etc.
Also extends thread names to provide more info (e.g. SendInitialDataToClient says what client the task is for).

11 files changed, 78 insertions, 65 deletions

diff --git a/OpenSim/Addons/Groups/Hypergrid/GroupsServiceHGConnectorModule.cs b/OpenSim/Addons/Groups/Hypergrid/GroupsServiceHGConnectorModule.cs
index a750d8d..9fbc1b3 100644
--- a/OpenSim/Addons/Groups/Hypergrid/GroupsServiceHGConnectorModule.cs
+++ b/OpenSim/Addons/Groups/Hypergrid/GroupsServiceHGConnectorModule.cs
@@ -32,6 +32,7 @@ using System.Reflection;
 using System.Text;
 
 using OpenSim.Framework;
+using OpenSim.Framework.Monitoring;
 using OpenSim.Framework.Servers;
 using OpenSim.Region.Framework.Scenes;
 using OpenSim.Region.Framework.Interfaces;
@@ -560,7 +561,7 @@ namespace OpenSim.Groups
 
                     // so we have the list of urls to send the notice to
                     // this may take a long time...
-                    Util.RunThreadNoTimeout(delegate
+                    Watchdog.RunInThread(delegate
                     {
                         foreach (string u in urls)
                         {
@@ -571,7 +572,7 @@ namespace OpenSim.Groups
                                     hasAttachment, attType, attName, attItemID, AgentUUIForOutside(attOwnerID));
                             }
                         }
-                    }, "AddGroupNotice", null);
+                    }, string.Format("AddGroupNotice (agent {0}, group {1})", RequestingAgentID, groupID) , null);
 
                     return true;
                 }
diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs
index 427ed7b..e9e7bd2 100644
--- a/OpenSim/Framework/Monitoring/Watchdog.cs
+++ b/OpenSim/Framework/Monitoring/Watchdog.cs
@@ -187,15 +187,16 @@ namespace OpenSim.Framework.Monitoring
         /// <param name="priority">Priority to run the thread at</param>
         /// <param name="isBackground">True to run this thread as a background thread, otherwise false</param>
         /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
+        /// <param name="log">If true then creation of thread is logged.</param>
         /// <returns>The newly created Thread object</returns>
         public static Thread StartThread(
-            ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout)
+            ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, bool log = true)
         {
-            return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS);
+            return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS, log);
         }
 
         /// <summary>
-        /// Start a new thread that is tracked by the watchdog timer
+        /// Start a new thread that is tracked by the watchdog
         /// </summary>
         /// <param name="start">The method that will be executed in a new thread</param>
         /// <param name="name">A name to give to the new thread</param>
@@ -208,10 +209,11 @@ namespace OpenSim.Framework.Monitoring
         /// Normally, this will just return some useful debugging information.
         /// </param>
         /// <param name="timeout">Number of milliseconds to wait until we issue a warning about timeout.</param>
+        /// <param name="log">If true then creation of thread is logged.</param>
         /// <returns>The newly created Thread object</returns>
         public static Thread StartThread(
             ThreadStart start, string name, ThreadPriority priority, bool isBackground,
-            bool alarmIfTimeout, Func<string> alarmMethod, int timeout)
+            bool alarmIfTimeout, Func<string> alarmMethod, int timeout, bool log = true)
         {
             Thread thread = new Thread(start);
             thread.Name = name;
@@ -222,8 +224,9 @@ namespace OpenSim.Framework.Monitoring
                 = new ThreadWatchdogInfo(thread, timeout)
                     { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod };
 
-            m_log.DebugFormat(
-                "[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
+            if (log)
+                m_log.DebugFormat(
+                    "[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
 
             lock (m_threads)
                 m_threads.Add(twi.Thread.ManagedThreadId, twi);
@@ -234,6 +237,39 @@ namespace OpenSim.Framework.Monitoring
         }
 
         /// <summary>
+        /// Run the callback in a new thread immediately.  If the thread exits with an exception log it but do
+        /// not propogate it.
+        /// </summary>
+        /// <param name="callback">Code for the thread to execute.</param>
+        /// <param name="name">Name of the thread</param>
+        /// <param name="obj">Object to pass to the thread.</param>
+        public static void RunInThread(WaitCallback callback, string name, object obj, bool log = false)
+        {
+            if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)           
+            {
+                Culture.SetCurrentCulture();
+                callback(obj);
+                return;
+            }
+
+            ThreadStart ts = new ThreadStart(delegate()
+            {
+                try
+                {
+                    Culture.SetCurrentCulture();
+                    callback(obj);
+                    Watchdog.RemoveThread(log:false);
+                }
+                catch (Exception e)
+                {
+                    m_log.Error(string.Format("[WATCHDOG]: Exception in thread {0}.", name), e);
+                }
+            });
+
+            StartThread(ts, name, ThreadPriority.Normal, true, false, log:log);
+        }
+
+        /// <summary>
         /// Marks the current thread as alive
         /// </summary>
         public static void UpdateThread()
@@ -244,24 +280,26 @@ namespace OpenSim.Framework.Monitoring
         /// <summary>
         /// Stops watchdog tracking on the current thread
         /// </summary>
+        /// <param name="log">If true then normal events in thread removal are not logged.</param>
         /// <returns>
         /// True if the thread was removed from the list of tracked
         /// threads, otherwise false
         /// </returns>
-        public static bool RemoveThread()
+        public static bool RemoveThread(bool log = true)
         {
-            return RemoveThread(Thread.CurrentThread.ManagedThreadId);
+            return RemoveThread(Thread.CurrentThread.ManagedThreadId, log);
         }
 
-        private static bool RemoveThread(int threadID)
+        private static bool RemoveThread(int threadID, bool log = true)
         {
             lock (m_threads)
             {
                 ThreadWatchdogInfo twi;
                 if (m_threads.TryGetValue(threadID, out twi))
                 {
-                    m_log.DebugFormat(
-                        "[WATCHDOG]: Removing thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
+                    if (log)
+                        m_log.DebugFormat(
+                            "[WATCHDOG]: Removing thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
 
                     twi.Cleanup();
                     m_threads.Remove(threadID);
diff --git a/OpenSim/Framework/Util.cs b/OpenSim/Framework/Util.cs
index 2c38571..fefa050 100644
--- a/OpenSim/Framework/Util.cs
+++ b/OpenSim/Framework/Util.cs
@@ -2400,36 +2400,6 @@ namespace OpenSim.Framework
         #endregion FireAndForget Threading Pattern
 
         /// <summary>
-        /// Run the callback on a different thread, outside the thread pool. This is used for tasks
-        /// that may take a long time.
-        /// </summary>
-        public static void RunThreadNoTimeout(WaitCallback callback, string name, object obj)
-        {
-            if (FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
-            {
-                Culture.SetCurrentCulture();
-                callback(obj);
-                return;
-            }
-
-            Thread t = new Thread(delegate()
-            {
-                try
-                {
-                    Culture.SetCurrentCulture();
-                    callback(obj);
-                }
-                catch (Exception e)
-                {
-                    m_log.Error("Exception in thread " + name, e);
-                }
-            });
-            
-            t.Name = name;
-            t.Start();
-        }
-
-        /// <summary>
         /// Environment.TickCount is an int but it counts all 32 bits so it goes positive
         /// and negative every 24.9 days. This trims down TickCount so it doesn't wrap
         /// for the callers. 
diff --git a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs
index f06d70d..9d6870f 100644
--- a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs
+++ b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs
@@ -43,6 +43,7 @@ using Mono.Addins;
 using OpenMetaverse;
 using OpenSim.Framework;
 using OpenSim.Framework.Console;
+using OpenSim.Framework.Monitoring;
 using OpenSim.Region.Framework.Interfaces;
 using OpenSim.Region.Framework.Scenes;
 using OpenSim.Services.Interfaces;
@@ -963,7 +964,8 @@ namespace OpenSim.Region.CoreModules.Asset
                     case "assets":
                         con.Output("Ensuring assets are cached for all scenes.");
 
-                        Util.RunThreadNoTimeout(delegate {
+                        Watchdog.RunInThread(delegate 
+                        {
                             int assetReferenceTotal = TouchAllSceneAssets(true);
                             con.OutputFormat("Completed check with {0} assets.", assetReferenceTotal);
                         }, "TouchAllSceneAssets", null);
diff --git a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs
index d15703e..a5b5aa2 100644
--- a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs
+++ b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs
@@ -34,6 +34,7 @@ using System.Xml;
 using log4net;
 using OpenMetaverse;
 using OpenSim.Framework;
+using OpenSim.Framework.Monitoring;
 using OpenSim.Framework.Serialization;
 using OpenSim.Framework.Serialization.External;
 using OpenSim.Region.CoreModules.World.Archiver;
@@ -356,7 +357,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
                             m_scene.UserAccountService, m_scene.RegionInfo.ScopeID,
                             options, ReceivedAllAssets);
 
-                    Util.RunThreadNoTimeout(o => ar.Execute(), "AssetsRequest", null);
+                    Watchdog.RunInThread(o => ar.Execute(), string.Format("AssetsRequest ({0})", m_scene.Name), null);
                 }
                 else
                 {
diff --git a/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs b/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs
index f80fab3..470ef02 100644
--- a/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs
+++ b/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs
@@ -33,6 +33,7 @@ using System;
 using System.Collections.Generic;
 using System.Reflection;
 using OpenSim.Framework;
+using OpenSim.Framework.Monitoring;
 using OpenSim.Data;
 using OpenSim.Server.Base;
 using OpenSim.Region.Framework.Interfaces;
@@ -183,12 +184,12 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsOut.Inventory
                 // Protect ourselves against the caller subsequently modifying the items list
                 List<InventoryItemBase> items = new List<InventoryItemBase>(invCol.Items);
 
-                Util.RunThreadNoTimeout(delegate
+                Watchdog.RunInThread(delegate
                 {
                     foreach (InventoryItemBase item in items)
                         if (!string.IsNullOrEmpty(item.CreatorData))
                             UserManager.AddUser(item.CreatorIdAsUuid, item.CreatorData);
-                }, "GetFolderContent", null);
+                }, string.Format("GetFolderContent (user {0}, folder {1})", userID, folderID), null);
             }
 
             return invCol;
diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs
index c8056d4..3e0c9f3 100644
--- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs
+++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs
@@ -36,6 +36,7 @@ using System.Xml;
 using log4net;
 using OpenMetaverse;
 using OpenSim.Framework;
+using OpenSim.Framework.Monitoring;
 using OpenSim.Framework.Serialization;
 using OpenSim.Framework.Serialization.External;
 using OpenSim.Region.CoreModules.World.Terrain;
@@ -371,7 +372,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
             // Start the scripts. We delayed this because we want the OAR to finish loading ASAP, so
             // that users can enter the scene. If we allow the scripts to start in the loop above
             // then they significantly increase the time until the OAR finishes loading.
-            Util.RunThreadNoTimeout(delegate(object o)
+            Watchdog.RunInThread(o =>
             {
                 Thread.Sleep(15000);
                 m_log.Info("[ARCHIVER]: Starting scripts in scene objects");
@@ -386,7 +387,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
 
                     sceneContext.SceneObjects.Clear();
                 }
-            }, "ReadArchiveStartScripts", null);
+            }, string.Format("ReadArchiveStartScripts (request {0})", m_requestId), null);
 
             m_log.InfoFormat("[ARCHIVER]: Successfully loaded archive");
 
diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs
index 448147a..803f24e 100644
--- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs
+++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs
@@ -36,6 +36,7 @@ using System.Xml;
 using log4net;
 using OpenMetaverse;
 using OpenSim.Framework;
+using OpenSim.Framework.Monitoring;
 using OpenSim.Framework.Serialization;
 using OpenSim.Region.CoreModules.World.Terrain;
 using OpenSim.Region.Framework.Interfaces;
@@ -199,7 +200,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
                             m_rootScene.AssetService, m_rootScene.UserAccountService,
                             m_rootScene.RegionInfo.ScopeID, options, ReceivedAllAssets);
 
-                    Util.RunThreadNoTimeout(o => ar.Execute(), "AssetsRequest", null);
+                    Watchdog.RunInThread(o => ar.Execute(), "Archive Assets Request", null);
 
                     // CloseArchive() will be called from ReceivedAllAssets()
                 }
diff --git a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
index 4f428c3..ec39bc0 100644
--- a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
+++ b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
@@ -33,6 +33,7 @@ using System.Timers;
 using log4net;
 using OpenMetaverse;
 using OpenSim.Framework;
+using OpenSim.Framework.Monitoring;
 using OpenSim.Framework.Serialization;
 using OpenSim.Framework.Serialization.External;
 using OpenSim.Services.Interfaces;
@@ -226,7 +227,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
             finally
             {
                 if (timedOut)
-                    Util.RunThreadNoTimeout(PerformAssetsRequestCallback, "AssetsRequestCallback", true);
+                    Watchdog.RunInThread(PerformAssetsRequestCallback, "Archive Assets Request Callback", true);
             }
         }
 
@@ -295,7 +296,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
                         
                         // We want to stop using the asset cache thread asap 
                         // as we now need to do the work of producing the rest of the archive
-                        Util.RunThreadNoTimeout(PerformAssetsRequestCallback, "AssetsRequestCallback", false);
+                        Watchdog.RunInThread(PerformAssetsRequestCallback, "Archive Assets Request Callback", false);
                     }
                     else
                     {
diff --git a/OpenSim/Region/Framework/Scenes/Scene.cs b/OpenSim/Region/Framework/Scenes/Scene.cs
index 5f0dbd7..28dbccb 100644
--- a/OpenSim/Region/Framework/Scenes/Scene.cs
+++ b/OpenSim/Region/Framework/Scenes/Scene.cs
@@ -1623,7 +1623,12 @@ namespace OpenSim.Region.Framework.Scenes
                     {
                         tmpMS = Util.EnvironmentTickCount();
                         m_cleaningTemps = true;
-                        Util.RunThreadNoTimeout(delegate { CleanTempObjects(); m_cleaningTemps = false;  }, "CleanTempObjects", null);
+
+                        Watchdog.RunInThread(
+                            delegate { CleanTempObjects(); m_cleaningTemps = false;  }, 
+                            string.Format("CleanTempObjects ({0})", Name), 
+                            null);
+
                         tempOnRezMS = Util.EnvironmentTickCountSubtract(tmpMS);
                     }
     
@@ -1803,7 +1808,7 @@ namespace OpenSim.Region.Framework.Scenes
             if (!m_backingup)
             {
                 m_backingup = true;
-                Util.RunThreadNoTimeout(BackupWaitCallback, "BackupWaitCallback", null);
+                Watchdog.RunInThread(o => Backup(false), string.Format("BackupWaitCallback ({0})", Name), null);
             }
         }
 
@@ -1814,14 +1819,6 @@ namespace OpenSim.Region.Framework.Scenes
         {
             m_eventManager.TriggerOnFrame();
         }
-
-        /// <summary>
-        /// Wrapper for Backup() that can be called with Util.RunThreadNoTimeout()
-        /// </summary>
-        private void BackupWaitCallback(object o)
-        {
-            Backup(false);
-        }
         
         /// <summary>
         /// Backup the scene.
diff --git a/OpenSim/Region/Framework/Scenes/ScenePresence.cs b/OpenSim/Region/Framework/Scenes/ScenePresence.cs
index 3c37de8..59b521d 100644
--- a/OpenSim/Region/Framework/Scenes/ScenePresence.cs
+++ b/OpenSim/Region/Framework/Scenes/ScenePresence.cs
@@ -37,6 +37,7 @@ using log4net;
 using Nini.Config;
 using OpenSim.Framework;
 using OpenSim.Framework.Client;
+using OpenSim.Framework.Monitoring;
 using OpenSim.Region.Framework.Interfaces;
 using OpenSim.Region.Framework.Scenes.Animation;
 using OpenSim.Region.Framework.Scenes.Types;
@@ -3362,7 +3363,7 @@ namespace OpenSim.Region.Framework.Scenes
             SentInitialDataToClient = true;
 
             // Send all scene object to the new client
-            Util.RunThreadNoTimeout(delegate
+            Watchdog.RunInThread(delegate
             {
 //                m_log.DebugFormat(
 //                    "[SCENE PRESENCE]: Sending initial data to {0} agent {1} in {2}, tp flags {3}", 
@@ -3380,8 +3381,7 @@ namespace OpenSim.Region.Framework.Scenes
                     if (e != null && e is SceneObjectGroup)
                         ((SceneObjectGroup)e).SendFullUpdateToClient(ControllingClient);
                 }
-
-            }, "SendInitialDataToClient", null);
+            }, string.Format("SendInitialDataToClient ({0} in {1})", Name, Scene.Name), null);
         }
 
         /// <summary>