Stop spurious scene loop startup timeout alarms for scenes with many prims.

On the first frame, all startup scene objects are added to the physics scene.
This can cause a considerable delay, so we don't start raising the alarm on scene loop timeouts until the second frame.
This commit also slightly changes the behaviour of timeout reporting.
Previously, a report was made for the very first timed out thread, ignoring all others until the next watchdog check.
Instead, we now report every timed out thread, though we still only do this once no matter how long the timeout.

10 files changed, 65 insertions, 26 deletions

diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs
index 2206feb..0062d4e 100644
--- a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs
+++ b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs
@@ -65,6 +65,7 @@ namespace OpenSim.Framework.Servers.HttpServer
                         String.Format("PollServiceWorkerThread{0}", i),
                         ThreadPriority.Normal,
                         false,
+                        true,
                         int.MaxValue);
             }
 
@@ -73,6 +74,7 @@ namespace OpenSim.Framework.Servers.HttpServer
                 "PollServiceWatcherThread",
                 ThreadPriority.Normal,
                 false,
+                true,
                 1000 * 60 * 10);
         }
 
diff --git a/OpenSim/Framework/Watchdog.cs b/OpenSim/Framework/Watchdog.cs
index 2dd6ebe..e443f0a 100644
--- a/OpenSim/Framework/Watchdog.cs
+++ b/OpenSim/Framework/Watchdog.cs
@@ -72,6 +72,11 @@ namespace OpenSim.Framework
             /// </summary>
             public bool IsTimedOut { get; set; }
 
+            /// <summary>
+            /// Will this thread trigger the alarm function if it has timed out?
+            /// </summary>
+            public bool AlarmIfTimeout { get; set; }
+
             public ThreadWatchdogInfo(Thread thread, int timeout)
             {
                 Thread = thread;
@@ -112,12 +117,13 @@ namespace OpenSim.Framework
         /// <param name="start">The method that will be executed in a new thread</param>
         /// <param name="name">A name to give to the new thread</param>
         /// <param name="priority">Priority to run the thread at</param>
-        /// <param name="isBackground">True to run this thread as a background
-        /// thread, otherwise false</param>
+        /// <param name="isBackground">True to run this thread as a background thread, otherwise false</param>
+        /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
         /// <returns>The newly created Thread object</returns>
-        public static Thread StartThread(ThreadStart start, string name, ThreadPriority priority, bool isBackground)
+        public static Thread StartThread(
+            ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout)
         {
-            return StartThread(start, name, priority, isBackground, WATCHDOG_TIMEOUT_MS);
+            return StartThread(start, name, priority, isBackground, alarmIfTimeout, WATCHDOG_TIMEOUT_MS);
         }
 
         /// <summary>
@@ -128,21 +134,21 @@ namespace OpenSim.Framework
         /// <param name="priority">Priority to run the thread at</param>
         /// <param name="isBackground">True to run this thread as a background
         /// thread, otherwise false</param>
-        /// <param name="timeout">
-        /// Number of milliseconds to wait until we issue a warning about timeout.
-        /// </para>
+        /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
+        /// <param name="timeout">Number of milliseconds to wait until we issue a warning about timeout.</param>
         /// <returns>The newly created Thread object</returns>
         public static Thread StartThread(
-            ThreadStart start, string name, ThreadPriority priority, bool isBackground, int timeout)
+            ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, int timeout)
         {
             Thread thread = new Thread(start);
             thread.Name = name;
             thread.Priority = priority;
             thread.IsBackground = isBackground;
             
-            ThreadWatchdogInfo twi = new ThreadWatchdogInfo(thread, timeout);
+            ThreadWatchdogInfo twi = new ThreadWatchdogInfo(thread, timeout) { AlarmIfTimeout = alarmIfTimeout };
 
-            m_log.Debug("[WATCHDOG]: Started tracking thread \"" + twi.Thread.Name + "\" (ID " + twi.Thread.ManagedThreadId + ")");
+            m_log.DebugFormat(
+                "[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
 
             lock (m_threads)
                 m_threads.Add(twi.Thread.ManagedThreadId, twi);
@@ -230,6 +236,26 @@ namespace OpenSim.Framework
                 return m_threads.Values.ToArray();
         }
 
+        /// <summary>
+        /// Return the current thread's watchdog info.
+        /// </summary>
+        /// <returns>The watchdog info.  null if the thread isn't being monitored.</returns>
+        public static ThreadWatchdogInfo GetCurrentThreadInfo()
+        {
+            lock (m_threads)
+            {
+                if (m_threads.ContainsKey(Thread.CurrentThread.ManagedThreadId))
+                    return m_threads[Thread.CurrentThread.ManagedThreadId];
+            }
+
+            return null;
+        }
+
+        /// <summary>
+        /// Check watched threads.  Fire alarm if appropriate.
+        /// </summary>
+        /// <param name="sender"></param>
+        /// <param name="e"></param>
         private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e)
         {
             WatchdogTimeout callback = OnWatchdogTimeout;
@@ -246,21 +272,18 @@ namespace OpenSim.Framework
                     {
                         if (threadInfo.Thread.ThreadState == ThreadState.Stopped)
                         {
-                            timedOut = threadInfo;
                             RemoveThread(threadInfo.Thread.ManagedThreadId);
-                            break;
+                            callback(threadInfo.Thread, threadInfo.LastTick);
                         }
                         else if (!threadInfo.IsTimedOut && now - threadInfo.LastTick >= threadInfo.Timeout)
                         {
                             threadInfo.IsTimedOut = true;
-                            timedOut = threadInfo;
-                            break;
+
+                            if (threadInfo.AlarmIfTimeout)
+                                callback(threadInfo.Thread, threadInfo.LastTick);
                         }
                     }
                 }
-
-                if (timedOut != null)
-                    callback(timedOut.Thread, timedOut.LastTick);
             }
 
             m_watchdogTimer.Start();
diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs
index 1e22fcc..fb6b11e 100644
--- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs
+++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs
@@ -244,8 +244,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             base.Start(m_recvBufferSize, m_asyncPacketHandling);
 
             // Start the packet processing threads
-            Watchdog.StartThread(IncomingPacketHandler, "Incoming Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false);
-            Watchdog.StartThread(OutgoingPacketHandler, "Outgoing Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false);
+            Watchdog.StartThread(
+                IncomingPacketHandler, "Incoming Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false, true);
+            Watchdog.StartThread(
+                OutgoingPacketHandler, "Outgoing Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false, true);
+
             m_elapsedMSSinceLastStatReport = Environment.TickCount;
         }
 
diff --git a/OpenSim/Region/CoreModules/InterGrid/OpenGridProtocolModule.cs b/OpenSim/Region/CoreModules/InterGrid/OpenGridProtocolModule.cs
index f367739..a6e2548 100644
--- a/OpenSim/Region/CoreModules/InterGrid/OpenGridProtocolModule.cs
+++ b/OpenSim/Region/CoreModules/InterGrid/OpenGridProtocolModule.cs
@@ -1210,7 +1210,7 @@ namespace OpenSim.Region.CoreModules.InterGrid
             if (homeScene.TryGetScenePresence(avatarId,out avatar))
             {
                 KillAUser ku = new KillAUser(avatar,mod);
-                Watchdog.StartThread(ku.ShutdownNoLogout, "OGPShutdown", ThreadPriority.Normal, true);
+                Watchdog.StartThread(ku.ShutdownNoLogout, "OGPShutdown", ThreadPriority.Normal, true, true);
             }
         }
 
diff --git a/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs b/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs
index b315d2c..74b047b 100644
--- a/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs
+++ b/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs
@@ -351,6 +351,7 @@ namespace OpenSim.Region.CoreModules.World.WorldMap
                 process,
                 string.Format("MapItemRequestThread ({0})", m_scene.RegionInfo.RegionName),
                 ThreadPriority.BelowNormal,
+                true,
                 true);
         }
 
diff --git a/OpenSim/Region/Framework/Scenes/Scene.cs b/OpenSim/Region/Framework/Scenes/Scene.cs
index cf6e6af..19d4bad 100644
--- a/OpenSim/Region/Framework/Scenes/Scene.cs
+++ b/OpenSim/Region/Framework/Scenes/Scene.cs
@@ -1140,7 +1140,7 @@ namespace OpenSim.Region.Framework.Scenes
 
             HeartbeatThread
                 = Watchdog.StartThread(
-                    Heartbeat, string.Format("Heartbeat ({0})", RegionInfo.RegionName), ThreadPriority.Normal, false);
+                    Heartbeat, string.Format("Heartbeat ({0})", RegionInfo.RegionName), ThreadPriority.Normal, false, false);
         }
 
         /// <summary>
@@ -1178,6 +1178,13 @@ namespace OpenSim.Region.Framework.Scenes
             try
             {
                 m_eventManager.TriggerOnRegionStarted(this);
+
+                // The first frame can take a very long time due to physics actors being added on startup.  Therefore,
+                // don't turn on the watchdog alarm for this thread until the second frame, in order to prevent false
+                // alarms for scenes with many objects.
+                Update();
+                Watchdog.GetCurrentThreadInfo().AlarmIfTimeout = true;
+
                 while (!shuttingdown)
                     Update();
 
@@ -1206,7 +1213,7 @@ namespace OpenSim.Region.Framework.Scenes
 
             ++Frame;
 
-//            m_log.DebugFormat("[SCENE]: Processing frame {0}", Frame);
+//            m_log.DebugFormat("[SCENE]: Processing frame {0} in {1}", Frame, RegionInfo.RegionName);
 
             try
             {
@@ -1418,7 +1425,6 @@ namespace OpenSim.Region.Framework.Scenes
                 entry.checkAtTargets();
         }
 
-
         /// <summary>
         /// Send out simstats data to all clients
         /// </summary>
diff --git a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs
index c928af7..d3c96e2 100644
--- a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs
+++ b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs
@@ -70,7 +70,7 @@ namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server
             m_client = client;
             m_scene = scene;
 
-            Watchdog.StartThread(InternalLoop, "IRCClientView", ThreadPriority.Normal, false);
+            Watchdog.StartThread(InternalLoop, "IRCClientView", ThreadPriority.Normal, false, true);
         }
 
         private void SendServerCommand(string command)
diff --git a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs
index eb39026..a7c5020 100644
--- a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs
+++ b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs
@@ -57,7 +57,7 @@ namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server
 
             m_listener.Start(50);
 
-            Watchdog.StartThread(ListenLoop, "IRCServer", ThreadPriority.Normal, false);
+            Watchdog.StartThread(ListenLoop, "IRCServer", ThreadPriority.Normal, false, true);
             m_baseScene = baseScene;
         }
 
diff --git a/OpenSim/Region/Physics/OdePlugin/ODEPrim.cs b/OpenSim/Region/Physics/OdePlugin/ODEPrim.cs
index 75364b7..97890ee 100644
--- a/OpenSim/Region/Physics/OdePlugin/ODEPrim.cs
+++ b/OpenSim/Region/Physics/OdePlugin/ODEPrim.cs
@@ -1474,6 +1474,8 @@ Console.WriteLine("CreateGeom:");
         /// </summary>
         private void changeadd()
         {
+//            m_log.DebugFormat("[ODE PRIM]: Adding prim {0}", Name);
+            
             int[] iprimspaceArrItem = _parent_scene.calculateSpaceArrayItemFromPos(_position);
             IntPtr targetspace = _parent_scene.calculateSpaceForGeom(_position);
 
diff --git a/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs b/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs
index ee32755..14edde4 100644
--- a/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs
+++ b/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs
@@ -137,7 +137,9 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api
             if (cmdHandlerThread == null)
             {
                 // Start the thread that will be doing the work
-                cmdHandlerThread = Watchdog.StartThread(CmdHandlerThreadLoop, "AsyncLSLCmdHandlerThread", ThreadPriority.Normal, true);
+                cmdHandlerThread
+                    = Watchdog.StartThread(
+                        CmdHandlerThreadLoop, "AsyncLSLCmdHandlerThread", ThreadPriority.Normal, true, true);
             }
         }