From f67f37074f3f7e0602b66aa66a044dd9fd107f6a Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 24 Feb 2012 05:02:33 +0000 Subject: Stop spurious scene loop startup timeout alarms for scenes with many prims. On the first frame, all startup scene objects are added to the physics scene. This can cause a considerable delay, so we don't start raising the alarm on scene loop timeouts until the second frame. This commit also slightly changes the behaviour of timeout reporting. Previously, a report was made for the very first timed out thread, ignoring all others until the next watchdog check. Instead, we now report every timed out thread, though we still only do this once no matter how long the timeout. --- .../HttpServer/PollServiceRequestManager.cs | 2 + OpenSim/Framework/Watchdog.cs | 57 +++++++++++++++------- .../Region/ClientStack/Linden/UDP/LLUDPServer.cs | 7 ++- .../InterGrid/OpenGridProtocolModule.cs | 2 +- .../CoreModules/World/WorldMap/WorldMapModule.cs | 1 + OpenSim/Region/Framework/Scenes/Scene.cs | 12 +++-- .../Server/IRCClientView.cs | 2 +- .../InternetRelayClientView/Server/IRCServer.cs | 2 +- OpenSim/Region/Physics/OdePlugin/ODEPrim.cs | 2 + .../Api/Implementation/AsyncCommandManager.cs | 4 +- 10 files changed, 65 insertions(+), 26 deletions(-) diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs index 2206feb..0062d4e 100644 --- a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs +++ b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs @@ -65,6 +65,7 @@ namespace OpenSim.Framework.Servers.HttpServer String.Format("PollServiceWorkerThread{0}", i), ThreadPriority.Normal, false, + true, int.MaxValue); } @@ -73,6 +74,7 @@ namespace OpenSim.Framework.Servers.HttpServer "PollServiceWatcherThread", ThreadPriority.Normal, false, + true, 1000 * 60 * 10); } diff --git a/OpenSim/Framework/Watchdog.cs b/OpenSim/Framework/Watchdog.cs index 2dd6ebe..e443f0a 100644 --- a/OpenSim/Framework/Watchdog.cs +++ b/OpenSim/Framework/Watchdog.cs @@ -72,6 +72,11 @@ namespace OpenSim.Framework /// public bool IsTimedOut { get; set; } + /// + /// Will this thread trigger the alarm function if it has timed out? + /// + public bool AlarmIfTimeout { get; set; } + public ThreadWatchdogInfo(Thread thread, int timeout) { Thread = thread; @@ -112,12 +117,13 @@ namespace OpenSim.Framework /// The method that will be executed in a new thread /// A name to give to the new thread /// Priority to run the thread at - /// True to run this thread as a background - /// thread, otherwise false + /// True to run this thread as a background thread, otherwise false + /// Trigger an alarm function is we have timed out /// The newly created Thread object - public static Thread StartThread(ThreadStart start, string name, ThreadPriority priority, bool isBackground) + public static Thread StartThread( + ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout) { - return StartThread(start, name, priority, isBackground, WATCHDOG_TIMEOUT_MS); + return StartThread(start, name, priority, isBackground, alarmIfTimeout, WATCHDOG_TIMEOUT_MS); } /// @@ -128,21 +134,21 @@ namespace OpenSim.Framework /// Priority to run the thread at /// True to run this thread as a background /// thread, otherwise false - /// - /// Number of milliseconds to wait until we issue a warning about timeout. - /// + /// Trigger an alarm function is we have timed out + /// Number of milliseconds to wait until we issue a warning about timeout. /// The newly created Thread object public static Thread StartThread( - ThreadStart start, string name, ThreadPriority priority, bool isBackground, int timeout) + ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, int timeout) { Thread thread = new Thread(start); thread.Name = name; thread.Priority = priority; thread.IsBackground = isBackground; - ThreadWatchdogInfo twi = new ThreadWatchdogInfo(thread, timeout); + ThreadWatchdogInfo twi = new ThreadWatchdogInfo(thread, timeout) { AlarmIfTimeout = alarmIfTimeout }; - m_log.Debug("[WATCHDOG]: Started tracking thread \"" + twi.Thread.Name + "\" (ID " + twi.Thread.ManagedThreadId + ")"); + m_log.DebugFormat( + "[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId); lock (m_threads) m_threads.Add(twi.Thread.ManagedThreadId, twi); @@ -230,6 +236,26 @@ namespace OpenSim.Framework return m_threads.Values.ToArray(); } + /// + /// Return the current thread's watchdog info. + /// + /// The watchdog info. null if the thread isn't being monitored. + public static ThreadWatchdogInfo GetCurrentThreadInfo() + { + lock (m_threads) + { + if (m_threads.ContainsKey(Thread.CurrentThread.ManagedThreadId)) + return m_threads[Thread.CurrentThread.ManagedThreadId]; + } + + return null; + } + + /// + /// Check watched threads. Fire alarm if appropriate. + /// + /// + /// private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e) { WatchdogTimeout callback = OnWatchdogTimeout; @@ -246,21 +272,18 @@ namespace OpenSim.Framework { if (threadInfo.Thread.ThreadState == ThreadState.Stopped) { - timedOut = threadInfo; RemoveThread(threadInfo.Thread.ManagedThreadId); - break; + callback(threadInfo.Thread, threadInfo.LastTick); } else if (!threadInfo.IsTimedOut && now - threadInfo.LastTick >= threadInfo.Timeout) { threadInfo.IsTimedOut = true; - timedOut = threadInfo; - break; + + if (threadInfo.AlarmIfTimeout) + callback(threadInfo.Thread, threadInfo.LastTick); } } } - - if (timedOut != null) - callback(timedOut.Thread, timedOut.LastTick); } m_watchdogTimer.Start(); diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 1e22fcc..fb6b11e 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -244,8 +244,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP base.Start(m_recvBufferSize, m_asyncPacketHandling); // Start the packet processing threads - Watchdog.StartThread(IncomingPacketHandler, "Incoming Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false); - Watchdog.StartThread(OutgoingPacketHandler, "Outgoing Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false); + Watchdog.StartThread( + IncomingPacketHandler, "Incoming Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false, true); + Watchdog.StartThread( + OutgoingPacketHandler, "Outgoing Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false, true); + m_elapsedMSSinceLastStatReport = Environment.TickCount; } diff --git a/OpenSim/Region/CoreModules/InterGrid/OpenGridProtocolModule.cs b/OpenSim/Region/CoreModules/InterGrid/OpenGridProtocolModule.cs index f367739..a6e2548 100644 --- a/OpenSim/Region/CoreModules/InterGrid/OpenGridProtocolModule.cs +++ b/OpenSim/Region/CoreModules/InterGrid/OpenGridProtocolModule.cs @@ -1210,7 +1210,7 @@ namespace OpenSim.Region.CoreModules.InterGrid if (homeScene.TryGetScenePresence(avatarId,out avatar)) { KillAUser ku = new KillAUser(avatar,mod); - Watchdog.StartThread(ku.ShutdownNoLogout, "OGPShutdown", ThreadPriority.Normal, true); + Watchdog.StartThread(ku.ShutdownNoLogout, "OGPShutdown", ThreadPriority.Normal, true, true); } } diff --git a/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs b/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs index b315d2c..74b047b 100644 --- a/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs +++ b/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs @@ -351,6 +351,7 @@ namespace OpenSim.Region.CoreModules.World.WorldMap process, string.Format("MapItemRequestThread ({0})", m_scene.RegionInfo.RegionName), ThreadPriority.BelowNormal, + true, true); } diff --git a/OpenSim/Region/Framework/Scenes/Scene.cs b/OpenSim/Region/Framework/Scenes/Scene.cs index cf6e6af..19d4bad 100644 --- a/OpenSim/Region/Framework/Scenes/Scene.cs +++ b/OpenSim/Region/Framework/Scenes/Scene.cs @@ -1140,7 +1140,7 @@ namespace OpenSim.Region.Framework.Scenes HeartbeatThread = Watchdog.StartThread( - Heartbeat, string.Format("Heartbeat ({0})", RegionInfo.RegionName), ThreadPriority.Normal, false); + Heartbeat, string.Format("Heartbeat ({0})", RegionInfo.RegionName), ThreadPriority.Normal, false, false); } /// @@ -1178,6 +1178,13 @@ namespace OpenSim.Region.Framework.Scenes try { m_eventManager.TriggerOnRegionStarted(this); + + // The first frame can take a very long time due to physics actors being added on startup. Therefore, + // don't turn on the watchdog alarm for this thread until the second frame, in order to prevent false + // alarms for scenes with many objects. + Update(); + Watchdog.GetCurrentThreadInfo().AlarmIfTimeout = true; + while (!shuttingdown) Update(); @@ -1206,7 +1213,7 @@ namespace OpenSim.Region.Framework.Scenes ++Frame; -// m_log.DebugFormat("[SCENE]: Processing frame {0}", Frame); +// m_log.DebugFormat("[SCENE]: Processing frame {0} in {1}", Frame, RegionInfo.RegionName); try { @@ -1418,7 +1425,6 @@ namespace OpenSim.Region.Framework.Scenes entry.checkAtTargets(); } - /// /// Send out simstats data to all clients /// diff --git a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs index c928af7..d3c96e2 100644 --- a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs +++ b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs @@ -70,7 +70,7 @@ namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server m_client = client; m_scene = scene; - Watchdog.StartThread(InternalLoop, "IRCClientView", ThreadPriority.Normal, false); + Watchdog.StartThread(InternalLoop, "IRCClientView", ThreadPriority.Normal, false, true); } private void SendServerCommand(string command) diff --git a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs index eb39026..a7c5020 100644 --- a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs +++ b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs @@ -57,7 +57,7 @@ namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server m_listener.Start(50); - Watchdog.StartThread(ListenLoop, "IRCServer", ThreadPriority.Normal, false); + Watchdog.StartThread(ListenLoop, "IRCServer", ThreadPriority.Normal, false, true); m_baseScene = baseScene; } diff --git a/OpenSim/Region/Physics/OdePlugin/ODEPrim.cs b/OpenSim/Region/Physics/OdePlugin/ODEPrim.cs index 75364b7..97890ee 100644 --- a/OpenSim/Region/Physics/OdePlugin/ODEPrim.cs +++ b/OpenSim/Region/Physics/OdePlugin/ODEPrim.cs @@ -1474,6 +1474,8 @@ Console.WriteLine("CreateGeom:"); /// private void changeadd() { +// m_log.DebugFormat("[ODE PRIM]: Adding prim {0}", Name); + int[] iprimspaceArrItem = _parent_scene.calculateSpaceArrayItemFromPos(_position); IntPtr targetspace = _parent_scene.calculateSpaceForGeom(_position); diff --git a/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs b/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs index ee32755..14edde4 100644 --- a/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs +++ b/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs @@ -137,7 +137,9 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api if (cmdHandlerThread == null) { // Start the thread that will be doing the work - cmdHandlerThread = Watchdog.StartThread(CmdHandlerThreadLoop, "AsyncLSLCmdHandlerThread", ThreadPriority.Normal, true); + cmdHandlerThread + = Watchdog.StartThread( + CmdHandlerThreadLoop, "AsyncLSLCmdHandlerThread", ThreadPriority.Normal, true, true); } } -- cgit v1.1