From ac7ccdf7d77810aef0a3ad70f1504fdb111dc0aa Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Mon, 26 Oct 2009 14:41:27 -0700 Subject: * Changed the watchdog timer to improve the speed of UpdateThread(), only track threads once the first call to UpdateThread() has been made, and allow re-tracking of threads that timed out but revived later * Added a commented out call to Watchdog.UpdateThread() in OdeScene. If it turns out that loading a large OAR file or some other operation is timing out the heartbeat thread, we'll need to uncomment it --- OpenSim/Framework/Watchdog.cs | 60 +++++++++++++-------------- OpenSim/Region/Framework/Scenes/Scene.cs | 2 + OpenSim/Region/Physics/OdePlugin/OdePlugin.cs | 8 +++- 3 files changed, 38 insertions(+), 32 deletions(-) diff --git a/OpenSim/Framework/Watchdog.cs b/OpenSim/Framework/Watchdog.cs index b905609..5d46905 100644 --- a/OpenSim/Framework/Watchdog.cs +++ b/OpenSim/Framework/Watchdog.cs @@ -28,6 +28,7 @@ using System; using System.Collections.Generic; using System.Threading; +using log4net; namespace OpenSim.Framework { @@ -66,6 +67,7 @@ namespace OpenSim.Framework /// stopped or has not called UpdateThread() in time public static event WatchdogTimeout OnWatchdogTimeout; + private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); private static Dictionary m_threads; private static System.Timers.Timer m_watchdogTimer; @@ -95,9 +97,6 @@ namespace OpenSim.Framework thread.IsBackground = isBackground; thread.Start(); - lock (m_threads) - m_threads.Add(thread.ManagedThreadId, new ThreadWatchdogInfo(thread)); - return thread; } @@ -110,24 +109,6 @@ namespace OpenSim.Framework } /// - /// Marks a thread as alive - /// - /// The ManagedThreadId of the thread to mark as - /// alive - public static void UpdateThread(int threadID) - { - ThreadWatchdogInfo threadInfo; - - lock (m_threads) - { - if (m_threads.TryGetValue(threadID, out threadInfo)) - { - threadInfo.LastTick = Environment.TickCount & Int32.MaxValue; - } - } - } - - /// /// Stops watchdog tracking on the current thread /// /// True if the thread was removed from the list of tracked @@ -137,19 +118,38 @@ namespace OpenSim.Framework return RemoveThread(Thread.CurrentThread.ManagedThreadId); } - /// - /// Stops watchdog tracking on a thread - /// - /// The ManagedThreadId of the thread to stop - /// tracking - /// True if the thread was removed from the list of tracked - /// threads, otherwise false - public static bool RemoveThread(int threadID) + private static void AddThread(ThreadWatchdogInfo threadInfo) + { + m_log.Debug("[WATCHDOG]: Started tracking thread \"" + threadInfo.Thread.Name + "\" (ID " + threadInfo.Thread.ManagedThreadId + ")"); + + lock (m_threads) + m_threads.Add(threadInfo.Thread.ManagedThreadId, threadInfo); + } + + private static bool RemoveThread(int threadID) { lock (m_threads) return m_threads.Remove(threadID); } + private static void UpdateThread(int threadID) + { + ThreadWatchdogInfo threadInfo; + + // Although TryGetValue is not a thread safe operation, we use a try/catch here instead + // of a lock for speed. Adding/removing threads is a very rare operation compared to + // UpdateThread(), and a single UpdateThread() failure here and there won't break + // anything + try + { + if (m_threads.TryGetValue(threadID, out threadInfo)) + threadInfo.LastTick = Environment.TickCount & Int32.MaxValue; + else + AddThread(new ThreadWatchdogInfo(Thread.CurrentThread)); + } + catch { } + } + private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e) { WatchdogTimeout callback = OnWatchdogTimeout; @@ -160,7 +160,7 @@ namespace OpenSim.Framework lock (m_threads) { - int now = Environment.TickCount; + int now = Environment.TickCount & Int32.MaxValue; foreach (ThreadWatchdogInfo threadInfo in m_threads.Values) { diff --git a/OpenSim/Region/Framework/Scenes/Scene.cs b/OpenSim/Region/Framework/Scenes/Scene.cs index f052c65..95d69a1 100644 --- a/OpenSim/Region/Framework/Scenes/Scene.cs +++ b/OpenSim/Region/Framework/Scenes/Scene.cs @@ -1025,6 +1025,7 @@ namespace OpenSim.Region.Framework.Scenes float physicsFPS = 0; frameMS = Environment.TickCount; + try { // Increment the frame counter @@ -1152,6 +1153,7 @@ namespace OpenSim.Region.Framework.Scenes if ((maintc < (m_timespan * 1000)) && maintc > 0) Thread.Sleep(maintc); + // Tell the watchdog that this thread is still alive Watchdog.UpdateThread(); } } diff --git a/OpenSim/Region/Physics/OdePlugin/OdePlugin.cs b/OpenSim/Region/Physics/OdePlugin/OdePlugin.cs index f979ce3..82392b1 100644 --- a/OpenSim/Region/Physics/OdePlugin/OdePlugin.cs +++ b/OpenSim/Region/Physics/OdePlugin/OdePlugin.cs @@ -2705,8 +2705,6 @@ namespace OpenSim.Region.Physics.OdePlugin { foreach (OdePrim prim in _taintedPrimL) { - - if (prim.m_taintremove) { //Console.WriteLine("Simulate calls RemovePrimThreadLocked"); @@ -2719,6 +2717,12 @@ namespace OpenSim.Region.Physics.OdePlugin } processedtaints = true; prim.m_collisionscore = 0; + + // This loop can block up the Heartbeat for a very long time on large regions. + // We need to let the Watchdog know that the Heartbeat is not dead + // NOTE: This is currently commented out, but if things like OAR loading are + // timing the heartbeat out we will need to uncomment it + //Watchdog.UpdateThread(); } if (SupportsNINJAJoints) -- cgit v1.1