From 5aec0ff207e9427b8756471eb003fd68859f67b1 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Wed, 25 Jul 2012 23:27:00 +0100 Subject: Move Watchdog and MemoryWatchdog classes into OpenSim.Framework.Monitoring with other monitoring code from OpenSim.Framework --- OpenSim/Framework/MemoryWatchdog.cs | 129 -------- OpenSim/Framework/Monitoring/MemoryWatchdog.cs | 129 ++++++++ OpenSim/Framework/Monitoring/Watchdog.cs | 334 +++++++++++++++++++++ OpenSim/Framework/Servers/BaseOpenSimServer.cs | 1 + .../Framework/Servers/HttpServer/BaseHttpServer.cs | 1 + .../HttpServer/PollServiceRequestManager.cs | 1 + .../Servers/HttpServer/PollServiceWorkerThread.cs | 1 + OpenSim/Framework/Watchdog.cs | 334 --------------------- .../InterGrid/OpenGridProtocolModule.cs | 1 + .../CoreModules/World/WorldMap/WorldMapModule.cs | 1 + OpenSim/Region/Framework/Scenes/Scene.cs | 1 + .../Server/IRCClientView.cs | 1 + .../InternetRelayClientView/Server/IRCServer.cs | 1 + .../OptionalModules/Avatar/Chat/IRCConnector.cs | 1 + .../Api/Implementation/AsyncCommandManager.cs | 1 + 15 files changed, 474 insertions(+), 463 deletions(-) delete mode 100644 OpenSim/Framework/MemoryWatchdog.cs create mode 100644 OpenSim/Framework/Monitoring/MemoryWatchdog.cs create mode 100644 OpenSim/Framework/Monitoring/Watchdog.cs delete mode 100644 OpenSim/Framework/Watchdog.cs (limited to 'OpenSim') diff --git a/OpenSim/Framework/MemoryWatchdog.cs b/OpenSim/Framework/MemoryWatchdog.cs deleted file mode 100644 index 1e93077..0000000 --- a/OpenSim/Framework/MemoryWatchdog.cs +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (c) Contributors, http://opensimulator.org/ - * See CONTRIBUTORS.TXT for a full list of copyright holders. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the OpenSimulator Project nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using System.Threading; -using log4net; - -namespace OpenSim.Framework -{ - /// - /// Experimental watchdog for memory usage. - /// - public static class MemoryWatchdog - { -// private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - - /// - /// Is this watchdog active? - /// - public static bool Enabled - { - get { return m_enabled; } - set - { -// m_log.DebugFormat("[MEMORY WATCHDOG]: Setting MemoryWatchdog.Enabled to {0}", value); - - if (value && !m_enabled) - UpdateLastRecord(GC.GetTotalMemory(false), Util.EnvironmentTickCount()); - - m_enabled = value; - } - } - private static bool m_enabled; - - /// - /// Average memory churn in bytes per millisecond. - /// - public static double AverageMemoryChurn - { - get { if (m_samples.Count > 0) return m_samples.Average(); else return 0; } - } - - /// - /// Maximum number of statistical samples. - /// - /// - /// At the moment this corresponds to 1 minute since the sampling rate is every 2.5 seconds as triggered from - /// the main Watchdog. - /// - private static int m_maxSamples = 24; - - /// - /// Time when the watchdog was last updated. - /// - private static int m_lastUpdateTick; - - /// - /// Memory used at time of last watchdog update. - /// - private static long m_lastUpdateMemory; - - /// - /// Memory churn rate per millisecond. - /// - private static double m_churnRatePerMillisecond; - - /// - /// Historical samples for calculating moving average. - /// - private static Queue m_samples = new Queue(m_maxSamples); - - public static void Update() - { - int now = Util.EnvironmentTickCount(); - long memoryNow = GC.GetTotalMemory(false); - long memoryDiff = memoryNow - m_lastUpdateMemory; - - if (memoryDiff >= 0) - { - if (m_samples.Count >= m_maxSamples) - m_samples.Dequeue(); - - double elapsed = Util.EnvironmentTickCountSubtract(now, m_lastUpdateTick); - - // This should never happen since it's not useful for updates to occur with no time elapsed, but - // protect ourselves from a divide-by-zero just in case. - if (elapsed == 0) - return; - - m_samples.Enqueue(memoryDiff / (double)elapsed); - } - - UpdateLastRecord(memoryNow, now); - } - - private static void UpdateLastRecord(long memoryNow, int timeNow) - { - m_lastUpdateMemory = memoryNow; - m_lastUpdateTick = timeNow; - } - } -} \ No newline at end of file diff --git a/OpenSim/Framework/Monitoring/MemoryWatchdog.cs b/OpenSim/Framework/Monitoring/MemoryWatchdog.cs new file mode 100644 index 0000000..6599613 --- /dev/null +++ b/OpenSim/Framework/Monitoring/MemoryWatchdog.cs @@ -0,0 +1,129 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Threading; +using log4net; + +namespace OpenSim.Framework.Monitoring +{ + /// + /// Experimental watchdog for memory usage. + /// + public static class MemoryWatchdog + { +// private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + /// + /// Is this watchdog active? + /// + public static bool Enabled + { + get { return m_enabled; } + set + { +// m_log.DebugFormat("[MEMORY WATCHDOG]: Setting MemoryWatchdog.Enabled to {0}", value); + + if (value && !m_enabled) + UpdateLastRecord(GC.GetTotalMemory(false), Util.EnvironmentTickCount()); + + m_enabled = value; + } + } + private static bool m_enabled; + + /// + /// Average memory churn in bytes per millisecond. + /// + public static double AverageMemoryChurn + { + get { if (m_samples.Count > 0) return m_samples.Average(); else return 0; } + } + + /// + /// Maximum number of statistical samples. + /// + /// + /// At the moment this corresponds to 1 minute since the sampling rate is every 2.5 seconds as triggered from + /// the main Watchdog. + /// + private static int m_maxSamples = 24; + + /// + /// Time when the watchdog was last updated. + /// + private static int m_lastUpdateTick; + + /// + /// Memory used at time of last watchdog update. + /// + private static long m_lastUpdateMemory; + + /// + /// Memory churn rate per millisecond. + /// + private static double m_churnRatePerMillisecond; + + /// + /// Historical samples for calculating moving average. + /// + private static Queue m_samples = new Queue(m_maxSamples); + + public static void Update() + { + int now = Util.EnvironmentTickCount(); + long memoryNow = GC.GetTotalMemory(false); + long memoryDiff = memoryNow - m_lastUpdateMemory; + + if (memoryDiff >= 0) + { + if (m_samples.Count >= m_maxSamples) + m_samples.Dequeue(); + + double elapsed = Util.EnvironmentTickCountSubtract(now, m_lastUpdateTick); + + // This should never happen since it's not useful for updates to occur with no time elapsed, but + // protect ourselves from a divide-by-zero just in case. + if (elapsed == 0) + return; + + m_samples.Enqueue(memoryDiff / (double)elapsed); + } + + UpdateLastRecord(memoryNow, now); + } + + private static void UpdateLastRecord(long memoryNow, int timeNow) + { + m_lastUpdateMemory = memoryNow; + m_lastUpdateTick = timeNow; + } + } +} \ No newline at end of file diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs new file mode 100644 index 0000000..d4cf02f --- /dev/null +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -0,0 +1,334 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using log4net; + +namespace OpenSim.Framework.Monitoring +{ + /// + /// Manages launching threads and keeping watch over them for timeouts + /// + public static class Watchdog + { + /// Timer interval in milliseconds for the watchdog timer + const double WATCHDOG_INTERVAL_MS = 2500.0d; + + /// Default timeout in milliseconds before a thread is considered dead + public const int DEFAULT_WATCHDOG_TIMEOUT_MS = 5000; + + [System.Diagnostics.DebuggerDisplay("{Thread.Name}")] + public class ThreadWatchdogInfo + { + public Thread Thread { get; private set; } + + /// + /// Approximate tick when this thread was started. + /// + /// + /// Not terribly good since this quickly wraps around. + /// + public int FirstTick { get; private set; } + + /// + /// Last time this heartbeat update was invoked + /// + public int LastTick { get; set; } + + /// + /// Number of milliseconds before we notify that the thread is having a problem. + /// + public int Timeout { get; set; } + + /// + /// Is this thread considered timed out? + /// + public bool IsTimedOut { get; set; } + + /// + /// Will this thread trigger the alarm function if it has timed out? + /// + public bool AlarmIfTimeout { get; set; } + + /// + /// Method execute if alarm goes off. If null then no alarm method is fired. + /// + public Func AlarmMethod { get; set; } + + public ThreadWatchdogInfo(Thread thread, int timeout) + { + Thread = thread; + Timeout = timeout; + FirstTick = Environment.TickCount & Int32.MaxValue; + LastTick = FirstTick; + } + } + + /// + /// This event is called whenever a tracked thread is + /// stopped or has not called UpdateThread() in time< + /// /summary> + public static event Action OnWatchdogTimeout; + + private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); + private static Dictionary m_threads; + private static System.Timers.Timer m_watchdogTimer; + + /// + /// Last time the watchdog thread ran. + /// + /// + /// Should run every WATCHDOG_INTERVAL_MS + /// + public static int LastWatchdogThreadTick { get; private set; } + + static Watchdog() + { + m_threads = new Dictionary(); + m_watchdogTimer = new System.Timers.Timer(WATCHDOG_INTERVAL_MS); + m_watchdogTimer.AutoReset = false; + m_watchdogTimer.Elapsed += WatchdogTimerElapsed; + + // Set now so we don't get alerted on the first run + LastWatchdogThreadTick = Environment.TickCount & Int32.MaxValue; + + m_watchdogTimer.Start(); + } + + /// + /// Start a new thread that is tracked by the watchdog timer. + /// + /// The method that will be executed in a new thread + /// A name to give to the new thread + /// Priority to run the thread at + /// True to run this thread as a background thread, otherwise false + /// Trigger an alarm function is we have timed out + /// The newly created Thread object + public static Thread StartThread( + ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout) + { + return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS); + } + + /// + /// Start a new thread that is tracked by the watchdog timer + /// + /// The method that will be executed in a new thread + /// A name to give to the new thread + /// Priority to run the thread at + /// True to run this thread as a background + /// thread, otherwise false + /// Trigger an alarm function is we have timed out + /// + /// Alarm method to call if alarmIfTimeout is true and there is a timeout. + /// Normally, this will just return some useful debugging information. + /// + /// Number of milliseconds to wait until we issue a warning about timeout. + /// The newly created Thread object + public static Thread StartThread( + ThreadStart start, string name, ThreadPriority priority, bool isBackground, + bool alarmIfTimeout, Func alarmMethod, int timeout) + { + Thread thread = new Thread(start); + thread.Name = name; + thread.Priority = priority; + thread.IsBackground = isBackground; + + ThreadWatchdogInfo twi + = new ThreadWatchdogInfo(thread, timeout) + { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod }; + + m_log.DebugFormat( + "[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId); + + lock (m_threads) + m_threads.Add(twi.Thread.ManagedThreadId, twi); + + thread.Start(); + + return thread; + } + + /// + /// Marks the current thread as alive + /// + public static void UpdateThread() + { + UpdateThread(Thread.CurrentThread.ManagedThreadId); + } + + /// + /// Stops watchdog tracking on the current thread + /// + /// + /// True if the thread was removed from the list of tracked + /// threads, otherwise false + /// + public static bool RemoveThread() + { + return RemoveThread(Thread.CurrentThread.ManagedThreadId); + } + + private static bool RemoveThread(int threadID) + { + lock (m_threads) + return m_threads.Remove(threadID); + } + + public static bool AbortThread(int threadID) + { + lock (m_threads) + { + if (m_threads.ContainsKey(threadID)) + { + ThreadWatchdogInfo twi = m_threads[threadID]; + twi.Thread.Abort(); + RemoveThread(threadID); + + return true; + } + else + { + return false; + } + } + } + + private static void UpdateThread(int threadID) + { + ThreadWatchdogInfo threadInfo; + + // Although TryGetValue is not a thread safe operation, we use a try/catch here instead + // of a lock for speed. Adding/removing threads is a very rare operation compared to + // UpdateThread(), and a single UpdateThread() failure here and there won't break + // anything + try + { + if (m_threads.TryGetValue(threadID, out threadInfo)) + { + threadInfo.LastTick = Environment.TickCount & Int32.MaxValue; + threadInfo.IsTimedOut = false; + } + else + { + m_log.WarnFormat("[WATCHDOG]: Asked to update thread {0} which is not being monitored", threadID); + } + } + catch { } + } + + /// + /// Get currently watched threads for diagnostic purposes + /// + /// + public static ThreadWatchdogInfo[] GetThreadsInfo() + { + lock (m_threads) + return m_threads.Values.ToArray(); + } + + /// + /// Return the current thread's watchdog info. + /// + /// The watchdog info. null if the thread isn't being monitored. + public static ThreadWatchdogInfo GetCurrentThreadInfo() + { + lock (m_threads) + { + if (m_threads.ContainsKey(Thread.CurrentThread.ManagedThreadId)) + return m_threads[Thread.CurrentThread.ManagedThreadId]; + } + + return null; + } + + /// + /// Check watched threads. Fire alarm if appropriate. + /// + /// + /// + private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e) + { + int now = Environment.TickCount & Int32.MaxValue; + int msElapsed = now - LastWatchdogThreadTick; + + if (msElapsed > WATCHDOG_INTERVAL_MS * 2) + m_log.WarnFormat( + "[WATCHDOG]: {0} ms since Watchdog last ran. Interval should be approximately {1} ms", + msElapsed, WATCHDOG_INTERVAL_MS); + + LastWatchdogThreadTick = Environment.TickCount & Int32.MaxValue; + + Action callback = OnWatchdogTimeout; + + if (callback != null) + { + List callbackInfos = null; + + lock (m_threads) + { + foreach (ThreadWatchdogInfo threadInfo in m_threads.Values) + { + if (threadInfo.Thread.ThreadState == ThreadState.Stopped) + { + RemoveThread(threadInfo.Thread.ManagedThreadId); + + if (callbackInfos == null) + callbackInfos = new List(); + + callbackInfos.Add(threadInfo); + } + else if (!threadInfo.IsTimedOut && now - threadInfo.LastTick >= threadInfo.Timeout) + { + threadInfo.IsTimedOut = true; + + if (threadInfo.AlarmIfTimeout) + { + if (callbackInfos == null) + callbackInfos = new List(); + + callbackInfos.Add(threadInfo); + } + } + } + } + + if (callbackInfos != null) + foreach (ThreadWatchdogInfo callbackInfo in callbackInfos) + callback(callbackInfo); + } + + if (MemoryWatchdog.Enabled) + MemoryWatchdog.Update(); + + m_watchdogTimer.Start(); + } + } +} \ No newline at end of file diff --git a/OpenSim/Framework/Servers/BaseOpenSimServer.cs b/OpenSim/Framework/Servers/BaseOpenSimServer.cs index ab8d95a..2a8ae38 100644 --- a/OpenSim/Framework/Servers/BaseOpenSimServer.cs +++ b/OpenSim/Framework/Servers/BaseOpenSimServer.cs @@ -40,6 +40,7 @@ using log4net.Core; using log4net.Repository; using OpenSim.Framework; using OpenSim.Framework.Console; +using OpenSim.Framework.Monitoring; using OpenSim.Framework.Servers; using OpenSim.Framework.Servers.HttpServer; using OpenSim.Framework.Monitoring; diff --git a/OpenSim/Framework/Servers/HttpServer/BaseHttpServer.cs b/OpenSim/Framework/Servers/HttpServer/BaseHttpServer.cs index 3de7f9c..f57ea76 100644 --- a/OpenSim/Framework/Servers/HttpServer/BaseHttpServer.cs +++ b/OpenSim/Framework/Servers/HttpServer/BaseHttpServer.cs @@ -45,6 +45,7 @@ using OpenMetaverse.StructuredData; using CoolHTTPListener = HttpServer.HttpListener; using HttpListener=System.Net.HttpListener; using LogPrio=HttpServer.LogPrio; +using OpenSim.Framework.Monitoring; namespace OpenSim.Framework.Servers.HttpServer { diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs index 3252251..8d50151 100644 --- a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs +++ b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs @@ -32,6 +32,7 @@ using System.Reflection; using log4net; using HttpServer; using OpenSim.Framework; +using OpenSim.Framework.Monitoring; namespace OpenSim.Framework.Servers.HttpServer { diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceWorkerThread.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceWorkerThread.cs index 35a8dee..5adbcd1 100644 --- a/OpenSim/Framework/Servers/HttpServer/PollServiceWorkerThread.cs +++ b/OpenSim/Framework/Servers/HttpServer/PollServiceWorkerThread.cs @@ -34,6 +34,7 @@ using HttpServer; using OpenMetaverse; using System.Reflection; using log4net; +using OpenSim.Framework.Monitoring; namespace OpenSim.Framework.Servers.HttpServer { diff --git a/OpenSim/Framework/Watchdog.cs b/OpenSim/Framework/Watchdog.cs deleted file mode 100644 index 54e3d1a..0000000 --- a/OpenSim/Framework/Watchdog.cs +++ /dev/null @@ -1,334 +0,0 @@ -/* - * Copyright (c) Contributors, http://opensimulator.org/ - * See CONTRIBUTORS.TXT for a full list of copyright holders. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the OpenSimulator Project nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using log4net; - -namespace OpenSim.Framework -{ - /// - /// Manages launching threads and keeping watch over them for timeouts - /// - public static class Watchdog - { - /// Timer interval in milliseconds for the watchdog timer - const double WATCHDOG_INTERVAL_MS = 2500.0d; - - /// Default timeout in milliseconds before a thread is considered dead - public const int DEFAULT_WATCHDOG_TIMEOUT_MS = 5000; - - [System.Diagnostics.DebuggerDisplay("{Thread.Name}")] - public class ThreadWatchdogInfo - { - public Thread Thread { get; private set; } - - /// - /// Approximate tick when this thread was started. - /// - /// - /// Not terribly good since this quickly wraps around. - /// - public int FirstTick { get; private set; } - - /// - /// Last time this heartbeat update was invoked - /// - public int LastTick { get; set; } - - /// - /// Number of milliseconds before we notify that the thread is having a problem. - /// - public int Timeout { get; set; } - - /// - /// Is this thread considered timed out? - /// - public bool IsTimedOut { get; set; } - - /// - /// Will this thread trigger the alarm function if it has timed out? - /// - public bool AlarmIfTimeout { get; set; } - - /// - /// Method execute if alarm goes off. If null then no alarm method is fired. - /// - public Func AlarmMethod { get; set; } - - public ThreadWatchdogInfo(Thread thread, int timeout) - { - Thread = thread; - Timeout = timeout; - FirstTick = Environment.TickCount & Int32.MaxValue; - LastTick = FirstTick; - } - } - - /// - /// This event is called whenever a tracked thread is - /// stopped or has not called UpdateThread() in time< - /// /summary> - public static event Action OnWatchdogTimeout; - - private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); - private static Dictionary m_threads; - private static System.Timers.Timer m_watchdogTimer; - - /// - /// Last time the watchdog thread ran. - /// - /// - /// Should run every WATCHDOG_INTERVAL_MS - /// - public static int LastWatchdogThreadTick { get; private set; } - - static Watchdog() - { - m_threads = new Dictionary(); - m_watchdogTimer = new System.Timers.Timer(WATCHDOG_INTERVAL_MS); - m_watchdogTimer.AutoReset = false; - m_watchdogTimer.Elapsed += WatchdogTimerElapsed; - - // Set now so we don't get alerted on the first run - LastWatchdogThreadTick = Environment.TickCount & Int32.MaxValue; - - m_watchdogTimer.Start(); - } - - /// - /// Start a new thread that is tracked by the watchdog timer. - /// - /// The method that will be executed in a new thread - /// A name to give to the new thread - /// Priority to run the thread at - /// True to run this thread as a background thread, otherwise false - /// Trigger an alarm function is we have timed out - /// The newly created Thread object - public static Thread StartThread( - ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout) - { - return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS); - } - - /// - /// Start a new thread that is tracked by the watchdog timer - /// - /// The method that will be executed in a new thread - /// A name to give to the new thread - /// Priority to run the thread at - /// True to run this thread as a background - /// thread, otherwise false - /// Trigger an alarm function is we have timed out - /// - /// Alarm method to call if alarmIfTimeout is true and there is a timeout. - /// Normally, this will just return some useful debugging information. - /// - /// Number of milliseconds to wait until we issue a warning about timeout. - /// The newly created Thread object - public static Thread StartThread( - ThreadStart start, string name, ThreadPriority priority, bool isBackground, - bool alarmIfTimeout, Func alarmMethod, int timeout) - { - Thread thread = new Thread(start); - thread.Name = name; - thread.Priority = priority; - thread.IsBackground = isBackground; - - ThreadWatchdogInfo twi - = new ThreadWatchdogInfo(thread, timeout) - { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod }; - - m_log.DebugFormat( - "[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId); - - lock (m_threads) - m_threads.Add(twi.Thread.ManagedThreadId, twi); - - thread.Start(); - - return thread; - } - - /// - /// Marks the current thread as alive - /// - public static void UpdateThread() - { - UpdateThread(Thread.CurrentThread.ManagedThreadId); - } - - /// - /// Stops watchdog tracking on the current thread - /// - /// - /// True if the thread was removed from the list of tracked - /// threads, otherwise false - /// - public static bool RemoveThread() - { - return RemoveThread(Thread.CurrentThread.ManagedThreadId); - } - - private static bool RemoveThread(int threadID) - { - lock (m_threads) - return m_threads.Remove(threadID); - } - - public static bool AbortThread(int threadID) - { - lock (m_threads) - { - if (m_threads.ContainsKey(threadID)) - { - ThreadWatchdogInfo twi = m_threads[threadID]; - twi.Thread.Abort(); - RemoveThread(threadID); - - return true; - } - else - { - return false; - } - } - } - - private static void UpdateThread(int threadID) - { - ThreadWatchdogInfo threadInfo; - - // Although TryGetValue is not a thread safe operation, we use a try/catch here instead - // of a lock for speed. Adding/removing threads is a very rare operation compared to - // UpdateThread(), and a single UpdateThread() failure here and there won't break - // anything - try - { - if (m_threads.TryGetValue(threadID, out threadInfo)) - { - threadInfo.LastTick = Environment.TickCount & Int32.MaxValue; - threadInfo.IsTimedOut = false; - } - else - { - m_log.WarnFormat("[WATCHDOG]: Asked to update thread {0} which is not being monitored", threadID); - } - } - catch { } - } - - /// - /// Get currently watched threads for diagnostic purposes - /// - /// - public static ThreadWatchdogInfo[] GetThreadsInfo() - { - lock (m_threads) - return m_threads.Values.ToArray(); - } - - /// - /// Return the current thread's watchdog info. - /// - /// The watchdog info. null if the thread isn't being monitored. - public static ThreadWatchdogInfo GetCurrentThreadInfo() - { - lock (m_threads) - { - if (m_threads.ContainsKey(Thread.CurrentThread.ManagedThreadId)) - return m_threads[Thread.CurrentThread.ManagedThreadId]; - } - - return null; - } - - /// - /// Check watched threads. Fire alarm if appropriate. - /// - /// - /// - private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e) - { - int now = Environment.TickCount & Int32.MaxValue; - int msElapsed = now - LastWatchdogThreadTick; - - if (msElapsed > WATCHDOG_INTERVAL_MS * 2) - m_log.WarnFormat( - "[WATCHDOG]: {0} ms since Watchdog last ran. Interval should be approximately {1} ms", - msElapsed, WATCHDOG_INTERVAL_MS); - - LastWatchdogThreadTick = Environment.TickCount & Int32.MaxValue; - - Action callback = OnWatchdogTimeout; - - if (callback != null) - { - List callbackInfos = null; - - lock (m_threads) - { - foreach (ThreadWatchdogInfo threadInfo in m_threads.Values) - { - if (threadInfo.Thread.ThreadState == ThreadState.Stopped) - { - RemoveThread(threadInfo.Thread.ManagedThreadId); - - if (callbackInfos == null) - callbackInfos = new List(); - - callbackInfos.Add(threadInfo); - } - else if (!threadInfo.IsTimedOut && now - threadInfo.LastTick >= threadInfo.Timeout) - { - threadInfo.IsTimedOut = true; - - if (threadInfo.AlarmIfTimeout) - { - if (callbackInfos == null) - callbackInfos = new List(); - - callbackInfos.Add(threadInfo); - } - } - } - } - - if (callbackInfos != null) - foreach (ThreadWatchdogInfo callbackInfo in callbackInfos) - callback(callbackInfo); - } - - if (MemoryWatchdog.Enabled) - MemoryWatchdog.Update(); - - m_watchdogTimer.Start(); - } - } -} \ No newline at end of file diff --git a/OpenSim/Region/CoreModules/InterGrid/OpenGridProtocolModule.cs b/OpenSim/Region/CoreModules/InterGrid/OpenGridProtocolModule.cs index a6e2548..4a76b00 100644 --- a/OpenSim/Region/CoreModules/InterGrid/OpenGridProtocolModule.cs +++ b/OpenSim/Region/CoreModules/InterGrid/OpenGridProtocolModule.cs @@ -40,6 +40,7 @@ using OpenMetaverse; using OpenMetaverse.StructuredData; using OpenSim.Framework; using OpenSim.Framework.Capabilities; +using OpenSim.Framework.Monitoring; using OpenSim.Framework.Servers; using OpenSim.Region.Framework.Interfaces; using OpenSim.Region.Framework.Scenes; diff --git a/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs b/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs index 724533b..dfba3ff 100644 --- a/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs +++ b/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs @@ -42,6 +42,7 @@ using OpenMetaverse.Imaging; using OpenMetaverse.StructuredData; using OpenSim.Framework; using OpenSim.Framework.Capabilities; +using OpenSim.Framework.Monitoring; using OpenSim.Framework.Servers; using OpenSim.Framework.Servers.HttpServer; using OpenSim.Region.Framework.Interfaces; diff --git a/OpenSim/Region/Framework/Scenes/Scene.cs b/OpenSim/Region/Framework/Scenes/Scene.cs index 20918bd..24f62e3 100644 --- a/OpenSim/Region/Framework/Scenes/Scene.cs +++ b/OpenSim/Region/Framework/Scenes/Scene.cs @@ -40,6 +40,7 @@ using OpenMetaverse; using OpenMetaverse.Packets; using OpenMetaverse.Imaging; using OpenSim.Framework; +using OpenSim.Framework.Monitoring; using OpenSim.Services.Interfaces; using OpenSim.Framework.Communications; using OpenSim.Framework.Console; diff --git a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs index 5043208..bae25cd 100644 --- a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs +++ b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs @@ -38,6 +38,7 @@ using OpenMetaverse; using OpenMetaverse.Packets; using OpenSim.Framework; using OpenSim.Framework.Client; +using OpenSim.Framework.Monitoring; using OpenSim.Region.Framework.Scenes; namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server diff --git a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs index a7c5020..9d27386 100644 --- a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs +++ b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs @@ -34,6 +34,7 @@ using System.Text; using System.Threading; using log4net; using OpenSim.Framework; +using OpenSim.Framework.Monitoring; using OpenSim.Region.Framework.Scenes; namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server diff --git a/OpenSim/Region/OptionalModules/Avatar/Chat/IRCConnector.cs b/OpenSim/Region/OptionalModules/Avatar/Chat/IRCConnector.cs index cd401a6..ca956fb 100644 --- a/OpenSim/Region/OptionalModules/Avatar/Chat/IRCConnector.cs +++ b/OpenSim/Region/OptionalModules/Avatar/Chat/IRCConnector.cs @@ -37,6 +37,7 @@ using OpenMetaverse; using log4net; using Nini.Config; using OpenSim.Framework; +using OpenSim.Framework.Monitoring; using OpenSim.Region.Framework.Interfaces; using OpenSim.Region.Framework.Scenes; diff --git a/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs b/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs index 5b22860..47a9cdc 100644 --- a/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs +++ b/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs @@ -31,6 +31,7 @@ using System.Collections.Generic; using System.Threading; using OpenMetaverse; using OpenSim.Framework; +using OpenSim.Framework.Monitoring; using OpenSim.Region.Framework.Interfaces; using OpenSim.Region.ScriptEngine.Interfaces; using OpenSim.Region.ScriptEngine.Shared; -- cgit v1.1