From 968cae6c17fab8aa8a15bfd8231b50873aa6e794 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 25 Oct 2011 20:49:46 +0100 Subject: Add "threads abort " simulator console command that allows us to abort a watchdog managed thread. This is for diagnostic purposes. --- OpenSim/Framework/Watchdog.cs | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) (limited to 'OpenSim/Framework/Watchdog.cs') diff --git a/OpenSim/Framework/Watchdog.cs b/OpenSim/Framework/Watchdog.cs index 0f34e83..c947ea6 100644 --- a/OpenSim/Framework/Watchdog.cs +++ b/OpenSim/Framework/Watchdog.cs @@ -112,8 +112,10 @@ namespace OpenSim.Framework /// /// Stops watchdog tracking on the current thread /// - /// True if the thread was removed from the list of tracked - /// threads, otherwise false + /// + /// True if the thread was removed from the list of tracked + /// threads, otherwise false + /// public static bool RemoveThread() { return RemoveThread(Thread.CurrentThread.ManagedThreadId); @@ -133,6 +135,25 @@ namespace OpenSim.Framework return m_threads.Remove(threadID); } + public static bool AbortThread(int threadID) + { + lock (m_threads) + { + if (m_threads.ContainsKey(threadID)) + { + ThreadWatchdogInfo twi = m_threads[threadID]; + twi.Thread.Abort(); + RemoveThread(threadID); + + return true; + } + else + { + return false; + } + } + } + private static void UpdateThread(int threadID) { ThreadWatchdogInfo threadInfo; -- cgit v1.1 From e549f2f443bdf1e711d02dbd7d1977e0c296b1df Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 25 Oct 2011 22:51:23 +0100 Subject: Add m_threads dictionary locking to Watchdog.GetThreads() --- OpenSim/Framework/Watchdog.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'OpenSim/Framework/Watchdog.cs') diff --git a/OpenSim/Framework/Watchdog.cs b/OpenSim/Framework/Watchdog.cs index c947ea6..3ef9ccb 100644 --- a/OpenSim/Framework/Watchdog.cs +++ b/OpenSim/Framework/Watchdog.cs @@ -178,7 +178,8 @@ namespace OpenSim.Framework /// public static ThreadWatchdogInfo[] GetThreads() { - return m_threads.Values.ToArray(); + lock (m_threads) + return m_threads.Values.ToArray(); } private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e) -- cgit v1.1 From 182908d216f27861d033f69398abff996caf8053 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 25 Oct 2011 23:16:01 +0100 Subject: In Watchdog, add ability to specific timeout for a thread. This also changes the point of registration to the StartThread() call rather than the first Update() --- OpenSim/Framework/Watchdog.cs | 72 +++++++++++++++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 16 deletions(-) (limited to 'OpenSim/Framework/Watchdog.cs') diff --git a/OpenSim/Framework/Watchdog.cs b/OpenSim/Framework/Watchdog.cs index 3ef9ccb..1374518 100644 --- a/OpenSim/Framework/Watchdog.cs +++ b/OpenSim/Framework/Watchdog.cs @@ -40,18 +40,30 @@ namespace OpenSim.Framework { /// Timer interval in milliseconds for the watchdog timer const double WATCHDOG_INTERVAL_MS = 2500.0d; + /// Maximum timeout in milliseconds before a thread is considered dead const int WATCHDOG_TIMEOUT_MS = 5000; [System.Diagnostics.DebuggerDisplay("{Thread.Name}")] public class ThreadWatchdogInfo { - public Thread Thread; - public int LastTick; + public Thread Thread { get; private set; } + public int LastTick { get; set; } + + /// + /// Number of seconds before we notify that the thread is having a problem. + /// + public int Timeout { get; set; } + + /// + /// Is this thread considered timed out? + /// + public bool IsTimedOut { get; set; } - public ThreadWatchdogInfo(Thread thread) + public ThreadWatchdogInfo(Thread thread, int timeout) { Thread = thread; + Timeout = timeout; LastTick = Environment.TickCount & Int32.MaxValue; } } @@ -82,7 +94,7 @@ namespace OpenSim.Framework } /// - /// Start a new thread that is tracked by the watchdog timer + /// Start a new thread that is tracked by the watchdog timer. /// /// The method that will be executed in a new thread /// A name to give to the new thread @@ -92,12 +104,37 @@ namespace OpenSim.Framework /// The newly created Thread object public static Thread StartThread(ThreadStart start, string name, ThreadPriority priority, bool isBackground) { + return StartThread(start, name, priority, isBackground, WATCHDOG_TIMEOUT_MS); + } + + /// + /// Start a new thread that is tracked by the watchdog timer + /// + /// The method that will be executed in a new thread + /// A name to give to the new thread + /// Priority to run the thread at + /// True to run this thread as a background + /// thread, otherwise false + /// + /// Number of seconds to wait until we issue a warning about timeout. + /// + /// The newly created Thread object + public static Thread StartThread( + ThreadStart start, string name, ThreadPriority priority, bool isBackground, int timeout) + { Thread thread = new Thread(start); thread.Name = name; thread.Priority = priority; thread.IsBackground = isBackground; thread.Start(); + ThreadWatchdogInfo twi = new ThreadWatchdogInfo(thread, timeout); + + m_log.Debug("[WATCHDOG]: Started tracking thread \"" + twi.Thread.Name + "\" (ID " + twi.Thread.ManagedThreadId + ")"); + + lock (m_threads) + m_threads.Add(twi.Thread.ManagedThreadId, twi); + return thread; } @@ -121,14 +158,6 @@ namespace OpenSim.Framework return RemoveThread(Thread.CurrentThread.ManagedThreadId); } - private static void AddThread(ThreadWatchdogInfo threadInfo) - { - m_log.Debug("[WATCHDOG]: Started tracking thread \"" + threadInfo.Thread.Name + "\" (ID " + threadInfo.Thread.ManagedThreadId + ")"); - - lock (m_threads) - m_threads.Add(threadInfo.Thread.ManagedThreadId, threadInfo); - } - private static bool RemoveThread(int threadID) { lock (m_threads) @@ -165,9 +194,14 @@ namespace OpenSim.Framework try { if (m_threads.TryGetValue(threadID, out threadInfo)) + { threadInfo.LastTick = Environment.TickCount & Int32.MaxValue; + threadInfo.IsTimedOut = false; + } else - AddThread(new ThreadWatchdogInfo(Thread.CurrentThread)); + { + m_log.WarnFormat("[WATCHDOG]: Asked to update thread {0} which is not being monitored", threadID); + } } catch { } } @@ -196,10 +230,16 @@ namespace OpenSim.Framework foreach (ThreadWatchdogInfo threadInfo in m_threads.Values) { - if (threadInfo.Thread.ThreadState == ThreadState.Stopped || now - threadInfo.LastTick >= WATCHDOG_TIMEOUT_MS) + if (threadInfo.Thread.ThreadState == ThreadState.Stopped) + { + timedOut = threadInfo; + RemoveThread(threadInfo.Thread.ManagedThreadId); + break; + } + else if (!threadInfo.IsTimedOut && now - threadInfo.LastTick >= threadInfo.Timeout) { + threadInfo.IsTimedOut = true; timedOut = threadInfo; - m_threads.Remove(threadInfo.Thread.ManagedThreadId); break; } } @@ -212,4 +252,4 @@ namespace OpenSim.Framework m_watchdogTimer.Start(); } } -} +} \ No newline at end of file -- cgit v1.1 From c275c229281c9a7113659457a310da775f26ec43 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 25 Oct 2011 23:26:21 +0100 Subject: Restart the event queue worker threads that I accidentally disabled earlier today in 8a0a78c. Also adds these to the watchdogs with very large timeouts (should really be infinite) --- OpenSim/Framework/Watchdog.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'OpenSim/Framework/Watchdog.cs') diff --git a/OpenSim/Framework/Watchdog.cs b/OpenSim/Framework/Watchdog.cs index 1374518..5ffa890 100644 --- a/OpenSim/Framework/Watchdog.cs +++ b/OpenSim/Framework/Watchdog.cs @@ -51,7 +51,7 @@ namespace OpenSim.Framework public int LastTick { get; set; } /// - /// Number of seconds before we notify that the thread is having a problem. + /// Number of milliseconds before we notify that the thread is having a problem. /// public int Timeout { get; set; } @@ -116,7 +116,7 @@ namespace OpenSim.Framework /// True to run this thread as a background /// thread, otherwise false /// - /// Number of seconds to wait until we issue a warning about timeout. + /// Number of milliseconds to wait until we issue a warning about timeout. /// /// The newly created Thread object public static Thread StartThread( -- cgit v1.1