From 98b46d48fefc7801b7c9e496000e9329f4266e5e Mon Sep 17 00:00:00 2001
From: Justin Clark-Casey (justincc)
Date: Thu, 7 Jun 2012 02:44:13 +0100
Subject: Allow the thread watchdog to accept an alarm method that is invoked
if the timeout is breached.
This alarm can then invoke this to log extra information.
This is used in LLUDPServer to show which client was being processed when incoming and outgoing udp watchdog alarms are triggered.
---
OpenSim/Framework/Watchdog.cs | 40 +++++++++++++++++++++++-----------------
1 file changed, 23 insertions(+), 17 deletions(-)
(limited to 'OpenSim/Framework/Watchdog.cs')
diff --git a/OpenSim/Framework/Watchdog.cs b/OpenSim/Framework/Watchdog.cs
index e93e50e..7552cd1 100644
--- a/OpenSim/Framework/Watchdog.cs
+++ b/OpenSim/Framework/Watchdog.cs
@@ -42,7 +42,7 @@ namespace OpenSim.Framework
const double WATCHDOG_INTERVAL_MS = 2500.0d;
/// Maximum timeout in milliseconds before a thread is considered dead
- const int WATCHDOG_TIMEOUT_MS = 5000;
+ public const int WATCHDOG_TIMEOUT_MS = 5000;
[System.Diagnostics.DebuggerDisplay("{Thread.Name}")]
public class ThreadWatchdogInfo
@@ -58,7 +58,7 @@ namespace OpenSim.Framework
public int FirstTick { get; private set; }
///
- /// First time this heartbeat update was invoked
+ /// Last time this heartbeat update was invoked
///
public int LastTick { get; set; }
@@ -77,6 +77,11 @@ namespace OpenSim.Framework
///
public bool AlarmIfTimeout { get; set; }
+ ///
+ /// Method execute if alarm goes off. If null then no alarm method is fired.
+ ///
+ public Func AlarmMethod { get; set; }
+
public ThreadWatchdogInfo(Thread thread, int timeout)
{
Thread = thread;
@@ -87,16 +92,10 @@ namespace OpenSim.Framework
}
///
- /// This event is called whenever a tracked thread is stopped or
- /// has not called UpdateThread() in time
- ///
- /// The thread that has been identified as dead
- /// The last time this thread called UpdateThread()
- public delegate void WatchdogTimeout(Thread thread, int lastTick);
-
- /// This event is called whenever a tracked thread is
- /// stopped or has not called UpdateThread() in time
- public static event WatchdogTimeout OnWatchdogTimeout;
+ /// This event is called whenever a tracked thread is
+ /// stopped or has not called UpdateThread() in time<
+ /// /summary>
+ public static event Action OnWatchdogTimeout;
private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
private static Dictionary m_threads;
@@ -123,7 +122,7 @@ namespace OpenSim.Framework
public static Thread StartThread(
ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout)
{
- return StartThread(start, name, priority, isBackground, alarmIfTimeout, WATCHDOG_TIMEOUT_MS);
+ return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, WATCHDOG_TIMEOUT_MS);
}
///
@@ -135,17 +134,24 @@ namespace OpenSim.Framework
/// True to run this thread as a background
/// thread, otherwise false
/// Trigger an alarm function is we have timed out
+ ///
+ /// Alarm method to call if alarmIfTimeout is true and there is a timeout.
+ /// Normally, this will just return some useful debugging information.
+ ///
/// Number of milliseconds to wait until we issue a warning about timeout.
/// The newly created Thread object
public static Thread StartThread(
- ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, int timeout)
+ ThreadStart start, string name, ThreadPriority priority, bool isBackground,
+ bool alarmIfTimeout, Func alarmMethod, int timeout)
{
Thread thread = new Thread(start);
thread.Name = name;
thread.Priority = priority;
thread.IsBackground = isBackground;
- ThreadWatchdogInfo twi = new ThreadWatchdogInfo(thread, timeout) { AlarmIfTimeout = alarmIfTimeout };
+ ThreadWatchdogInfo twi
+ = new ThreadWatchdogInfo(thread, timeout)
+ { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod };
m_log.DebugFormat(
"[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
@@ -258,7 +264,7 @@ namespace OpenSim.Framework
///
private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e)
{
- WatchdogTimeout callback = OnWatchdogTimeout;
+ Action callback = OnWatchdogTimeout;
if (callback != null)
{
@@ -296,7 +302,7 @@ namespace OpenSim.Framework
if (callbackInfos != null)
foreach (ThreadWatchdogInfo callbackInfo in callbackInfos)
- callback(callbackInfo.Thread, callbackInfo.LastTick);
+ callback(callbackInfo);
}
m_watchdogTimer.Start();
--
cgit v1.1
From 5691a8b860b22939e2b608d6d8400b5260b25cf3 Mon Sep 17 00:00:00 2001
From: Justin Clark-Casey (justincc)
Date: Wed, 4 Jul 2012 00:15:03 +0100
Subject: refactor: rename Watchdog.WATCHDOG_TIMEOUT_MS to
DEFAULT_WATCHDOG_TIMEOUT_MS to reflect what it actually is
---
OpenSim/Framework/Watchdog.cs | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
(limited to 'OpenSim/Framework/Watchdog.cs')
diff --git a/OpenSim/Framework/Watchdog.cs b/OpenSim/Framework/Watchdog.cs
index 7552cd1..2b3a719 100644
--- a/OpenSim/Framework/Watchdog.cs
+++ b/OpenSim/Framework/Watchdog.cs
@@ -41,8 +41,8 @@ namespace OpenSim.Framework
/// Timer interval in milliseconds for the watchdog timer
const double WATCHDOG_INTERVAL_MS = 2500.0d;
- /// Maximum timeout in milliseconds before a thread is considered dead
- public const int WATCHDOG_TIMEOUT_MS = 5000;
+ /// Default timeout in milliseconds before a thread is considered dead
+ public const int DEFAULT_WATCHDOG_TIMEOUT_MS = 5000;
[System.Diagnostics.DebuggerDisplay("{Thread.Name}")]
public class ThreadWatchdogInfo
@@ -122,7 +122,7 @@ namespace OpenSim.Framework
public static Thread StartThread(
ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout)
{
- return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, WATCHDOG_TIMEOUT_MS);
+ return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS);
}
///
--
cgit v1.1
From 7e73f609e5f61040358ec247d371306165945ca1 Mon Sep 17 00:00:00 2001
From: Justin Clark-Casey (justincc)
Date: Thu, 5 Jul 2012 23:15:59 +0100
Subject: Log warning if time between invocations of the watchdog thread is
twice the timer setting.
This is to help detect situations where thread timeout warnings are being generated because of general machine issues rather than deadlock, network or other problems.
---
OpenSim/Framework/Watchdog.cs | 24 ++++++++++++++++++++++--
1 file changed, 22 insertions(+), 2 deletions(-)
(limited to 'OpenSim/Framework/Watchdog.cs')
diff --git a/OpenSim/Framework/Watchdog.cs b/OpenSim/Framework/Watchdog.cs
index 2b3a719..8a74f53 100644
--- a/OpenSim/Framework/Watchdog.cs
+++ b/OpenSim/Framework/Watchdog.cs
@@ -101,12 +101,24 @@ namespace OpenSim.Framework
private static Dictionary m_threads;
private static System.Timers.Timer m_watchdogTimer;
+ ///
+ /// Last time the watchdog thread ran.
+ ///
+ ///
+ /// Should run every WATCHDOG_INTERVAL_MS
+ ///
+ public static int LastWatchdogThreadTick { get; private set; }
+
static Watchdog()
{
m_threads = new Dictionary();
m_watchdogTimer = new System.Timers.Timer(WATCHDOG_INTERVAL_MS);
m_watchdogTimer.AutoReset = false;
m_watchdogTimer.Elapsed += WatchdogTimerElapsed;
+
+ // Set now so we don't get alerted on the first run
+ LastWatchdogThreadTick = Environment.TickCount & Int32.MaxValue;
+
m_watchdogTimer.Start();
}
@@ -264,6 +276,16 @@ namespace OpenSim.Framework
///
private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e)
{
+ int now = Environment.TickCount & Int32.MaxValue;
+ int msElapsed = now - LastWatchdogThreadTick;
+
+ if (msElapsed > WATCHDOG_INTERVAL_MS * 2)
+ m_log.WarnFormat(
+ "[WATCHDOG]: {0} ms since Watchdog last ran. Interval should be approximately {1} ms",
+ msElapsed, WATCHDOG_INTERVAL_MS);
+
+ LastWatchdogThreadTick = Environment.TickCount & Int32.MaxValue;
+
Action callback = OnWatchdogTimeout;
if (callback != null)
@@ -272,8 +294,6 @@ namespace OpenSim.Framework
lock (m_threads)
{
- int now = Environment.TickCount & Int32.MaxValue;
-
foreach (ThreadWatchdogInfo threadInfo in m_threads.Values)
{
if (threadInfo.Thread.ThreadState == ThreadState.Stopped)
--
cgit v1.1