Implemented a Watchdog class. Do not manually create Thread objects anymore, use Watchdog.StartThread(). While your thread is running call Watchdog.UpdateThread(). When it is shutting down call Watchdog.RemoveThread(). Most of the threads in OpenSim have been updated

3 files changed, 185 insertions, 12 deletions

diff --git a/OpenSim/Framework/Servers/HttpServer/BaseHttpServer.cs b/OpenSim/Framework/Servers/HttpServer/BaseHttpServer.cs
index 85d7be2..bec5ed3 100644
--- a/OpenSim/Framework/Servers/HttpServer/BaseHttpServer.cs
+++ b/OpenSim/Framework/Servers/HttpServer/BaseHttpServer.cs
@@ -1559,15 +1559,7 @@ namespace OpenSim.Framework.Servers.HttpServer
         public void Start()
         {
             m_log.Info("[HTTPD]: Starting up HTTP Server");
-
-            //m_workerThread = new Thread(new ThreadStart(StartHTTP));
-            //m_workerThread.Name = "HttpThread";
-            //m_workerThread.IsBackground = true;
-            //m_workerThread.Start();
-            //ThreadTracker.Add(m_workerThread);
             StartHTTP();
-            
-
         }
 
         private void StartHTTP()
diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs
index 1c54581..e7a64f7 100644
--- a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs
+++ b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs
@@ -50,9 +50,7 @@ namespace OpenSim.Framework.Servers.HttpServer
             m_WorkerThreadCount = pWorkerThreadCount;
             m_workerThreads = new Thread[m_WorkerThreadCount];
             m_PollServiceWorkerThreads = new PollServiceWorkerThread[m_WorkerThreadCount];
-            m_watcherThread = new Thread(ThreadStart);
 
-            
             //startup worker threads
             for (uint i=0;i<m_WorkerThreadCount;i++)
             {
@@ -65,11 +63,11 @@ namespace OpenSim.Framework.Servers.HttpServer
                 m_workerThreads[i].Start();
                 
             }
+
             //start watcher threads
+            m_watcherThread = new Thread(ThreadStart);
             m_watcherThread.Name = "PollServiceWatcherThread";
             m_watcherThread.Start();
-
-
         }
 
         internal void ReQueueEvent(PollServiceHttpRequest req)
diff --git a/OpenSim/Framework/Watchdog.cs b/OpenSim/Framework/Watchdog.cs
new file mode 100644
index 0000000..b905609
--- /dev/null
+++ b/OpenSim/Framework/Watchdog.cs
@@ -0,0 +1,183 @@
+/*
+ * Copyright (c) Contributors, http://opensimulator.org/
+ * See CONTRIBUTORS.TXT for a full list of copyright holders.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the OpenSimulator Project nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+
+namespace OpenSim.Framework
+{
+    /// <summary>
+    /// Manages launching threads and keeping watch over them for timeouts
+    /// </summary>
+    public static class Watchdog
+    {
+        /// <summary>Timer interval in milliseconds for the watchdog timer</summary>
+        const double WATCHDOG_INTERVAL_MS = 2500.0d;
+        /// <summary>Maximum timeout in milliseconds before a thread is considered dead</summary>
+        const int WATCHDOG_TIMEOUT_MS = 5000;
+
+        [System.Diagnostics.DebuggerDisplay("{Thread.Name}")]
+        private class ThreadWatchdogInfo
+        {
+            public Thread Thread;
+            public int LastTick;
+
+            public ThreadWatchdogInfo(Thread thread)
+            {
+                Thread = thread;
+                LastTick = Environment.TickCount & Int32.MaxValue;
+            }
+        }
+
+        /// <summary>
+        /// This event is called whenever a tracked thread is stopped or
+        /// has not called UpdateThread() in time
+        /// </summary>
+        /// <param name="thread">The thread that has been identified as dead</param>
+        /// <param name="lastTick">The last time this thread called UpdateThread()</param>
+        public delegate void WatchdogTimeout(Thread thread, int lastTick);
+
+        /// <summary>This event is called whenever a tracked thread is
+        /// stopped or has not called UpdateThread() in time</summary>
+        public static event WatchdogTimeout OnWatchdogTimeout;
+
+        private static Dictionary<int, ThreadWatchdogInfo> m_threads;
+        private static System.Timers.Timer m_watchdogTimer;
+
+        static Watchdog()
+        {
+            m_threads = new Dictionary<int, ThreadWatchdogInfo>();
+            m_watchdogTimer = new System.Timers.Timer(WATCHDOG_INTERVAL_MS);
+            m_watchdogTimer.AutoReset = false;
+            m_watchdogTimer.Elapsed += WatchdogTimerElapsed;
+            m_watchdogTimer.Start();
+        }
+
+        /// <summary>
+        /// Start a new thread that is tracked by the watchdog timer
+        /// </summary>
+        /// <param name="start">The method that will be executed in a new thread</param>
+        /// <param name="name">A name to give to the new thread</param>
+        /// <param name="priority">Priority to run the thread at</param>
+        /// <param name="isBackground">True to run this thread as a background
+        /// thread, otherwise false</param>
+        /// <returns>The newly created Thread object</returns>
+        public static Thread StartThread(ThreadStart start, string name, ThreadPriority priority, bool isBackground)
+        {
+            Thread thread = new Thread(start);
+            thread.Name = name;
+            thread.Priority = priority;
+            thread.IsBackground = isBackground;
+            thread.Start();
+
+            lock (m_threads)
+                m_threads.Add(thread.ManagedThreadId, new ThreadWatchdogInfo(thread));
+
+            return thread;
+        }
+
+        /// <summary>
+        /// Marks the current thread as alive
+        /// </summary>
+        public static void UpdateThread()
+        {
+            UpdateThread(Thread.CurrentThread.ManagedThreadId);
+        }
+
+        /// <summary>
+        /// Marks a thread as alive
+        /// </summary>
+        /// <param name="threadID">The ManagedThreadId of the thread to mark as
+        /// alive</param>
+        public static void UpdateThread(int threadID)
+        {
+            ThreadWatchdogInfo threadInfo;
+
+            lock (m_threads)
+            {
+                if (m_threads.TryGetValue(threadID, out threadInfo))
+                {
+                    threadInfo.LastTick = Environment.TickCount & Int32.MaxValue;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Stops watchdog tracking on the current thread
+        /// </summary>
+        /// <returns>True if the thread was removed from the list of tracked
+        /// threads, otherwise false</returns>
+        public static bool RemoveThread()
+        {
+            return RemoveThread(Thread.CurrentThread.ManagedThreadId);
+        }
+
+        /// <summary>
+        /// Stops watchdog tracking on a thread
+        /// </summary>
+        /// <param name="threadID">The ManagedThreadId of the thread to stop
+        /// tracking</param>
+        /// <returns>True if the thread was removed from the list of tracked
+        /// threads, otherwise false</returns>
+        public static bool RemoveThread(int threadID)
+        {
+            lock (m_threads)
+                return m_threads.Remove(threadID);
+        }
+
+        private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e)
+        {
+            WatchdogTimeout callback = OnWatchdogTimeout;
+
+            if (callback != null)
+            {
+                ThreadWatchdogInfo timedOut = null;
+
+                lock (m_threads)
+                {
+                    int now = Environment.TickCount;
+
+                    foreach (ThreadWatchdogInfo threadInfo in m_threads.Values)
+                    {
+                        if (threadInfo.Thread.ThreadState == ThreadState.Stopped || now - threadInfo.LastTick >= WATCHDOG_TIMEOUT_MS)
+                        {
+                            timedOut = threadInfo;
+                            m_threads.Remove(threadInfo.Thread.ManagedThreadId);
+                            break;
+                        }
+                    }
+                }
+
+                if (timedOut != null)
+                    callback(timedOut.Thread, timedOut.LastTick);
+            }
+
+            m_watchdogTimer.Start();
+        }
+    }
+}