From b2ed348aa2746fbf034b713d006e40366c479d5a Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Thu, 22 Oct 2009 12:33:23 -0700 Subject: Implemented a Watchdog class. Do not manually create Thread objects anymore, use Watchdog.StartThread(). While your thread is running call Watchdog.UpdateThread(). When it is shutting down call Watchdog.RemoveThread(). Most of the threads in OpenSim have been updated --- .../Client/MXP/PacketHandler/MXPPacketServer.cs | 5 +- .../Framework/Servers/HttpServer/BaseHttpServer.cs | 8 - .../HttpServer/PollServiceRequestManager.cs | 6 +- OpenSim/Framework/Watchdog.cs | 183 +++++++++++++++++++++ .../UserServer.Modules/MessageServersConnector.cs | 8 +- OpenSim/Region/Application/OpenSim.cs | 11 ++ .../Region/ClientStack/LindenUDP/LLUDPServer.cs | 19 ++- .../InterGrid/OpenGridProtocolModule.cs | 15 +- .../CoreModules/World/Archiver/AssetsRequest.cs | 8 +- .../CoreModules/World/WorldMap/WorldMapModule.cs | 13 +- OpenSim/Region/Framework/Scenes/Scene.cs | 20 +-- .../Server/IRCClientView.cs | 10 +- .../InternetRelayClientView/Server/IRCServer.cs | 7 +- .../ContentManagementSystem/CMController.cs | 10 +- .../DotNetEngine/EventQueueThreadClass.cs | 12 +- .../ScriptEngine/DotNetEngine/MaintenanceThread.cs | 83 +++++----- .../Api/Implementation/AsyncCommandManager.cs | 8 +- 17 files changed, 306 insertions(+), 120 deletions(-) create mode 100644 OpenSim/Framework/Watchdog.cs (limited to 'OpenSim') diff --git a/OpenSim/Client/MXP/PacketHandler/MXPPacketServer.cs b/OpenSim/Client/MXP/PacketHandler/MXPPacketServer.cs index ba9c653..63381a4 100644 --- a/OpenSim/Client/MXP/PacketHandler/MXPPacketServer.cs +++ b/OpenSim/Client/MXP/PacketHandler/MXPPacketServer.cs @@ -85,10 +85,7 @@ namespace OpenSim.Client.MXP.PacketHandler m_transmitter = new Transmitter(port); - m_clientThread = new Thread(StartListener); - m_clientThread.Name = "MXPThread"; - m_clientThread.IsBackground = true; - m_clientThread.Start(); + StartListener(); } public void StartListener() diff --git a/OpenSim/Framework/Servers/HttpServer/BaseHttpServer.cs b/OpenSim/Framework/Servers/HttpServer/BaseHttpServer.cs index 85d7be2..bec5ed3 100644 --- a/OpenSim/Framework/Servers/HttpServer/BaseHttpServer.cs +++ b/OpenSim/Framework/Servers/HttpServer/BaseHttpServer.cs @@ -1559,15 +1559,7 @@ namespace OpenSim.Framework.Servers.HttpServer public void Start() { m_log.Info("[HTTPD]: Starting up HTTP Server"); - - //m_workerThread = new Thread(new ThreadStart(StartHTTP)); - //m_workerThread.Name = "HttpThread"; - //m_workerThread.IsBackground = true; - //m_workerThread.Start(); - //ThreadTracker.Add(m_workerThread); StartHTTP(); - - } private void StartHTTP() diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs index 1c54581..e7a64f7 100644 --- a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs +++ b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs @@ -50,9 +50,7 @@ namespace OpenSim.Framework.Servers.HttpServer m_WorkerThreadCount = pWorkerThreadCount; m_workerThreads = new Thread[m_WorkerThreadCount]; m_PollServiceWorkerThreads = new PollServiceWorkerThread[m_WorkerThreadCount]; - m_watcherThread = new Thread(ThreadStart); - //startup worker threads for (uint i=0;i + /// Manages launching threads and keeping watch over them for timeouts + /// + public static class Watchdog + { + /// Timer interval in milliseconds for the watchdog timer + const double WATCHDOG_INTERVAL_MS = 2500.0d; + /// Maximum timeout in milliseconds before a thread is considered dead + const int WATCHDOG_TIMEOUT_MS = 5000; + + [System.Diagnostics.DebuggerDisplay("{Thread.Name}")] + private class ThreadWatchdogInfo + { + public Thread Thread; + public int LastTick; + + public ThreadWatchdogInfo(Thread thread) + { + Thread = thread; + LastTick = Environment.TickCount & Int32.MaxValue; + } + } + + /// + /// This event is called whenever a tracked thread is stopped or + /// has not called UpdateThread() in time + /// + /// The thread that has been identified as dead + /// The last time this thread called UpdateThread() + public delegate void WatchdogTimeout(Thread thread, int lastTick); + + /// This event is called whenever a tracked thread is + /// stopped or has not called UpdateThread() in time + public static event WatchdogTimeout OnWatchdogTimeout; + + private static Dictionary m_threads; + private static System.Timers.Timer m_watchdogTimer; + + static Watchdog() + { + m_threads = new Dictionary(); + m_watchdogTimer = new System.Timers.Timer(WATCHDOG_INTERVAL_MS); + m_watchdogTimer.AutoReset = false; + m_watchdogTimer.Elapsed += WatchdogTimerElapsed; + m_watchdogTimer.Start(); + } + + /// + /// Start a new thread that is tracked by the watchdog timer + /// + /// The method that will be executed in a new thread + /// A name to give to the new thread + /// Priority to run the thread at + /// True to run this thread as a background + /// thread, otherwise false + /// The newly created Thread object + public static Thread StartThread(ThreadStart start, string name, ThreadPriority priority, bool isBackground) + { + Thread thread = new Thread(start); + thread.Name = name; + thread.Priority = priority; + thread.IsBackground = isBackground; + thread.Start(); + + lock (m_threads) + m_threads.Add(thread.ManagedThreadId, new ThreadWatchdogInfo(thread)); + + return thread; + } + + /// + /// Marks the current thread as alive + /// + public static void UpdateThread() + { + UpdateThread(Thread.CurrentThread.ManagedThreadId); + } + + /// + /// Marks a thread as alive + /// + /// The ManagedThreadId of the thread to mark as + /// alive + public static void UpdateThread(int threadID) + { + ThreadWatchdogInfo threadInfo; + + lock (m_threads) + { + if (m_threads.TryGetValue(threadID, out threadInfo)) + { + threadInfo.LastTick = Environment.TickCount & Int32.MaxValue; + } + } + } + + /// + /// Stops watchdog tracking on the current thread + /// + /// True if the thread was removed from the list of tracked + /// threads, otherwise false + public static bool RemoveThread() + { + return RemoveThread(Thread.CurrentThread.ManagedThreadId); + } + + /// + /// Stops watchdog tracking on a thread + /// + /// The ManagedThreadId of the thread to stop + /// tracking + /// True if the thread was removed from the list of tracked + /// threads, otherwise false + public static bool RemoveThread(int threadID) + { + lock (m_threads) + return m_threads.Remove(threadID); + } + + private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e) + { + WatchdogTimeout callback = OnWatchdogTimeout; + + if (callback != null) + { + ThreadWatchdogInfo timedOut = null; + + lock (m_threads) + { + int now = Environment.TickCount; + + foreach (ThreadWatchdogInfo threadInfo in m_threads.Values) + { + if (threadInfo.Thread.ThreadState == ThreadState.Stopped || now - threadInfo.LastTick >= WATCHDOG_TIMEOUT_MS) + { + timedOut = threadInfo; + m_threads.Remove(threadInfo.Thread.ManagedThreadId); + break; + } + } + } + + if (timedOut != null) + callback(timedOut.Thread, timedOut.LastTick); + } + + m_watchdogTimer.Start(); + } + } +} diff --git a/OpenSim/Grid/UserServer.Modules/MessageServersConnector.cs b/OpenSim/Grid/UserServer.Modules/MessageServersConnector.cs index 8ce353c..f24cef6 100644 --- a/OpenSim/Grid/UserServer.Modules/MessageServersConnector.cs +++ b/OpenSim/Grid/UserServer.Modules/MessageServersConnector.cs @@ -78,8 +78,6 @@ namespace OpenSim.Grid.UserServer.Modules private OpenSim.Framework.BlockingQueue m_NotifyQueue = new OpenSim.Framework.BlockingQueue(); - Thread m_NotifyThread; - private IGridServiceCore m_core; public event AgentLocationDelegate OnAgentLocation; @@ -96,8 +94,8 @@ namespace OpenSim.Grid.UserServer.Modules { m_core = core; m_core.RegisterInterface(this); - m_NotifyThread = new Thread(new ThreadStart(NotifyQueueRunner)); - m_NotifyThread.Start(); + + Watchdog.StartThread(NotifyQueueRunner, "NotifyQueueRunner", ThreadPriority.Normal, true); } public void PostInitialise() @@ -427,6 +425,8 @@ namespace OpenSim.Grid.UserServer.Modules { TellMessageServersAboutUserLogoffInternal(presence.agentID); } + + Watchdog.UpdateThread(); } } diff --git a/OpenSim/Region/Application/OpenSim.cs b/OpenSim/Region/Application/OpenSim.cs index 26298e7..0366d94 100644 --- a/OpenSim/Region/Application/OpenSim.cs +++ b/OpenSim/Region/Application/OpenSim.cs @@ -172,6 +172,9 @@ namespace OpenSim m_scriptTimer.Elapsed += RunAutoTimerScript; } + // Hook up to the watchdog timer + Watchdog.OnWatchdogTimeout += WatchdogTimeoutHandler; + PrintFileToConsole("startuplogo.txt"); // For now, start at the 'root' level by default @@ -384,6 +387,14 @@ namespace OpenSim } } + private void WatchdogTimeoutHandler(System.Threading.Thread thread, int lastTick) + { + int now = Environment.TickCount & Int32.MaxValue; + + m_log.ErrorFormat("[WATCHDOG]: Timeout detected for thread \"{0}\". ThreadState={1}. Last tick was {2}ms ago", + thread.Name, thread.ThreadState, now - lastTick); + } + #region Console Commands /// diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs index a9f4b2c..734471e 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs @@ -187,14 +187,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP base.Start(m_recvBufferSize, m_asyncPacketHandling); - // Start the incoming packet processing thread - Thread incomingThread = new Thread(IncomingPacketHandler); - incomingThread.Name = "Incoming Packets (" + m_scene.RegionInfo.RegionName + ")"; - incomingThread.Start(); - - Thread outgoingThread = new Thread(OutgoingPacketHandler); - outgoingThread.Name = "Outgoing Packets (" + m_scene.RegionInfo.RegionName + ")"; - outgoingThread.Start(); + // Start the packet processing threads + Watchdog.StartThread(IncomingPacketHandler, "Incoming Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false); + Watchdog.StartThread(OutgoingPacketHandler, "Outgoing Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false); } public new void Stop() @@ -775,11 +770,15 @@ namespace OpenSim.Region.ClientStack.LindenUDP { m_log.Error("[LLUDPSERVER]: Error in the incoming packet handler loop: " + ex.Message, ex); } + + Watchdog.UpdateThread(); } if (packetInbox.Count > 0) m_log.Warn("[LLUDPSERVER]: IncomingPacketHandler is shutting down, dropping " + packetInbox.Count + " packets"); packetInbox.Clear(); + + Watchdog.RemoveThread(); } private void OutgoingPacketHandler() @@ -842,12 +841,16 @@ namespace OpenSim.Region.ClientStack.LindenUDP // token bucket could get more tokens if (!m_packetSent) Thread.Sleep((int)TickCountResolution); + + Watchdog.UpdateThread(); } catch (Exception ex) { m_log.Error("[LLUDPSERVER]: OutgoingPacketHandler loop threw an exception: " + ex.Message, ex); } } + + Watchdog.RemoveThread(); } private void ClientOutgoingPacketHandler(IClientAPI client) diff --git a/OpenSim/Region/CoreModules/InterGrid/OpenGridProtocolModule.cs b/OpenSim/Region/CoreModules/InterGrid/OpenGridProtocolModule.cs index d636b1c..62500a2 100644 --- a/OpenSim/Region/CoreModules/InterGrid/OpenGridProtocolModule.cs +++ b/OpenSim/Region/CoreModules/InterGrid/OpenGridProtocolModule.cs @@ -1208,10 +1208,7 @@ namespace OpenSim.Region.CoreModules.InterGrid if (homeScene.TryGetAvatar(avatarId,out avatar)) { KillAUser ku = new KillAUser(avatar,mod); - Thread ta = new Thread(ku.ShutdownNoLogout); - ta.IsBackground = true; - ta.Name = "ShutdownThread"; - ta.Start(); + Watchdog.StartThread(ku.ShutdownNoLogout, "OGPShutdown", ThreadPriority.Normal, true); } } @@ -1261,7 +1258,13 @@ namespace OpenSim.Region.CoreModules.InterGrid avToBeKilled.ControllingClient.SendLogoutPacketWhenClosing = false; - Thread.Sleep(30000); + int sleepMS = 30000; + while (sleepMS > 0) + { + Watchdog.UpdateThread(); + Thread.Sleep(1000); + sleepMS -= 1000; + } // test for child agent because they might have come back if (avToBeKilled.IsChildAgent) @@ -1270,6 +1273,8 @@ namespace OpenSim.Region.CoreModules.InterGrid avToBeKilled.ControllingClient.Close(); } } + + Watchdog.RemoveThread(); } } diff --git a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs index fe9c8d9..c9fce91 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs @@ -128,7 +128,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver if (m_repliesRequired == 0) { m_requestState = RequestState.Completed; - PerformAssetsRequestCallback(); + PerformAssetsRequestCallback(null); return; } @@ -246,9 +246,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver // We want to stop using the asset cache thread asap // as we now need to do the work of producing the rest of the archive - Thread newThread = new Thread(PerformAssetsRequestCallback); - newThread.Name = "OpenSimulator archiving thread post assets receipt"; - newThread.Start(); + Util.FireAndForget(PerformAssetsRequestCallback); } else { @@ -265,7 +263,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// Perform the callback on the original requester of the assets /// - protected void PerformAssetsRequestCallback() + protected void PerformAssetsRequestCallback(object o) { try { diff --git a/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs b/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs index 4e40084..a4bcbad 100644 --- a/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs +++ b/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs @@ -74,7 +74,6 @@ namespace OpenSim.Region.CoreModules.World.WorldMap private Dictionary m_blacklistedregions = new Dictionary(); private Dictionary m_cachedRegionMapItemsAddress = new Dictionary(); private List m_rootAgents = new List(); - private Thread mapItemReqThread; private volatile bool threadrunning = false; //private int CacheRegionsDistance = 256; @@ -338,13 +337,10 @@ namespace OpenSim.Region.CoreModules.World.WorldMap { if (threadrunning) return; threadrunning = true; + m_log.Debug("[WORLD MAP]: Starting remote MapItem request thread"); - mapItemReqThread = new Thread(new ThreadStart(process)); - mapItemReqThread.IsBackground = true; - mapItemReqThread.Name = "MapItemRequestThread"; - mapItemReqThread.Priority = ThreadPriority.BelowNormal; - mapItemReqThread.SetApartmentState(ApartmentState.MTA); - mapItemReqThread.Start(); + + Watchdog.StartThread(process, "MapItemRequestThread", ThreadPriority.BelowNormal, true); } /// @@ -461,6 +457,8 @@ namespace OpenSim.Region.CoreModules.World.WorldMap OSDMap response = RequestMapItemsAsync("", st.agentID, st.flags, st.EstateID, st.godlike, st.itemtype, st.regionhandle); RequestMapItemsCompleted(response); } + + Watchdog.UpdateThread(); } } catch (Exception e) @@ -469,6 +467,7 @@ namespace OpenSim.Region.CoreModules.World.WorldMap } threadrunning = false; + Watchdog.RemoveThread(); } /// diff --git a/OpenSim/Region/Framework/Scenes/Scene.cs b/OpenSim/Region/Framework/Scenes/Scene.cs index 5005ac9..4b87f92 100644 --- a/OpenSim/Region/Framework/Scenes/Scene.cs +++ b/OpenSim/Region/Framework/Scenes/Scene.cs @@ -81,8 +81,6 @@ namespace OpenSim.Region.Framework.Scenes protected Timer m_restartWaitTimer = new Timer(); - protected Thread m_updateEntitiesThread; - public SimStatsReporter StatsReporter; protected List m_regionRestartNotifyList = new List(); @@ -945,11 +943,8 @@ namespace OpenSim.Region.Framework.Scenes HeartbeatThread = null; } m_lastUpdate = Environment.TickCount; - HeartbeatThread = new Thread(new ParameterizedThreadStart(Heartbeat)); - HeartbeatThread.SetApartmentState(ApartmentState.MTA); - HeartbeatThread.Name = string.Format("Heartbeat for region {0}", RegionInfo.RegionName); - HeartbeatThread.Priority = ThreadPriority.AboveNormal; - HeartbeatThread.Start(); + + HeartbeatThread = Watchdog.StartThread(Heartbeat, "Heartbeat for region " + RegionInfo.RegionName, ThreadPriority.Normal, false); } /// @@ -976,12 +971,13 @@ namespace OpenSim.Region.Framework.Scenes /// /// Performs per-frame updates regularly /// - /// - /// - private void Heartbeat(object sender) + private void Heartbeat() { if (!Monitor.TryEnter(m_heartbeatLock)) + { + Watchdog.RemoveThread(); return; + } try { @@ -998,6 +994,8 @@ namespace OpenSim.Region.Framework.Scenes Monitor.Pulse(m_heartbeatLock); Monitor.Exit(m_heartbeatLock); } + + Watchdog.RemoveThread(); } /// @@ -1146,6 +1144,8 @@ namespace OpenSim.Region.Framework.Scenes if ((maintc < (m_timespan * 1000)) && maintc > 0) Thread.Sleep(maintc); + + Watchdog.UpdateThread(); } } diff --git a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs index df03b8d..4b0d01a 100644 --- a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs +++ b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs @@ -67,9 +67,8 @@ namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server { m_client = client; m_scene = scene; - - Thread loopThread = new Thread(InternalLoop); - loopThread.Start(); + + Watchdog.StartThread(InternalLoop, "IRCClientView", ThreadPriority.Normal, false); } private void SendServerCommand(string command) @@ -102,7 +101,7 @@ namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server { try { - string strbuf = ""; + string strbuf = String.Empty; while (m_connected && m_client.Connected) { @@ -140,6 +139,7 @@ namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server } Thread.Sleep(0); + Watchdog.UpdateThread(); } } catch (IOException) @@ -156,6 +156,8 @@ namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server m_log.Warn("[IRCd] Disconnected client."); } + + Watchdog.RemoveThread(); } private void ProcessInMessage(string message, string command) diff --git a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs index 91ce9f1..eb39026 100644 --- a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs +++ b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs @@ -33,6 +33,7 @@ using System.Reflection; using System.Text; using System.Threading; using log4net; +using OpenSim.Framework; using OpenSim.Region.Framework.Scenes; namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server @@ -56,8 +57,7 @@ namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server m_listener.Start(50); - Thread thread = new Thread(ListenLoop); - thread.Start(); + Watchdog.StartThread(ListenLoop, "IRCServer", ThreadPriority.Normal, false); m_baseScene = baseScene; } @@ -72,7 +72,10 @@ namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server while (m_running) { AcceptClient(m_listener.AcceptTcpClient()); + Watchdog.UpdateThread(); } + + Watchdog.RemoveThread(); } private void AcceptClient(TcpClient client) diff --git a/OpenSim/Region/OptionalModules/ContentManagementSystem/CMController.cs b/OpenSim/Region/OptionalModules/ContentManagementSystem/CMController.cs index 16fe9e9..8d6c41d 100644 --- a/OpenSim/Region/OptionalModules/ContentManagementSystem/CMController.cs +++ b/OpenSim/Region/OptionalModules/ContentManagementSystem/CMController.cs @@ -86,7 +86,6 @@ namespace OpenSim.Region.OptionalModules.ContentManagement /// Hashtable m_sceneList = Hashtable.Synchronized(new Hashtable()); State m_state = State.NONE; - Thread m_thread = null; CMView m_view = null; #endregion Fields @@ -148,10 +147,7 @@ namespace OpenSim.Region.OptionalModules.ContentManagement lock (this) { m_estateModule = scene.RequestModuleInterface(); - m_thread = new Thread(MainLoop); - m_thread.Name = "Content Management"; - m_thread.IsBackground = true; - m_thread.Start(); + Watchdog.StartThread(MainLoop, "Content Management", ThreadPriority.Normal, true); m_state = State.NONE; } } @@ -200,6 +196,8 @@ namespace OpenSim.Region.OptionalModules.ContentManagement m_log.Debug("[CONTENT MANAGEMENT] MAIN LOOP -- uuuuuuuuuh, what?"); break; } + + Watchdog.UpdateThread(); } } catch (Exception e) @@ -209,6 +207,8 @@ namespace OpenSim.Region.OptionalModules.ContentManagement "[CONTENT MANAGEMENT]: Content management thread terminating with exception. PLEASE REBOOT YOUR SIM - CONTENT MANAGEMENT WILL NOT BE AVAILABLE UNTIL YOU DO. Exception is {0}", e); } + + Watchdog.RemoveThread(); } /// diff --git a/OpenSim/Region/ScriptEngine/DotNetEngine/EventQueueThreadClass.cs b/OpenSim/Region/ScriptEngine/DotNetEngine/EventQueueThreadClass.cs index 583d2ff..51fd41a 100644 --- a/OpenSim/Region/ScriptEngine/DotNetEngine/EventQueueThreadClass.cs +++ b/OpenSim/Region/ScriptEngine/DotNetEngine/EventQueueThreadClass.cs @@ -132,12 +132,7 @@ namespace OpenSim.Region.ScriptEngine.DotNetEngine /// private void Start() { - EventQueueThread = new Thread(EventQueueThreadLoop); - EventQueueThread.IsBackground = true; - - EventQueueThread.Priority = MyThreadPriority; - EventQueueThread.Name = "EventQueueManagerThread_" + ThreadCount; - EventQueueThread.Start(); + EventQueueThread = Watchdog.StartThread(EventQueueThreadLoop, "EventQueueManagerThread_" + ThreadCount, MyThreadPriority, true); // Look at this... Don't you wish everyone did that solid // coding everywhere? :P @@ -184,6 +179,7 @@ namespace OpenSim.Region.ScriptEngine.DotNetEngine while (true) { DoProcessQueue(); + Watchdog.UpdateThread(); } } catch (ThreadAbortException) @@ -214,6 +210,8 @@ namespace OpenSim.Region.ScriptEngine.DotNetEngine m_log.ErrorFormat("[{0}]: Exception {1} thrown", ScriptEngineName, e.GetType().ToString()); throw e; } + + Watchdog.UpdateThread(); } } catch (ThreadAbortException) @@ -226,6 +224,8 @@ namespace OpenSim.Region.ScriptEngine.DotNetEngine "[{0}]: Event queue thread terminating with exception. PLEASE REBOOT YOUR SIM - SCRIPT EVENTS WILL NOT WORK UNTIL YOU DO. Exception is {1}", ScriptEngineName, e); } + + Watchdog.RemoveThread(); } public void DoProcessQueue() diff --git a/OpenSim/Region/ScriptEngine/DotNetEngine/MaintenanceThread.cs b/OpenSim/Region/ScriptEngine/DotNetEngine/MaintenanceThread.cs index 7ffdb1a..87fdf1f 100644 --- a/OpenSim/Region/ScriptEngine/DotNetEngine/MaintenanceThread.cs +++ b/OpenSim/Region/ScriptEngine/DotNetEngine/MaintenanceThread.cs @@ -93,10 +93,7 @@ namespace OpenSim.Region.ScriptEngine.DotNetEngine { if (MaintenanceThreadThread == null) { - MaintenanceThreadThread = new Thread(MaintenanceLoop); - MaintenanceThreadThread.Name = "ScriptMaintenanceThread"; - MaintenanceThreadThread.IsBackground = true; - MaintenanceThreadThread.Start(); + MaintenanceThreadThread = Watchdog.StartThread(MaintenanceLoop, "ScriptMaintenanceThread", ThreadPriority.Normal, true); } } @@ -164,56 +161,54 @@ namespace OpenSim.Region.ScriptEngine.DotNetEngine MaintenanceLoopTicks_ScriptLoadUnload_Count++; MaintenanceLoopTicks_Other_Count++; - - //lock (ScriptEngine.ScriptEngines) - //{ - foreach (ScriptEngine m_ScriptEngine in new ArrayList(ScriptEngine.ScriptEngines)) + foreach (ScriptEngine m_ScriptEngine in new ArrayList(ScriptEngine.ScriptEngines)) + { + // lastScriptEngine = m_ScriptEngine; + // Re-reading config every x seconds + if (MaintenanceLoopTicks_Other_Count >= MaintenanceLoopTicks_Other) { - // lastScriptEngine = m_ScriptEngine; - // Re-reading config every x seconds - if (MaintenanceLoopTicks_Other_Count >= MaintenanceLoopTicks_Other) + MaintenanceLoopTicks_Other_ResetCount = true; + if (m_ScriptEngine.RefreshConfigFilens > 0) { - MaintenanceLoopTicks_Other_ResetCount = true; - if (m_ScriptEngine.RefreshConfigFilens > 0) + // Check if its time to re-read config + if (DateTime.Now.Ticks - Last_ReReadConfigFilens > + m_ScriptEngine.RefreshConfigFilens) { - // Check if its time to re-read config - if (DateTime.Now.Ticks - Last_ReReadConfigFilens > - m_ScriptEngine.RefreshConfigFilens) - { - //m_log.Debug("Time passed: " + (DateTime.Now.Ticks - Last_ReReadConfigFilens) + ">" + m_ScriptEngine.RefreshConfigFilens); - // Its time to re-read config file - m_ScriptEngine.ReadConfig(); - Last_ReReadConfigFilens = DateTime.Now.Ticks; // Reset time - } + //m_log.Debug("Time passed: " + (DateTime.Now.Ticks - Last_ReReadConfigFilens) + ">" + m_ScriptEngine.RefreshConfigFilens); + // Its time to re-read config file + m_ScriptEngine.ReadConfig(); + Last_ReReadConfigFilens = DateTime.Now.Ticks; // Reset time + } - // Adjust number of running script threads if not correct - if (m_ScriptEngine.m_EventQueueManager != null) - m_ScriptEngine.m_EventQueueManager.AdjustNumberOfScriptThreads(); + // Adjust number of running script threads if not correct + if (m_ScriptEngine.m_EventQueueManager != null) + m_ScriptEngine.m_EventQueueManager.AdjustNumberOfScriptThreads(); - // Check if any script has exceeded its max execution time - if (EventQueueManager.EnforceMaxExecutionTime) + // Check if any script has exceeded its max execution time + if (EventQueueManager.EnforceMaxExecutionTime) + { + // We are enforcing execution time + if (DateTime.Now.Ticks - Last_maxFunctionExecutionTimens > + EventQueueManager.maxFunctionExecutionTimens) { - // We are enforcing execution time - if (DateTime.Now.Ticks - Last_maxFunctionExecutionTimens > - EventQueueManager.maxFunctionExecutionTimens) - { - // Its time to check again - m_ScriptEngine.m_EventQueueManager.CheckScriptMaxExecTime(); // Do check - Last_maxFunctionExecutionTimens = DateTime.Now.Ticks; // Reset time - } + // Its time to check again + m_ScriptEngine.m_EventQueueManager.CheckScriptMaxExecTime(); // Do check + Last_maxFunctionExecutionTimens = DateTime.Now.Ticks; // Reset time } } } - if (MaintenanceLoopTicks_ScriptLoadUnload_Count >= MaintenanceLoopTicks_ScriptLoadUnload) - { - MaintenanceLoopTicks_ScriptLoadUnload_ResetCount = true; - // LOAD / UNLOAD SCRIPTS - if (m_ScriptEngine.m_ScriptManager != null) - m_ScriptEngine.m_ScriptManager.DoScriptLoadUnload(); - } } - //} + if (MaintenanceLoopTicks_ScriptLoadUnload_Count >= MaintenanceLoopTicks_ScriptLoadUnload) + { + MaintenanceLoopTicks_ScriptLoadUnload_ResetCount = true; + // LOAD / UNLOAD SCRIPTS + if (m_ScriptEngine.m_ScriptManager != null) + m_ScriptEngine.m_ScriptManager.DoScriptLoadUnload(); + } + } + + Watchdog.UpdateThread(); } } catch(ThreadAbortException) @@ -225,6 +220,8 @@ namespace OpenSim.Region.ScriptEngine.DotNetEngine m_log.ErrorFormat("Exception in MaintenanceLoopThread. Thread will recover after 5 sec throttle. Exception: {0}", ex.ToString()); } } + + Watchdog.RemoveThread(); } #endregion diff --git a/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs b/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs index 1607d34..9d97cb2 100644 --- a/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs +++ b/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs @@ -137,11 +137,7 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api if (cmdHandlerThread == null) { // Start the thread that will be doing the work - cmdHandlerThread = new Thread(CmdHandlerThreadLoop); - cmdHandlerThread.Name = "AsyncLSLCmdHandlerThread"; - cmdHandlerThread.Priority = ThreadPriority.BelowNormal; - cmdHandlerThread.IsBackground = true; - cmdHandlerThread.Start(); + cmdHandlerThread = Watchdog.StartThread(CmdHandlerThreadLoop, "AsyncLSLCmdHandlerThread", ThreadPriority.Normal, true); } } @@ -185,6 +181,8 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api Thread.Sleep(cmdHandlerThreadCycleSleepms); DoOneCmdHandlerPass(); + + Watchdog.UpdateThread(); } } catch -- cgit v1.1