From 98b46d48fefc7801b7c9e496000e9329f4266e5e Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 7 Jun 2012 02:44:13 +0100 Subject: Allow the thread watchdog to accept an alarm method that is invoked if the timeout is breached. This alarm can then invoke this to log extra information. This is used in LLUDPServer to show which client was being processed when incoming and outgoing udp watchdog alarms are triggered. --- .../HttpServer/PollServiceRequestManager.cs | 2 + OpenSim/Framework/Watchdog.cs | 40 +++++++------ OpenSim/Region/Application/OpenSim.cs | 10 +++- .../Region/ClientStack/Linden/UDP/LLUDPServer.cs | 69 ++++++++++++++++++++-- 4 files changed, 96 insertions(+), 25 deletions(-) (limited to 'OpenSim') diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs index 0062d4e..f96fd1f 100644 --- a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs +++ b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs @@ -66,6 +66,7 @@ namespace OpenSim.Framework.Servers.HttpServer ThreadPriority.Normal, false, true, + null, int.MaxValue); } @@ -75,6 +76,7 @@ namespace OpenSim.Framework.Servers.HttpServer ThreadPriority.Normal, false, true, + null, 1000 * 60 * 10); } diff --git a/OpenSim/Framework/Watchdog.cs b/OpenSim/Framework/Watchdog.cs index e93e50e..7552cd1 100644 --- a/OpenSim/Framework/Watchdog.cs +++ b/OpenSim/Framework/Watchdog.cs @@ -42,7 +42,7 @@ namespace OpenSim.Framework const double WATCHDOG_INTERVAL_MS = 2500.0d; /// Maximum timeout in milliseconds before a thread is considered dead - const int WATCHDOG_TIMEOUT_MS = 5000; + public const int WATCHDOG_TIMEOUT_MS = 5000; [System.Diagnostics.DebuggerDisplay("{Thread.Name}")] public class ThreadWatchdogInfo @@ -58,7 +58,7 @@ namespace OpenSim.Framework public int FirstTick { get; private set; } /// - /// First time this heartbeat update was invoked + /// Last time this heartbeat update was invoked /// public int LastTick { get; set; } @@ -77,6 +77,11 @@ namespace OpenSim.Framework /// public bool AlarmIfTimeout { get; set; } + /// + /// Method execute if alarm goes off. If null then no alarm method is fired. + /// + public Func AlarmMethod { get; set; } + public ThreadWatchdogInfo(Thread thread, int timeout) { Thread = thread; @@ -87,16 +92,10 @@ namespace OpenSim.Framework } /// - /// This event is called whenever a tracked thread is stopped or - /// has not called UpdateThread() in time - /// - /// The thread that has been identified as dead - /// The last time this thread called UpdateThread() - public delegate void WatchdogTimeout(Thread thread, int lastTick); - - /// This event is called whenever a tracked thread is - /// stopped or has not called UpdateThread() in time - public static event WatchdogTimeout OnWatchdogTimeout; + /// This event is called whenever a tracked thread is + /// stopped or has not called UpdateThread() in time< + /// /summary> + public static event Action OnWatchdogTimeout; private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); private static Dictionary m_threads; @@ -123,7 +122,7 @@ namespace OpenSim.Framework public static Thread StartThread( ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout) { - return StartThread(start, name, priority, isBackground, alarmIfTimeout, WATCHDOG_TIMEOUT_MS); + return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, WATCHDOG_TIMEOUT_MS); } /// @@ -135,17 +134,24 @@ namespace OpenSim.Framework /// True to run this thread as a background /// thread, otherwise false /// Trigger an alarm function is we have timed out + /// + /// Alarm method to call if alarmIfTimeout is true and there is a timeout. + /// Normally, this will just return some useful debugging information. + /// /// Number of milliseconds to wait until we issue a warning about timeout. /// The newly created Thread object public static Thread StartThread( - ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, int timeout) + ThreadStart start, string name, ThreadPriority priority, bool isBackground, + bool alarmIfTimeout, Func alarmMethod, int timeout) { Thread thread = new Thread(start); thread.Name = name; thread.Priority = priority; thread.IsBackground = isBackground; - ThreadWatchdogInfo twi = new ThreadWatchdogInfo(thread, timeout) { AlarmIfTimeout = alarmIfTimeout }; + ThreadWatchdogInfo twi + = new ThreadWatchdogInfo(thread, timeout) + { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod }; m_log.DebugFormat( "[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId); @@ -258,7 +264,7 @@ namespace OpenSim.Framework /// private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e) { - WatchdogTimeout callback = OnWatchdogTimeout; + Action callback = OnWatchdogTimeout; if (callback != null) { @@ -296,7 +302,7 @@ namespace OpenSim.Framework if (callbackInfos != null) foreach (ThreadWatchdogInfo callbackInfo in callbackInfos) - callback(callbackInfo.Thread, callbackInfo.LastTick); + callback(callbackInfo); } m_watchdogTimer.Start(); diff --git a/OpenSim/Region/Application/OpenSim.cs b/OpenSim/Region/Application/OpenSim.cs index 76ac827..caba236 100644 --- a/OpenSim/Region/Application/OpenSim.cs +++ b/OpenSim/Region/Application/OpenSim.cs @@ -438,12 +438,16 @@ namespace OpenSim } } - private void WatchdogTimeoutHandler(System.Threading.Thread thread, int lastTick) + private void WatchdogTimeoutHandler(Watchdog.ThreadWatchdogInfo twi) { int now = Environment.TickCount & Int32.MaxValue; - m_log.ErrorFormat("[WATCHDOG]: Timeout detected for thread \"{0}\". ThreadState={1}. Last tick was {2}ms ago", - thread.Name, thread.ThreadState, now - lastTick); + m_log.ErrorFormat( + "[WATCHDOG]: Timeout detected for thread \"{0}\". ThreadState={1}. Last tick was {2}ms ago. {3}", + twi.Thread.Name, + twi.Thread.ThreadState, + now - twi.LastTick, + twi.AlarmMethod != null ? string.Format("Data: {0}", twi.AlarmMethod()) : ""); } #region Console Commands diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 32ba590..e37adb8 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -163,6 +163,16 @@ namespace OpenSim.Region.ClientStack.LindenUDP private int m_malformedCount = 0; // Guard against a spamming attack + /// + /// Record current outgoing client for monitoring purposes. + /// + private IClientAPI m_currentOutgoingClient; + + /// + /// Recording current incoming client for monitoring purposes. + /// + private IClientAPI m_currentIncomingClient; + public LLUDPServer(IPAddress listenIP, ref uint port, int proxyPortOffsetParm, bool allow_alternate_port, IConfigSource configSource, AgentCircuitManager circuitManager) : base(listenIP, (int)port) { @@ -244,19 +254,56 @@ namespace OpenSim.Region.ClientStack.LindenUDP if (m_scene == null) throw new InvalidOperationException("[LLUDPSERVER]: Cannot LLUDPServer.Start() without an IScene reference"); - m_log.Info("[LLUDPSERVER]: Starting the LLUDP server in " + (m_asyncPacketHandling ? "asynchronous" : "synchronous") + " mode"); + m_log.InfoFormat( + "[LLUDPSERVER]: Starting the LLUDP server in {0} mode", + m_asyncPacketHandling ? "asynchronous" : "synchronous"); base.Start(m_recvBufferSize, m_asyncPacketHandling); // Start the packet processing threads Watchdog.StartThread( - IncomingPacketHandler, "Incoming Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false, true); + IncomingPacketHandler, + string.Format("Incoming Packets ({0})", m_scene.RegionInfo.RegionName), + ThreadPriority.Normal, + false, + true, + GetWatchdogIncomingAlarmData, + Watchdog.WATCHDOG_TIMEOUT_MS); + Watchdog.StartThread( - OutgoingPacketHandler, "Outgoing Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false, true); + OutgoingPacketHandler, + string.Format("Outgoing Packets ({0})", m_scene.RegionInfo.RegionName), + ThreadPriority.Normal, + false, + true, + GetWatchdogOutgoingAlarmData, + Watchdog.WATCHDOG_TIMEOUT_MS); m_elapsedMSSinceLastStatReport = Environment.TickCount; } + /// + /// If the outgoing UDP thread times out, then return client that was being processed to help with debugging. + /// + /// + private string GetWatchdogIncomingAlarmData() + { + return string.Format( + "Client is {0}", + m_currentIncomingClient != null ? m_currentIncomingClient.Name : "none"); + } + + /// + /// If the outgoing UDP thread times out, then return client that was being processed to help with debugging. + /// + /// + private string GetWatchdogOutgoingAlarmData() + { + return string.Format( + "Client is {0}", + m_currentOutgoingClient != null ? m_currentOutgoingClient.Name : "none"); + } + public new void Stop() { m_log.Info("[LLUDPSERVER]: Shutting down the LLUDP server for " + m_scene.RegionInfo.RegionName); @@ -1173,6 +1220,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP // client. m_packetSent will be set to true if a packet is sent m_scene.ForEachClient(clientPacketHandler); + m_currentOutgoingClient = null; + // If nothing was sent, sleep for the minimum amount of time before a // token bucket could get more tokens if (!m_packetSent) @@ -1191,6 +1240,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP private void ClientOutgoingPacketHandler(IClientAPI client) { + m_currentOutgoingClient = client; + try { if (client is LLClientView) @@ -1216,8 +1267,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP } catch (Exception ex) { - m_log.Error("[LLUDPSERVER]: OutgoingPacketHandler iteration for " + client.Name + - " threw an exception: " + ex.Message, ex); + m_log.Error( + string.Format("[LLUDPSERVER]: OutgoingPacketHandler iteration for {0} threw ", client.Name), ex); } } @@ -1243,6 +1294,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP { nticks++; watch1.Start(); + m_currentOutgoingClient = client; + try { if (client is LLClientView) @@ -1344,6 +1397,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP // Make sure this client is still alive if (m_scene.TryGetClient(udpClient.AgentID, out client)) { + m_currentIncomingClient = client; + try { // Process this packet @@ -1361,6 +1416,10 @@ namespace OpenSim.Region.ClientStack.LindenUDP m_log.ErrorFormat("[LLUDPSERVER]: Client packet handler for {0} for packet {1} threw an exception", udpClient.AgentID, packet.Type); m_log.Error(e.Message, e); } + finally + { + m_currentIncomingClient = null; + } } else { -- cgit v1.1 From 7550b97e65fa873b42624648747cec3e12a8e689 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 7 Jun 2012 04:00:29 +0100 Subject: Log warning if we try to remove a UDP client that has already been removed. --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'OpenSim') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index e37adb8..09bb52c 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1114,6 +1114,12 @@ namespace OpenSim.Region.ClientStack.LindenUDP client.IsLoggingOut = true; client.Close(); } + else + { + m_log.WarnFormat( + "[LLUDPSERVER]: Tried to remove client with id {0} but not such client in {1}", + udpClient.AgentID, m_scene.RegionInfo.RegionName); + } } private void IncomingPacketHandler() -- cgit v1.1 From a1e857932a2019c3e93e1b708486f37e9bb86b26 Mon Sep 17 00:00:00 2001 From: BlueWall Date: Thu, 7 Jun 2012 10:15:40 -0400 Subject: Make change to fix Windows builds --- OpenSim/Framework/Servers/HttpServer/OSHttpRequest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim') diff --git a/OpenSim/Framework/Servers/HttpServer/OSHttpRequest.cs b/OpenSim/Framework/Servers/HttpServer/OSHttpRequest.cs index fc8daf3..3171759 100644 --- a/OpenSim/Framework/Servers/HttpServer/OSHttpRequest.cs +++ b/OpenSim/Framework/Servers/HttpServer/OSHttpRequest.cs @@ -107,7 +107,7 @@ namespace OpenSim.Framework.Servers.HttpServer public bool IsSecured { - get { return _context.Secured; } + get { return _context.IsSecured; } } public bool KeepAlive -- cgit v1.1 From de87e4871b6eb3852e8f7175f637adaf14a74ccf Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 7 Jun 2012 23:35:21 +0100 Subject: Don't send kill object messages to clients when a child agent is closed. --- OpenSim/Region/Framework/Scenes/Scene.cs | 39 +++++++++++++++++--------------- 1 file changed, 21 insertions(+), 18 deletions(-) (limited to 'OpenSim') diff --git a/OpenSim/Region/Framework/Scenes/Scene.cs b/OpenSim/Region/Framework/Scenes/Scene.cs index ae35cb9..f291022 100644 --- a/OpenSim/Region/Framework/Scenes/Scene.cs +++ b/OpenSim/Region/Framework/Scenes/Scene.cs @@ -3274,26 +3274,29 @@ namespace OpenSim.Region.Framework.Scenes { m_eventManager.TriggerOnRemovePresence(agentID); - if (AttachmentsModule != null && !isChildAgent && avatar.PresenceType != PresenceType.Npc) + if (!isChildAgent) { - IUserManagement uMan = RequestModuleInterface(); - // Don't save attachments for HG visitors, it - // messes up their inventory. When a HG visitor logs - // out on a foreign grid, their attachments will be - // reloaded in the state they were in when they left - // the home grid. This is best anyway as the visited - // grid may use an incompatible script engine. - if (uMan == null || uMan.IsLocalGridUser(avatar.UUID)) - AttachmentsModule.SaveChangedAttachments(avatar, false); - } - - ForEachClient( - delegate(IClientAPI client) + if (AttachmentsModule != null && avatar.PresenceType != PresenceType.Npc) { - //We can safely ignore null reference exceptions. It means the avatar is dead and cleaned up anyway - try { client.SendKillObject(avatar.RegionHandle, new List { avatar.LocalId }); } - catch (NullReferenceException) { } - }); + IUserManagement uMan = RequestModuleInterface(); + // Don't save attachments for HG visitors, it + // messes up their inventory. When a HG visitor logs + // out on a foreign grid, their attachments will be + // reloaded in the state they were in when they left + // the home grid. This is best anyway as the visited + // grid may use an incompatible script engine. + if (uMan == null || uMan.IsLocalGridUser(avatar.UUID)) + AttachmentsModule.SaveChangedAttachments(avatar, false); + } + + ForEachClient( + delegate(IClientAPI client) + { + //We can safely ignore null reference exceptions. It means the avatar is dead and cleaned up anyway + try { client.SendKillObject(avatar.RegionHandle, new List { avatar.LocalId }); } + catch (NullReferenceException) { } + }); + } IAgentAssetTransactions agentTransactions = this.RequestModuleInterface(); if (agentTransactions != null) -- cgit v1.1 From 0c5fefacb415fb916ffece839c95c155ec718254 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 7 Jun 2012 23:51:04 +0100 Subject: Record the fact that child agents can have asset transactions. Also change code to grab the agent asset transaction module once. --- .../AssetTransaction/AssetTransactionModule.cs | 15 ++++++----- .../InventoryAccess/InventoryAccessModule.cs | 2 +- OpenSim/Region/Framework/Scenes/Scene.Inventory.cs | 29 +++++++++------------- OpenSim/Region/Framework/Scenes/Scene.cs | 12 ++++----- 4 files changed, 26 insertions(+), 32 deletions(-) (limited to 'OpenSim') diff --git a/OpenSim/Region/CoreModules/Agent/AssetTransaction/AssetTransactionModule.cs b/OpenSim/Region/CoreModules/Agent/AssetTransaction/AssetTransactionModule.cs index 874693e..7081989 100644 --- a/OpenSim/Region/CoreModules/Agent/AssetTransaction/AssetTransactionModule.cs +++ b/OpenSim/Region/CoreModules/Agent/AssetTransaction/AssetTransactionModule.cs @@ -42,8 +42,7 @@ namespace OpenSim.Region.CoreModules.Agent.AssetTransaction public class AssetTransactionModule : INonSharedRegionModule, IAgentAssetTransactions { -// private static readonly ILog m_log = LogManager.GetLogger( -// MethodBase.GetCurrentMethod().DeclaringType); + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); protected Scene m_Scene; private bool m_dumpAssetsToFile = false; @@ -209,15 +208,15 @@ namespace OpenSim.Region.CoreModules.Agent.AssetTransaction /// and comes through this method. /// /// + /// /// /// - public void HandleTaskItemUpdateFromTransaction(IClientAPI remoteClient, - SceneObjectPart part, UUID transactionID, - TaskInventoryItem item) + public void HandleTaskItemUpdateFromTransaction( + IClientAPI remoteClient, SceneObjectPart part, UUID transactionID, TaskInventoryItem item) { -// m_log.DebugFormat( -// "[TRANSACTIONS MANAGER] Called HandleTaskItemUpdateFromTransaction with item {0}", -// item.Name); + m_log.DebugFormat( + "[TRANSACTIONS MANAGER] Called HandleTaskItemUpdateFromTransaction with item {0} in {1} for {2} in {3}", + item.Name, part.Name, remoteClient.Name, m_Scene.RegionInfo.RegionName); AgentAssetTransactions transactions = GetUserTransactions(remoteClient.AgentId); diff --git a/OpenSim/Region/CoreModules/Framework/InventoryAccess/InventoryAccessModule.cs b/OpenSim/Region/CoreModules/Framework/InventoryAccess/InventoryAccessModule.cs index 69767c1..7d51eed 100644 --- a/OpenSim/Region/CoreModules/Framework/InventoryAccess/InventoryAccessModule.cs +++ b/OpenSim/Region/CoreModules/Framework/InventoryAccess/InventoryAccessModule.cs @@ -216,7 +216,7 @@ namespace OpenSim.Region.CoreModules.Framework.InventoryAccess } else { - IAgentAssetTransactions agentTransactions = m_Scene.RequestModuleInterface(); + IAgentAssetTransactions agentTransactions = m_Scene.AgentTransactionsModule; if (agentTransactions != null) { agentTransactions.HandleItemCreationFromTransaction( diff --git a/OpenSim/Region/Framework/Scenes/Scene.Inventory.cs b/OpenSim/Region/Framework/Scenes/Scene.Inventory.cs index 79c9309..b59fd05 100644 --- a/OpenSim/Region/Framework/Scenes/Scene.Inventory.cs +++ b/OpenSim/Region/Framework/Scenes/Scene.Inventory.cs @@ -440,10 +440,9 @@ namespace OpenSim.Region.Framework.Scenes } else { - IAgentAssetTransactions agentTransactions = this.RequestModuleInterface(); - if (agentTransactions != null) + if (AgentTransactionsModule != null) { - agentTransactions.HandleItemUpdateFromTransaction(remoteClient, transactionID, item); + AgentTransactionsModule.HandleItemUpdateFromTransaction(remoteClient, transactionID, item); } } } @@ -1532,21 +1531,17 @@ namespace OpenSim.Region.Framework.Scenes // Only look for an uploaded updated asset if we are passed a transaction ID. This is only the // case for updates uploded through UDP. Updates uploaded via a capability (e.g. a script update) // will not pass in a transaction ID in the update message. - if (transactionID != UUID.Zero) + if (transactionID != UUID.Zero && AgentTransactionsModule != null) { - IAgentAssetTransactions agentTransactions = this.RequestModuleInterface(); - if (agentTransactions != null) - { - agentTransactions.HandleTaskItemUpdateFromTransaction( - remoteClient, part, transactionID, currentItem); - - if ((InventoryType)itemInfo.InvType == InventoryType.Notecard) - remoteClient.SendAgentAlertMessage("Notecard saved", false); - else if ((InventoryType)itemInfo.InvType == InventoryType.LSL) - remoteClient.SendAgentAlertMessage("Script saved", false); - else - remoteClient.SendAgentAlertMessage("Item saved", false); - } + AgentTransactionsModule.HandleTaskItemUpdateFromTransaction( + remoteClient, part, transactionID, currentItem); + + if ((InventoryType)itemInfo.InvType == InventoryType.Notecard) + remoteClient.SendAgentAlertMessage("Notecard saved", false); + else if ((InventoryType)itemInfo.InvType == InventoryType.LSL) + remoteClient.SendAgentAlertMessage("Script saved", false); + else + remoteClient.SendAgentAlertMessage("Item saved", false); } // Base ALWAYS has move diff --git a/OpenSim/Region/Framework/Scenes/Scene.cs b/OpenSim/Region/Framework/Scenes/Scene.cs index f291022..939c8fa 100644 --- a/OpenSim/Region/Framework/Scenes/Scene.cs +++ b/OpenSim/Region/Framework/Scenes/Scene.cs @@ -499,6 +499,7 @@ namespace OpenSim.Region.Framework.Scenes public IAttachmentsModule AttachmentsModule { get; set; } public IEntityTransferModule EntityTransferModule { get; private set; } + public IAgentAssetTransactions AgentTransactionsModule { get; private set; } public IAvatarFactoryModule AvatarFactory { @@ -1241,6 +1242,7 @@ namespace OpenSim.Region.Framework.Scenes m_capsModule = RequestModuleInterface(); EntityTransferModule = RequestModuleInterface(); m_groupsModule = RequestModuleInterface(); + AgentTransactionsModule = RequestModuleInterface(); } #endregion @@ -3297,12 +3299,10 @@ namespace OpenSim.Region.Framework.Scenes catch (NullReferenceException) { } }); } - - IAgentAssetTransactions agentTransactions = this.RequestModuleInterface(); - if (agentTransactions != null) - { - agentTransactions.RemoveAgentAssetTransactions(agentID); - } + + // It's possible for child agents to have transactions if changes are being made cross-border. + if (AgentTransactionsModule != null) + AgentTransactionsModule.RemoveAgentAssetTransactions(agentID); } finally { -- cgit v1.1