9 files changed, 149 insertions, 75 deletions

diff --git a/OpenSim/Framework/Servers/HttpServer/OSHttpRequest.cs b/OpenSim/Framework/Servers/HttpServer/OSHttpRequest.cs
index fc8daf3..3171759 100644
--- a/OpenSim/Framework/Servers/HttpServer/OSHttpRequest.cs
+++ b/OpenSim/Framework/Servers/HttpServer/OSHttpRequest.cs
@@ -107,7 +107,7 @@ namespace OpenSim.Framework.Servers.HttpServer
 
         public bool IsSecured
         {
-            get { return _context.Secured; }
+            get { return _context.IsSecured; }
         }
 
         public bool KeepAlive
diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs
index 0062d4e..f96fd1f 100644
--- a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs
+++ b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs
@@ -66,6 +66,7 @@ namespace OpenSim.Framework.Servers.HttpServer
                         ThreadPriority.Normal,
                         false,
                         true,
+                        null,
                         int.MaxValue);
             }
 
@@ -75,6 +76,7 @@ namespace OpenSim.Framework.Servers.HttpServer
                 ThreadPriority.Normal,
                 false,
                 true,
+                null,
                 1000 * 60 * 10);
         }
 
diff --git a/OpenSim/Framework/Watchdog.cs b/OpenSim/Framework/Watchdog.cs
index 881b6aa..68bf477 100644
--- a/OpenSim/Framework/Watchdog.cs
+++ b/OpenSim/Framework/Watchdog.cs
@@ -42,7 +42,7 @@ namespace OpenSim.Framework
         const double WATCHDOG_INTERVAL_MS = 2500.0d;
 
         /// <summary>Maximum timeout in milliseconds before a thread is considered dead</summary>
-        const int WATCHDOG_TIMEOUT_MS = 5000;
+        public const int WATCHDOG_TIMEOUT_MS = 5000;
 
         [System.Diagnostics.DebuggerDisplay("{Thread.Name}")]
         public class ThreadWatchdogInfo
@@ -58,7 +58,7 @@ namespace OpenSim.Framework
             public int FirstTick { get; private set; }
 
             /// <summary>
-            /// First time this heartbeat update was invoked
+            /// Last time this heartbeat update was invoked
             /// </summary>
             public int LastTick { get; set; }
 
@@ -77,6 +77,11 @@ namespace OpenSim.Framework
             /// </summary>
             public bool AlarmIfTimeout { get; set; }
 
+            /// <summary>
+            /// Method execute if alarm goes off.  If null then no alarm method is fired.
+            /// </summary>
+            public Func<string> AlarmMethod { get; set; }
+
             public ThreadWatchdogInfo(Thread thread, int timeout)
             {
                 Thread = thread;
@@ -87,16 +92,10 @@ namespace OpenSim.Framework
         }
 
         /// <summary>
-        /// This event is called whenever a tracked thread is stopped or
-        /// has not called UpdateThread() in time
-        /// </summary>
-        /// <param name="thread">The thread that has been identified as dead</param>
-        /// <param name="lastTick">The last time this thread called UpdateThread()</param>
-        public delegate void WatchdogTimeout(Thread thread, int lastTick);
-
-        /// <summary>This event is called whenever a tracked thread is
-        /// stopped or has not called UpdateThread() in time</summary>
-        public static event WatchdogTimeout OnWatchdogTimeout;
+        /// This event is called whenever a tracked thread is
+        /// stopped or has not called UpdateThread() in time<
+        /// /summary>
+        public static event Action<ThreadWatchdogInfo> OnWatchdogTimeout;
 
         private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
         private static Dictionary<int, ThreadWatchdogInfo> m_threads;
@@ -123,7 +122,7 @@ namespace OpenSim.Framework
         public static Thread StartThread(
             ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout)
         {
-            return StartThread(start, name, priority, isBackground, alarmIfTimeout, WATCHDOG_TIMEOUT_MS);
+            return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, WATCHDOG_TIMEOUT_MS);
         }
 
         /// <summary>
@@ -135,17 +134,24 @@ namespace OpenSim.Framework
         /// <param name="isBackground">True to run this thread as a background
         /// thread, otherwise false</param>
         /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
+        /// <param name="alarmMethod">
+        /// Alarm method to call if alarmIfTimeout is true and there is a timeout.
+        /// Normally, this will just return some useful debugging information.
+        /// </param>
         /// <param name="timeout">Number of milliseconds to wait until we issue a warning about timeout.</param>
         /// <returns>The newly created Thread object</returns>
         public static Thread StartThread(
-            ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, int timeout)
+            ThreadStart start, string name, ThreadPriority priority, bool isBackground,
+            bool alarmIfTimeout, Func<string> alarmMethod, int timeout)
         {
             Thread thread = new Thread(start);
             thread.Name = name;
             thread.Priority = priority;
             thread.IsBackground = isBackground;
             
-            ThreadWatchdogInfo twi = new ThreadWatchdogInfo(thread, timeout) { AlarmIfTimeout = alarmIfTimeout };
+            ThreadWatchdogInfo twi
+                = new ThreadWatchdogInfo(thread, timeout)
+                    { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod };
 
             m_log.DebugFormat(
                 "[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
@@ -258,7 +264,7 @@ namespace OpenSim.Framework
         /// <param name="e"></param>
         private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e)
         {
-            WatchdogTimeout callback = OnWatchdogTimeout;
+            Action<ThreadWatchdogInfo> callback = OnWatchdogTimeout;
 
             if (callback != null)
             {
@@ -296,7 +302,7 @@ namespace OpenSim.Framework
 
                 if (callbackInfos != null)
                     foreach (ThreadWatchdogInfo callbackInfo in callbackInfos)
-                        callback(callbackInfo.Thread, callbackInfo.LastTick);
+                        callback(callbackInfo);
             }
 
             m_watchdogTimer.Start();
diff --git a/OpenSim/Region/Application/OpenSim.cs b/OpenSim/Region/Application/OpenSim.cs
index 28f9481..ac0e3e1 100644
--- a/OpenSim/Region/Application/OpenSim.cs
+++ b/OpenSim/Region/Application/OpenSim.cs
@@ -438,12 +438,16 @@ namespace OpenSim
             }
         }
 
-        private void WatchdogTimeoutHandler(System.Threading.Thread thread, int lastTick)
+        private void WatchdogTimeoutHandler(Watchdog.ThreadWatchdogInfo twi)
         {
             int now = Environment.TickCount & Int32.MaxValue;
 
-            m_log.ErrorFormat("[WATCHDOG]: Timeout detected for thread \"{0}\". ThreadState={1}. Last tick was {2}ms ago",
-                thread.Name, thread.ThreadState, now - lastTick);
+            m_log.ErrorFormat(
+                "[WATCHDOG]: Timeout detected for thread \"{0}\". ThreadState={1}. Last tick was {2}ms ago.  {3}",
+                twi.Thread.Name,
+                twi.Thread.ThreadState,
+                now - twi.LastTick,
+                twi.AlarmMethod != null ? string.Format("Data: {0}", twi.AlarmMethod()) : "");
         }
 
         #region Console Commands
diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs
index d310d65..3bdde3b 100644
--- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs
+++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs
@@ -163,6 +163,16 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
         private int m_malformedCount = 0; // Guard against a spamming attack
 
+        /// <summary>
+        /// Record current outgoing client for monitoring purposes.
+        /// </summary>
+        private IClientAPI m_currentOutgoingClient;
+
+        /// <summary>
+        /// Recording current incoming client for monitoring purposes.
+        /// </summary>
+        private IClientAPI m_currentIncomingClient;
+
         public LLUDPServer(IPAddress listenIP, ref uint port, int proxyPortOffsetParm, bool allow_alternate_port, IConfigSource configSource, AgentCircuitManager circuitManager)
             : base(listenIP, (int)port)
         {
@@ -244,19 +254,56 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             if (m_scene == null)
                 throw new InvalidOperationException("[LLUDPSERVER]: Cannot LLUDPServer.Start() without an IScene reference");
 
-            m_log.Info("[LLUDPSERVER]: Starting the LLUDP server in " + (m_asyncPacketHandling ? "asynchronous" : "synchronous") + " mode");
+            m_log.InfoFormat(
+                "[LLUDPSERVER]: Starting the LLUDP server in {0} mode",
+                m_asyncPacketHandling ? "asynchronous" : "synchronous");
 
             base.Start(m_recvBufferSize, m_asyncPacketHandling);
 
             // Start the packet processing threads
             Watchdog.StartThread(
-                IncomingPacketHandler, "Incoming Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false, true);
+                IncomingPacketHandler,
+                string.Format("Incoming Packets ({0})", m_scene.RegionInfo.RegionName),
+                ThreadPriority.Normal,
+                false,
+                true,
+                GetWatchdogIncomingAlarmData,
+                Watchdog.WATCHDOG_TIMEOUT_MS);
+
             Watchdog.StartThread(
-                OutgoingPacketHandler, "Outgoing Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false, true);
+                OutgoingPacketHandler,
+                string.Format("Outgoing Packets ({0})", m_scene.RegionInfo.RegionName),
+                ThreadPriority.Normal,
+                false,
+                true,
+                GetWatchdogOutgoingAlarmData,
+                Watchdog.WATCHDOG_TIMEOUT_MS);
 
             m_elapsedMSSinceLastStatReport = Environment.TickCount;
         }
 
+        /// <summary>
+        /// If the outgoing UDP thread times out, then return client that was being processed to help with debugging.
+        /// </summary>
+        /// <returns></returns>
+        private string GetWatchdogIncomingAlarmData()
+        {
+            return string.Format(
+                "Client is {0}",
+                m_currentIncomingClient != null ? m_currentIncomingClient.Name : "none");
+        }
+
+        /// <summary>
+        /// If the outgoing UDP thread times out, then return client that was being processed to help with debugging.
+        /// </summary>
+        /// <returns></returns>
+        private string GetWatchdogOutgoingAlarmData()
+        {
+            return string.Format(
+                "Client is {0}",
+                m_currentOutgoingClient != null ? m_currentOutgoingClient.Name : "none");
+        }
+
         public new void Stop()
         {
             m_log.Info("[LLUDPSERVER]: Shutting down the LLUDP server for " + m_scene.RegionInfo.RegionName);
@@ -1067,6 +1114,12 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                 client.IsLoggingOut = true;
                 client.Close(false);
             }
+            else
+            {
+                m_log.WarnFormat(
+                    "[LLUDPSERVER]: Tried to remove client with id {0} but not such client in {1}",
+                    udpClient.AgentID, m_scene.RegionInfo.RegionName);
+            }
         }
 
         private void IncomingPacketHandler()
@@ -1175,6 +1228,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     // client. m_packetSent will be set to true if a packet is sent
                     m_scene.ForEachClient(clientPacketHandler);
 
+                    m_currentOutgoingClient = null;
+
                     // If nothing was sent, sleep for the minimum amount of time before a
                     // token bucket could get more tokens
                     if (!m_packetSent)
@@ -1193,6 +1248,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
         private void ClientOutgoingPacketHandler(IClientAPI client)
         {
+            m_currentOutgoingClient = client;
+
             try
             {
                 if (client is LLClientView)
@@ -1218,8 +1275,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             }
             catch (Exception ex)
             {
-                m_log.Error("[LLUDPSERVER]: OutgoingPacketHandler iteration for " + client.Name +
-                    " threw an exception: " + ex.Message, ex);
+                m_log.Error(
+                    string.Format("[LLUDPSERVER]: OutgoingPacketHandler iteration for {0} threw ", client.Name), ex);
             }
         }
 
@@ -1245,6 +1302,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         {
             nticks++;
             watch1.Start();
+            m_currentOutgoingClient = client;
+
             try
             {
                 if (client is LLClientView)
@@ -1346,6 +1405,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             // Make sure this client is still alive
             if (m_scene.TryGetClient(udpClient.AgentID, out client))
             {
+                m_currentIncomingClient = client;
+
                 try
                 {
                     // Process this packet
@@ -1363,6 +1424,10 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     m_log.ErrorFormat("[LLUDPSERVER]: Client packet handler for {0} for packet {1} threw an exception", udpClient.AgentID, packet.Type);
                     m_log.Error(e.Message, e);
                 }
+                finally
+                {
+                    m_currentIncomingClient = null;
+                }
             }
             else
             {
diff --git a/OpenSim/Region/CoreModules/Agent/AssetTransaction/AssetTransactionModule.cs b/OpenSim/Region/CoreModules/Agent/AssetTransaction/AssetTransactionModule.cs
index 874693e..7081989 100644
--- a/OpenSim/Region/CoreModules/Agent/AssetTransaction/AssetTransactionModule.cs
+++ b/OpenSim/Region/CoreModules/Agent/AssetTransaction/AssetTransactionModule.cs
@@ -42,8 +42,7 @@ namespace OpenSim.Region.CoreModules.Agent.AssetTransaction
     public class AssetTransactionModule : INonSharedRegionModule,
             IAgentAssetTransactions
     {
-//        private static readonly ILog m_log = LogManager.GetLogger(
-//                MethodBase.GetCurrentMethod().DeclaringType);
+        private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
         
         protected Scene m_Scene;
         private bool m_dumpAssetsToFile = false;
@@ -209,15 +208,15 @@ namespace OpenSim.Region.CoreModules.Agent.AssetTransaction
         /// and comes through this method.
         /// </summary>
         /// <param name="remoteClient"></param>
+        /// <param name="part"></param>
         /// <param name="transactionID"></param>
         /// <param name="item"></param>
-        public void HandleTaskItemUpdateFromTransaction(IClientAPI remoteClient,
-                SceneObjectPart part, UUID transactionID,
-                TaskInventoryItem item)
+        public void HandleTaskItemUpdateFromTransaction(
+            IClientAPI remoteClient, SceneObjectPart part, UUID transactionID, TaskInventoryItem item)
         {
-//            m_log.DebugFormat(
-//                "[TRANSACTIONS MANAGER] Called HandleTaskItemUpdateFromTransaction with item {0}",
-//                item.Name);
+            m_log.DebugFormat(
+                "[TRANSACTIONS MANAGER] Called HandleTaskItemUpdateFromTransaction with item {0} in {1} for {2} in {3}",
+                item.Name, part.Name, remoteClient.Name, m_Scene.RegionInfo.RegionName);
 
             AgentAssetTransactions transactions =
                     GetUserTransactions(remoteClient.AgentId);
diff --git a/OpenSim/Region/CoreModules/Framework/InventoryAccess/InventoryAccessModule.cs b/OpenSim/Region/CoreModules/Framework/InventoryAccess/InventoryAccessModule.cs
index 31820e0..d30c2e2 100644
--- a/OpenSim/Region/CoreModules/Framework/InventoryAccess/InventoryAccessModule.cs
+++ b/OpenSim/Region/CoreModules/Framework/InventoryAccess/InventoryAccessModule.cs
@@ -216,7 +216,7 @@ namespace OpenSim.Region.CoreModules.Framework.InventoryAccess
             }
             else
             {
-                IAgentAssetTransactions agentTransactions = m_Scene.RequestModuleInterface<IAgentAssetTransactions>();
+                IAgentAssetTransactions agentTransactions = m_Scene.AgentTransactionsModule;
                 if (agentTransactions != null)
                 {
                     agentTransactions.HandleItemCreationFromTransaction(
diff --git a/OpenSim/Region/Framework/Scenes/Scene.Inventory.cs b/OpenSim/Region/Framework/Scenes/Scene.Inventory.cs
index 270b01b..e1281a6 100644
--- a/OpenSim/Region/Framework/Scenes/Scene.Inventory.cs
+++ b/OpenSim/Region/Framework/Scenes/Scene.Inventory.cs
@@ -435,10 +435,9 @@ namespace OpenSim.Region.Framework.Scenes
                 }
                 else
                 {
-                    IAgentAssetTransactions agentTransactions = this.RequestModuleInterface<IAgentAssetTransactions>();
-                    if (agentTransactions != null)
+                    if (AgentTransactionsModule != null)
                     {
-                        agentTransactions.HandleItemUpdateFromTransaction(remoteClient, transactionID, item);
+                        AgentTransactionsModule.HandleItemUpdateFromTransaction(remoteClient, transactionID, item);
                     }
                 }
             }
@@ -1566,21 +1565,17 @@ namespace OpenSim.Region.Framework.Scenes
                     // Only look for an uploaded updated asset if we are passed a transaction ID.  This is only the
                     // case for updates uploded through UDP.  Updates uploaded via a capability (e.g. a script update)
                     // will not pass in a transaction ID in the update message.
-                    if (transactionID != UUID.Zero)
+                    if (transactionID != UUID.Zero && AgentTransactionsModule != null)
                     {
-                        IAgentAssetTransactions agentTransactions = this.RequestModuleInterface<IAgentAssetTransactions>();
-                        if (agentTransactions != null)
-                        {
-                            agentTransactions.HandleTaskItemUpdateFromTransaction(
-                                remoteClient, part, transactionID, currentItem);
-    
-//                            if ((InventoryType)itemInfo.InvType == InventoryType.Notecard)
-//                                remoteClient.SendAgentAlertMessage("Notecard saved", false);
-//                            else if ((InventoryType)itemInfo.InvType == InventoryType.LSL)
-//                                remoteClient.SendAgentAlertMessage("Script saved", false);
-//                            else
-//                                remoteClient.SendAgentAlertMessage("Item saved", false);
-                        }
+                        AgentTransactionsModule.HandleTaskItemUpdateFromTransaction(
+                            remoteClient, part, transactionID, currentItem);
+
+//                        if ((InventoryType)itemInfo.InvType == InventoryType.Notecard)
+//                            remoteClient.SendAgentAlertMessage("Notecard saved", false);
+//                        else if ((InventoryType)itemInfo.InvType == InventoryType.LSL)
+//                            remoteClient.SendAgentAlertMessage("Script saved", false);
+//                        else
+//                            remoteClient.SendAgentAlertMessage("Item saved", false);
                     }
 
                     // Base ALWAYS has move
diff --git a/OpenSim/Region/Framework/Scenes/Scene.cs b/OpenSim/Region/Framework/Scenes/Scene.cs
index 9c80d3e..78fbd3b 100644
--- a/OpenSim/Region/Framework/Scenes/Scene.cs
+++ b/OpenSim/Region/Framework/Scenes/Scene.cs
@@ -521,6 +521,7 @@ namespace OpenSim.Region.Framework.Scenes
 
         public IAttachmentsModule AttachmentsModule { get; set; }
         public IEntityTransferModule EntityTransferModule { get; private set; }
+        public IAgentAssetTransactions AgentTransactionsModule { get; private set; }
 
         public IAvatarFactoryModule AvatarFactory
         {
@@ -1289,6 +1290,7 @@ namespace OpenSim.Region.Framework.Scenes
             m_capsModule = RequestModuleInterface<ICapabilitiesModule>();
             EntityTransferModule = RequestModuleInterface<IEntityTransferModule>();
             m_groupsModule = RequestModuleInterface<IGroupsModule>();
+            AgentTransactionsModule = RequestModuleInterface<IAgentAssetTransactions>();
         }
 
         #endregion
@@ -3428,32 +3430,33 @@ namespace OpenSim.Region.Framework.Scenes
                 {
                     m_eventManager.TriggerOnRemovePresence(agentID);
     
-                    if (AttachmentsModule != null && !isChildAgent && avatar.PresenceType != PresenceType.Npc)
+                    if (!isChildAgent)
                     {
-                        IUserManagement uMan = RequestModuleInterface<IUserManagement>(); 
-                        // Don't save attachments for HG visitors, it
-                        // messes up their inventory. When a HG visitor logs
-                        // out on a foreign grid, their attachments will be
-                        // reloaded in the state they were in when they left
-                        // the home grid. This is best anyway as the visited
-                        // grid may use an incompatible script engine.
-                        if (uMan == null || uMan.IsLocalGridUser(avatar.UUID))
-                            AttachmentsModule.SaveChangedAttachments(avatar, false);
-                    }
-    
-                    ForEachClient(
-                        delegate(IClientAPI client)
+                        if (AttachmentsModule != null && avatar.PresenceType != PresenceType.Npc)
                         {
-                            //We can safely ignore null reference exceptions.  It means the avatar is dead and cleaned up anyway
-                            try { client.SendKillObject(avatar.RegionHandle, new List<uint> { avatar.LocalId }); }
-                            catch (NullReferenceException) { }
-                        });
-    
-                    IAgentAssetTransactions agentTransactions = this.RequestModuleInterface<IAgentAssetTransactions>();
-                    if (agentTransactions != null)
-                    {
-                        agentTransactions.RemoveAgentAssetTransactions(agentID);
+                            IUserManagement uMan = RequestModuleInterface<IUserManagement>();
+                            // Don't save attachments for HG visitors, it
+                            // messes up their inventory. When a HG visitor logs
+                            // out on a foreign grid, their attachments will be
+                            // reloaded in the state they were in when they left
+                            // the home grid. This is best anyway as the visited
+                            // grid may use an incompatible script engine.
+                            if (uMan == null || uMan.IsLocalGridUser(avatar.UUID))
+                                AttachmentsModule.SaveChangedAttachments(avatar, false);
+                        }
+
+                        ForEachClient(
+                            delegate(IClientAPI client)
+                            {
+                                //We can safely ignore null reference exceptions.  It means the avatar is dead and cleaned up anyway
+                                try { client.SendKillObject(avatar.RegionHandle, new List<uint> { avatar.LocalId }); }
+                                catch (NullReferenceException) { }
+                            });
                     }
+
+                    // It's possible for child agents to have transactions if changes are being made cross-border.
+                    if (AgentTransactionsModule != null)
+                        AgentTransactionsModule.RemoveAgentAssetTransactions(agentID);
                 }
                 finally
                 {