Add an async inventory details sender to respond to FetchInventory packets.

If a user with a very large inventory right-clicks on their "My Inventory" folder, viewer 1 code will send a massive number of Fetchinventory requests.
Even though each is handled asynchronously via a pool thread, the sheer frequency of requests overwhelms the pool and freezes inbound packet handling.
This change makes the first Fetchinventory thread also handle subsequent requests, freeing up the other threads.
Further efficiencies could be made by handling all the items in a particular FetchInventory request together, rather than separately.

5 files changed, 165 insertions, 28 deletions

diff --git a/OpenSim/Framework/IClientAPI.cs b/OpenSim/Framework/IClientAPI.cs
index 659d42f..f6e2977 100644
--- a/OpenSim/Framework/IClientAPI.cs
+++ b/OpenSim/Framework/IClientAPI.cs
@@ -737,7 +737,7 @@ namespace OpenSim.Framework
         bool IsActive { get; set; }
 
         /// <value>
-        /// Determines whether the client is logging out or not.
+        /// Determines whether the client is or has been removed from a given scene
         /// </value>
         bool IsLoggingOut { get; set; }
         
diff --git a/OpenSim/Region/Framework/Scenes/AsyncInventorySender.cs b/OpenSim/Region/Framework/Scenes/AsyncInventorySender.cs
new file mode 100644
index 0000000..06cd14b
--- /dev/null
+++ b/OpenSim/Region/Framework/Scenes/AsyncInventorySender.cs
@@ -0,0 +1,156 @@
+/*
+ * Copyright (c) Contributors, http://opensimulator.org/
+ * See CONTRIBUTORS.TXT for a full list of copyright holders.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the OpenSimulator Project nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+using System;
+using System.Collections.Generic;
+using System.Reflection;
+using System.Threading;
+using log4net;
+using OpenMetaverse;
+using OpenSim.Framework;
+using OpenSim.Region.Framework.Interfaces;
+
+namespace OpenSim.Region.Framework.Scenes
+{
+    class FetchHolder
+    {
+        public IClientAPI Client { get; private set; }
+        public UUID ItemID { get; private set; }
+
+        public FetchHolder(IClientAPI client, UUID itemID)
+        {
+            Client = client;
+            ItemID = itemID;
+        }
+    }
+
+    /// <summary>
+    /// Send FetchInventoryReply information to clients asynchronously on a single thread rather than asynchronously via
+    /// multiple threads.
+    /// </summary>
+    /// <remarks>
+    /// If the main root inventory is right-clicked on a version 1 viewer for a user with a large inventory, a very
+    /// very large number of FetchInventory requests are sent to the simulator.  Each is handled on a separate thread
+    /// by the IClientAPI, but the sheer number of requests overwhelms the number of threads available and ends up
+    /// freezing the inbound packet handling.
+    ///
+    /// This class makes the first FetchInventory packet thread process the queue.  If new requests come
+    /// in while it is processing, then the subsequent threads just add the requests and leave it to the original
+    /// thread to process them.
+    ///
+    /// This might slow down outbound packets but these are limited by the IClientAPI outbound queues
+    /// anyway.
+    ///
+    /// It might be possible to ignore FetchInventory requests altogether, particularly as they are redundant wrt to
+    /// FetchInventoryDescendents requests, but this would require more investigation.
+    /// </remarks>
+    public class AsyncInventorySender
+    {
+        private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
+
+        protected Scene m_scene;
+
+        /// <summary>
+        /// Queues fetch requests
+        /// </summary>
+        Queue<FetchHolder> m_fetchHolder = new Queue<FetchHolder>();
+
+        /// <summary>
+        /// Signal whether a queue is currently being processed or not.
+        /// </summary>
+        protected volatile bool m_processing;
+
+        public AsyncInventorySender(Scene scene)
+        {
+            m_processing = false;
+            m_scene = scene;
+        }
+
+        /// <summary>
+        /// Handle a fetch inventory request from the client
+        /// </summary>
+        /// <param name="remoteClient"></param>
+        /// <param name="itemID"></param>
+        /// <param name="ownerID"></param>
+        public void HandleFetchInventory(IClientAPI remoteClient, UUID itemID, UUID ownerID)
+        {
+            lock (m_fetchHolder)
+            {
+//                m_log.DebugFormat(
+//                    "[ASYNC INVENTORY SENDER]: Putting request from {0} for {1} on queue", remoteClient.Name, itemID);
+
+                m_fetchHolder.Enqueue(new FetchHolder(remoteClient, itemID));
+            }
+
+            if (!m_processing)
+            {
+                m_processing = true;
+                ProcessQueue();
+            }
+        }
+
+        /// <summary>
+        /// Process the queue of fetches
+        /// </summary>
+        protected void ProcessQueue()
+        {
+            FetchHolder fh = null;
+
+            while (true)
+            {
+                lock (m_fetchHolder)
+                {
+//                    m_log.DebugFormat("[ASYNC INVENTORY SENDER]: {0} items left to process", m_fetchHolder.Count);
+
+                    if (m_fetchHolder.Count == 0)
+                    {
+                        m_processing = false;
+                        return;
+                    }
+                    else
+                    {
+                        fh = m_fetchHolder.Dequeue();
+                    }
+                }
+
+                if (fh.Client.IsLoggingOut)
+                    continue;
+
+//                m_log.DebugFormat(
+//                    "[ASYNC INVENTORY SENDER]: Handling request from {0} for {1} on queue", fh.Client.Name, fh.ItemID);
+
+                InventoryItemBase item = new InventoryItemBase(fh.ItemID, fh.Client.AgentId);
+                item = m_scene.InventoryService.GetItem(item);
+    
+                if (item != null)
+                    fh.Client.SendInventoryItemDetails(item.Owner, item);
+    
+                 // TODO: Possibly log any failure
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/OpenSim/Region/Framework/Scenes/Scene.Inventory.cs b/OpenSim/Region/Framework/Scenes/Scene.Inventory.cs
index 13085e3..30421d4 100644
--- a/OpenSim/Region/Framework/Scenes/Scene.Inventory.cs
+++ b/OpenSim/Region/Framework/Scenes/Scene.Inventory.cs
@@ -52,6 +52,11 @@ namespace OpenSim.Region.Framework.Scenes
         protected AsyncSceneObjectGroupDeleter m_asyncSceneObjectDeleter;
 
         /// <summary>
+        /// Allows inventory details to be sent to clients asynchronously
+        /// </summary>
+        protected AsyncInventorySender m_asyncInventorySender;
+
+        /// <summary>
         /// Start all the scripts in the scene which should be started.
         /// </summary>
         public void CreateScriptInstances()
diff --git a/OpenSim/Region/Framework/Scenes/Scene.PacketHandlers.cs b/OpenSim/Region/Framework/Scenes/Scene.PacketHandlers.cs
index e2d7208..44472b2 100644
--- a/OpenSim/Region/Framework/Scenes/Scene.PacketHandlers.cs
+++ b/OpenSim/Region/Framework/Scenes/Scene.PacketHandlers.cs
@@ -461,31 +461,6 @@ namespace OpenSim.Region.Framework.Scenes
                 }
             );
         }
-
-
-        /// <summary>
-        /// Handle a fetch inventory request from the client
-        /// </summary>
-        /// <param name="remoteClient"></param>
-        /// <param name="itemID"></param>
-        /// <param name="ownerID"></param>
-        public void HandleFetchInventory(IClientAPI remoteClient, UUID itemID, UUID ownerID)
-        {
-            if (LibraryService != null && LibraryService.LibraryRootFolder != null && ownerID == LibraryService.LibraryRootFolder.Owner)
-            {
-                //m_log.Debug("request info for library item");
-                return;
-            }
-            
-            InventoryItemBase item = new InventoryItemBase(itemID, remoteClient.AgentId);
-            item = InventoryService.GetItem(item);
-            
-            if (item != null)
-            {
-                remoteClient.SendInventoryItemDetails(ownerID, item);
-            }
-            // else shouldn't we send an alert message?
-        }
         
         /// <summary>
         /// Tell the client about the various child items and folders contained in the requested folder.
diff --git a/OpenSim/Region/Framework/Scenes/Scene.cs b/OpenSim/Region/Framework/Scenes/Scene.cs
index ad41e88..eeb881f 100644
--- a/OpenSim/Region/Framework/Scenes/Scene.cs
+++ b/OpenSim/Region/Framework/Scenes/Scene.cs
@@ -583,6 +583,8 @@ namespace OpenSim.Region.Framework.Scenes
             m_asyncSceneObjectDeleter = new AsyncSceneObjectGroupDeleter(this);
             m_asyncSceneObjectDeleter.Enabled = true;
 
+            m_asyncInventorySender = new AsyncInventorySender(this);
+
             #region Region Settings
 
             // Load region settings
@@ -2760,14 +2762,13 @@ namespace OpenSim.Region.Framework.Scenes
 
         public virtual void SubscribeToClientInventoryEvents(IClientAPI client)
         {
-            
             client.OnLinkInventoryItem += HandleLinkInventoryItem;
             client.OnCreateNewInventoryFolder += HandleCreateInventoryFolder;
             client.OnUpdateInventoryFolder += HandleUpdateInventoryFolder;
             client.OnMoveInventoryFolder += HandleMoveInventoryFolder; // 2; //!!
             client.OnFetchInventoryDescendents += HandleFetchInventoryDescendents;
             client.OnPurgeInventoryDescendents += HandlePurgeInventoryDescendents; // 2; //!!
-            client.OnFetchInventory += HandleFetchInventory;
+            client.OnFetchInventory += m_asyncInventorySender.HandleFetchInventory;
             client.OnUpdateInventoryItem += UpdateInventoryItemAsset;
             client.OnCopyInventoryItem += CopyInventoryItem;
             client.OnMoveInventoryItem += MoveInventoryItem;