Thank you dslake for diagnosing and fixing a race condition in OGS1SecureInventoryServer (mantis #3483). The provided patch was slightly modified to narrow the locking scope to smaller portions of the functions. Applied the same locking to HGInventoryService, which suffered from the same race condition.

2 files changed, 146 insertions, 132 deletions

diff --git a/OpenSim/Region/Communications/Hypergrid/HGInventoryService.cs b/OpenSim/Region/Communications/Hypergrid/HGInventoryService.cs
index 156137e..98f5f68 100644
--- a/OpenSim/Region/Communications/Hypergrid/HGInventoryService.cs
+++ b/OpenSim/Region/Communications/Hypergrid/HGInventoryService.cs
@@ -77,52 +77,54 @@ namespace OpenSim.Region.Communications.Hypergrid
             }
 
             // grid/hypergrid mode
-            if (!m_RequestingInventory.ContainsKey(userID))
+            lock (m_RequestingInventory)
             {
-                m_RequestingInventory.Add(userID, callback);
-
-                string invServer = GetUserInventoryURI(userID);
-                m_log.InfoFormat(
-                    "[HGrid INVENTORY SERVICE]: Requesting inventory from {0}/GetInventory/ for user {1} ({2})",
-                    /*_inventoryServerUrl*/ invServer, userID, userID.Guid);
-
-                try
+                if (!m_RequestingInventory.ContainsKey(userID))
                 {
-
-                    //RestSessionObjectPosterResponse<Guid, InventoryCollection> requester
-                    //    = new RestSessionObjectPosterResponse<Guid, InventoryCollection>();
-                    //requester.ResponseCallback = InventoryResponse;
-
-                    //requester.BeginPostObject(invServer + "/GetInventory/", userID.Guid, session_id.ToString(), userID.ToString());
-                    GetInventoryDelegate d = GetInventoryAsync;
-                    d.BeginInvoke(invServer, userID, session_id, GetInventoryCompleted, d);
-
+                    m_RequestingInventory.Add(userID, callback);
                 }
-                catch (WebException e)
+                else
                 {
-                    if (StatsManager.SimExtraStats != null)
-                        StatsManager.SimExtraStats.AddInventoryServiceRetrievalFailure();
-
-                    m_log.ErrorFormat("[HGrid INVENTORY SERVICE]: Request inventory operation failed, {0} {1}",
-                        e.Source, e.Message);
-
-                    // Well, let's synthesize one
-                    InventoryCollection icol = new InventoryCollection();
-                    icol.UserID = userID;
-                    icol.Items = new List<InventoryItemBase>();
-                    icol.Folders = new List<InventoryFolderBase>();
-                    InventoryFolderBase rootFolder = new InventoryFolderBase();
-                    rootFolder.ID = UUID.Random();
-                    rootFolder.Owner = userID;
-                    icol.Folders.Add(rootFolder);
-                    InventoryResponse(icol);
+                    m_log.ErrorFormat("[HGrid INVENTORY SERVICE]: RequestInventoryForUser() - could  not find user profile for {0}", userID);
+                    return;
                 }
             }
-            else
+            string invServer = GetUserInventoryURI(userID);
+            m_log.InfoFormat(
+                "[HGrid INVENTORY SERVICE]: Requesting inventory from {0}/GetInventory/ for user {1} ({2})",
+                /*_inventoryServerUrl*/ invServer, userID, userID.Guid);
+
+            try
             {
-                m_log.ErrorFormat("[HGrid INVENTORY SERVICE]: RequestInventoryForUser() - could  not find user profile for {0}", userID);
+
+                //RestSessionObjectPosterResponse<Guid, InventoryCollection> requester
+                //    = new RestSessionObjectPosterResponse<Guid, InventoryCollection>();
+                //requester.ResponseCallback = InventoryResponse;
+
+                //requester.BeginPostObject(invServer + "/GetInventory/", userID.Guid, session_id.ToString(), userID.ToString());
+                GetInventoryDelegate d = GetInventoryAsync;
+                d.BeginInvoke(invServer, userID, session_id, GetInventoryCompleted, d);
+
             }
-            
+            catch (WebException e)
+            {
+                if (StatsManager.SimExtraStats != null)
+                    StatsManager.SimExtraStats.AddInventoryServiceRetrievalFailure();
+
+                m_log.ErrorFormat("[HGrid INVENTORY SERVICE]: Request inventory operation failed, {0} {1}",
+                    e.Source, e.Message);
+
+                // Well, let's synthesize one
+                InventoryCollection icol = new InventoryCollection();
+                icol.UserID = userID;
+                icol.Items = new List<InventoryItemBase>();
+                icol.Folders = new List<InventoryFolderBase>();
+                InventoryFolderBase rootFolder = new InventoryFolderBase();
+                rootFolder.ID = UUID.Random();
+                rootFolder.Owner = userID;
+                icol.Folders.Add(rootFolder);
+                InventoryResponse(icol);
+            }            
         }
 
         private delegate InventoryCollection GetInventoryDelegate(string url, UUID userID, UUID sessionID);
@@ -447,62 +449,66 @@ namespace OpenSim.Region.Communications.Hypergrid
         private void InventoryResponse(InventoryCollection response)
         {
             UUID userID = response.UserID;
-            if (m_RequestingInventory.ContainsKey(userID))
+            InventoryReceiptCallback callback = null;
+            lock (m_RequestingInventory)
             {
-                m_log.InfoFormat("[HGrid INVENTORY SERVICE]: " +
-                                 "Received inventory response for user {0} containing {1} folders and {2} items",
-                                 userID, response.Folders.Count, response.Items.Count);
+                if (m_RequestingInventory.ContainsKey(userID))
+                {
+                    m_log.InfoFormat("[HGrid INVENTORY SERVICE]: " +
+                                     "Received inventory response for user {0} containing {1} folders and {2} items",
+                                     userID, response.Folders.Count, response.Items.Count);
+                    callback = m_RequestingInventory[userID];
+                    m_RequestingInventory.Remove(userID);
+                }
+                else
+                {
+                    m_log.WarnFormat(
+                        "[HGrid INVENTORY SERVICE]: " +
+                        "Received inventory response for {0} for which we do not have a record of requesting!",
+                        userID);
+                    return;
+                }
+            }
 
-                InventoryFolderImpl rootFolder = null;
-                InventoryReceiptCallback callback = m_RequestingInventory[userID];
+            InventoryFolderImpl rootFolder = null;
 
-                ICollection<InventoryFolderImpl> folders = new List<InventoryFolderImpl>();
-                ICollection<InventoryItemBase> items = new List<InventoryItemBase>();
+            ICollection<InventoryFolderImpl> folders = new List<InventoryFolderImpl>();
+            ICollection<InventoryItemBase> items = new List<InventoryItemBase>();
 
-                foreach (InventoryFolderBase folder in response.Folders)
+            foreach (InventoryFolderBase folder in response.Folders)
+            {
+                if (folder.ParentID == UUID.Zero)
                 {
-                    if (folder.ParentID == UUID.Zero)
-                    {
-                        rootFolder = new InventoryFolderImpl(folder);
-                        folders.Add(rootFolder);
+                    rootFolder = new InventoryFolderImpl(folder);
+                    folders.Add(rootFolder);
 
-                        break;
-                    }
+                    break;
                 }
+            }
 
-                if (rootFolder != null)
+            if (rootFolder != null)
+            {
+                foreach (InventoryFolderBase folder in response.Folders)
                 {
-                    foreach (InventoryFolderBase folder in response.Folders)
-                    {
-                        if (folder.ID != rootFolder.ID)
-                        {
-                            folders.Add(new InventoryFolderImpl(folder));
-                        }
-                    }
-
-                    foreach (InventoryItemBase item in response.Items)
+                    if (folder.ID != rootFolder.ID)
                     {
-                        items.Add(item);
+                        folders.Add(new InventoryFolderImpl(folder));
                     }
                 }
-                else
+
+                foreach (InventoryItemBase item in response.Items)
                 {
-                    m_log.ErrorFormat("[HGrid INVENTORY SERVICE]: Did not get back an inventory containing a root folder for user {0}", userID);
+                    items.Add(item);
                 }
-
-                m_RequestingInventory.Remove(userID);
-                callback(folders, items);
-
             }
             else
             {
-                m_log.WarnFormat(
-                    "[HGrid INVENTORY SERVICE]: " +
-                    "Received inventory response for {0} for which we do not have a record of requesting!",
-                    userID);
+                m_log.ErrorFormat("[HGrid INVENTORY SERVICE]: Did not get back an inventory containing a root folder for user {0}", userID);
             }
-        }
 
+            callback(folders, items);
+
+        }
 
         private bool IsLocalStandaloneUser(UUID userID)
         {
diff --git a/OpenSim/Region/Communications/OGS1/OGS1SecureInventoryService.cs b/OpenSim/Region/Communications/OGS1/OGS1SecureInventoryService.cs
index 03fb0d5..e24f820 100644
--- a/OpenSim/Region/Communications/OGS1/OGS1SecureInventoryService.cs
+++ b/OpenSim/Region/Communications/OGS1/OGS1SecureInventoryService.cs
@@ -69,34 +69,36 @@ namespace OpenSim.Region.Communications.OGS1
         /// <param name="callback"></param>
         public void RequestInventoryForUser(UUID userID, UUID session_id, InventoryReceiptCallback callback)
         {
-            if (!m_RequestingInventory.ContainsKey(userID))
+            lock (m_RequestingInventory)
             {
-                m_RequestingInventory.Add(userID, callback);
-
-                try
+                if (!m_RequestingInventory.ContainsKey(userID))
+                    m_RequestingInventory.Add(userID, callback);
+                else
                 {
-                    m_log.InfoFormat(
-                        "[OGS1 INVENTORY SERVICE]: Requesting inventory from {0}/GetInventory/ for user {1}",
-                        _inventoryServerUrl, userID);
+                    m_log.ErrorFormat("[OGS1 INVENTORY SERVICE]: RequestInventoryForUser() - could you not find user profile for {0}", userID);
+                    return;
+                }
+            }
 
-                    RestSessionObjectPosterResponse<Guid, InventoryCollection> requester
-                        = new RestSessionObjectPosterResponse<Guid, InventoryCollection>();
-                    requester.ResponseCallback = InventoryResponse;
+            try
+            {
+                m_log.InfoFormat(
+                    "[OGS1 INVENTORY SERVICE]: Requesting inventory from {0}/GetInventory/ for user {1}",
+                    _inventoryServerUrl, userID);
 
-                    requester.BeginPostObject(_inventoryServerUrl + "/GetInventory/", userID.Guid, session_id.ToString(), userID.ToString());
-                }
-                catch (WebException e)
-                {
-                    if (StatsManager.SimExtraStats != null)
-                        StatsManager.SimExtraStats.AddInventoryServiceRetrievalFailure();
+                RestSessionObjectPosterResponse<Guid, InventoryCollection> requester
+                    = new RestSessionObjectPosterResponse<Guid, InventoryCollection>();
+                requester.ResponseCallback = InventoryResponse;
 
-                    m_log.ErrorFormat("[OGS1 INVENTORY SERVICE]: Request inventory operation failed, {0} {1}",
-                        e.Source, e.Message);
-                }
+                requester.BeginPostObject(_inventoryServerUrl + "/GetInventory/", userID.Guid, session_id.ToString(), userID.ToString());
             }
-            else
+            catch (WebException e)
             {
-                m_log.ErrorFormat("[OGS1 INVENTORY SERVICE]: RequestInventoryForUser() - could you not find user profile for {0}", userID);
+                if (StatsManager.SimExtraStats != null)
+                    StatsManager.SimExtraStats.AddInventoryServiceRetrievalFailure();
+
+                m_log.ErrorFormat("[OGS1 INVENTORY SERVICE]: Request inventory operation failed, {0} {1}",
+                    e.Source, e.Message);
             }
         }
 
@@ -107,60 +109,66 @@ namespace OpenSim.Region.Communications.OGS1
         private void InventoryResponse(InventoryCollection response)
         {
             UUID userID = response.UserID;
-            if (m_RequestingInventory.ContainsKey(userID))
+            InventoryReceiptCallback callback = null;
+            lock (m_RequestingInventory)
             {
-                m_log.InfoFormat("[OGS1 INVENTORY SERVICE]: " +
-                                 "Received inventory response for user {0} containing {1} folders and {2} items",
-                                 userID, response.Folders.Count, response.Items.Count);
+                if (m_RequestingInventory.ContainsKey(userID))
+                {
+                    callback = m_RequestingInventory[userID];
+                    m_RequestingInventory.Remove(userID);
+                }
+                else
+                {
+                    m_log.WarnFormat(
+                        "[OGS1 INVENTORY SERVICE]: " +
+                        "Received inventory response for {0} for which we do not have a record of requesting!",
+                        userID);
+                    return;
+                }
+            }
 
-                InventoryFolderImpl rootFolder = null;
-                InventoryReceiptCallback callback = m_RequestingInventory[userID];
+            m_log.InfoFormat("[OGS1 INVENTORY SERVICE]: " +
+                             "Received inventory response for user {0} containing {1} folders and {2} items",
+                             userID, response.Folders.Count, response.Items.Count);
 
-                ICollection<InventoryFolderImpl> folders = new List<InventoryFolderImpl>();
-                ICollection<InventoryItemBase> items = new List<InventoryItemBase>();
+            InventoryFolderImpl rootFolder = null;
 
-                foreach (InventoryFolderBase folder in response.Folders)
+            ICollection<InventoryFolderImpl> folders = new List<InventoryFolderImpl>();
+            ICollection<InventoryItemBase> items = new List<InventoryItemBase>();
+
+            foreach (InventoryFolderBase folder in response.Folders)
+            {
+                if (folder.ParentID == UUID.Zero)
                 {
-                    if (folder.ParentID == UUID.Zero)
-                    {
-                        rootFolder = new InventoryFolderImpl(folder);
-                        folders.Add(rootFolder);
+                    rootFolder = new InventoryFolderImpl(folder);
+                    folders.Add(rootFolder);
 
-                        break;
-                    }
+                    break;
                 }
+            }
 
-                if (rootFolder != null)
+            if (rootFolder != null)
+            {
+                foreach (InventoryFolderBase folder in response.Folders)
                 {
-                    foreach (InventoryFolderBase folder in response.Folders)
+                    if (folder.ID != rootFolder.ID)
                     {
-                        if (folder.ID != rootFolder.ID)
-                        {
-                            folders.Add(new InventoryFolderImpl(folder));
-                        }
-                    }
-
-                    foreach (InventoryItemBase item in response.Items)
-                    {
-                        items.Add(item);
+                        folders.Add(new InventoryFolderImpl(folder));
                     }
                 }
-                else
+
+                foreach (InventoryItemBase item in response.Items)
                 {
-                    m_log.ErrorFormat("[OGS1 INVENTORY SERVICE]: Did not get back an inventory containing a root folder for user {0}", userID);
+                    items.Add(item);
                 }
-
-                callback(folders, items);
-
-                m_RequestingInventory.Remove(userID);
             }
             else
             {
-                m_log.WarnFormat(
-                    "[OGS1 INVENTORY SERVICE]: " +
-                    "Received inventory response for {0} for which we do not have a record of requesting!",
-                    userID);
+                m_log.ErrorFormat("[OGS1 INVENTORY SERVICE]: Did not get back an inventory containing a root folder for user {0}", userID);
             }
+
+            callback(folders, items);
+
         }
 
         /// <summary>