* When saving an oar, save assets when immediately received rather than storing them all up in memory

* Hopefully this will remove out of memory problems when saving large oars on machines without much memory
* It may also speed up saving of large oars

5 files changed, 132 insertions, 132 deletions

diff --git a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs
index b5c0d86..fec2425 100644
--- a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs
+++ b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs
@@ -56,7 +56,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
         private InventoryArchiverModule m_module;
         private CachedUserInfo m_userInfo;
         private string m_invPath;        
-        protected TarArchiveWriter m_archive;
+        protected TarArchiveWriter m_archiveWriter;
         protected UuidGatherer m_assetGatherer;
 
         /// <value>
@@ -100,17 +100,14 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
             m_assetGatherer = new UuidGatherer(m_module.CommsManager.AssetCache);
         }
 
-        protected void ReceivedAllAssets(IDictionary<UUID, AssetBase> assetsFound, ICollection<UUID> assetsNotFoundUuids)
+        protected void ReceivedAllAssets(ICollection<UUID> assetsFoundUuids, ICollection<UUID> assetsNotFoundUuids)
         {
-            AssetsArchiver assetsArchiver = new AssetsArchiver(assetsFound);
-            assetsArchiver.Archive(m_archive);
-
             Exception reportedException = null;
             bool succeeded = true;
 
             try
             {
-                m_archive.Close();
+                m_archiveWriter.Close();
             }
             catch (IOException e)
             {
@@ -133,7 +130,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
             saveItem.CreatorId = OspResolver.MakeOspa(saveItem.CreatorIdAsUuid, m_module.CommsManager);
 
             string serialization = UserInventoryItemSerializer.Serialize(saveItem);
-            m_archive.WriteFile(filename, serialization);
+            m_archiveWriter.WriteFile(filename, serialization);
 
             m_assetGatherer.GatherAssetUuids(saveItem.AssetID, (AssetType)saveItem.AssetType, m_assetUuids);
         }
@@ -156,7 +153,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
                         inventoryFolder.ID);
 
                 // We need to make sure that we record empty folders            
-                m_archive.WriteDir(path);
+                m_archiveWriter.WriteDir(path);
             }
 
             List<InventoryFolderImpl> childFolders = inventoryFolder.RequestListOfFolderImpls();
@@ -265,7 +262,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
                 inventoryItem = m_userInfo.RootFolder.FindItemByPath(m_invPath);
             }
 
-            m_archive = new TarArchiveWriter(m_saveStream);
+            m_archiveWriter = new TarArchiveWriter(m_saveStream);
 
             if (null == inventoryFolder)
             {
@@ -298,7 +295,9 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
             }
 
             SaveUsers();
-            new AssetsRequest(m_assetUuids.Keys, m_module.CommsManager.AssetCache, ReceivedAllAssets).Execute();
+            new AssetsRequest(
+                new AssetsArchiver(m_archiveWriter), m_assetUuids.Keys, 
+                m_module.CommsManager.AssetCache, ReceivedAllAssets).Execute();
         }
 
         /// <summary>
@@ -316,7 +315,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
 
                 if (creator != null)
                 {
-                    m_archive.WriteFile(
+                    m_archiveWriter.WriteFile(
                         ArchiveConstants.USERS_PATH + creator.UserProfile.Name + ".xml",
                         UserProfileSerializer.Serialize(creator.UserProfile));
                 }
diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs
index 943d9d1..b167ce2 100644
--- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs
+++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs
@@ -44,7 +44,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver
     /// <summary>
     /// Method called when all the necessary assets for an archive request have been received.
     /// </summary>
-    public delegate void AssetsRequestCallback(IDictionary<UUID, AssetBase> assetsFound, ICollection<UUID> assetsNotFoundUuids);
+    public delegate void AssetsRequestCallback(
+        ICollection<UUID> assetsFoundUuids, ICollection<UUID> assetsNotFoundUuids);
 
     /// <summary>
     /// Execute the write of an archive once we have received all the necessary data
@@ -57,7 +58,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
         protected IRegionSerialiserModule m_serialiser;
         protected List<SceneObjectGroup> m_sceneObjects;
         protected Scene m_scene;
-        protected Stream m_saveStream;
+        protected TarArchiveWriter m_archiveWriter;
         protected Guid m_requestId;
 
         public ArchiveWriteRequestExecution(
@@ -65,19 +66,19 @@ namespace OpenSim.Region.CoreModules.World.Archiver
              ITerrainModule terrainModule,
              IRegionSerialiserModule serialiser,
              Scene scene,
-             Stream saveStream,
+             TarArchiveWriter archiveWriter,
              Guid requestId)
         {
             m_sceneObjects = sceneObjects;
             m_terrainModule = terrainModule;
             m_serialiser = serialiser;
             m_scene = scene;
-            m_saveStream = saveStream;
+            m_archiveWriter = archiveWriter;
             m_requestId = requestId;
         }
 
         protected internal void ReceivedAllAssets(
-            IDictionary<UUID, AssetBase> assetsFound, ICollection<UUID> assetsNotFoundUuids)
+            ICollection<UUID> assetsFoundUuids, ICollection<UUID> assetsNotFoundUuids)
         {
             foreach (UUID uuid in assetsNotFoundUuids)
             {
@@ -86,21 +87,19 @@ namespace OpenSim.Region.CoreModules.World.Archiver
 
             m_log.InfoFormat(
                 "[ARCHIVER]: Received {0} of {1} assets requested",
-                assetsFound.Count, assetsFound.Count + assetsNotFoundUuids.Count);
+                assetsFoundUuids.Count, assetsFoundUuids.Count + assetsNotFoundUuids.Count);
 
             m_log.InfoFormat("[ARCHIVER]: Creating archive file.  This may take some time.");
 
-            TarArchiveWriter archive = new TarArchiveWriter(m_saveStream);
-
             // Write out control file
-            archive.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, Create0p2ControlFile());
+            m_archiveWriter.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, Create0p2ControlFile());
 
             m_log.InfoFormat("[ARCHIVER]: Added control file to archive.");
 
             // Write out region settings
             string settingsPath
                 = String.Format("{0}{1}.xml", ArchiveConstants.SETTINGS_PATH, m_scene.RegionInfo.RegionName);
-            archive.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings));
+            m_archiveWriter.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings));
 
             m_log.InfoFormat("[ARCHIVER]: Added region settings to archive.");
 
@@ -110,7 +109,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
 
             MemoryStream ms = new MemoryStream();
             m_terrainModule.SaveToStream(terrainPath, ms);
-            archive.WriteFile(terrainPath, ms.ToArray());
+            m_archiveWriter.WriteFile(terrainPath, ms.ToArray());
             ms.Close();
 
             m_log.InfoFormat("[ARCHIVER]: Added terrain information to archive.");
@@ -130,16 +129,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver
                         Math.Round(position.X), Math.Round(position.Y), Math.Round(position.Z),
                         sceneObject.UUID);
 
-                archive.WriteFile(filename, serializedObject);
+                m_archiveWriter.WriteFile(filename, serializedObject);
             }
 
             m_log.InfoFormat("[ARCHIVER]: Added scene objects to archive.");
 
-            // Write out assets
-            AssetsArchiver assetsArchiver = new AssetsArchiver(assetsFound);
-            assetsArchiver.Archive(archive);
-
-            archive.Close();
+            m_archiveWriter.Close();
 
             m_log.InfoFormat("[ARCHIVER]: Wrote out OpenSimulator archive for {0}", m_scene.RegionInfo.RegionName);
 
diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs
index a6ad24c..0a882eb 100644
--- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs
+++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs
@@ -35,6 +35,7 @@ using System.Threading;
 using log4net;
 using OpenMetaverse;
 using OpenSim.Framework;
+using OpenSim.Framework.Serialization;
 using OpenSim.Region.CoreModules.World.Terrain;
 using OpenSim.Region.Framework.Interfaces;
 using OpenSim.Region.Framework.Scenes;
@@ -126,6 +127,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver
             if (regionSettings.TerrainTexture4 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_4)
                 assetUuids[regionSettings.TerrainTexture4] = 1;
 
+            TarArchiveWriter archiveWriter = new TarArchiveWriter(m_saveStream);
+            
             // Asynchronously request all the assets required to perform this archive operation
             ArchiveWriteRequestExecution awre
                 = new ArchiveWriteRequestExecution(
@@ -133,10 +136,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver
                     m_scene.RequestModuleInterface<ITerrainModule>(),
                     m_scene.RequestModuleInterface<IRegionSerialiserModule>(),
                     m_scene,
-                    m_saveStream,
-                    m_requestId);
+                    archiveWriter,
+                    m_requestId);           
             
-            new AssetsRequest(assetUuids.Keys, m_scene.CommsManager.AssetCache, awre.ReceivedAllAssets).Execute();
+            new AssetsRequest(
+                new AssetsArchiver(archiveWriter), assetUuids.Keys, 
+                m_scene.CommsManager.AssetCache, awre.ReceivedAllAssets).Execute();
         }
     }
 }
diff --git a/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs b/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs
index 29d7c5e..0c01cae 100644
--- a/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs
+++ b/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs
@@ -46,114 +46,106 @@ namespace OpenSim.Region.CoreModules.World.Archiver
         /// <value>
         /// Post a message to the log every x assets as a progress bar
         /// </value>
-        private static int LOG_ASSET_LOAD_NOTIFICATION_INTERVAL = 50;
+        protected static int LOG_ASSET_LOAD_NOTIFICATION_INTERVAL = 50;
 
-        /// <summary>
-        /// Archive assets
-        /// </summary>
-        protected IDictionary<UUID, AssetBase> m_assets;
+        /// <value>
+        /// Keep a count of the number of assets written so that we can provide status updates
+        /// </value>
+        protected int m_assetsWritten; 
+        
+        protected TarArchiveWriter m_archiveWriter;
 
-        public AssetsArchiver(IDictionary<UUID, AssetBase> assets)
+        public AssetsArchiver(TarArchiveWriter archiveWriter)
         {
-            m_assets = assets;
+            m_archiveWriter = archiveWriter;
         }
 
         /// <summary>
         /// Archive the assets given to this archiver to the given archive.
         /// </summary>
         /// <param name="archive"></param>
-        public void Archive(TarArchiveWriter archive)
-        {
+        public void WriteAsset(AssetBase asset)
+        {            
             //WriteMetadata(archive);
-            WriteData(archive);
+            WriteData(asset);
         }
 
         /// <summary>
         /// Write an assets metadata file to the given archive
         /// </summary>
         /// <param name="archive"></param>
-        protected void WriteMetadata(TarArchiveWriter archive)
-        {
-            StringWriter sw = new StringWriter();
-            XmlTextWriter xtw = new XmlTextWriter(sw);
-
-            xtw.Formatting = Formatting.Indented;
-            xtw.WriteStartDocument();
-
-            xtw.WriteStartElement("assets");
-
-            foreach (UUID uuid in m_assets.Keys)
-            {
-                AssetBase asset = m_assets[uuid];
-
-                if (asset != null)
-                {
-                    xtw.WriteStartElement("asset");
-
-                    string extension = string.Empty;
-
-                    if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type))
-                    {
-                        extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type];
-                    }
-
-                    xtw.WriteElementString("filename", uuid.ToString() + extension);
-
-                    xtw.WriteElementString("name", asset.Name);
-                    xtw.WriteElementString("description", asset.Description);
-                    xtw.WriteElementString("asset-type", asset.Type.ToString());
-
-                    xtw.WriteEndElement();
-                }
-            }
-
-            xtw.WriteEndElement();
-
-            xtw.WriteEndDocument();
-
-            archive.WriteFile("assets.xml", sw.ToString());
-        }
+//        protected void WriteMetadata(TarArchiveWriter archive)
+//        {
+//            StringWriter sw = new StringWriter();
+//            XmlTextWriter xtw = new XmlTextWriter(sw);
+//
+//            xtw.Formatting = Formatting.Indented;
+//            xtw.WriteStartDocument();
+//
+//            xtw.WriteStartElement("assets");
+//
+//            foreach (UUID uuid in m_assets.Keys)
+//            {
+//                AssetBase asset = m_assets[uuid];
+//
+//                if (asset != null)
+//                {
+//                    xtw.WriteStartElement("asset");
+//
+//                    string extension = string.Empty;
+//
+//                    if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type))
+//                    {
+//                        extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type];
+//                    }
+//
+//                    xtw.WriteElementString("filename", uuid.ToString() + extension);
+//
+//                    xtw.WriteElementString("name", asset.Name);
+//                    xtw.WriteElementString("description", asset.Description);
+//                    xtw.WriteElementString("asset-type", asset.Type.ToString());
+//
+//                    xtw.WriteEndElement();
+//                }
+//            }
+//
+//            xtw.WriteEndElement();
+//
+//            xtw.WriteEndDocument();
+//
+//            archive.WriteFile("assets.xml", sw.ToString());
+//        }
 
         /// <summary>
         /// Write asset data files to the given archive
         /// </summary>
-        /// <param name="archive"></param>
-        protected void WriteData(TarArchiveWriter archive)
+        /// <param name="asset"></param>
+        protected void WriteData(AssetBase asset)
         {
             // It appears that gtar, at least, doesn't need the intermediate directory entries in the tar
             //archive.AddDir("assets");
 
-            int assetsAdded = 0;
+            string extension = string.Empty;
 
-            foreach (UUID uuid in m_assets.Keys)
+            if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type))
             {
-                AssetBase asset = m_assets[uuid];
-
-                string extension = string.Empty;
-
-                if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type))
-                {
-                    extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type];
-                }
-                else
-                {
-                    m_log.ErrorFormat(
-                        "[ARCHIVER]: Unrecognized asset type {0} with uuid {1}.  This asset will be saved but not reloaded",
-                        asset.Type, asset.ID);
-                }
-
-                archive.WriteFile(
-                    ArchiveConstants.ASSETS_PATH + uuid.ToString() + extension,
-                    asset.Data);
+                extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type];
+            }
+            else
+            {
+                m_log.ErrorFormat(
+                    "[ARCHIVER]: Unrecognized asset type {0} with uuid {1}.  This asset will be saved but not reloaded",
+                    asset.Type, asset.ID);
+            }
 
-                assetsAdded++;
+            m_archiveWriter.WriteFile(
+                ArchiveConstants.ASSETS_PATH + asset.FullID.ToString() + extension,
+                asset.Data);
 
-                if (assetsAdded % LOG_ASSET_LOAD_NOTIFICATION_INTERVAL == 0)
-                    m_log.InfoFormat("[ARCHIVER]: Added {0} assets to archive", assetsAdded);
-            }
+            m_assetsWritten++;
 
-            if (assetsAdded % LOG_ASSET_LOAD_NOTIFICATION_INTERVAL != 0)
-                m_log.InfoFormat("[ARCHIVER]: Added {0} assets to archive", assetsAdded);
+            if (m_assetsWritten % LOG_ASSET_LOAD_NOTIFICATION_INTERVAL == 0)
+                m_log.InfoFormat("[ARCHIVER]: Added {0} assets to archive", m_assetsWritten);
         }
     }
 }
diff --git a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
index 0a0bb4c..d806a9c 100644
--- a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
+++ b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
@@ -32,6 +32,7 @@ using System.Threading;
 using log4net;
 using OpenMetaverse;
 using OpenSim.Framework;
+using OpenSim.Framework.Serialization;
 
 namespace OpenSim.Region.CoreModules.World.Archiver
 {
@@ -42,39 +43,43 @@ namespace OpenSim.Region.CoreModules.World.Archiver
     {
         private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
 
-        /// <summary>
+        /// <value>
         /// uuids to request
-        /// </summary>
+        /// </value>
         protected ICollection<UUID> m_uuids;
 
-        /// <summary>
+        /// <value>
         /// Callback used when all the assets requested have been received.
-        /// </summary>
+        /// </value>
         protected AssetsRequestCallback m_assetsRequestCallback;
 
-        /// <summary>
-        /// Assets retrieved in this request
-        /// </summary>
-        protected Dictionary<UUID, AssetBase> m_assets = new Dictionary<UUID, AssetBase>();
-
-        /// <summary>
+        /// <value>
+        /// List of assets that were found.  This will be passed back to the requester.
+        /// </value>
+        protected List<UUID> m_foundAssetUuids = new List<UUID>();
+        
+        /// <value>
         /// Maintain a list of assets that could not be found.  This will be passed back to the requester.
-        /// </summary>
+        /// </value>
         protected List<UUID> m_notFoundAssetUuids = new List<UUID>();
 
-        /// <summary>
+        /// <value>
         /// Record the number of asset replies required so we know when we've finished
-        /// </summary>
+        /// </value>
         private int m_repliesRequired;
 
-        /// <summary>
+        /// <value>
         /// Asset cache used to request the assets
-        /// </summary>
+        /// </value>
         protected IAssetCache m_assetCache;
 
+        protected AssetsArchiver m_assetsArchiver;
+
         protected internal AssetsRequest(
-            ICollection<UUID> uuids, IAssetCache assetCache, AssetsRequestCallback assetsRequestCallback)
+            AssetsArchiver assetsArchiver, ICollection<UUID> uuids, 
+            IAssetCache assetCache, AssetsRequestCallback assetsRequestCallback)
         {
+            m_assetsArchiver = assetsArchiver;
             m_uuids = uuids;
             m_assetsRequestCallback = assetsRequestCallback;
             m_assetCache = assetCache;
@@ -87,7 +92,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
             
             // We can stop here if there are no assets to fetch
             if (m_repliesRequired == 0)
-                m_assetsRequestCallback(m_assets, m_notFoundAssetUuids);
+                m_assetsRequestCallback(m_foundAssetUuids, m_notFoundAssetUuids);
 
             foreach (UUID uuid in m_uuids)
             {
@@ -106,21 +111,25 @@ namespace OpenSim.Region.CoreModules.World.Archiver
             
             if (asset != null)
             {
+                // Make sure that we don't run out of memory by hogging assets in the cache
                 m_assetCache.ExpireAsset(assetID);
-                m_assets[assetID] = asset;                
+                
+                m_foundAssetUuids.Add(assetID);
+                m_assetsArchiver.WriteAsset(asset);
             }
             else
             {
                 m_notFoundAssetUuids.Add(assetID);
             }
 
-            if (m_assets.Count + m_notFoundAssetUuids.Count == m_repliesRequired)
+            if (m_foundAssetUuids.Count + m_notFoundAssetUuids.Count == m_repliesRequired)
             {
                 m_log.DebugFormat(
                     "[ARCHIVER]: Successfully received {0} assets and notification of {1} missing assets", 
-                    m_assets.Count, m_notFoundAssetUuids.Count);
+                    m_foundAssetUuids.Count, m_notFoundAssetUuids.Count);
                 
-                // We want to stop using the asset cache thread asap as we now need to do the actual work of producing the archive
+                // We want to stop using the asset cache thread asap 
+                // as we now need to do the work of producing the rest of the archive
                 Thread newThread = new Thread(PerformAssetsRequestCallback);
                 newThread.Name = "OpenSimulator archiving thread post assets receipt";
                 newThread.Start();
@@ -134,7 +143,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
         {
             try
             {
-                m_assetsRequestCallback(m_assets, m_notFoundAssetUuids);
+                m_assetsRequestCallback(m_foundAssetUuids, m_notFoundAssetUuids);
             }
             catch (Exception e)
             {