From 62771560448edbc2112d427acd4b37780dfe2154 Mon Sep 17 00:00:00 2001 From: Justin Clarke Casey Date: Thu, 14 May 2009 20:37:54 +0000 Subject: * When saving an oar, save assets when immediately received rather than storing them all up in memory * Hopefully this will remove out of memory problems when saving large oars on machines without much memory * It may also speed up saving of large oars --- .../World/Archiver/ArchiveWriteRequestExecution.cs | 29 ++-- .../Archiver/ArchiveWriteRequestPreparation.cs | 11 +- .../CoreModules/World/Archiver/AssetsArchiver.cs | 150 ++++++++++----------- .../CoreModules/World/Archiver/AssetsRequest.cs | 53 +++++--- 4 files changed, 122 insertions(+), 121 deletions(-) (limited to 'OpenSim/Region/CoreModules/World') diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs index 943d9d1..b167ce2 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs @@ -44,7 +44,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// Method called when all the necessary assets for an archive request have been received. /// - public delegate void AssetsRequestCallback(IDictionary assetsFound, ICollection assetsNotFoundUuids); + public delegate void AssetsRequestCallback( + ICollection assetsFoundUuids, ICollection assetsNotFoundUuids); /// /// Execute the write of an archive once we have received all the necessary data @@ -57,7 +58,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver protected IRegionSerialiserModule m_serialiser; protected List m_sceneObjects; protected Scene m_scene; - protected Stream m_saveStream; + protected TarArchiveWriter m_archiveWriter; protected Guid m_requestId; public ArchiveWriteRequestExecution( @@ -65,19 +66,19 @@ namespace OpenSim.Region.CoreModules.World.Archiver ITerrainModule terrainModule, IRegionSerialiserModule serialiser, Scene scene, - Stream saveStream, + TarArchiveWriter archiveWriter, Guid requestId) { m_sceneObjects = sceneObjects; m_terrainModule = terrainModule; m_serialiser = serialiser; m_scene = scene; - m_saveStream = saveStream; + m_archiveWriter = archiveWriter; m_requestId = requestId; } protected internal void ReceivedAllAssets( - IDictionary assetsFound, ICollection assetsNotFoundUuids) + ICollection assetsFoundUuids, ICollection assetsNotFoundUuids) { foreach (UUID uuid in assetsNotFoundUuids) { @@ -86,21 +87,19 @@ namespace OpenSim.Region.CoreModules.World.Archiver m_log.InfoFormat( "[ARCHIVER]: Received {0} of {1} assets requested", - assetsFound.Count, assetsFound.Count + assetsNotFoundUuids.Count); + assetsFoundUuids.Count, assetsFoundUuids.Count + assetsNotFoundUuids.Count); m_log.InfoFormat("[ARCHIVER]: Creating archive file. This may take some time."); - TarArchiveWriter archive = new TarArchiveWriter(m_saveStream); - // Write out control file - archive.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, Create0p2ControlFile()); + m_archiveWriter.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, Create0p2ControlFile()); m_log.InfoFormat("[ARCHIVER]: Added control file to archive."); // Write out region settings string settingsPath = String.Format("{0}{1}.xml", ArchiveConstants.SETTINGS_PATH, m_scene.RegionInfo.RegionName); - archive.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings)); + m_archiveWriter.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings)); m_log.InfoFormat("[ARCHIVER]: Added region settings to archive."); @@ -110,7 +109,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver MemoryStream ms = new MemoryStream(); m_terrainModule.SaveToStream(terrainPath, ms); - archive.WriteFile(terrainPath, ms.ToArray()); + m_archiveWriter.WriteFile(terrainPath, ms.ToArray()); ms.Close(); m_log.InfoFormat("[ARCHIVER]: Added terrain information to archive."); @@ -130,16 +129,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver Math.Round(position.X), Math.Round(position.Y), Math.Round(position.Z), sceneObject.UUID); - archive.WriteFile(filename, serializedObject); + m_archiveWriter.WriteFile(filename, serializedObject); } m_log.InfoFormat("[ARCHIVER]: Added scene objects to archive."); - // Write out assets - AssetsArchiver assetsArchiver = new AssetsArchiver(assetsFound); - assetsArchiver.Archive(archive); - - archive.Close(); + m_archiveWriter.Close(); m_log.InfoFormat("[ARCHIVER]: Wrote out OpenSimulator archive for {0}", m_scene.RegionInfo.RegionName); diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs index a6ad24c..0a882eb 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs @@ -35,6 +35,7 @@ using System.Threading; using log4net; using OpenMetaverse; using OpenSim.Framework; +using OpenSim.Framework.Serialization; using OpenSim.Region.CoreModules.World.Terrain; using OpenSim.Region.Framework.Interfaces; using OpenSim.Region.Framework.Scenes; @@ -126,6 +127,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver if (regionSettings.TerrainTexture4 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_4) assetUuids[regionSettings.TerrainTexture4] = 1; + TarArchiveWriter archiveWriter = new TarArchiveWriter(m_saveStream); + // Asynchronously request all the assets required to perform this archive operation ArchiveWriteRequestExecution awre = new ArchiveWriteRequestExecution( @@ -133,10 +136,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver m_scene.RequestModuleInterface(), m_scene.RequestModuleInterface(), m_scene, - m_saveStream, - m_requestId); + archiveWriter, + m_requestId); - new AssetsRequest(assetUuids.Keys, m_scene.CommsManager.AssetCache, awre.ReceivedAllAssets).Execute(); + new AssetsRequest( + new AssetsArchiver(archiveWriter), assetUuids.Keys, + m_scene.CommsManager.AssetCache, awre.ReceivedAllAssets).Execute(); } } } diff --git a/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs b/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs index 29d7c5e..0c01cae 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs @@ -46,114 +46,106 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// Post a message to the log every x assets as a progress bar /// - private static int LOG_ASSET_LOAD_NOTIFICATION_INTERVAL = 50; + protected static int LOG_ASSET_LOAD_NOTIFICATION_INTERVAL = 50; - /// - /// Archive assets - /// - protected IDictionary m_assets; + /// + /// Keep a count of the number of assets written so that we can provide status updates + /// + protected int m_assetsWritten; + + protected TarArchiveWriter m_archiveWriter; - public AssetsArchiver(IDictionary assets) + public AssetsArchiver(TarArchiveWriter archiveWriter) { - m_assets = assets; + m_archiveWriter = archiveWriter; } /// /// Archive the assets given to this archiver to the given archive. /// /// - public void Archive(TarArchiveWriter archive) - { + public void WriteAsset(AssetBase asset) + { //WriteMetadata(archive); - WriteData(archive); + WriteData(asset); } /// /// Write an assets metadata file to the given archive /// /// - protected void WriteMetadata(TarArchiveWriter archive) - { - StringWriter sw = new StringWriter(); - XmlTextWriter xtw = new XmlTextWriter(sw); - - xtw.Formatting = Formatting.Indented; - xtw.WriteStartDocument(); - - xtw.WriteStartElement("assets"); - - foreach (UUID uuid in m_assets.Keys) - { - AssetBase asset = m_assets[uuid]; - - if (asset != null) - { - xtw.WriteStartElement("asset"); - - string extension = string.Empty; - - if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type)) - { - extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type]; - } - - xtw.WriteElementString("filename", uuid.ToString() + extension); - - xtw.WriteElementString("name", asset.Name); - xtw.WriteElementString("description", asset.Description); - xtw.WriteElementString("asset-type", asset.Type.ToString()); - - xtw.WriteEndElement(); - } - } - - xtw.WriteEndElement(); - - xtw.WriteEndDocument(); - - archive.WriteFile("assets.xml", sw.ToString()); - } +// protected void WriteMetadata(TarArchiveWriter archive) +// { +// StringWriter sw = new StringWriter(); +// XmlTextWriter xtw = new XmlTextWriter(sw); +// +// xtw.Formatting = Formatting.Indented; +// xtw.WriteStartDocument(); +// +// xtw.WriteStartElement("assets"); +// +// foreach (UUID uuid in m_assets.Keys) +// { +// AssetBase asset = m_assets[uuid]; +// +// if (asset != null) +// { +// xtw.WriteStartElement("asset"); +// +// string extension = string.Empty; +// +// if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type)) +// { +// extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type]; +// } +// +// xtw.WriteElementString("filename", uuid.ToString() + extension); +// +// xtw.WriteElementString("name", asset.Name); +// xtw.WriteElementString("description", asset.Description); +// xtw.WriteElementString("asset-type", asset.Type.ToString()); +// +// xtw.WriteEndElement(); +// } +// } +// +// xtw.WriteEndElement(); +// +// xtw.WriteEndDocument(); +// +// archive.WriteFile("assets.xml", sw.ToString()); +// } /// /// Write asset data files to the given archive /// - /// - protected void WriteData(TarArchiveWriter archive) + /// + protected void WriteData(AssetBase asset) { // It appears that gtar, at least, doesn't need the intermediate directory entries in the tar //archive.AddDir("assets"); - int assetsAdded = 0; + string extension = string.Empty; - foreach (UUID uuid in m_assets.Keys) + if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type)) { - AssetBase asset = m_assets[uuid]; - - string extension = string.Empty; - - if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type)) - { - extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type]; - } - else - { - m_log.ErrorFormat( - "[ARCHIVER]: Unrecognized asset type {0} with uuid {1}. This asset will be saved but not reloaded", - asset.Type, asset.ID); - } - - archive.WriteFile( - ArchiveConstants.ASSETS_PATH + uuid.ToString() + extension, - asset.Data); + extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type]; + } + else + { + m_log.ErrorFormat( + "[ARCHIVER]: Unrecognized asset type {0} with uuid {1}. This asset will be saved but not reloaded", + asset.Type, asset.ID); + } - assetsAdded++; + m_archiveWriter.WriteFile( + ArchiveConstants.ASSETS_PATH + asset.FullID.ToString() + extension, + asset.Data); - if (assetsAdded % LOG_ASSET_LOAD_NOTIFICATION_INTERVAL == 0) - m_log.InfoFormat("[ARCHIVER]: Added {0} assets to archive", assetsAdded); - } + m_assetsWritten++; - if (assetsAdded % LOG_ASSET_LOAD_NOTIFICATION_INTERVAL != 0) - m_log.InfoFormat("[ARCHIVER]: Added {0} assets to archive", assetsAdded); + if (m_assetsWritten % LOG_ASSET_LOAD_NOTIFICATION_INTERVAL == 0) + m_log.InfoFormat("[ARCHIVER]: Added {0} assets to archive", m_assetsWritten); } } } diff --git a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs index 0a0bb4c..d806a9c 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs @@ -32,6 +32,7 @@ using System.Threading; using log4net; using OpenMetaverse; using OpenSim.Framework; +using OpenSim.Framework.Serialization; namespace OpenSim.Region.CoreModules.World.Archiver { @@ -42,39 +43,43 @@ namespace OpenSim.Region.CoreModules.World.Archiver { private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - /// + /// /// uuids to request - /// + /// protected ICollection m_uuids; - /// + /// /// Callback used when all the assets requested have been received. - /// + /// protected AssetsRequestCallback m_assetsRequestCallback; - /// - /// Assets retrieved in this request - /// - protected Dictionary m_assets = new Dictionary(); - - /// + /// + /// List of assets that were found. This will be passed back to the requester. + /// + protected List m_foundAssetUuids = new List(); + + /// /// Maintain a list of assets that could not be found. This will be passed back to the requester. - /// + /// protected List m_notFoundAssetUuids = new List(); - /// + /// /// Record the number of asset replies required so we know when we've finished - /// + /// private int m_repliesRequired; - /// + /// /// Asset cache used to request the assets - /// + /// protected IAssetCache m_assetCache; + protected AssetsArchiver m_assetsArchiver; + protected internal AssetsRequest( - ICollection uuids, IAssetCache assetCache, AssetsRequestCallback assetsRequestCallback) + AssetsArchiver assetsArchiver, ICollection uuids, + IAssetCache assetCache, AssetsRequestCallback assetsRequestCallback) { + m_assetsArchiver = assetsArchiver; m_uuids = uuids; m_assetsRequestCallback = assetsRequestCallback; m_assetCache = assetCache; @@ -87,7 +92,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver // We can stop here if there are no assets to fetch if (m_repliesRequired == 0) - m_assetsRequestCallback(m_assets, m_notFoundAssetUuids); + m_assetsRequestCallback(m_foundAssetUuids, m_notFoundAssetUuids); foreach (UUID uuid in m_uuids) { @@ -106,21 +111,25 @@ namespace OpenSim.Region.CoreModules.World.Archiver if (asset != null) { + // Make sure that we don't run out of memory by hogging assets in the cache m_assetCache.ExpireAsset(assetID); - m_assets[assetID] = asset; + + m_foundAssetUuids.Add(assetID); + m_assetsArchiver.WriteAsset(asset); } else { m_notFoundAssetUuids.Add(assetID); } - if (m_assets.Count + m_notFoundAssetUuids.Count == m_repliesRequired) + if (m_foundAssetUuids.Count + m_notFoundAssetUuids.Count == m_repliesRequired) { m_log.DebugFormat( "[ARCHIVER]: Successfully received {0} assets and notification of {1} missing assets", - m_assets.Count, m_notFoundAssetUuids.Count); + m_foundAssetUuids.Count, m_notFoundAssetUuids.Count); - // We want to stop using the asset cache thread asap as we now need to do the actual work of producing the archive + // We want to stop using the asset cache thread asap + // as we now need to do the work of producing the rest of the archive Thread newThread = new Thread(PerformAssetsRequestCallback); newThread.Name = "OpenSimulator archiving thread post assets receipt"; newThread.Start(); @@ -134,7 +143,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver { try { - m_assetsRequestCallback(m_assets, m_notFoundAssetUuids); + m_assetsRequestCallback(m_foundAssetUuids, m_notFoundAssetUuids); } catch (Exception e) { -- cgit v1.1