From 85774de2310141f4311bc3df1946d44df9ddde59 Mon Sep 17 00:00:00 2001 From: Justin Clarke Casey Date: Fri, 6 Mar 2009 20:12:08 +0000 Subject: * Improve memory usage when writing OARs * This should make saving large OARs a somewhat better experience * However, the problem where saving an archive pulls large numbers of assets into the asset cache isn't yet resolved * This patch also removes lots of archive writing spam that crept in --- .../Archiver/InventoryArchiveWriteRequest.cs | 12 +-- .../World/Archiver/ArchiveWriteRequestExecution.cs | 12 +-- .../CoreModules/World/Archiver/AssetsArchiver.cs | 4 +- .../CoreModules/World/Archiver/TarArchiveReader.cs | 10 +-- .../CoreModules/World/Archiver/TarArchiveWriter.cs | 98 ++++++++++------------ .../World/Archiver/Tests/ArchiverTests.cs | 8 +- 6 files changed, 68 insertions(+), 76 deletions(-) (limited to 'OpenSim/Region/CoreModules') diff --git a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs index 357ed40..b0f2742 100644 --- a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs +++ b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs @@ -45,7 +45,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver { private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - protected TarArchiveWriter archive = new TarArchiveWriter(); + protected TarArchiveWriter m_archive; protected UuidGatherer m_assetGatherer; protected Dictionary assetUuids = new Dictionary(); @@ -87,14 +87,14 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver protected void ReceivedAllAssets(IDictionary assetsFound, ICollection assetsNotFoundUuids) { AssetsArchiver assetsArchiver = new AssetsArchiver(assetsFound); - assetsArchiver.Archive(archive); + assetsArchiver.Archive(m_archive); Exception reportedException = null; bool succeeded = true; try { - archive.WriteTar(m_saveStream); + m_archive.Close(); } catch (IOException e) { @@ -172,7 +172,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver writer.WriteEndElement(); - archive.AddFile(filename, sw.ToString()); + m_archive.WriteFile(filename, sw.ToString()); m_assetGatherer.GatherAssetUuids(inventoryItem.AssetID, (AssetType)inventoryItem.AssetType, assetUuids); } @@ -185,7 +185,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver inventoryFolder.Name, InventoryArchiveConstants.INVENTORY_NODE_NAME_COMPONENT_SEPARATOR, inventoryFolder.ID); - archive.AddDir(path); + m_archive.WriteDir(path); List childFolders = inventoryFolder.RequestListOfFolderImpls(); List items = inventoryFolder.RequestListOfItems(); @@ -279,6 +279,8 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver { inventoryItem = m_userInfo.RootFolder.FindItemByPath(m_invPath); } + + m_archive = new TarArchiveWriter(m_saveStream); if (null == inventoryFolder) { diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs index b5aa0f3..cc4eadb 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs @@ -85,17 +85,17 @@ namespace OpenSim.Region.CoreModules.World.Archiver m_log.InfoFormat("[ARCHIVER]: Creating archive file. This may take some time."); - TarArchiveWriter archive = new TarArchiveWriter(); + TarArchiveWriter archive = new TarArchiveWriter(m_saveStream); // Write out control file - archive.AddFile(ArchiveConstants.CONTROL_FILE_PATH, Create0p2ControlFile()); + archive.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, Create0p2ControlFile()); m_log.InfoFormat("[ARCHIVER]: Added control file to archive."); // Write out region settings string settingsPath = String.Format("{0}{1}.xml", ArchiveConstants.SETTINGS_PATH, m_scene.RegionInfo.RegionName); - archive.AddFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings)); + archive.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings)); m_log.InfoFormat("[ARCHIVER]: Added region settings to archive."); @@ -105,7 +105,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver MemoryStream ms = new MemoryStream(); m_terrainModule.SaveToStream(terrainPath, ms); - archive.AddFile(terrainPath, ms.ToArray()); + archive.WriteFile(terrainPath, ms.ToArray()); ms.Close(); m_log.InfoFormat("[ARCHIVER]: Added terrain information to archive."); @@ -125,7 +125,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver Math.Round(position.X), Math.Round(position.Y), Math.Round(position.Z), sceneObject.UUID); - archive.AddFile(filename, serializedObject); + archive.WriteFile(filename, serializedObject); } m_log.InfoFormat("[ARCHIVER]: Added scene objects to archive."); @@ -134,7 +134,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver AssetsArchiver assetsArchiver = new AssetsArchiver(assetsFound); assetsArchiver.Archive(archive); - archive.WriteTar(m_saveStream); + archive.Close(); m_log.InfoFormat("[ARCHIVER]: Wrote out OpenSimulator archive for {0}", m_scene.RegionInfo.RegionName); diff --git a/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs b/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs index e863188..8debfcf 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs @@ -110,7 +110,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver xtw.WriteEndDocument(); - archive.AddFile("assets.xml", sw.ToString()); + archive.WriteFile("assets.xml", sw.ToString()); } /// @@ -141,7 +141,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver asset.Type, asset.ID); } - archive.AddFile( + archive.WriteFile( ArchiveConstants.ASSETS_PATH + uuid.ToString() + extension, asset.Data); diff --git a/OpenSim/Region/CoreModules/World/Archiver/TarArchiveReader.cs b/OpenSim/Region/CoreModules/World/Archiver/TarArchiveReader.cs index 19d2629..070f597 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/TarArchiveReader.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/TarArchiveReader.cs @@ -38,7 +38,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// public class TarArchiveReader { - private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + //private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); public enum TarEntryType { @@ -113,14 +113,14 @@ namespace OpenSim.Region.CoreModules.World.Archiver { int longNameLength = ConvertOctalBytesToDecimal(header, 124, 11); tarHeader.FilePath = m_asciiEncoding.GetString(ReadData(longNameLength)); - m_log.DebugFormat("[TAR ARCHIVE READER]: Got long file name {0}", tarHeader.FilePath); + //m_log.DebugFormat("[TAR ARCHIVE READER]: Got long file name {0}", tarHeader.FilePath); header = m_br.ReadBytes(512); } else { tarHeader.FilePath = m_asciiEncoding.GetString(header, 0, 100); tarHeader.FilePath = tarHeader.FilePath.Trim(m_nullCharArray); - m_log.DebugFormat("[TAR ARCHIVE READER]: Got short file name {0}", tarHeader.FilePath); + //m_log.DebugFormat("[TAR ARCHIVE READER]: Got short file name {0}", tarHeader.FilePath); } tarHeader.FileSize = ConvertOctalBytesToDecimal(header, 124, 11); @@ -168,14 +168,14 @@ namespace OpenSim.Region.CoreModules.World.Archiver { byte[] data = m_br.ReadBytes(fileSize); - m_log.DebugFormat("[TAR ARCHIVE READER]: fileSize {0}", fileSize); + //m_log.DebugFormat("[TAR ARCHIVE READER]: fileSize {0}", fileSize); // Read the rest of the empty padding in the 512 byte block if (fileSize % 512 != 0) { int paddingLeft = 512 - (fileSize % 512); - m_log.DebugFormat("[TAR ARCHIVE READER]: Reading {0} padding bytes", paddingLeft); + //m_log.DebugFormat("[TAR ARCHIVE READER]: Reading {0} padding bytes", paddingLeft); m_br.ReadBytes(paddingLeft); } diff --git a/OpenSim/Region/CoreModules/World/Archiver/TarArchiveWriter.cs b/OpenSim/Region/CoreModules/World/Archiver/TarArchiveWriter.cs index 09ea7ec..506fcc5 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/TarArchiveWriter.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/TarArchiveWriter.cs @@ -39,65 +39,80 @@ namespace OpenSim.Region.CoreModules.World.Archiver { //private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - protected Dictionary m_files = new Dictionary(); - protected static ASCIIEncoding m_asciiEncoding = new ASCIIEncoding(); /// - /// Add a directory to the tar archive. We can only handle one path level right now! + /// Binary writer for the underlying stream + /// + protected BinaryWriter m_bw; + + public TarArchiveWriter(Stream s) + { + m_bw = new BinaryWriter(s); + } + + /// + /// Write a directory entry to the tar archive. We can only handle one path level right now! /// /// - public void AddDir(string dirName) + public void WriteDir(string dirName) { // Directories are signalled by a final / if (!dirName.EndsWith("/")) dirName += "/"; - AddFile(dirName, new byte[0]); + WriteFile(dirName, new byte[0]); } - + /// - /// Add a file to the tar archive + /// Write a file to the tar archive /// /// /// - public void AddFile(string filePath, string data) + public void WriteFile(string filePath, string data) { - AddFile(filePath, m_asciiEncoding.GetBytes(data)); + WriteFile(filePath, m_asciiEncoding.GetBytes(data)); } /// - /// Add a file to the tar archive + /// Write a file to the tar archive /// /// /// - public void AddFile(string filePath, byte[] data) + public void WriteFile(string filePath, byte[] data) { - m_files[filePath] = data; + if (filePath.Length > 100) + WriteEntry("././@LongLink", m_asciiEncoding.GetBytes(filePath), 'L'); + + char fileType; + + if (filePath.EndsWith("/")) + { + fileType = '5'; + } + else + { + fileType = '0'; + } + + WriteEntry(filePath, data, fileType); } /// - /// Write the raw tar archive data to a stream. The stream will be closed on completion. + /// Finish writing the raw tar archive data to a stream. The stream will be closed on completion. /// /// Stream to which to write the data /// - public void WriteTar(Stream s) + public void Close() { - BinaryWriter bw = new BinaryWriter(s); - - foreach (string filePath in m_files.Keys) - { - WriteFile(bw, filePath, m_files[filePath]); - } - //m_log.Debug("[TAR ARCHIVE WRITER]: Writing final consecutive 0 blocks"); // Write two consecutive 0 blocks to end the archive byte[] finalZeroPadding = new byte[1024]; - bw.Write(finalZeroPadding); + m_bw.Write(finalZeroPadding); - bw.Flush(); - bw.Close(); + m_bw.Flush(); + m_bw.Close(); } public static byte[] ConvertDecimalToPaddedOctalBytes(int d, int padding) @@ -119,39 +134,14 @@ namespace OpenSim.Region.CoreModules.World.Archiver return oBytes; } - - /// - /// Write a particular file of data - /// - /// - /// - protected void WriteFile(BinaryWriter bw, string filePath, byte[] data) - { - if (filePath.Length > 100) - WriteEntry(bw, "././@LongLink", m_asciiEncoding.GetBytes(filePath), 'L'); - - char fileType; - - if (filePath.EndsWith("/")) - { - fileType = '5'; - } - else - { - fileType = '0'; - } - - WriteEntry(bw, filePath, data, fileType); - } /// - /// Write a particular file of data + /// Write a particular entry /// - /// /// /// /// - protected void WriteEntry(BinaryWriter bw, string filePath, byte[] data, char fileType) + protected void WriteEntry(string filePath, byte[] data, char fileType) { byte[] header = new byte[512]; @@ -208,10 +198,10 @@ namespace OpenSim.Region.CoreModules.World.Archiver header[154] = 0; // Write out header - bw.Write(header); + m_bw.Write(header); // Write out data - bw.Write(data); + m_bw.Write(data); if (data.Length % 512 != 0) { @@ -220,7 +210,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver //m_log.DebugFormat("[TAR ARCHIVE WRITER]: Padding data with {0} bytes", paddingRequired); byte[] padding = new byte[paddingRequired]; - bw.Write(padding); + m_bw.Write(padding); } } } diff --git a/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs b/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs index e6146cf..d4c138a 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs @@ -173,9 +173,9 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests //log4net.Config.XmlConfigurator.Configure(); MemoryStream archiveWriteStream = new MemoryStream(); - TarArchiveWriter tar = new TarArchiveWriter(); + TarArchiveWriter tar = new TarArchiveWriter(archiveWriteStream); - tar.AddFile(ArchiveConstants.CONTROL_FILE_PATH, ArchiveWriteRequestExecution.Create0p2ControlFile()); + tar.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, ArchiveWriteRequestExecution.Create0p2ControlFile()); string part1Name = "object1"; PrimitiveBaseShape shape = PrimitiveBaseShape.CreateCylinder(); @@ -194,9 +194,9 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests part1Name, Math.Round(groupPosition.X), Math.Round(groupPosition.Y), Math.Round(groupPosition.Z), part1.UUID); - tar.AddFile(ArchiveConstants.OBJECTS_PATH + object1FileName, object1.ToXmlString2()); + tar.WriteFile(ArchiveConstants.OBJECTS_PATH + object1FileName, object1.ToXmlString2()); - tar.WriteTar(archiveWriteStream); + tar.Close(); MemoryStream archiveReadStream = new MemoryStream(archiveWriteStream.ToArray()); -- cgit v1.1