From ce468215d576cc301a261d85bee9baa68a246ce6 Mon Sep 17 00:00:00 2001 From: Oren Hurvitz Date: Tue, 24 Jul 2012 19:48:08 +0300 Subject: Support multi-region OAR files Merged ArchiveWriteRequestPreparation.cs and ArchiveWriteRequestExecution.cs. This simplifies the code, and it's faster to write each scene to the archive as it's found rather than all at once at the end. --- .../Archiver/ArchiveWriteRequestPreparation.cs | 492 ++++++++++++++------- 1 file changed, 344 insertions(+), 148 deletions(-) (limited to 'OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs') diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs index 4edaaca..2c34f4b 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs @@ -43,6 +43,7 @@ using OpenSim.Region.Framework.Scenes; using Ionic.Zlib; using GZipStream = Ionic.Zlib.GZipStream; using CompressionMode = Ionic.Zlib.CompressionMode; +using OpenSim.Framework.Serialization.External; namespace OpenSim.Region.CoreModules.World.Archiver { @@ -61,17 +62,29 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// The maximum major version of OAR that we can write. /// - public static int MAX_MAJOR_VERSION = 0; + public static int MAX_MAJOR_VERSION = 1; + + /// + /// Whether we're saving a multi-region archive. + /// + public bool MultiRegionFormat { get; set; } /// /// Determine whether this archive will save assets. Default is true. /// public bool SaveAssets { get; set; } - protected ArchiverModule m_module; - protected Scene m_scene; + /// + /// Determines which objects will be included in the archive, according to their permissions. + /// Default is null, meaning no permission checks. + /// + public string CheckPermissions { get; set; } + + protected Scene m_rootScene; protected Stream m_saveStream; + protected TarArchiveWriter m_archiveWriter; protected Guid m_requestId; + protected Dictionary m_options; /// /// Constructor @@ -82,7 +95,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// If there was a problem opening a stream for the file specified by the savePath /// - public ArchiveWriteRequestPreparation(ArchiverModule module, string savePath, Guid requestId) : this(module, requestId) + public ArchiveWriteRequestPreparation(Scene scene, string savePath, Guid requestId) : this(scene, requestId) { try { @@ -100,26 +113,23 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// Constructor. /// - /// Calling module + /// The root scene to archive /// The stream to which to save data. /// The id associated with this request - public ArchiveWriteRequestPreparation(ArchiverModule module, Stream saveStream, Guid requestId) : this(module, requestId) + public ArchiveWriteRequestPreparation(Scene scene, Stream saveStream, Guid requestId) : this(scene, requestId) { m_saveStream = saveStream; } - protected ArchiveWriteRequestPreparation(ArchiverModule module, Guid requestId) + protected ArchiveWriteRequestPreparation(Scene scene, Guid requestId) { - m_module = module; - - // FIXME: This is only here for regression test purposes since they do not supply a module. Need to fix - // this. - if (m_module != null) - m_scene = m_module.Scene; - + m_rootScene = scene; m_requestId = requestId; + m_archiveWriter = null; + MultiRegionFormat = false; SaveAssets = true; + CheckPermissions = null; } /// @@ -128,126 +138,157 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// if there was an io problem with creating the file public void ArchiveRegion(Dictionary options) { + m_options = options; + + if (options.ContainsKey("all") && (bool)options["all"]) + MultiRegionFormat = true; + if (options.ContainsKey("noassets") && (bool)options["noassets"]) SaveAssets = false; + Object temp; + if (options.TryGetValue("checkPermissions", out temp)) + CheckPermissions = (string)temp; + + + // Find the regions to archive + ArchiveScenesGroup scenesGroup = new ArchiveScenesGroup(); + if (MultiRegionFormat) + { + m_log.InfoFormat("[ARCHIVER]: Saving {0} regions", SceneManager.Instance.Scenes.Count); + SceneManager.Instance.ForEachScene(delegate(Scene scene) + { + scenesGroup.AddScene(scene); + }); + } + else + { + scenesGroup.AddScene(m_rootScene); + } + scenesGroup.CalcSceneLocations(); + + + m_archiveWriter = new TarArchiveWriter(m_saveStream); + try { + // Write out control file. It should be first so that it will be found ASAP when loading the file. + m_archiveWriter.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, CreateControlFile(scenesGroup)); + m_log.InfoFormat("[ARCHIVER]: Added control file to archive."); + + // Archive the regions + Dictionary assetUuids = new Dictionary(); - - EntityBase[] entities = m_scene.GetEntities(); - List sceneObjects = new List(); - - string checkPermissions = null; - int numObjectsSkippedPermissions = 0; - Object temp; - if (options.TryGetValue("checkPermissions", out temp)) - checkPermissions = (string)temp; - - // Filter entities so that we only have scene objects. - // FIXME: Would be nicer to have this as a proper list in SceneGraph, since lots of methods - // end up having to do this - foreach (EntityBase entity in entities) + + scenesGroup.ForEachScene(delegate(Scene scene) { - if (entity is SceneObjectGroup) - { - SceneObjectGroup sceneObject = (SceneObjectGroup)entity; + string regionDir = MultiRegionFormat ? scenesGroup.GetRegionDir(scene.RegionInfo.RegionID) : ""; + ArchiveOneRegion(scene, regionDir, assetUuids); + }); - if (!sceneObject.IsDeleted && !sceneObject.IsAttachment) - { - if (!CanUserArchiveObject(m_scene.RegionInfo.EstateSettings.EstateOwner, sceneObject, checkPermissions)) - { - // The user isn't allowed to copy/transfer this object, so it will not be included in the OAR. - ++numObjectsSkippedPermissions; - } - else - { - sceneObjects.Add(sceneObject); - } - } - } - } + // Archive the assets if (SaveAssets) { - UuidGatherer assetGatherer = new UuidGatherer(m_scene.AssetService); + m_log.DebugFormat("[ARCHIVER]: Saving {0} assets", assetUuids.Count); - foreach (SceneObjectGroup sceneObject in sceneObjects) - { - assetGatherer.GatherAssetUuids(sceneObject, assetUuids); - } + // Asynchronously request all the assets required to perform this archive operation + AssetsRequest ar + = new AssetsRequest( + new AssetsArchiver(m_archiveWriter), assetUuids, + m_rootScene.AssetService, m_rootScene.UserAccountService, + m_rootScene.RegionInfo.ScopeID, options, ReceivedAllAssets); - m_log.DebugFormat( - "[ARCHIVER]: {0} scene objects to serialize requiring save of {1} assets", - sceneObjects.Count, assetUuids.Count); + Util.FireAndForget(o => ar.Execute()); + + // CloseArchive() will be called from ReceivedAllAssets() } else { m_log.DebugFormat("[ARCHIVER]: Not saving assets since --noassets was specified"); + CloseArchive(string.Empty); } + } + catch (Exception e) + { + CloseArchive(e.Message); + throw; + } + } - if (numObjectsSkippedPermissions > 0) - { - m_log.DebugFormat( - "[ARCHIVER]: {0} scene objects skipped due to lack of permissions", - numObjectsSkippedPermissions); - } - // Make sure that we also request terrain texture assets - RegionSettings regionSettings = m_scene.RegionInfo.RegionSettings; - - if (regionSettings.TerrainTexture1 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_1) - assetUuids[regionSettings.TerrainTexture1] = AssetType.Texture; - - if (regionSettings.TerrainTexture2 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_2) - assetUuids[regionSettings.TerrainTexture2] = AssetType.Texture; - - if (regionSettings.TerrainTexture3 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_3) - assetUuids[regionSettings.TerrainTexture3] = AssetType.Texture; - - if (regionSettings.TerrainTexture4 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_4) - assetUuids[regionSettings.TerrainTexture4] = AssetType.Texture; - - TarArchiveWriter archiveWriter = new TarArchiveWriter(m_saveStream); - - // Asynchronously request all the assets required to perform this archive operation - ArchiveWriteRequestExecution awre - = new ArchiveWriteRequestExecution( - sceneObjects, - m_scene.RequestModuleInterface(), - m_scene.RequestModuleInterface(), - m_scene, - archiveWriter, - m_requestId, - options); - - m_log.InfoFormat("[ARCHIVER]: Creating archive file. This may take some time."); - - // Write out control file. This has to be done first so that subsequent loaders will see this file first - // XXX: I know this is a weak way of doing it since external non-OAR aware tar executables will not do this - archiveWriter.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, CreateControlFile(options)); - m_log.InfoFormat("[ARCHIVER]: Added control file to archive."); + private void ArchiveOneRegion(Scene scene, string regionDir, Dictionary assetUuids) + { + m_log.InfoFormat("[ARCHIVER]: Writing region {0}", scene.RegionInfo.RegionName); - if (SaveAssets) + EntityBase[] entities = scene.GetEntities(); + List sceneObjects = new List(); + + int numObjectsSkippedPermissions = 0; + + // Filter entities so that we only have scene objects. + // FIXME: Would be nicer to have this as a proper list in SceneGraph, since lots of methods + // end up having to do this + IPermissionsModule permissionsModule = scene.RequestModuleInterface(); + foreach (EntityBase entity in entities) + { + if (entity is SceneObjectGroup) { - AssetsRequest ar - = new AssetsRequest( - new AssetsArchiver(archiveWriter), assetUuids, - m_scene.AssetService, m_scene.UserAccountService, - m_scene.RegionInfo.ScopeID, options, awre.ReceivedAllAssets); + SceneObjectGroup sceneObject = (SceneObjectGroup)entity; - Util.FireAndForget(o => ar.Execute()); + if (!sceneObject.IsDeleted && !sceneObject.IsAttachment) + { + if (!CanUserArchiveObject(scene.RegionInfo.EstateSettings.EstateOwner, sceneObject, CheckPermissions, permissionsModule)) + { + // The user isn't allowed to copy/transfer this object, so it will not be included in the OAR. + ++numObjectsSkippedPermissions; + } + else + { + sceneObjects.Add(sceneObject); + } + } } - else + } + + if (SaveAssets) + { + UuidGatherer assetGatherer = new UuidGatherer(scene.AssetService); + int prevAssets = assetUuids.Count; + + foreach (SceneObjectGroup sceneObject in sceneObjects) { - awre.ReceivedAllAssets(new List(), new List()); + assetGatherer.GatherAssetUuids(sceneObject, assetUuids); } + + m_log.DebugFormat( + "[ARCHIVER]: {0} scene objects to serialize requiring save of {1} assets", + sceneObjects.Count, assetUuids.Count - prevAssets); } - catch (Exception) + + if (numObjectsSkippedPermissions > 0) { - m_saveStream.Close(); - throw; - } + m_log.DebugFormat( + "[ARCHIVER]: {0} scene objects skipped due to lack of permissions", + numObjectsSkippedPermissions); + } + + // Make sure that we also request terrain texture assets + RegionSettings regionSettings = scene.RegionInfo.RegionSettings; + + if (regionSettings.TerrainTexture1 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_1) + assetUuids[regionSettings.TerrainTexture1] = AssetType.Texture; + + if (regionSettings.TerrainTexture2 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_2) + assetUuids[regionSettings.TerrainTexture2] = AssetType.Texture; + + if (regionSettings.TerrainTexture3 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_3) + assetUuids[regionSettings.TerrainTexture3] = AssetType.Texture; + + if (regionSettings.TerrainTexture4 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_4) + assetUuids[regionSettings.TerrainTexture4] = AssetType.Texture; + + Save(scene, sceneObjects, regionDir); } /// @@ -256,14 +297,14 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// The user /// The object group /// Which permissions to check: "C" = Copy, "T" = Transfer + /// The scene's permissions module /// Whether the user is allowed to export the object to an OAR - private bool CanUserArchiveObject(UUID user, SceneObjectGroup objGroup, string checkPermissions) + private bool CanUserArchiveObject(UUID user, SceneObjectGroup objGroup, string checkPermissions, IPermissionsModule permissionsModule) { if (checkPermissions == null) return true; - IPermissionsModule module = m_scene.RequestModuleInterface(); - if (module == null) + if (permissionsModule == null) return true; // this shouldn't happen // Check whether the user is permitted to export all of the parts in the SOG. If any @@ -275,7 +316,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver foreach (SceneObjectPart obj in objGroup.Parts) { uint perm; - PermissionClass permissionClass = module.GetPermissionClass(user, obj); + PermissionClass permissionClass = permissionsModule.GetPermissionClass(user, obj); switch (permissionClass) { case PermissionClass.Owner: @@ -330,16 +371,30 @@ namespace OpenSim.Region.CoreModules.World.Archiver } /// - /// Create the control file for the most up to date archive + /// Create the control file. /// /// - public string CreateControlFile(Dictionary options) + public string CreateControlFile(ArchiveScenesGroup scenesGroup) { - int majorVersion = MAX_MAJOR_VERSION, minorVersion = 8; + int majorVersion; + int minorVersion; + + if (MultiRegionFormat) + { + majorVersion = MAX_MAJOR_VERSION; + minorVersion = 0; + } + else + { + // To support older versions of OpenSim, we continue to create single-region OARs + // using the old file format. In the future this format will be discontinued. + majorVersion = 0; + minorVersion = 8; + } // -// if (options.ContainsKey("version")) +// if (m_options.ContainsKey("version")) // { -// string[] parts = options["version"].ToString().Split('.'); +// string[] parts = m_options["version"].ToString().Split('.'); // if (parts.Length >= 1) // { // majorVersion = Int32.Parse(parts[0]); @@ -368,10 +423,10 @@ namespace OpenSim.Region.CoreModules.World.Archiver // } m_log.InfoFormat("[ARCHIVER]: Creating version {0}.{1} OAR", majorVersion, minorVersion); - //if (majorVersion == 1) - //{ - // m_log.WarnFormat("[ARCHIVER]: Please be aware that version 1.0 OARs are not compatible with OpenSim 0.7.0.2 and earlier. Please use the --version=0 option if you want to produce a compatible OAR"); - //} + if (majorVersion == 1) + { + m_log.WarnFormat("[ARCHIVER]: Please be aware that version 1.0 OARs are not compatible with OpenSim versions prior to 0.7.4. Do not use the --all option if you want to produce a compatible OAR"); + } String s; @@ -389,50 +444,191 @@ namespace OpenSim.Region.CoreModules.World.Archiver DateTime now = DateTime.UtcNow; TimeSpan t = now - new DateTime(1970, 1, 1); xtw.WriteElementString("datetime", ((int)t.TotalSeconds).ToString()); - xtw.WriteElementString("id", UUID.Random().ToString()); + if (!MultiRegionFormat) + xtw.WriteElementString("id", m_rootScene.RegionInfo.RegionID.ToString()); xtw.WriteEndElement(); + + xtw.WriteElementString("assets_included", SaveAssets.ToString()); - xtw.WriteStartElement("region_info"); + if (MultiRegionFormat) + { + WriteRegionsManifest(scenesGroup, xtw); + } + else + { + xtw.WriteStartElement("region_info"); + WriteRegionInfo(m_rootScene, xtw); + xtw.WriteEndElement(); + } - bool isMegaregion; - Vector2 size; - IRegionCombinerModule rcMod = null; + xtw.WriteEndElement(); + + xtw.Flush(); + } - // FIXME: This is only here for regression test purposes since they do not supply a module. Need to fix - // this, possibly by doing control file creation somewhere else. - if (m_module != null) - rcMod = m_module.RegionCombinerModule; + s = sw.ToString(); + } - if (rcMod != null) - isMegaregion = rcMod.IsRootForMegaregion(m_scene.RegionInfo.RegionID); - else - isMegaregion = false; + return s; + } - if (isMegaregion) - size = rcMod.GetSizeOfMegaregion(m_scene.RegionInfo.RegionID); - else - size = new Vector2((float)Constants.RegionSize, (float)Constants.RegionSize); + /// + /// Writes the list of regions included in a multi-region OAR. + /// + private static void WriteRegionsManifest(ArchiveScenesGroup scenesGroup, XmlTextWriter xtw) + { + xtw.WriteStartElement("regions"); - xtw.WriteElementString("is_megaregion", isMegaregion.ToString()); - xtw.WriteElementString("size_in_meters", string.Format("{0},{1}", size.X, size.Y)); + // Write the regions in order: rows from South to North, then regions from West to East. + // The list of regions can have "holes"; we write empty elements in their position. - xtw.WriteEndElement(); - - xtw.WriteElementString("assets_included", SaveAssets.ToString()); + for (uint y = (uint)scenesGroup.Rect.Top; y < scenesGroup.Rect.Bottom; ++y) + { + SortedDictionary row; + if (scenesGroup.Regions.TryGetValue(y, out row)) + { + xtw.WriteStartElement("row"); + + for (uint x = (uint)scenesGroup.Rect.Left; x < scenesGroup.Rect.Right; ++x) + { + Scene scene; + if (row.TryGetValue(x, out scene)) + { + xtw.WriteStartElement("region"); + xtw.WriteElementString("id", scene.RegionInfo.RegionID.ToString()); + xtw.WriteElementString("dir", scenesGroup.GetRegionDir(scene.RegionInfo.RegionID)); + WriteRegionInfo(scene, xtw); + xtw.WriteEndElement(); + } + else + { + // Write a placeholder for a missing region + xtw.WriteElementString("region", ""); + } + } xtw.WriteEndElement(); - - xtw.Flush(); } + else + { + // Write a placeholder for a missing row + xtw.WriteElementString("row", ""); + } + } - s = sw.ToString(); + xtw.WriteEndElement(); // "regions" + } + + protected static void WriteRegionInfo(Scene scene, XmlTextWriter xtw) + { + bool isMegaregion; + Vector2 size; + + IRegionCombinerModule rcMod = scene.RequestModuleInterface(); + + if (rcMod != null) + isMegaregion = rcMod.IsRootForMegaregion(scene.RegionInfo.RegionID); + else + isMegaregion = false; + + if (isMegaregion) + size = rcMod.GetSizeOfMegaregion(scene.RegionInfo.RegionID); + else + size = new Vector2((float)Constants.RegionSize, (float)Constants.RegionSize); + + xtw.WriteElementString("is_megaregion", isMegaregion.ToString()); + xtw.WriteElementString("size_in_meters", string.Format("{0},{1}", size.X, size.Y)); + } + + + protected void Save(Scene scene, List sceneObjects, string regionDir) + { + if (regionDir != string.Empty) + regionDir = ArchiveConstants.REGIONS_PATH + regionDir + "/"; + + m_log.InfoFormat("[ARCHIVER]: Adding region settings to archive."); + + // Write out region settings + string settingsPath = String.Format("{0}{1}{2}.xml", + regionDir, ArchiveConstants.SETTINGS_PATH, scene.RegionInfo.RegionName); + m_archiveWriter.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(scene.RegionInfo.RegionSettings)); + + m_log.InfoFormat("[ARCHIVER]: Adding parcel settings to archive."); + + // Write out land data (aka parcel) settings + List landObjects = scene.LandChannel.AllParcels(); + foreach (ILandObject lo in landObjects) + { + LandData landData = lo.LandData; + string landDataPath = String.Format("{0}{1}{2}.xml", + regionDir, ArchiveConstants.LANDDATA_PATH, landData.GlobalID.ToString()); + m_archiveWriter.WriteFile(landDataPath, LandDataSerializer.Serialize(landData, m_options)); } -// if (m_scene != null) -// Console.WriteLine( -// "[ARCHIVE WRITE REQUEST PREPARATION]: Control file for {0} is: {1}", m_scene.RegionInfo.RegionName, s); + m_log.InfoFormat("[ARCHIVER]: Adding terrain information to archive."); - return s; + // Write out terrain + string terrainPath = String.Format("{0}{1}{2}.r32", + regionDir, ArchiveConstants.TERRAINS_PATH, scene.RegionInfo.RegionName); + + MemoryStream ms = new MemoryStream(); + scene.RequestModuleInterface().SaveToStream(terrainPath, ms); + m_archiveWriter.WriteFile(terrainPath, ms.ToArray()); + ms.Close(); + + m_log.InfoFormat("[ARCHIVER]: Adding scene objects to archive."); + + // Write out scene object metadata + IRegionSerialiserModule serializer = scene.RequestModuleInterface(); + foreach (SceneObjectGroup sceneObject in sceneObjects) + { + //m_log.DebugFormat("[ARCHIVER]: Saving {0} {1}, {2}", entity.Name, entity.UUID, entity.GetType()); + + string serializedObject = serializer.SerializeGroupToXml2(sceneObject, m_options); + string objectPath = string.Format("{0}{1}", regionDir, ArchiveHelpers.CreateObjectPath(sceneObject)); + m_archiveWriter.WriteFile(objectPath, serializedObject); + } } + + protected void ReceivedAllAssets( + ICollection assetsFoundUuids, ICollection assetsNotFoundUuids) + { + foreach (UUID uuid in assetsNotFoundUuids) + { + m_log.DebugFormat("[ARCHIVER]: Could not find asset {0}", uuid); + } + + // m_log.InfoFormat( + // "[ARCHIVER]: Received {0} of {1} assets requested", + // assetsFoundUuids.Count, assetsFoundUuids.Count + assetsNotFoundUuids.Count); + + CloseArchive(String.Empty); + } + + + /// + /// Closes the archive and notifies that we're done. + /// + /// The error that occurred, or empty for success + protected void CloseArchive(string errorMessage) + { + try + { + if (m_archiveWriter != null) + m_archiveWriter.Close(); + m_saveStream.Close(); + } + catch (Exception e) + { + m_log.Error(string.Format("[ARCHIVER]: Error closing archive: {0} ", e.Message), e); + if (errorMessage == string.Empty) + errorMessage = e.Message; + } + + m_log.InfoFormat("[ARCHIVER]: Finished writing out OAR for {0}", m_rootScene.RegionInfo.RegionName); + + m_rootScene.EventManager.TriggerOarFileSaved(m_requestId, errorMessage); + } + } } -- cgit v1.1