From ce468215d576cc301a261d85bee9baa68a246ce6 Mon Sep 17 00:00:00 2001 From: Oren Hurvitz Date: Tue, 24 Jul 2012 19:48:08 +0300 Subject: Support multi-region OAR files Merged ArchiveWriteRequestPreparation.cs and ArchiveWriteRequestExecution.cs. This simplifies the code, and it's faster to write each scene to the archive as it's found rather than all at once at the end. --- .../World/Archiver/ArchiveReadRequest.cs | 365 +++++++++++---- .../World/Archiver/ArchiveScenesGroup.cs | 176 ++++++++ .../World/Archiver/ArchiveWriteRequestExecution.cs | 153 ------- .../Archiver/ArchiveWriteRequestPreparation.cs | 492 ++++++++++++++------- .../CoreModules/World/Archiver/ArchiverModule.cs | 5 +- .../CoreModules/World/Archiver/AssetsRequest.cs | 6 + .../World/Archiver/DearchiveScenesGroup.cs | 232 ++++++++++ .../World/Archiver/Tests/ArchiverTests.cs | 23 +- 8 files changed, 1057 insertions(+), 395 deletions(-) create mode 100644 OpenSim/Region/CoreModules/World/Archiver/ArchiveScenesGroup.cs delete mode 100644 OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs create mode 100644 OpenSim/Region/CoreModules/World/Archiver/DearchiveScenesGroup.cs (limited to 'OpenSim/Region/CoreModules/World') diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs index 433166d..a6923ef 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs @@ -43,6 +43,7 @@ using OpenSim.Region.Framework.Interfaces; using OpenSim.Region.Framework.Scenes; using OpenSim.Region.Framework.Scenes.Serialization; using OpenSim.Services.Interfaces; +using System.Threading; namespace OpenSim.Region.CoreModules.World.Archiver { @@ -52,7 +53,30 @@ namespace OpenSim.Region.CoreModules.World.Archiver public class ArchiveReadRequest { private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + /// + /// Contains data used while dearchiving a single scene. + /// + private class DearchiveContext + { + public Scene Scene { get; set; } + + public List SerialisedSceneObjects { get; set; } + + public List SerialisedParcels { get; set; } + + public List SceneObjects { get; set; } + + public DearchiveContext(Scene scene) + { + Scene = scene; + SerialisedSceneObjects = new List(); + SerialisedParcels = new List(); + SceneObjects = new List(); + } + } + /// /// The maximum major version of OAR that we can read. Minor versions shouldn't need a max number since version /// bumps here should be compatible. @@ -62,9 +86,10 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// Has the control file been loaded for this archive? /// - public bool ControlFileLoaded { get; private set; } + public bool ControlFileLoaded { get; private set; } - protected Scene m_scene; + protected string m_loadPath; + protected Scene m_rootScene; protected Stream m_loadStream; protected Guid m_requestId; protected string m_errorMessage; @@ -91,7 +116,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver { if (m_UserMan == null) { - m_UserMan = m_scene.RequestModuleInterface(); + m_UserMan = m_rootScene.RequestModuleInterface(); } return m_UserMan; } @@ -104,10 +129,14 @@ namespace OpenSim.Region.CoreModules.World.Archiver private IGroupsModule m_groupsModule; + private IAssetService m_assetService = null; + + public ArchiveReadRequest(Scene scene, string loadPath, bool merge, bool skipAssets, Guid requestId) { - m_scene = scene; + m_rootScene = scene; + m_loadPath = loadPath; try { m_loadStream = new GZipStream(ArchiveHelpers.GetStream(loadPath), CompressionMode.Decompress); @@ -128,12 +157,14 @@ namespace OpenSim.Region.CoreModules.World.Archiver // Zero can never be a valid user id m_validUserUuids[UUID.Zero] = false; - m_groupsModule = m_scene.RequestModuleInterface(); + m_groupsModule = m_rootScene.RequestModuleInterface(); + m_assetService = m_rootScene.AssetService; } public ArchiveReadRequest(Scene scene, Stream loadStream, bool merge, bool skipAssets, Guid requestId) { - m_scene = scene; + m_rootScene = scene; + m_loadPath = null; m_loadStream = loadStream; m_merge = merge; m_skipAssets = skipAssets; @@ -142,7 +173,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver // Zero can never be a valid user id m_validUserUuids[UUID.Zero] = false; - m_groupsModule = m_scene.RequestModuleInterface(); + m_groupsModule = m_rootScene.RequestModuleInterface(); + m_assetService = m_rootScene.AssetService; } /// @@ -150,25 +182,25 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// public void DearchiveRegion() { - // The same code can handle dearchiving 0.1 and 0.2 OpenSim Archive versions - DearchiveRegion0DotStar(); - } - - private void DearchiveRegion0DotStar() - { int successfulAssetRestores = 0; int failedAssetRestores = 0; - List serialisedSceneObjects = new List(); - List serialisedParcels = new List(); - string filePath = "NONE"; - TarArchiveReader archive = new TarArchiveReader(m_loadStream); + DearchiveScenesInfo dearchivedScenes; + + // We dearchive all the scenes at once, because the files in the TAR archive might be mixed. + // Therefore, we have to keep track of the dearchive context of all the scenes. + Dictionary sceneContexts = new Dictionary(); + + string fullPath = "NONE"; + TarArchiveReader archive = null; byte[] data; TarArchiveReader.TarEntryType entryType; - + try { - while ((data = archive.ReadEntry(out filePath, out entryType)) != null) + FindAndLoadControlFile(out archive, out dearchivedScenes); + + while ((data = archive.ReadEntry(out fullPath, out entryType)) != null) { //m_log.DebugFormat( // "[ARCHIVER]: Successfully read {0} ({1} bytes)", filePath, data.Length); @@ -176,9 +208,30 @@ namespace OpenSim.Region.CoreModules.World.Archiver if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY == entryType) continue; + + // Find the scene that this file belongs to + + Scene scene; + string filePath; + if (!dearchivedScenes.GetRegionFromPath(fullPath, out scene, out filePath)) + continue; // this file belongs to a region that we're not loading + + DearchiveContext sceneContext = null; + if (scene != null) + { + if (!sceneContexts.TryGetValue(scene.RegionInfo.RegionID, out sceneContext)) + { + sceneContext = new DearchiveContext(scene); + sceneContexts.Add(scene.RegionInfo.RegionID, sceneContext); + } + } + + + // Process the file + if (filePath.StartsWith(ArchiveConstants.OBJECTS_PATH)) { - serialisedSceneObjects.Add(Encoding.UTF8.GetString(data)); + sceneContext.SerialisedSceneObjects.Add(Encoding.UTF8.GetString(data)); } else if (filePath.StartsWith(ArchiveConstants.ASSETS_PATH) && !m_skipAssets) { @@ -192,19 +245,19 @@ namespace OpenSim.Region.CoreModules.World.Archiver } else if (!m_merge && filePath.StartsWith(ArchiveConstants.TERRAINS_PATH)) { - LoadTerrain(filePath, data); + LoadTerrain(scene, filePath, data); } else if (!m_merge && filePath.StartsWith(ArchiveConstants.SETTINGS_PATH)) { - LoadRegionSettings(filePath, data); + LoadRegionSettings(scene, filePath, data, dearchivedScenes); } else if (!m_merge && filePath.StartsWith(ArchiveConstants.LANDDATA_PATH)) { - serialisedParcels.Add(Encoding.UTF8.GetString(data)); + sceneContext.SerialisedParcels.Add(Encoding.UTF8.GetString(data)); } else if (filePath == ArchiveConstants.CONTROL_FILE_PATH) { - LoadControlFile(filePath, data); + // Ignore, because we already read the control file } } @@ -212,15 +265,16 @@ namespace OpenSim.Region.CoreModules.World.Archiver } catch (Exception e) { - m_log.ErrorFormat( - "[ARCHIVER]: Aborting load with error in archive file {0}. {1}", filePath, e); + m_log.Error( + String.Format("[ARCHIVER]: Aborting load with error in archive file {0} ", fullPath), e); m_errorMessage += e.ToString(); - m_scene.EventManager.TriggerOarFileLoaded(m_requestId, m_errorMessage); + m_rootScene.EventManager.TriggerOarFileLoaded(m_requestId, new List(), m_errorMessage); return; } finally { - archive.Close(); + if (archive != null) + archive.Close(); } if (!m_skipAssets) @@ -234,32 +288,143 @@ namespace OpenSim.Region.CoreModules.World.Archiver } } - if (!m_merge) + foreach (DearchiveContext sceneContext in sceneContexts.Values) { - m_log.Info("[ARCHIVER]: Clearing all existing scene objects"); - m_scene.DeleteAllSceneObjects(); + m_log.InfoFormat("[ARCHIVER:] Loading region {0}", sceneContext.Scene.RegionInfo.RegionName); + + if (!m_merge) + { + m_log.Info("[ARCHIVER]: Clearing all existing scene objects"); + sceneContext.Scene.DeleteAllSceneObjects(); + } + + try + { + LoadParcels(sceneContext.Scene, sceneContext.SerialisedParcels); + LoadObjects(sceneContext.Scene, sceneContext.SerialisedSceneObjects, sceneContext.SceneObjects); + + // Inform any interested parties that the region has changed. We waited until now so that all + // of the region's objects will be loaded when we send this notification. + IEstateModule estateModule = sceneContext.Scene.RequestModuleInterface(); + if (estateModule != null) + estateModule.TriggerRegionInfoChange(); + } + catch (Exception e) + { + m_log.Error("[ARCHIVER]: Error loading parcels or objects ", e); + m_errorMessage += e.ToString(); + m_rootScene.EventManager.TriggerOarFileLoaded(m_requestId, new List(), m_errorMessage); + return; + } } - LoadParcels(serialisedParcels); - LoadObjects(serialisedSceneObjects); + // Start the scripts. We delayed this because we want the OAR to finish loading ASAP, so + // that users can enter the scene. If we allow the scripts to start in the loop above + // then they significantly increase the time until the OAR finishes loading. + Util.FireAndForget(delegate(object o) + { + Thread.Sleep(15000); + m_log.Info("Starting scripts in scene objects"); + + foreach (DearchiveContext sceneContext in sceneContexts.Values) + { + foreach (SceneObjectGroup sceneObject in sceneContext.SceneObjects) + { + sceneObject.CreateScriptInstances(0, false, sceneContext.Scene.DefaultScriptEngine, 0); // StateSource.RegionStart + sceneObject.ResumeScripts(); + } + + sceneContext.SceneObjects.Clear(); + } + }); m_log.InfoFormat("[ARCHIVER]: Successfully loaded archive"); - m_scene.EventManager.TriggerOarFileLoaded(m_requestId, m_errorMessage); + m_rootScene.EventManager.TriggerOarFileLoaded(m_requestId, dearchivedScenes.GetLoadedScenes(), m_errorMessage); + } + + /// + /// Searches through the files in the archive for the control file, and reads it. + /// We must read the control file first, in order to know which regions are available. + /// + /// + /// In most cases the control file *is* first, since that's how we create archives. However, + /// it's possible that someone rewrote the archive externally so we can't rely on this fact. + /// + /// + /// + private void FindAndLoadControlFile(out TarArchiveReader archive, out DearchiveScenesInfo dearchivedScenes) + { + archive = new TarArchiveReader(m_loadStream); + dearchivedScenes = new DearchiveScenesInfo(); + + string filePath; + byte[] data; + TarArchiveReader.TarEntryType entryType; + bool firstFile = true; + + while ((data = archive.ReadEntry(out filePath, out entryType)) != null) + { + if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY == entryType) + continue; + + if (filePath == ArchiveConstants.CONTROL_FILE_PATH) + { + LoadControlFile(filePath, data, dearchivedScenes); + + // Find which scenes are available in the simulator + ArchiveScenesGroup simulatorScenes = new ArchiveScenesGroup(); + SceneManager.Instance.ForEachScene(delegate(Scene scene2) + { + simulatorScenes.AddScene(scene2); + }); + simulatorScenes.CalcSceneLocations(); + dearchivedScenes.SetSimulatorScenes(m_rootScene, simulatorScenes); + + // If the control file wasn't the first file then reset the read pointer + if (!firstFile) + { + m_log.Warn("Control file wasn't the first file in the archive"); + if (m_loadStream.CanSeek) + { + m_loadStream.Seek(0, SeekOrigin.Begin); + } + else if (m_loadPath != null) + { + archive.Close(); + archive = null; + m_loadStream.Close(); + m_loadStream = null; + m_loadStream = new GZipStream(ArchiveHelpers.GetStream(m_loadPath), CompressionMode.Decompress); + archive = new TarArchiveReader(m_loadStream); + } + else + { + // There isn't currently a scenario where this happens, but it's best to add a check just in case + throw new Exception("Error reading archive: control file wasn't the first file, and the input stream doesn't allow seeking"); + } + } + + return; + } + + firstFile = false; + } + + throw new Exception("Control file not found"); } /// /// Load serialized scene objects. /// - /// - protected void LoadObjects(List serialisedSceneObjects) + protected void LoadObjects(Scene scene, List serialisedSceneObjects, List sceneObjects) { // Reload serialized prims m_log.InfoFormat("[ARCHIVER]: Loading {0} scene objects. Please wait.", serialisedSceneObjects.Count); - UUID oldTelehubUUID = m_scene.RegionInfo.RegionSettings.TelehubObject; + UUID oldTelehubUUID = scene.RegionInfo.RegionSettings.TelehubObject; - IRegionSerialiserModule serialiser = m_scene.RequestModuleInterface(); + IRegionSerialiserModule serialiser = scene.RequestModuleInterface(); int sceneObjectsLoadedCount = 0; foreach (string serialisedSceneObject in serialisedSceneObjects) @@ -280,7 +445,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver SceneObjectGroup sceneObject = serialiser.DeserializeGroupFromXml2(serialisedSceneObject); - bool isTelehub = (sceneObject.UUID == oldTelehubUUID); + bool isTelehub = (sceneObject.UUID == oldTelehubUUID) && (oldTelehubUUID != UUID.Zero); // For now, give all incoming scene objects new uuids. This will allow scenes to be cloned // on the same region server and multiple examples a single object archive to be imported @@ -290,8 +455,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver if (isTelehub) { // Change the Telehub Object to the new UUID - m_scene.RegionInfo.RegionSettings.TelehubObject = sceneObject.UUID; - m_scene.RegionInfo.RegionSettings.Save(); + scene.RegionInfo.RegionSettings.TelehubObject = sceneObject.UUID; + scene.RegionInfo.RegionSettings.Save(); oldTelehubUUID = UUID.Zero; } @@ -301,17 +466,17 @@ namespace OpenSim.Region.CoreModules.World.Archiver { if (part.CreatorData == null || part.CreatorData == string.Empty) { - if (!ResolveUserUuid(part.CreatorID)) - part.CreatorID = m_scene.RegionInfo.EstateSettings.EstateOwner; + if (!ResolveUserUuid(scene, part.CreatorID)) + part.CreatorID = scene.RegionInfo.EstateSettings.EstateOwner; } if (UserManager != null) UserManager.AddUser(part.CreatorID, part.CreatorData); - if (!ResolveUserUuid(part.OwnerID)) - part.OwnerID = m_scene.RegionInfo.EstateSettings.EstateOwner; + if (!ResolveUserUuid(scene, part.OwnerID)) + part.OwnerID = scene.RegionInfo.EstateSettings.EstateOwner; - if (!ResolveUserUuid(part.LastOwnerID)) - part.LastOwnerID = m_scene.RegionInfo.EstateSettings.EstateOwner; + if (!ResolveUserUuid(scene, part.LastOwnerID)) + part.LastOwnerID = scene.RegionInfo.EstateSettings.EstateOwner; if (!ResolveGroupUuid(part.GroupID)) part.GroupID = UUID.Zero; @@ -328,15 +493,15 @@ namespace OpenSim.Region.CoreModules.World.Archiver TaskInventoryDictionary inv = part.TaskInventory; foreach (KeyValuePair kvp in inv) { - if (!ResolveUserUuid(kvp.Value.OwnerID)) + if (!ResolveUserUuid(scene, kvp.Value.OwnerID)) { - kvp.Value.OwnerID = m_scene.RegionInfo.EstateSettings.EstateOwner; + kvp.Value.OwnerID = scene.RegionInfo.EstateSettings.EstateOwner; } if (kvp.Value.CreatorData == null || kvp.Value.CreatorData == string.Empty) { - if (!ResolveUserUuid(kvp.Value.CreatorID)) - kvp.Value.CreatorID = m_scene.RegionInfo.EstateSettings.EstateOwner; + if (!ResolveUserUuid(scene, kvp.Value.CreatorID)) + kvp.Value.CreatorID = scene.RegionInfo.EstateSettings.EstateOwner; } if (UserManager != null) @@ -348,10 +513,10 @@ namespace OpenSim.Region.CoreModules.World.Archiver } } - if (m_scene.AddRestoredSceneObject(sceneObject, true, false)) + if (scene.AddRestoredSceneObject(sceneObject, true, false)) { sceneObjectsLoadedCount++; - sceneObject.CreateScriptInstances(0, false, m_scene.DefaultScriptEngine, 0); + sceneObject.CreateScriptInstances(0, false, scene.DefaultScriptEngine, 0); sceneObject.ResumeScripts(); } } @@ -366,16 +531,17 @@ namespace OpenSim.Region.CoreModules.World.Archiver if (oldTelehubUUID != UUID.Zero) { m_log.WarnFormat("Telehub object not found: {0}", oldTelehubUUID); - m_scene.RegionInfo.RegionSettings.TelehubObject = UUID.Zero; - m_scene.RegionInfo.RegionSettings.ClearSpawnPoints(); + scene.RegionInfo.RegionSettings.TelehubObject = UUID.Zero; + scene.RegionInfo.RegionSettings.ClearSpawnPoints(); } } /// /// Load serialized parcels. /// + /// /// - protected void LoadParcels(List serialisedParcels) + protected void LoadParcels(Scene scene, List serialisedParcels) { // Reload serialized parcels m_log.InfoFormat("[ARCHIVER]: Loading {0} parcels. Please wait.", serialisedParcels.Count); @@ -386,8 +552,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver // Validate User and Group UUID's - if (!ResolveUserUuid(parcel.OwnerID)) - parcel.OwnerID = m_scene.RegionInfo.EstateSettings.EstateOwner; + if (!ResolveUserUuid(scene, parcel.OwnerID)) + parcel.OwnerID = m_rootScene.RegionInfo.EstateSettings.EstateOwner; if (!ResolveGroupUuid(parcel.GroupID)) { @@ -398,7 +564,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver List accessList = new List(); foreach (LandAccessEntry entry in parcel.ParcelAccessList) { - if (ResolveUserUuid(entry.AgentID)) + if (ResolveUserUuid(scene, entry.AgentID)) accessList.Add(entry); // else, drop this access rule } @@ -414,23 +580,24 @@ namespace OpenSim.Region.CoreModules.World.Archiver if (!m_merge) { bool setupDefaultParcel = (landData.Count == 0); - m_scene.LandChannel.Clear(setupDefaultParcel); + scene.LandChannel.Clear(setupDefaultParcel); } - m_scene.EventManager.TriggerIncomingLandDataFromStorage(landData); + scene.EventManager.TriggerIncomingLandDataFromStorage(landData); m_log.InfoFormat("[ARCHIVER]: Restored {0} parcels.", landData.Count); } /// /// Look up the given user id to check whether it's one that is valid for this grid. /// + /// /// /// - private bool ResolveUserUuid(UUID uuid) + private bool ResolveUserUuid(Scene scene, UUID uuid) { if (!m_validUserUuids.ContainsKey(uuid)) { - UserAccount account = m_scene.UserAccountService.GetUserAccount(m_scene.RegionInfo.ScopeID, uuid); + UserAccount account = scene.UserAccountService.GetUserAccount(scene.RegionInfo.ScopeID, uuid); m_validUserUuids.Add(uuid, account != null); } @@ -485,7 +652,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver string extension = filename.Substring(i); string uuid = filename.Remove(filename.Length - extension.Length); - if (m_scene.AssetService.GetMetadata(uuid) != null) + if (m_assetService.GetMetadata(uuid) != null) { // m_log.DebugFormat("[ARCHIVER]: found existing asset {0}",uuid); return true; @@ -505,7 +672,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver // We're relying on the asset service to do the sensible thing and not store the asset if it already // exists. - m_scene.AssetService.Store(asset); + m_assetService.Store(asset); /** * Create layers on decode for image assets. This is likely to significantly increase the time to load archives so @@ -533,12 +700,14 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// Load region settings data /// + /// /// /// + /// /// /// true if settings were loaded successfully, false otherwise /// - private bool LoadRegionSettings(string settingsPath, byte[] data) + private bool LoadRegionSettings(Scene scene, string settingsPath, byte[] data, DearchiveScenesInfo dearchivedScenes) { RegionSettings loadedRegionSettings; @@ -554,7 +723,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver return false; } - RegionSettings currentRegionSettings = m_scene.RegionInfo.RegionSettings; + RegionSettings currentRegionSettings = scene.RegionInfo.RegionSettings; currentRegionSettings.AgentLimit = loadedRegionSettings.AgentLimit; currentRegionSettings.AllowDamage = loadedRegionSettings.AllowDamage; @@ -591,12 +760,14 @@ namespace OpenSim.Region.CoreModules.World.Archiver foreach (SpawnPoint sp in loadedRegionSettings.SpawnPoints()) currentRegionSettings.AddSpawnPoint(sp); + currentRegionSettings.LoadedCreationDateTime = dearchivedScenes.LoadedCreationDateTime; + currentRegionSettings.LoadedCreationID = dearchivedScenes.GetOriginalRegionID(scene.RegionInfo.RegionID).ToString(); + currentRegionSettings.Save(); - m_scene.TriggerEstateSunUpdate(); + scene.TriggerEstateSunUpdate(); - IEstateModule estateModule = m_scene.RequestModuleInterface(); - + IEstateModule estateModule = scene.RequestModuleInterface(); if (estateModule != null) estateModule.sendRegionHandshakeToAll(); @@ -606,14 +777,15 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// Load terrain data /// + /// /// /// /// /// true if terrain was resolved successfully, false otherwise. /// - private bool LoadTerrain(string terrainPath, byte[] data) + private bool LoadTerrain(Scene scene, string terrainPath, byte[] data) { - ITerrainModule terrainModule = m_scene.RequestModuleInterface(); + ITerrainModule terrainModule = scene.RequestModuleInterface(); MemoryStream ms = new MemoryStream(data); terrainModule.LoadFromStream(terrainPath, ms); @@ -629,17 +801,18 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// /// - public void LoadControlFile(string path, byte[] data) + /// + public DearchiveScenesInfo LoadControlFile(string path, byte[] data, DearchiveScenesInfo dearchivedScenes) { XmlNamespaceManager nsmgr = new XmlNamespaceManager(new NameTable()); XmlParserContext context = new XmlParserContext(null, nsmgr, null, XmlSpace.None); XmlTextReader xtr = new XmlTextReader(Encoding.ASCII.GetString(data), XmlNodeType.Document, context); - RegionSettings currentRegionSettings = m_scene.RegionInfo.RegionSettings; + // Loaded metadata will be empty if no information exists in the archive + dearchivedScenes.LoadedCreationDateTime = 0; + dearchivedScenes.DefaultOriginalID = ""; - // Loaded metadata will empty if no information exists in the archive - currentRegionSettings.LoadedCreationDateTime = 0; - currentRegionSettings.LoadedCreationID = ""; + bool multiRegion = false; while (xtr.Read()) { @@ -665,18 +838,44 @@ namespace OpenSim.Region.CoreModules.World.Archiver { int value; if (Int32.TryParse(xtr.ReadElementContentAsString(), out value)) - currentRegionSettings.LoadedCreationDateTime = value; + dearchivedScenes.LoadedCreationDateTime = value; } - else if (xtr.Name.ToString() == "id") + else if (xtr.Name.ToString() == "row") + { + multiRegion = true; + dearchivedScenes.StartRow(); + } + else if (xtr.Name.ToString() == "region") { - currentRegionSettings.LoadedCreationID = xtr.ReadElementContentAsString(); + dearchivedScenes.StartRegion(); + } + else if (xtr.Name.ToString() == "id") + { + string id = xtr.ReadElementContentAsString(); + dearchivedScenes.DefaultOriginalID = id; + if (multiRegion) + dearchivedScenes.SetRegionOriginalID(id); + } + else if (xtr.Name.ToString() == "dir") + { + dearchivedScenes.SetRegionDirectory(xtr.ReadElementContentAsString()); } } } - - currentRegionSettings.Save(); - + + dearchivedScenes.MultiRegionFormat = multiRegion; + if (!multiRegion) + { + // Add the single scene + dearchivedScenes.StartRow(); + dearchivedScenes.StartRegion(); + dearchivedScenes.SetRegionOriginalID(dearchivedScenes.DefaultOriginalID); + dearchivedScenes.SetRegionDirectory(""); + } + ControlFileLoaded = true; + + return dearchivedScenes; } } } \ No newline at end of file diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveScenesGroup.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveScenesGroup.cs new file mode 100644 index 0000000..a66ed88 --- /dev/null +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveScenesGroup.cs @@ -0,0 +1,176 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using OpenSim.Region.Framework.Scenes; +using OpenMetaverse; +using System.Drawing; + +namespace OpenSim.Region.CoreModules.World.Archiver +{ + /// + /// A group of regions arranged in a rectangle, possibly with holes. + /// + /// + /// The regions usually (but not necessarily) belong to an archive file, in which case we + /// store additional information used to create the archive (e.g., each region's + /// directory within the archive). + /// + public class ArchiveScenesGroup + { + /// + /// All the regions. The outer dictionary contains rows (key: Y coordinate). + /// The inner dictionaries contain each row's regions (key: X coordinate). + /// + public SortedDictionary> Regions { get; set; } + + /// + /// The subdirectory where each region is stored in the archive. + /// + protected Dictionary m_regionDirs; + + /// + /// The grid coordinates of the regions' bounding box. + /// + public Rectangle Rect { get; set; } + + + public ArchiveScenesGroup() + { + Regions = new SortedDictionary>(); + m_regionDirs = new Dictionary(); + Rect = new Rectangle(0, 0, 0, 0); + } + + public void AddScene(Scene scene) + { + uint x = scene.RegionInfo.RegionLocX; + uint y = scene.RegionInfo.RegionLocY; + + SortedDictionary row; + if (!Regions.TryGetValue(y, out row)) + { + row = new SortedDictionary(); + Regions[y] = row; + } + + row[x] = scene; + } + + /// + /// Called after all the scenes have been added. Performs calculations that require + /// knowledge of all the scenes. + /// + public void CalcSceneLocations() + { + if (Regions.Count == 0) + return; + + // Find the bounding rectangle + + uint firstY = Regions.First().Key; + uint lastY = Regions.Last().Key; + + uint? firstX = null; + uint? lastX = null; + + foreach (SortedDictionary row in Regions.Values) + { + uint curFirstX = row.First().Key; + uint curLastX = row.Last().Key; + + firstX = (firstX == null) ? curFirstX : (firstX < curFirstX) ? firstX : curFirstX; + lastX = (lastX == null) ? curLastX : (lastX > curLastX) ? lastX : curLastX; + } + + Rect = new Rectangle((int)firstX, (int)firstY, (int)(lastY - firstY + 1), (int)(lastX - firstX + 1)); + + + // Calculate the subdirectory in which each region will be stored in the archive + + m_regionDirs.Clear(); + ForEachScene(delegate(Scene scene) + { + // We add the region's coordinates to ensure uniqueness even if multiple regions have the same name + string path = string.Format("{0}_{1}_{2}", + scene.RegionInfo.RegionLocX - Rect.X + 1, + scene.RegionInfo.RegionLocY - Rect.Y + 1, + scene.RegionInfo.RegionName.Replace(' ', '_')); + m_regionDirs[scene.RegionInfo.RegionID] = path; + }); + } + + /// + /// Returns the subdirectory where the region is stored. + /// + /// + /// + public string GetRegionDir(UUID regionID) + { + return m_regionDirs[regionID]; + } + + /// + /// Performs an action on all the scenes in this order: rows from South to North, + /// and within each row West to East. + /// + /// + public void ForEachScene(Action action) + { + foreach (SortedDictionary row in Regions.Values) + { + foreach (Scene scene in row.Values) + { + action(scene); + } + } + } + + /// + /// Returns the scene at position 'location'. + /// + /// A location in the grid + /// The scene at this location + /// Whether the scene was found + public bool TryGetScene(Point location, out Scene scene) + { + SortedDictionary row; + if (Regions.TryGetValue((uint)location.Y, out row)) + { + if (row.TryGetValue((uint)location.X, out scene)) + return true; + } + + scene = null; + return false; + } + + } +} diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs deleted file mode 100644 index 0780d86..0000000 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright (c) Contributors, http://opensimulator.org/ - * See CONTRIBUTORS.TXT for a full list of copyright holders. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the OpenSimulator Project nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -using System; -using System.Collections.Generic; -using System.IO; -using System.Reflection; -using System.Xml; -using log4net; -using OpenMetaverse; -using OpenSim.Framework; -using OpenSim.Framework.Serialization; -using OpenSim.Framework.Serialization.External; -using OpenSim.Region.CoreModules.World.Terrain; -using OpenSim.Region.Framework.Interfaces; -using OpenSim.Region.Framework.Scenes; - -namespace OpenSim.Region.CoreModules.World.Archiver -{ - /// - /// Method called when all the necessary assets for an archive request have been received. - /// - public delegate void AssetsRequestCallback( - ICollection assetsFoundUuids, ICollection assetsNotFoundUuids); - - /// - /// Execute the write of an archive once we have received all the necessary data - /// - public class ArchiveWriteRequestExecution - { - private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - - protected ITerrainModule m_terrainModule; - protected IRegionSerialiserModule m_serialiser; - protected List m_sceneObjects; - protected Scene m_scene; - protected TarArchiveWriter m_archiveWriter; - protected Guid m_requestId; - protected Dictionary m_options; - - public ArchiveWriteRequestExecution( - List sceneObjects, - ITerrainModule terrainModule, - IRegionSerialiserModule serialiser, - Scene scene, - TarArchiveWriter archiveWriter, - Guid requestId, - Dictionary options) - { - m_sceneObjects = sceneObjects; - m_terrainModule = terrainModule; - m_serialiser = serialiser; - m_scene = scene; - m_archiveWriter = archiveWriter; - m_requestId = requestId; - m_options = options; - } - - protected internal void ReceivedAllAssets( - ICollection assetsFoundUuids, ICollection assetsNotFoundUuids) - { - try - { - Save(assetsFoundUuids, assetsNotFoundUuids); - } - finally - { - m_archiveWriter.Close(); - } - - m_log.InfoFormat("[ARCHIVER]: Finished writing out OAR for {0}", m_scene.RegionInfo.RegionName); - - m_scene.EventManager.TriggerOarFileSaved(m_requestId, String.Empty); - } - - protected internal void Save(ICollection assetsFoundUuids, ICollection assetsNotFoundUuids) - { - foreach (UUID uuid in assetsNotFoundUuids) - { - m_log.DebugFormat("[ARCHIVER]: Could not find asset {0}", uuid); - } - -// m_log.InfoFormat( -// "[ARCHIVER]: Received {0} of {1} assets requested", -// assetsFoundUuids.Count, assetsFoundUuids.Count + assetsNotFoundUuids.Count); - - m_log.InfoFormat("[ARCHIVER]: Adding region settings to archive."); - - // Write out region settings - string settingsPath - = String.Format("{0}{1}.xml", ArchiveConstants.SETTINGS_PATH, m_scene.RegionInfo.RegionName); - m_archiveWriter.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings)); - - m_log.InfoFormat("[ARCHIVER]: Adding parcel settings to archive."); - - // Write out land data (aka parcel) settings - ListlandObjects = m_scene.LandChannel.AllParcels(); - foreach (ILandObject lo in landObjects) - { - LandData landData = lo.LandData; - string landDataPath = String.Format("{0}{1}.xml", ArchiveConstants.LANDDATA_PATH, - landData.GlobalID.ToString()); - m_archiveWriter.WriteFile(landDataPath, LandDataSerializer.Serialize(landData, m_options)); - } - - m_log.InfoFormat("[ARCHIVER]: Adding terrain information to archive."); - - // Write out terrain - string terrainPath - = String.Format("{0}{1}.r32", ArchiveConstants.TERRAINS_PATH, m_scene.RegionInfo.RegionName); - - MemoryStream ms = new MemoryStream(); - m_terrainModule.SaveToStream(terrainPath, ms); - m_archiveWriter.WriteFile(terrainPath, ms.ToArray()); - ms.Close(); - - m_log.InfoFormat("[ARCHIVER]: Adding scene objects to archive."); - - // Write out scene object metadata - foreach (SceneObjectGroup sceneObject in m_sceneObjects) - { - //m_log.DebugFormat("[ARCHIVER]: Saving {0} {1}, {2}", entity.Name, entity.UUID, entity.GetType()); - - string serializedObject = m_serialiser.SerializeGroupToXml2(sceneObject, m_options); - m_archiveWriter.WriteFile(ArchiveHelpers.CreateObjectPath(sceneObject), serializedObject); - } - } - } -} \ No newline at end of file diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs index 4edaaca..2c34f4b 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs @@ -43,6 +43,7 @@ using OpenSim.Region.Framework.Scenes; using Ionic.Zlib; using GZipStream = Ionic.Zlib.GZipStream; using CompressionMode = Ionic.Zlib.CompressionMode; +using OpenSim.Framework.Serialization.External; namespace OpenSim.Region.CoreModules.World.Archiver { @@ -61,17 +62,29 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// The maximum major version of OAR that we can write. /// - public static int MAX_MAJOR_VERSION = 0; + public static int MAX_MAJOR_VERSION = 1; + + /// + /// Whether we're saving a multi-region archive. + /// + public bool MultiRegionFormat { get; set; } /// /// Determine whether this archive will save assets. Default is true. /// public bool SaveAssets { get; set; } - protected ArchiverModule m_module; - protected Scene m_scene; + /// + /// Determines which objects will be included in the archive, according to their permissions. + /// Default is null, meaning no permission checks. + /// + public string CheckPermissions { get; set; } + + protected Scene m_rootScene; protected Stream m_saveStream; + protected TarArchiveWriter m_archiveWriter; protected Guid m_requestId; + protected Dictionary m_options; /// /// Constructor @@ -82,7 +95,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// If there was a problem opening a stream for the file specified by the savePath /// - public ArchiveWriteRequestPreparation(ArchiverModule module, string savePath, Guid requestId) : this(module, requestId) + public ArchiveWriteRequestPreparation(Scene scene, string savePath, Guid requestId) : this(scene, requestId) { try { @@ -100,26 +113,23 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// Constructor. /// - /// Calling module + /// The root scene to archive /// The stream to which to save data. /// The id associated with this request - public ArchiveWriteRequestPreparation(ArchiverModule module, Stream saveStream, Guid requestId) : this(module, requestId) + public ArchiveWriteRequestPreparation(Scene scene, Stream saveStream, Guid requestId) : this(scene, requestId) { m_saveStream = saveStream; } - protected ArchiveWriteRequestPreparation(ArchiverModule module, Guid requestId) + protected ArchiveWriteRequestPreparation(Scene scene, Guid requestId) { - m_module = module; - - // FIXME: This is only here for regression test purposes since they do not supply a module. Need to fix - // this. - if (m_module != null) - m_scene = m_module.Scene; - + m_rootScene = scene; m_requestId = requestId; + m_archiveWriter = null; + MultiRegionFormat = false; SaveAssets = true; + CheckPermissions = null; } /// @@ -128,126 +138,157 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// if there was an io problem with creating the file public void ArchiveRegion(Dictionary options) { + m_options = options; + + if (options.ContainsKey("all") && (bool)options["all"]) + MultiRegionFormat = true; + if (options.ContainsKey("noassets") && (bool)options["noassets"]) SaveAssets = false; + Object temp; + if (options.TryGetValue("checkPermissions", out temp)) + CheckPermissions = (string)temp; + + + // Find the regions to archive + ArchiveScenesGroup scenesGroup = new ArchiveScenesGroup(); + if (MultiRegionFormat) + { + m_log.InfoFormat("[ARCHIVER]: Saving {0} regions", SceneManager.Instance.Scenes.Count); + SceneManager.Instance.ForEachScene(delegate(Scene scene) + { + scenesGroup.AddScene(scene); + }); + } + else + { + scenesGroup.AddScene(m_rootScene); + } + scenesGroup.CalcSceneLocations(); + + + m_archiveWriter = new TarArchiveWriter(m_saveStream); + try { + // Write out control file. It should be first so that it will be found ASAP when loading the file. + m_archiveWriter.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, CreateControlFile(scenesGroup)); + m_log.InfoFormat("[ARCHIVER]: Added control file to archive."); + + // Archive the regions + Dictionary assetUuids = new Dictionary(); - - EntityBase[] entities = m_scene.GetEntities(); - List sceneObjects = new List(); - - string checkPermissions = null; - int numObjectsSkippedPermissions = 0; - Object temp; - if (options.TryGetValue("checkPermissions", out temp)) - checkPermissions = (string)temp; - - // Filter entities so that we only have scene objects. - // FIXME: Would be nicer to have this as a proper list in SceneGraph, since lots of methods - // end up having to do this - foreach (EntityBase entity in entities) + + scenesGroup.ForEachScene(delegate(Scene scene) { - if (entity is SceneObjectGroup) - { - SceneObjectGroup sceneObject = (SceneObjectGroup)entity; + string regionDir = MultiRegionFormat ? scenesGroup.GetRegionDir(scene.RegionInfo.RegionID) : ""; + ArchiveOneRegion(scene, regionDir, assetUuids); + }); - if (!sceneObject.IsDeleted && !sceneObject.IsAttachment) - { - if (!CanUserArchiveObject(m_scene.RegionInfo.EstateSettings.EstateOwner, sceneObject, checkPermissions)) - { - // The user isn't allowed to copy/transfer this object, so it will not be included in the OAR. - ++numObjectsSkippedPermissions; - } - else - { - sceneObjects.Add(sceneObject); - } - } - } - } + // Archive the assets if (SaveAssets) { - UuidGatherer assetGatherer = new UuidGatherer(m_scene.AssetService); + m_log.DebugFormat("[ARCHIVER]: Saving {0} assets", assetUuids.Count); - foreach (SceneObjectGroup sceneObject in sceneObjects) - { - assetGatherer.GatherAssetUuids(sceneObject, assetUuids); - } + // Asynchronously request all the assets required to perform this archive operation + AssetsRequest ar + = new AssetsRequest( + new AssetsArchiver(m_archiveWriter), assetUuids, + m_rootScene.AssetService, m_rootScene.UserAccountService, + m_rootScene.RegionInfo.ScopeID, options, ReceivedAllAssets); - m_log.DebugFormat( - "[ARCHIVER]: {0} scene objects to serialize requiring save of {1} assets", - sceneObjects.Count, assetUuids.Count); + Util.FireAndForget(o => ar.Execute()); + + // CloseArchive() will be called from ReceivedAllAssets() } else { m_log.DebugFormat("[ARCHIVER]: Not saving assets since --noassets was specified"); + CloseArchive(string.Empty); } + } + catch (Exception e) + { + CloseArchive(e.Message); + throw; + } + } - if (numObjectsSkippedPermissions > 0) - { - m_log.DebugFormat( - "[ARCHIVER]: {0} scene objects skipped due to lack of permissions", - numObjectsSkippedPermissions); - } - // Make sure that we also request terrain texture assets - RegionSettings regionSettings = m_scene.RegionInfo.RegionSettings; - - if (regionSettings.TerrainTexture1 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_1) - assetUuids[regionSettings.TerrainTexture1] = AssetType.Texture; - - if (regionSettings.TerrainTexture2 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_2) - assetUuids[regionSettings.TerrainTexture2] = AssetType.Texture; - - if (regionSettings.TerrainTexture3 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_3) - assetUuids[regionSettings.TerrainTexture3] = AssetType.Texture; - - if (regionSettings.TerrainTexture4 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_4) - assetUuids[regionSettings.TerrainTexture4] = AssetType.Texture; - - TarArchiveWriter archiveWriter = new TarArchiveWriter(m_saveStream); - - // Asynchronously request all the assets required to perform this archive operation - ArchiveWriteRequestExecution awre - = new ArchiveWriteRequestExecution( - sceneObjects, - m_scene.RequestModuleInterface(), - m_scene.RequestModuleInterface(), - m_scene, - archiveWriter, - m_requestId, - options); - - m_log.InfoFormat("[ARCHIVER]: Creating archive file. This may take some time."); - - // Write out control file. This has to be done first so that subsequent loaders will see this file first - // XXX: I know this is a weak way of doing it since external non-OAR aware tar executables will not do this - archiveWriter.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, CreateControlFile(options)); - m_log.InfoFormat("[ARCHIVER]: Added control file to archive."); + private void ArchiveOneRegion(Scene scene, string regionDir, Dictionary assetUuids) + { + m_log.InfoFormat("[ARCHIVER]: Writing region {0}", scene.RegionInfo.RegionName); - if (SaveAssets) + EntityBase[] entities = scene.GetEntities(); + List sceneObjects = new List(); + + int numObjectsSkippedPermissions = 0; + + // Filter entities so that we only have scene objects. + // FIXME: Would be nicer to have this as a proper list in SceneGraph, since lots of methods + // end up having to do this + IPermissionsModule permissionsModule = scene.RequestModuleInterface(); + foreach (EntityBase entity in entities) + { + if (entity is SceneObjectGroup) { - AssetsRequest ar - = new AssetsRequest( - new AssetsArchiver(archiveWriter), assetUuids, - m_scene.AssetService, m_scene.UserAccountService, - m_scene.RegionInfo.ScopeID, options, awre.ReceivedAllAssets); + SceneObjectGroup sceneObject = (SceneObjectGroup)entity; - Util.FireAndForget(o => ar.Execute()); + if (!sceneObject.IsDeleted && !sceneObject.IsAttachment) + { + if (!CanUserArchiveObject(scene.RegionInfo.EstateSettings.EstateOwner, sceneObject, CheckPermissions, permissionsModule)) + { + // The user isn't allowed to copy/transfer this object, so it will not be included in the OAR. + ++numObjectsSkippedPermissions; + } + else + { + sceneObjects.Add(sceneObject); + } + } } - else + } + + if (SaveAssets) + { + UuidGatherer assetGatherer = new UuidGatherer(scene.AssetService); + int prevAssets = assetUuids.Count; + + foreach (SceneObjectGroup sceneObject in sceneObjects) { - awre.ReceivedAllAssets(new List(), new List()); + assetGatherer.GatherAssetUuids(sceneObject, assetUuids); } + + m_log.DebugFormat( + "[ARCHIVER]: {0} scene objects to serialize requiring save of {1} assets", + sceneObjects.Count, assetUuids.Count - prevAssets); } - catch (Exception) + + if (numObjectsSkippedPermissions > 0) { - m_saveStream.Close(); - throw; - } + m_log.DebugFormat( + "[ARCHIVER]: {0} scene objects skipped due to lack of permissions", + numObjectsSkippedPermissions); + } + + // Make sure that we also request terrain texture assets + RegionSettings regionSettings = scene.RegionInfo.RegionSettings; + + if (regionSettings.TerrainTexture1 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_1) + assetUuids[regionSettings.TerrainTexture1] = AssetType.Texture; + + if (regionSettings.TerrainTexture2 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_2) + assetUuids[regionSettings.TerrainTexture2] = AssetType.Texture; + + if (regionSettings.TerrainTexture3 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_3) + assetUuids[regionSettings.TerrainTexture3] = AssetType.Texture; + + if (regionSettings.TerrainTexture4 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_4) + assetUuids[regionSettings.TerrainTexture4] = AssetType.Texture; + + Save(scene, sceneObjects, regionDir); } /// @@ -256,14 +297,14 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// The user /// The object group /// Which permissions to check: "C" = Copy, "T" = Transfer + /// The scene's permissions module /// Whether the user is allowed to export the object to an OAR - private bool CanUserArchiveObject(UUID user, SceneObjectGroup objGroup, string checkPermissions) + private bool CanUserArchiveObject(UUID user, SceneObjectGroup objGroup, string checkPermissions, IPermissionsModule permissionsModule) { if (checkPermissions == null) return true; - IPermissionsModule module = m_scene.RequestModuleInterface(); - if (module == null) + if (permissionsModule == null) return true; // this shouldn't happen // Check whether the user is permitted to export all of the parts in the SOG. If any @@ -275,7 +316,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver foreach (SceneObjectPart obj in objGroup.Parts) { uint perm; - PermissionClass permissionClass = module.GetPermissionClass(user, obj); + PermissionClass permissionClass = permissionsModule.GetPermissionClass(user, obj); switch (permissionClass) { case PermissionClass.Owner: @@ -330,16 +371,30 @@ namespace OpenSim.Region.CoreModules.World.Archiver } /// - /// Create the control file for the most up to date archive + /// Create the control file. /// /// - public string CreateControlFile(Dictionary options) + public string CreateControlFile(ArchiveScenesGroup scenesGroup) { - int majorVersion = MAX_MAJOR_VERSION, minorVersion = 8; + int majorVersion; + int minorVersion; + + if (MultiRegionFormat) + { + majorVersion = MAX_MAJOR_VERSION; + minorVersion = 0; + } + else + { + // To support older versions of OpenSim, we continue to create single-region OARs + // using the old file format. In the future this format will be discontinued. + majorVersion = 0; + minorVersion = 8; + } // -// if (options.ContainsKey("version")) +// if (m_options.ContainsKey("version")) // { -// string[] parts = options["version"].ToString().Split('.'); +// string[] parts = m_options["version"].ToString().Split('.'); // if (parts.Length >= 1) // { // majorVersion = Int32.Parse(parts[0]); @@ -368,10 +423,10 @@ namespace OpenSim.Region.CoreModules.World.Archiver // } m_log.InfoFormat("[ARCHIVER]: Creating version {0}.{1} OAR", majorVersion, minorVersion); - //if (majorVersion == 1) - //{ - // m_log.WarnFormat("[ARCHIVER]: Please be aware that version 1.0 OARs are not compatible with OpenSim 0.7.0.2 and earlier. Please use the --version=0 option if you want to produce a compatible OAR"); - //} + if (majorVersion == 1) + { + m_log.WarnFormat("[ARCHIVER]: Please be aware that version 1.0 OARs are not compatible with OpenSim versions prior to 0.7.4. Do not use the --all option if you want to produce a compatible OAR"); + } String s; @@ -389,50 +444,191 @@ namespace OpenSim.Region.CoreModules.World.Archiver DateTime now = DateTime.UtcNow; TimeSpan t = now - new DateTime(1970, 1, 1); xtw.WriteElementString("datetime", ((int)t.TotalSeconds).ToString()); - xtw.WriteElementString("id", UUID.Random().ToString()); + if (!MultiRegionFormat) + xtw.WriteElementString("id", m_rootScene.RegionInfo.RegionID.ToString()); xtw.WriteEndElement(); + + xtw.WriteElementString("assets_included", SaveAssets.ToString()); - xtw.WriteStartElement("region_info"); + if (MultiRegionFormat) + { + WriteRegionsManifest(scenesGroup, xtw); + } + else + { + xtw.WriteStartElement("region_info"); + WriteRegionInfo(m_rootScene, xtw); + xtw.WriteEndElement(); + } - bool isMegaregion; - Vector2 size; - IRegionCombinerModule rcMod = null; + xtw.WriteEndElement(); + + xtw.Flush(); + } - // FIXME: This is only here for regression test purposes since they do not supply a module. Need to fix - // this, possibly by doing control file creation somewhere else. - if (m_module != null) - rcMod = m_module.RegionCombinerModule; + s = sw.ToString(); + } - if (rcMod != null) - isMegaregion = rcMod.IsRootForMegaregion(m_scene.RegionInfo.RegionID); - else - isMegaregion = false; + return s; + } - if (isMegaregion) - size = rcMod.GetSizeOfMegaregion(m_scene.RegionInfo.RegionID); - else - size = new Vector2((float)Constants.RegionSize, (float)Constants.RegionSize); + /// + /// Writes the list of regions included in a multi-region OAR. + /// + private static void WriteRegionsManifest(ArchiveScenesGroup scenesGroup, XmlTextWriter xtw) + { + xtw.WriteStartElement("regions"); - xtw.WriteElementString("is_megaregion", isMegaregion.ToString()); - xtw.WriteElementString("size_in_meters", string.Format("{0},{1}", size.X, size.Y)); + // Write the regions in order: rows from South to North, then regions from West to East. + // The list of regions can have "holes"; we write empty elements in their position. - xtw.WriteEndElement(); - - xtw.WriteElementString("assets_included", SaveAssets.ToString()); + for (uint y = (uint)scenesGroup.Rect.Top; y < scenesGroup.Rect.Bottom; ++y) + { + SortedDictionary row; + if (scenesGroup.Regions.TryGetValue(y, out row)) + { + xtw.WriteStartElement("row"); + + for (uint x = (uint)scenesGroup.Rect.Left; x < scenesGroup.Rect.Right; ++x) + { + Scene scene; + if (row.TryGetValue(x, out scene)) + { + xtw.WriteStartElement("region"); + xtw.WriteElementString("id", scene.RegionInfo.RegionID.ToString()); + xtw.WriteElementString("dir", scenesGroup.GetRegionDir(scene.RegionInfo.RegionID)); + WriteRegionInfo(scene, xtw); + xtw.WriteEndElement(); + } + else + { + // Write a placeholder for a missing region + xtw.WriteElementString("region", ""); + } + } xtw.WriteEndElement(); - - xtw.Flush(); } + else + { + // Write a placeholder for a missing row + xtw.WriteElementString("row", ""); + } + } - s = sw.ToString(); + xtw.WriteEndElement(); // "regions" + } + + protected static void WriteRegionInfo(Scene scene, XmlTextWriter xtw) + { + bool isMegaregion; + Vector2 size; + + IRegionCombinerModule rcMod = scene.RequestModuleInterface(); + + if (rcMod != null) + isMegaregion = rcMod.IsRootForMegaregion(scene.RegionInfo.RegionID); + else + isMegaregion = false; + + if (isMegaregion) + size = rcMod.GetSizeOfMegaregion(scene.RegionInfo.RegionID); + else + size = new Vector2((float)Constants.RegionSize, (float)Constants.RegionSize); + + xtw.WriteElementString("is_megaregion", isMegaregion.ToString()); + xtw.WriteElementString("size_in_meters", string.Format("{0},{1}", size.X, size.Y)); + } + + + protected void Save(Scene scene, List sceneObjects, string regionDir) + { + if (regionDir != string.Empty) + regionDir = ArchiveConstants.REGIONS_PATH + regionDir + "/"; + + m_log.InfoFormat("[ARCHIVER]: Adding region settings to archive."); + + // Write out region settings + string settingsPath = String.Format("{0}{1}{2}.xml", + regionDir, ArchiveConstants.SETTINGS_PATH, scene.RegionInfo.RegionName); + m_archiveWriter.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(scene.RegionInfo.RegionSettings)); + + m_log.InfoFormat("[ARCHIVER]: Adding parcel settings to archive."); + + // Write out land data (aka parcel) settings + List landObjects = scene.LandChannel.AllParcels(); + foreach (ILandObject lo in landObjects) + { + LandData landData = lo.LandData; + string landDataPath = String.Format("{0}{1}{2}.xml", + regionDir, ArchiveConstants.LANDDATA_PATH, landData.GlobalID.ToString()); + m_archiveWriter.WriteFile(landDataPath, LandDataSerializer.Serialize(landData, m_options)); } -// if (m_scene != null) -// Console.WriteLine( -// "[ARCHIVE WRITE REQUEST PREPARATION]: Control file for {0} is: {1}", m_scene.RegionInfo.RegionName, s); + m_log.InfoFormat("[ARCHIVER]: Adding terrain information to archive."); - return s; + // Write out terrain + string terrainPath = String.Format("{0}{1}{2}.r32", + regionDir, ArchiveConstants.TERRAINS_PATH, scene.RegionInfo.RegionName); + + MemoryStream ms = new MemoryStream(); + scene.RequestModuleInterface().SaveToStream(terrainPath, ms); + m_archiveWriter.WriteFile(terrainPath, ms.ToArray()); + ms.Close(); + + m_log.InfoFormat("[ARCHIVER]: Adding scene objects to archive."); + + // Write out scene object metadata + IRegionSerialiserModule serializer = scene.RequestModuleInterface(); + foreach (SceneObjectGroup sceneObject in sceneObjects) + { + //m_log.DebugFormat("[ARCHIVER]: Saving {0} {1}, {2}", entity.Name, entity.UUID, entity.GetType()); + + string serializedObject = serializer.SerializeGroupToXml2(sceneObject, m_options); + string objectPath = string.Format("{0}{1}", regionDir, ArchiveHelpers.CreateObjectPath(sceneObject)); + m_archiveWriter.WriteFile(objectPath, serializedObject); + } } + + protected void ReceivedAllAssets( + ICollection assetsFoundUuids, ICollection assetsNotFoundUuids) + { + foreach (UUID uuid in assetsNotFoundUuids) + { + m_log.DebugFormat("[ARCHIVER]: Could not find asset {0}", uuid); + } + + // m_log.InfoFormat( + // "[ARCHIVER]: Received {0} of {1} assets requested", + // assetsFoundUuids.Count, assetsFoundUuids.Count + assetsNotFoundUuids.Count); + + CloseArchive(String.Empty); + } + + + /// + /// Closes the archive and notifies that we're done. + /// + /// The error that occurred, or empty for success + protected void CloseArchive(string errorMessage) + { + try + { + if (m_archiveWriter != null) + m_archiveWriter.Close(); + m_saveStream.Close(); + } + catch (Exception e) + { + m_log.Error(string.Format("[ARCHIVER]: Error closing archive: {0} ", e.Message), e); + if (errorMessage == string.Empty) + errorMessage = e.Message; + } + + m_log.InfoFormat("[ARCHIVER]: Finished writing out OAR for {0}", m_rootScene.RegionInfo.RegionName); + + m_rootScene.EventManager.TriggerOarFileSaved(m_requestId, errorMessage); + } + } } diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiverModule.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiverModule.cs index bf3b124..26535a9 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiverModule.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiverModule.cs @@ -146,6 +146,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver ops.Add("noassets", delegate(string v) { options["noassets"] = v != null; }); ops.Add("publish", v => options["wipe-owners"] = v != null); ops.Add("perm=", delegate(string v) { options["checkPermissions"] = v; }); + ops.Add("all", delegate(string v) { options["all"] = v != null; }); List mainParams = ops.Parse(cmdparams); @@ -169,7 +170,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver m_log.InfoFormat( "[ARCHIVER]: Writing archive for region {0} to {1}", Scene.RegionInfo.RegionName, savePath); - new ArchiveWriteRequestPreparation(this, savePath, requestId).ArchiveRegion(options); + new ArchiveWriteRequestPreparation(Scene, savePath, requestId).ArchiveRegion(options); } public void ArchiveRegion(Stream saveStream) @@ -184,7 +185,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver public void ArchiveRegion(Stream saveStream, Guid requestId, Dictionary options) { - new ArchiveWriteRequestPreparation(this, saveStream, requestId).ArchiveRegion(options); + new ArchiveWriteRequestPreparation(Scene, saveStream, requestId).ArchiveRegion(options); } public void DearchiveRegion(string loadPath) diff --git a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs index a073cb9..5787279 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs @@ -46,6 +46,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver { private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + /// + /// Method called when all the necessary assets for an archive request have been received. + /// + public delegate void AssetsRequestCallback( + ICollection assetsFoundUuids, ICollection assetsNotFoundUuids); + enum RequestState { Initial, diff --git a/OpenSim/Region/CoreModules/World/Archiver/DearchiveScenesGroup.cs b/OpenSim/Region/CoreModules/World/Archiver/DearchiveScenesGroup.cs new file mode 100644 index 0000000..3dcc020 --- /dev/null +++ b/OpenSim/Region/CoreModules/World/Archiver/DearchiveScenesGroup.cs @@ -0,0 +1,232 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using OpenSim.Region.Framework.Scenes; +using OpenMetaverse; +using System.Drawing; +using log4net; +using System.Reflection; +using OpenSim.Framework.Serialization; + +namespace OpenSim.Region.CoreModules.World.Archiver +{ + /// + /// The regions included in an OAR file. + /// + public class DearchiveScenesInfo + { + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + /// + /// One region in the archive. + /// + public class RegionInfo + { + /// + /// The subdirectory in which the region is stored. + /// + public string Directory { get; set; } + + /// + /// The region's coordinates (relative to the South-West corner of the block). + /// + public Point Location { get; set; } + + /// + /// The UUID of the original scene from which this archived region was saved. + /// + public string OriginalID { get; set; } + + /// + /// The scene in the current simulator into which this region is loaded. + /// If null then the region doesn't have a corresponding scene, and it won't be loaded. + /// + public Scene Scene { get; set; } + } + + /// + /// Whether this archive uses the multi-region format. + /// + public Boolean MultiRegionFormat { get; set; } + + /// + /// Maps (Region directory -> region) + /// + protected Dictionary m_directory2region = new Dictionary(); + + /// + /// Maps (UUID of the scene in the simulator where the region will be loaded -> region) + /// + protected Dictionary m_newId2region = new Dictionary(); + + public int LoadedCreationDateTime { get; set; } + public string DefaultOriginalID { get; set; } + + // These variables are used while reading the archive control file + protected int? m_curY = null; + protected int? m_curX = null; + protected RegionInfo m_curRegion; + + + public DearchiveScenesInfo() + { + MultiRegionFormat = false; + } + + + // The following methods are used while reading the archive control file + + public void StartRow() + { + m_curY = (m_curY == null) ? 0 : m_curY + 1; + m_curX = null; + } + + public void StartRegion() + { + m_curX = (m_curX == null) ? 0 : m_curX + 1; + // Note: this doesn't mean we have a real region in this location; this could just be a "hole" + } + + public void SetRegionOriginalID(string id) + { + m_curRegion = new RegionInfo(); + m_curRegion.Location = new Point((int)m_curX, (int)m_curY); + m_curRegion.OriginalID = id; + // 'curRegion' will be saved in 'm_directory2region' when SetRegionDir() is called + } + + public void SetRegionDirectory(string directory) + { + m_curRegion.Directory = directory; + m_directory2region[directory] = m_curRegion; + } + + + /// + /// Sets all the scenes present in the simulator. + /// + /// + /// This method matches regions in the archive to scenes in the simulator according to + /// their relative position. We only load regions if there's an existing Scene in the + /// grid location where the region should be loaded. + /// + /// The scene where the Load OAR operation was run + /// All the scenes in the simulator + public void SetSimulatorScenes(Scene rootScene, ArchiveScenesGroup simulatorScenes) + { + foreach (RegionInfo archivedRegion in m_directory2region.Values) + { + Point location = new Point((int)rootScene.RegionInfo.RegionLocX, (int)rootScene.RegionInfo.RegionLocY); + location.Offset(archivedRegion.Location); + + Scene scene; + if (simulatorScenes.TryGetScene(location, out scene)) + { + archivedRegion.Scene = scene; + m_newId2region[scene.RegionInfo.RegionID] = archivedRegion; + } + else + { + m_log.WarnFormat("[ARCHIVER]: Not loading archived region {0} because there's no existing region at location {1},{2}", + archivedRegion.Directory, location.X, location.Y); + } + } + } + + /// + /// Returns the archived region according to the path of a file in the archive. + /// Also, converts the full path into a path that is relative to the region's directory. + /// + /// The path of a file in the archive + /// The corresponding Scene, or null if none + /// The path relative to the region's directory. (Or the original + /// path, if this file doesn't belong to a region.) + /// True: use this file; False: skip it + public bool GetRegionFromPath(string fullPath, out Scene scene, out string relativePath) + { + scene = null; + relativePath = fullPath; + + if (!MultiRegionFormat) + { + if (m_newId2region.Count > 0) + scene = m_newId2region.First().Value.Scene; + return true; + } + + if (!fullPath.StartsWith(ArchiveConstants.REGIONS_PATH)) + return true; // this file doesn't belong to a region + + string[] parts = fullPath.Split(new Char[] { '/' }, 3); + if (parts.Length != 3) + return false; + string regionDirectory = parts[1]; + relativePath = parts[2]; + + RegionInfo region; + if (m_directory2region.TryGetValue(regionDirectory, out region)) + { + scene = region.Scene; + return (scene != null); + } + else + { + return false; + } + } + + /// + /// Returns the original UUID of a region (from the simulator where the OAR was saved), + /// given the UUID of the scene it was loaded into in the current simulator. + /// + /// + /// + public string GetOriginalRegionID(UUID newID) + { + RegionInfo region; + if (m_newId2region.TryGetValue(newID, out region)) + return region.OriginalID; + else + return DefaultOriginalID; + } + + /// + /// Returns the scenes that have been (or will be) loaded. + /// + /// + public List GetLoadedScenes() + { + return m_newId2region.Keys.ToList(); + } + + } +} diff --git a/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs b/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs index 904110e..cfdfd8c 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs @@ -47,6 +47,7 @@ using ArchiveConstants = OpenSim.Framework.Serialization.ArchiveConstants; using TarArchiveReader = OpenSim.Framework.Serialization.TarArchiveReader; using TarArchiveWriter = OpenSim.Framework.Serialization.TarArchiveWriter; using RegionSettings = OpenSim.Framework.RegionSettings; +using OpenSim.Region.Framework.Interfaces; namespace OpenSim.Region.CoreModules.World.Archiver.Tests { @@ -70,9 +71,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests m_scene = new SceneHelpers().SetupScene(); SceneHelpers.SetupSceneModules(m_scene, m_archiverModule, serialiserModule, terrainModule); + + new SceneManager(); + SceneManager.Instance.Add(m_scene); } - - private void LoadCompleted(Guid requestId, string errorMessage) + + private void LoadCompleted(Guid requestId, List loadedScenes, string errorMessage) { lock (this) { @@ -186,7 +190,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests Assert.That(filePath, Is.EqualTo(ArchiveConstants.CONTROL_FILE_PATH)); ArchiveReadRequest arr = new ArchiveReadRequest(m_scene, (Stream)null, false, false, Guid.Empty); - arr.LoadControlFile(filePath, data); + arr.LoadControlFile(filePath, data, new DearchiveScenesInfo()); Assert.That(arr.ControlFileLoaded, Is.True); @@ -270,7 +274,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests Assert.That(filePath, Is.EqualTo(ArchiveConstants.CONTROL_FILE_PATH)); ArchiveReadRequest arr = new ArchiveReadRequest(m_scene, (Stream)null, false, false, Guid.Empty); - arr.LoadControlFile(filePath, data); + arr.LoadControlFile(filePath, data, new DearchiveScenesInfo()); Assert.That(arr.ControlFileLoaded, Is.True); @@ -307,7 +311,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests tar.WriteFile( ArchiveConstants.CONTROL_FILE_PATH, - new ArchiveWriteRequestPreparation(null, (Stream)null, Guid.Empty).CreateControlFile(new Dictionary())); + new ArchiveWriteRequestPreparation(m_scene, (Stream)null, Guid.Empty).CreateControlFile(new ArchiveScenesGroup())); SceneObjectGroup sog1 = SceneHelpers.CreateSceneObject(1, ownerId, "obj1-", 0x11); SceneObjectPart sop2 @@ -362,11 +366,10 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests // Also check that direct entries which will also have a file entry containing that directory doesn't // upset load tar.WriteDir(ArchiveConstants.TERRAINS_PATH); - + tar.WriteFile( ArchiveConstants.CONTROL_FILE_PATH, - new ArchiveWriteRequestPreparation(null, (Stream)null, Guid.Empty).CreateControlFile(new Dictionary())); - + new ArchiveWriteRequestPreparation(m_scene, (Stream)null, Guid.Empty).CreateControlFile(new ArchiveScenesGroup())); SceneObjectPart part1 = CreateSceneObjectPart1(); part1.SitTargetOrientation = new Quaternion(0.2f, 0.3f, 0.4f, 0.5f); @@ -519,6 +522,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests TestScene scene2 = new SceneHelpers().SetupScene(); SceneHelpers.SetupSceneModules(scene2, archiverModule, serialiserModule, terrainModule); + SceneManager.Instance.Add(scene2); + // Make sure there's a valid owner for the owner we saved (this should have been wiped if the code is // behaving correctly UserAccountHelpers.CreateUserWithInventory(scene2, objectOwner); @@ -554,7 +559,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests tar.WriteDir(ArchiveConstants.TERRAINS_PATH); tar.WriteFile( ArchiveConstants.CONTROL_FILE_PATH, - new ArchiveWriteRequestPreparation(null, (Stream)null, Guid.Empty).CreateControlFile(new Dictionary())); + new ArchiveWriteRequestPreparation(m_scene, (Stream)null, Guid.Empty).CreateControlFile(new ArchiveScenesGroup())); RegionSettings rs = new RegionSettings(); rs.AgentLimit = 17; -- cgit v1.1 From d7e6fe488d5258ec92230b6c652969f2d0420376 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 7 Sep 2012 23:30:54 +0100 Subject: Rename ArchiveWriteRequestPreparatio nto ArchiveWriteRequest since after the multi-OAR patch there is now only one class that handles this operation. Adapation of 0004-Renamed-ArchiveWriteRequestPreparation-to-ArchiveWri.patch in http://opensimulator.org/mantis/view.php?id=6105 since that did not directly apply --- .../World/Archiver/ArchiveWriteRequest.cs | 634 +++++++++++++++++++++ .../Archiver/ArchiveWriteRequestPreparation.cs | 634 --------------------- .../CoreModules/World/Archiver/ArchiverModule.cs | 4 +- .../World/Archiver/Tests/ArchiverTests.cs | 6 +- 4 files changed, 639 insertions(+), 639 deletions(-) create mode 100644 OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs delete mode 100644 OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs (limited to 'OpenSim/Region/CoreModules/World') diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs new file mode 100644 index 0000000..d751b1c --- /dev/null +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs @@ -0,0 +1,634 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Reflection; +using System.Text.RegularExpressions; +using System.Threading; +using System.Xml; +using log4net; +using OpenMetaverse; +using OpenSim.Framework; +using OpenSim.Framework.Serialization; +using OpenSim.Region.CoreModules.World.Terrain; +using OpenSim.Region.Framework.Interfaces; +using OpenSim.Region.Framework.Scenes; +using Ionic.Zlib; +using GZipStream = Ionic.Zlib.GZipStream; +using CompressionMode = Ionic.Zlib.CompressionMode; +using OpenSim.Framework.Serialization.External; + +namespace OpenSim.Region.CoreModules.World.Archiver +{ + /// + /// Prepare to write out an archive. + /// + public class ArchiveWriteRequest + { + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + /// + /// The minimum major version of OAR that we can write. + /// + public static int MIN_MAJOR_VERSION = 0; + + /// + /// The maximum major version of OAR that we can write. + /// + public static int MAX_MAJOR_VERSION = 1; + + /// + /// Whether we're saving a multi-region archive. + /// + public bool MultiRegionFormat { get; set; } + + /// + /// Determine whether this archive will save assets. Default is true. + /// + public bool SaveAssets { get; set; } + + /// + /// Determines which objects will be included in the archive, according to their permissions. + /// Default is null, meaning no permission checks. + /// + public string CheckPermissions { get; set; } + + protected Scene m_rootScene; + protected Stream m_saveStream; + protected TarArchiveWriter m_archiveWriter; + protected Guid m_requestId; + protected Dictionary m_options; + + /// + /// Constructor + /// + /// Calling module + /// The path to which to save data. + /// The id associated with this request + /// + /// If there was a problem opening a stream for the file specified by the savePath + /// + public ArchiveWriteRequest(Scene scene, string savePath, Guid requestId) : this(scene, requestId) + { + try + { + m_saveStream = new GZipStream(new FileStream(savePath, FileMode.Create), CompressionMode.Compress, CompressionLevel.BestCompression); + } + catch (EntryPointNotFoundException e) + { + m_log.ErrorFormat( + "[ARCHIVER]: Mismatch between Mono and zlib1g library version when trying to create compression stream." + + "If you've manually installed Mono, have you appropriately updated zlib1g as well?"); + m_log.ErrorFormat("{0} {1}", e.Message, e.StackTrace); + } + } + + /// + /// Constructor. + /// + /// The root scene to archive + /// The stream to which to save data. + /// The id associated with this request + public ArchiveWriteRequest(Scene scene, Stream saveStream, Guid requestId) : this(scene, requestId) + { + m_saveStream = saveStream; + } + + protected ArchiveWriteRequest(Scene scene, Guid requestId) + { + m_rootScene = scene; + m_requestId = requestId; + m_archiveWriter = null; + + MultiRegionFormat = false; + SaveAssets = true; + CheckPermissions = null; + } + + /// + /// Archive the region requested. + /// + /// if there was an io problem with creating the file + public void ArchiveRegion(Dictionary options) + { + m_options = options; + + if (options.ContainsKey("all") && (bool)options["all"]) + MultiRegionFormat = true; + + if (options.ContainsKey("noassets") && (bool)options["noassets"]) + SaveAssets = false; + + Object temp; + if (options.TryGetValue("checkPermissions", out temp)) + CheckPermissions = (string)temp; + + + // Find the regions to archive + ArchiveScenesGroup scenesGroup = new ArchiveScenesGroup(); + if (MultiRegionFormat) + { + m_log.InfoFormat("[ARCHIVER]: Saving {0} regions", SceneManager.Instance.Scenes.Count); + SceneManager.Instance.ForEachScene(delegate(Scene scene) + { + scenesGroup.AddScene(scene); + }); + } + else + { + scenesGroup.AddScene(m_rootScene); + } + scenesGroup.CalcSceneLocations(); + + + m_archiveWriter = new TarArchiveWriter(m_saveStream); + + try + { + // Write out control file. It should be first so that it will be found ASAP when loading the file. + m_archiveWriter.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, CreateControlFile(scenesGroup)); + m_log.InfoFormat("[ARCHIVER]: Added control file to archive."); + + // Archive the regions + + Dictionary assetUuids = new Dictionary(); + + scenesGroup.ForEachScene(delegate(Scene scene) + { + string regionDir = MultiRegionFormat ? scenesGroup.GetRegionDir(scene.RegionInfo.RegionID) : ""; + ArchiveOneRegion(scene, regionDir, assetUuids); + }); + + // Archive the assets + + if (SaveAssets) + { + m_log.DebugFormat("[ARCHIVER]: Saving {0} assets", assetUuids.Count); + + // Asynchronously request all the assets required to perform this archive operation + AssetsRequest ar + = new AssetsRequest( + new AssetsArchiver(m_archiveWriter), assetUuids, + m_rootScene.AssetService, m_rootScene.UserAccountService, + m_rootScene.RegionInfo.ScopeID, options, ReceivedAllAssets); + + Util.FireAndForget(o => ar.Execute()); + + // CloseArchive() will be called from ReceivedAllAssets() + } + else + { + m_log.DebugFormat("[ARCHIVER]: Not saving assets since --noassets was specified"); + CloseArchive(string.Empty); + } + } + catch (Exception e) + { + CloseArchive(e.Message); + throw; + } + } + + + private void ArchiveOneRegion(Scene scene, string regionDir, Dictionary assetUuids) + { + m_log.InfoFormat("[ARCHIVER]: Writing region {0}", scene.RegionInfo.RegionName); + + EntityBase[] entities = scene.GetEntities(); + List sceneObjects = new List(); + + int numObjectsSkippedPermissions = 0; + + // Filter entities so that we only have scene objects. + // FIXME: Would be nicer to have this as a proper list in SceneGraph, since lots of methods + // end up having to do this + IPermissionsModule permissionsModule = scene.RequestModuleInterface(); + foreach (EntityBase entity in entities) + { + if (entity is SceneObjectGroup) + { + SceneObjectGroup sceneObject = (SceneObjectGroup)entity; + + if (!sceneObject.IsDeleted && !sceneObject.IsAttachment) + { + if (!CanUserArchiveObject(scene.RegionInfo.EstateSettings.EstateOwner, sceneObject, CheckPermissions, permissionsModule)) + { + // The user isn't allowed to copy/transfer this object, so it will not be included in the OAR. + ++numObjectsSkippedPermissions; + } + else + { + sceneObjects.Add(sceneObject); + } + } + } + } + + if (SaveAssets) + { + UuidGatherer assetGatherer = new UuidGatherer(scene.AssetService); + int prevAssets = assetUuids.Count; + + foreach (SceneObjectGroup sceneObject in sceneObjects) + { + assetGatherer.GatherAssetUuids(sceneObject, assetUuids); + } + + m_log.DebugFormat( + "[ARCHIVER]: {0} scene objects to serialize requiring save of {1} assets", + sceneObjects.Count, assetUuids.Count - prevAssets); + } + + if (numObjectsSkippedPermissions > 0) + { + m_log.DebugFormat( + "[ARCHIVER]: {0} scene objects skipped due to lack of permissions", + numObjectsSkippedPermissions); + } + + // Make sure that we also request terrain texture assets + RegionSettings regionSettings = scene.RegionInfo.RegionSettings; + + if (regionSettings.TerrainTexture1 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_1) + assetUuids[regionSettings.TerrainTexture1] = AssetType.Texture; + + if (regionSettings.TerrainTexture2 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_2) + assetUuids[regionSettings.TerrainTexture2] = AssetType.Texture; + + if (regionSettings.TerrainTexture3 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_3) + assetUuids[regionSettings.TerrainTexture3] = AssetType.Texture; + + if (regionSettings.TerrainTexture4 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_4) + assetUuids[regionSettings.TerrainTexture4] = AssetType.Texture; + + Save(scene, sceneObjects, regionDir); + } + + /// + /// Checks whether the user has permission to export an object group to an OAR. + /// + /// The user + /// The object group + /// Which permissions to check: "C" = Copy, "T" = Transfer + /// The scene's permissions module + /// Whether the user is allowed to export the object to an OAR + private bool CanUserArchiveObject(UUID user, SceneObjectGroup objGroup, string checkPermissions, IPermissionsModule permissionsModule) + { + if (checkPermissions == null) + return true; + + if (permissionsModule == null) + return true; // this shouldn't happen + + // Check whether the user is permitted to export all of the parts in the SOG. If any + // part can't be exported then the entire SOG can't be exported. + + bool permitted = true; + //int primNumber = 1; + + foreach (SceneObjectPart obj in objGroup.Parts) + { + uint perm; + PermissionClass permissionClass = permissionsModule.GetPermissionClass(user, obj); + switch (permissionClass) + { + case PermissionClass.Owner: + perm = obj.BaseMask; + break; + case PermissionClass.Group: + perm = obj.GroupMask | obj.EveryoneMask; + break; + case PermissionClass.Everyone: + default: + perm = obj.EveryoneMask; + break; + } + + bool canCopy = (perm & (uint)PermissionMask.Copy) != 0; + bool canTransfer = (perm & (uint)PermissionMask.Transfer) != 0; + + // Special case: if Everyone can copy the object then this implies it can also be + // Transferred. + // However, if the user is the Owner then we don't check EveryoneMask, because it seems that the mask + // always (incorrectly) includes the Copy bit set in this case. But that's a mistake: the viewer + // does NOT show that the object has Everyone-Copy permissions, and doesn't allow it to be copied. + if (permissionClass != PermissionClass.Owner) + canTransfer |= (obj.EveryoneMask & (uint)PermissionMask.Copy) != 0; + + bool partPermitted = true; + if (checkPermissions.Contains("C") && !canCopy) + partPermitted = false; + if (checkPermissions.Contains("T") && !canTransfer) + partPermitted = false; + + // If the user is the Creator of the object then it can always be included in the OAR + bool creator = (obj.CreatorID.Guid == user.Guid); + if (creator) + partPermitted = true; + + //string name = (objGroup.PrimCount == 1) ? objGroup.Name : string.Format("{0} ({1}/{2})", obj.Name, primNumber, objGroup.PrimCount); + //m_log.DebugFormat("[ARCHIVER]: Object permissions: {0}: Base={1:X4}, Owner={2:X4}, Everyone={3:X4}, permissionClass={4}, checkPermissions={5}, canCopy={6}, canTransfer={7}, creator={8}, permitted={9}", + // name, obj.BaseMask, obj.OwnerMask, obj.EveryoneMask, + // permissionClass, checkPermissions, canCopy, canTransfer, creator, partPermitted); + + if (!partPermitted) + { + permitted = false; + break; + } + + //++primNumber; + } + + return permitted; + } + + /// + /// Create the control file. + /// + /// + public string CreateControlFile(ArchiveScenesGroup scenesGroup) + { + int majorVersion; + int minorVersion; + + if (MultiRegionFormat) + { + majorVersion = MAX_MAJOR_VERSION; + minorVersion = 0; + } + else + { + // To support older versions of OpenSim, we continue to create single-region OARs + // using the old file format. In the future this format will be discontinued. + majorVersion = 0; + minorVersion = 8; + } +// +// if (m_options.ContainsKey("version")) +// { +// string[] parts = m_options["version"].ToString().Split('.'); +// if (parts.Length >= 1) +// { +// majorVersion = Int32.Parse(parts[0]); +// +// if (parts.Length >= 2) +// minorVersion = Int32.Parse(parts[1]); +// } +// } +// +// if (majorVersion < MIN_MAJOR_VERSION || majorVersion > MAX_MAJOR_VERSION) +// { +// throw new Exception( +// string.Format( +// "OAR version number for save must be between {0} and {1}", +// MIN_MAJOR_VERSION, MAX_MAJOR_VERSION)); +// } +// else if (majorVersion == MAX_MAJOR_VERSION) +// { +// // Force 1.0 +// minorVersion = 0; +// } +// else if (majorVersion == MIN_MAJOR_VERSION) +// { +// // Force 0.4 +// minorVersion = 4; +// } + + m_log.InfoFormat("[ARCHIVER]: Creating version {0}.{1} OAR", majorVersion, minorVersion); + if (majorVersion == 1) + { + m_log.WarnFormat("[ARCHIVER]: Please be aware that version 1.0 OARs are not compatible with OpenSim versions prior to 0.7.4. Do not use the --all option if you want to produce a compatible OAR"); + } + + String s; + + using (StringWriter sw = new StringWriter()) + { + using (XmlTextWriter xtw = new XmlTextWriter(sw)) + { + xtw.Formatting = Formatting.Indented; + xtw.WriteStartDocument(); + xtw.WriteStartElement("archive"); + xtw.WriteAttributeString("major_version", majorVersion.ToString()); + xtw.WriteAttributeString("minor_version", minorVersion.ToString()); + + xtw.WriteStartElement("creation_info"); + DateTime now = DateTime.UtcNow; + TimeSpan t = now - new DateTime(1970, 1, 1); + xtw.WriteElementString("datetime", ((int)t.TotalSeconds).ToString()); + if (!MultiRegionFormat) + xtw.WriteElementString("id", m_rootScene.RegionInfo.RegionID.ToString()); + xtw.WriteEndElement(); + + xtw.WriteElementString("assets_included", SaveAssets.ToString()); + + if (MultiRegionFormat) + { + WriteRegionsManifest(scenesGroup, xtw); + } + else + { + xtw.WriteStartElement("region_info"); + WriteRegionInfo(m_rootScene, xtw); + xtw.WriteEndElement(); + } + + xtw.WriteEndElement(); + + xtw.Flush(); + } + + s = sw.ToString(); + } + + return s; + } + + /// + /// Writes the list of regions included in a multi-region OAR. + /// + private static void WriteRegionsManifest(ArchiveScenesGroup scenesGroup, XmlTextWriter xtw) + { + xtw.WriteStartElement("regions"); + + // Write the regions in order: rows from South to North, then regions from West to East. + // The list of regions can have "holes"; we write empty elements in their position. + + for (uint y = (uint)scenesGroup.Rect.Top; y < scenesGroup.Rect.Bottom; ++y) + { + SortedDictionary row; + if (scenesGroup.Regions.TryGetValue(y, out row)) + { + xtw.WriteStartElement("row"); + + for (uint x = (uint)scenesGroup.Rect.Left; x < scenesGroup.Rect.Right; ++x) + { + Scene scene; + if (row.TryGetValue(x, out scene)) + { + xtw.WriteStartElement("region"); + xtw.WriteElementString("id", scene.RegionInfo.RegionID.ToString()); + xtw.WriteElementString("dir", scenesGroup.GetRegionDir(scene.RegionInfo.RegionID)); + WriteRegionInfo(scene, xtw); + xtw.WriteEndElement(); + } + else + { + // Write a placeholder for a missing region + xtw.WriteElementString("region", ""); + } + } + + xtw.WriteEndElement(); + } + else + { + // Write a placeholder for a missing row + xtw.WriteElementString("row", ""); + } + } + + xtw.WriteEndElement(); // "regions" + } + + protected static void WriteRegionInfo(Scene scene, XmlTextWriter xtw) + { + bool isMegaregion; + Vector2 size; + + IRegionCombinerModule rcMod = scene.RequestModuleInterface(); + + if (rcMod != null) + isMegaregion = rcMod.IsRootForMegaregion(scene.RegionInfo.RegionID); + else + isMegaregion = false; + + if (isMegaregion) + size = rcMod.GetSizeOfMegaregion(scene.RegionInfo.RegionID); + else + size = new Vector2((float)Constants.RegionSize, (float)Constants.RegionSize); + + xtw.WriteElementString("is_megaregion", isMegaregion.ToString()); + xtw.WriteElementString("size_in_meters", string.Format("{0},{1}", size.X, size.Y)); + } + + + protected void Save(Scene scene, List sceneObjects, string regionDir) + { + if (regionDir != string.Empty) + regionDir = ArchiveConstants.REGIONS_PATH + regionDir + "/"; + + m_log.InfoFormat("[ARCHIVER]: Adding region settings to archive."); + + // Write out region settings + string settingsPath = String.Format("{0}{1}{2}.xml", + regionDir, ArchiveConstants.SETTINGS_PATH, scene.RegionInfo.RegionName); + m_archiveWriter.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(scene.RegionInfo.RegionSettings)); + + m_log.InfoFormat("[ARCHIVER]: Adding parcel settings to archive."); + + // Write out land data (aka parcel) settings + List landObjects = scene.LandChannel.AllParcels(); + foreach (ILandObject lo in landObjects) + { + LandData landData = lo.LandData; + string landDataPath = String.Format("{0}{1}{2}.xml", + regionDir, ArchiveConstants.LANDDATA_PATH, landData.GlobalID.ToString()); + m_archiveWriter.WriteFile(landDataPath, LandDataSerializer.Serialize(landData, m_options)); + } + + m_log.InfoFormat("[ARCHIVER]: Adding terrain information to archive."); + + // Write out terrain + string terrainPath = String.Format("{0}{1}{2}.r32", + regionDir, ArchiveConstants.TERRAINS_PATH, scene.RegionInfo.RegionName); + + MemoryStream ms = new MemoryStream(); + scene.RequestModuleInterface().SaveToStream(terrainPath, ms); + m_archiveWriter.WriteFile(terrainPath, ms.ToArray()); + ms.Close(); + + m_log.InfoFormat("[ARCHIVER]: Adding scene objects to archive."); + + // Write out scene object metadata + IRegionSerialiserModule serializer = scene.RequestModuleInterface(); + foreach (SceneObjectGroup sceneObject in sceneObjects) + { + //m_log.DebugFormat("[ARCHIVER]: Saving {0} {1}, {2}", entity.Name, entity.UUID, entity.GetType()); + + string serializedObject = serializer.SerializeGroupToXml2(sceneObject, m_options); + string objectPath = string.Format("{0}{1}", regionDir, ArchiveHelpers.CreateObjectPath(sceneObject)); + m_archiveWriter.WriteFile(objectPath, serializedObject); + } + } + + protected void ReceivedAllAssets( + ICollection assetsFoundUuids, ICollection assetsNotFoundUuids) + { + foreach (UUID uuid in assetsNotFoundUuids) + { + m_log.DebugFormat("[ARCHIVER]: Could not find asset {0}", uuid); + } + + // m_log.InfoFormat( + // "[ARCHIVER]: Received {0} of {1} assets requested", + // assetsFoundUuids.Count, assetsFoundUuids.Count + assetsNotFoundUuids.Count); + + CloseArchive(String.Empty); + } + + + /// + /// Closes the archive and notifies that we're done. + /// + /// The error that occurred, or empty for success + protected void CloseArchive(string errorMessage) + { + try + { + if (m_archiveWriter != null) + m_archiveWriter.Close(); + m_saveStream.Close(); + } + catch (Exception e) + { + m_log.Error(string.Format("[ARCHIVER]: Error closing archive: {0} ", e.Message), e); + if (errorMessage == string.Empty) + errorMessage = e.Message; + } + + m_log.InfoFormat("[ARCHIVER]: Finished writing out OAR for {0}", m_rootScene.RegionInfo.RegionName); + + m_rootScene.EventManager.TriggerOarFileSaved(m_requestId, errorMessage); + } + + } +} diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs deleted file mode 100644 index 2c34f4b..0000000 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs +++ /dev/null @@ -1,634 +0,0 @@ -/* - * Copyright (c) Contributors, http://opensimulator.org/ - * See CONTRIBUTORS.TXT for a full list of copyright holders. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the OpenSimulator Project nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -using System; -using System.Collections.Generic; -using System.IO; -using System.IO.Compression; -using System.Reflection; -using System.Text.RegularExpressions; -using System.Threading; -using System.Xml; -using log4net; -using OpenMetaverse; -using OpenSim.Framework; -using OpenSim.Framework.Serialization; -using OpenSim.Region.CoreModules.World.Terrain; -using OpenSim.Region.Framework.Interfaces; -using OpenSim.Region.Framework.Scenes; -using Ionic.Zlib; -using GZipStream = Ionic.Zlib.GZipStream; -using CompressionMode = Ionic.Zlib.CompressionMode; -using OpenSim.Framework.Serialization.External; - -namespace OpenSim.Region.CoreModules.World.Archiver -{ - /// - /// Prepare to write out an archive. - /// - public class ArchiveWriteRequestPreparation - { - private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - - /// - /// The minimum major version of OAR that we can write. - /// - public static int MIN_MAJOR_VERSION = 0; - - /// - /// The maximum major version of OAR that we can write. - /// - public static int MAX_MAJOR_VERSION = 1; - - /// - /// Whether we're saving a multi-region archive. - /// - public bool MultiRegionFormat { get; set; } - - /// - /// Determine whether this archive will save assets. Default is true. - /// - public bool SaveAssets { get; set; } - - /// - /// Determines which objects will be included in the archive, according to their permissions. - /// Default is null, meaning no permission checks. - /// - public string CheckPermissions { get; set; } - - protected Scene m_rootScene; - protected Stream m_saveStream; - protected TarArchiveWriter m_archiveWriter; - protected Guid m_requestId; - protected Dictionary m_options; - - /// - /// Constructor - /// - /// Calling module - /// The path to which to save data. - /// The id associated with this request - /// - /// If there was a problem opening a stream for the file specified by the savePath - /// - public ArchiveWriteRequestPreparation(Scene scene, string savePath, Guid requestId) : this(scene, requestId) - { - try - { - m_saveStream = new GZipStream(new FileStream(savePath, FileMode.Create), CompressionMode.Compress, CompressionLevel.BestCompression); - } - catch (EntryPointNotFoundException e) - { - m_log.ErrorFormat( - "[ARCHIVER]: Mismatch between Mono and zlib1g library version when trying to create compression stream." - + "If you've manually installed Mono, have you appropriately updated zlib1g as well?"); - m_log.ErrorFormat("{0} {1}", e.Message, e.StackTrace); - } - } - - /// - /// Constructor. - /// - /// The root scene to archive - /// The stream to which to save data. - /// The id associated with this request - public ArchiveWriteRequestPreparation(Scene scene, Stream saveStream, Guid requestId) : this(scene, requestId) - { - m_saveStream = saveStream; - } - - protected ArchiveWriteRequestPreparation(Scene scene, Guid requestId) - { - m_rootScene = scene; - m_requestId = requestId; - m_archiveWriter = null; - - MultiRegionFormat = false; - SaveAssets = true; - CheckPermissions = null; - } - - /// - /// Archive the region requested. - /// - /// if there was an io problem with creating the file - public void ArchiveRegion(Dictionary options) - { - m_options = options; - - if (options.ContainsKey("all") && (bool)options["all"]) - MultiRegionFormat = true; - - if (options.ContainsKey("noassets") && (bool)options["noassets"]) - SaveAssets = false; - - Object temp; - if (options.TryGetValue("checkPermissions", out temp)) - CheckPermissions = (string)temp; - - - // Find the regions to archive - ArchiveScenesGroup scenesGroup = new ArchiveScenesGroup(); - if (MultiRegionFormat) - { - m_log.InfoFormat("[ARCHIVER]: Saving {0} regions", SceneManager.Instance.Scenes.Count); - SceneManager.Instance.ForEachScene(delegate(Scene scene) - { - scenesGroup.AddScene(scene); - }); - } - else - { - scenesGroup.AddScene(m_rootScene); - } - scenesGroup.CalcSceneLocations(); - - - m_archiveWriter = new TarArchiveWriter(m_saveStream); - - try - { - // Write out control file. It should be first so that it will be found ASAP when loading the file. - m_archiveWriter.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, CreateControlFile(scenesGroup)); - m_log.InfoFormat("[ARCHIVER]: Added control file to archive."); - - // Archive the regions - - Dictionary assetUuids = new Dictionary(); - - scenesGroup.ForEachScene(delegate(Scene scene) - { - string regionDir = MultiRegionFormat ? scenesGroup.GetRegionDir(scene.RegionInfo.RegionID) : ""; - ArchiveOneRegion(scene, regionDir, assetUuids); - }); - - // Archive the assets - - if (SaveAssets) - { - m_log.DebugFormat("[ARCHIVER]: Saving {0} assets", assetUuids.Count); - - // Asynchronously request all the assets required to perform this archive operation - AssetsRequest ar - = new AssetsRequest( - new AssetsArchiver(m_archiveWriter), assetUuids, - m_rootScene.AssetService, m_rootScene.UserAccountService, - m_rootScene.RegionInfo.ScopeID, options, ReceivedAllAssets); - - Util.FireAndForget(o => ar.Execute()); - - // CloseArchive() will be called from ReceivedAllAssets() - } - else - { - m_log.DebugFormat("[ARCHIVER]: Not saving assets since --noassets was specified"); - CloseArchive(string.Empty); - } - } - catch (Exception e) - { - CloseArchive(e.Message); - throw; - } - } - - - private void ArchiveOneRegion(Scene scene, string regionDir, Dictionary assetUuids) - { - m_log.InfoFormat("[ARCHIVER]: Writing region {0}", scene.RegionInfo.RegionName); - - EntityBase[] entities = scene.GetEntities(); - List sceneObjects = new List(); - - int numObjectsSkippedPermissions = 0; - - // Filter entities so that we only have scene objects. - // FIXME: Would be nicer to have this as a proper list in SceneGraph, since lots of methods - // end up having to do this - IPermissionsModule permissionsModule = scene.RequestModuleInterface(); - foreach (EntityBase entity in entities) - { - if (entity is SceneObjectGroup) - { - SceneObjectGroup sceneObject = (SceneObjectGroup)entity; - - if (!sceneObject.IsDeleted && !sceneObject.IsAttachment) - { - if (!CanUserArchiveObject(scene.RegionInfo.EstateSettings.EstateOwner, sceneObject, CheckPermissions, permissionsModule)) - { - // The user isn't allowed to copy/transfer this object, so it will not be included in the OAR. - ++numObjectsSkippedPermissions; - } - else - { - sceneObjects.Add(sceneObject); - } - } - } - } - - if (SaveAssets) - { - UuidGatherer assetGatherer = new UuidGatherer(scene.AssetService); - int prevAssets = assetUuids.Count; - - foreach (SceneObjectGroup sceneObject in sceneObjects) - { - assetGatherer.GatherAssetUuids(sceneObject, assetUuids); - } - - m_log.DebugFormat( - "[ARCHIVER]: {0} scene objects to serialize requiring save of {1} assets", - sceneObjects.Count, assetUuids.Count - prevAssets); - } - - if (numObjectsSkippedPermissions > 0) - { - m_log.DebugFormat( - "[ARCHIVER]: {0} scene objects skipped due to lack of permissions", - numObjectsSkippedPermissions); - } - - // Make sure that we also request terrain texture assets - RegionSettings regionSettings = scene.RegionInfo.RegionSettings; - - if (regionSettings.TerrainTexture1 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_1) - assetUuids[regionSettings.TerrainTexture1] = AssetType.Texture; - - if (regionSettings.TerrainTexture2 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_2) - assetUuids[regionSettings.TerrainTexture2] = AssetType.Texture; - - if (regionSettings.TerrainTexture3 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_3) - assetUuids[regionSettings.TerrainTexture3] = AssetType.Texture; - - if (regionSettings.TerrainTexture4 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_4) - assetUuids[regionSettings.TerrainTexture4] = AssetType.Texture; - - Save(scene, sceneObjects, regionDir); - } - - /// - /// Checks whether the user has permission to export an object group to an OAR. - /// - /// The user - /// The object group - /// Which permissions to check: "C" = Copy, "T" = Transfer - /// The scene's permissions module - /// Whether the user is allowed to export the object to an OAR - private bool CanUserArchiveObject(UUID user, SceneObjectGroup objGroup, string checkPermissions, IPermissionsModule permissionsModule) - { - if (checkPermissions == null) - return true; - - if (permissionsModule == null) - return true; // this shouldn't happen - - // Check whether the user is permitted to export all of the parts in the SOG. If any - // part can't be exported then the entire SOG can't be exported. - - bool permitted = true; - //int primNumber = 1; - - foreach (SceneObjectPart obj in objGroup.Parts) - { - uint perm; - PermissionClass permissionClass = permissionsModule.GetPermissionClass(user, obj); - switch (permissionClass) - { - case PermissionClass.Owner: - perm = obj.BaseMask; - break; - case PermissionClass.Group: - perm = obj.GroupMask | obj.EveryoneMask; - break; - case PermissionClass.Everyone: - default: - perm = obj.EveryoneMask; - break; - } - - bool canCopy = (perm & (uint)PermissionMask.Copy) != 0; - bool canTransfer = (perm & (uint)PermissionMask.Transfer) != 0; - - // Special case: if Everyone can copy the object then this implies it can also be - // Transferred. - // However, if the user is the Owner then we don't check EveryoneMask, because it seems that the mask - // always (incorrectly) includes the Copy bit set in this case. But that's a mistake: the viewer - // does NOT show that the object has Everyone-Copy permissions, and doesn't allow it to be copied. - if (permissionClass != PermissionClass.Owner) - canTransfer |= (obj.EveryoneMask & (uint)PermissionMask.Copy) != 0; - - bool partPermitted = true; - if (checkPermissions.Contains("C") && !canCopy) - partPermitted = false; - if (checkPermissions.Contains("T") && !canTransfer) - partPermitted = false; - - // If the user is the Creator of the object then it can always be included in the OAR - bool creator = (obj.CreatorID.Guid == user.Guid); - if (creator) - partPermitted = true; - - //string name = (objGroup.PrimCount == 1) ? objGroup.Name : string.Format("{0} ({1}/{2})", obj.Name, primNumber, objGroup.PrimCount); - //m_log.DebugFormat("[ARCHIVER]: Object permissions: {0}: Base={1:X4}, Owner={2:X4}, Everyone={3:X4}, permissionClass={4}, checkPermissions={5}, canCopy={6}, canTransfer={7}, creator={8}, permitted={9}", - // name, obj.BaseMask, obj.OwnerMask, obj.EveryoneMask, - // permissionClass, checkPermissions, canCopy, canTransfer, creator, partPermitted); - - if (!partPermitted) - { - permitted = false; - break; - } - - //++primNumber; - } - - return permitted; - } - - /// - /// Create the control file. - /// - /// - public string CreateControlFile(ArchiveScenesGroup scenesGroup) - { - int majorVersion; - int minorVersion; - - if (MultiRegionFormat) - { - majorVersion = MAX_MAJOR_VERSION; - minorVersion = 0; - } - else - { - // To support older versions of OpenSim, we continue to create single-region OARs - // using the old file format. In the future this format will be discontinued. - majorVersion = 0; - minorVersion = 8; - } -// -// if (m_options.ContainsKey("version")) -// { -// string[] parts = m_options["version"].ToString().Split('.'); -// if (parts.Length >= 1) -// { -// majorVersion = Int32.Parse(parts[0]); -// -// if (parts.Length >= 2) -// minorVersion = Int32.Parse(parts[1]); -// } -// } -// -// if (majorVersion < MIN_MAJOR_VERSION || majorVersion > MAX_MAJOR_VERSION) -// { -// throw new Exception( -// string.Format( -// "OAR version number for save must be between {0} and {1}", -// MIN_MAJOR_VERSION, MAX_MAJOR_VERSION)); -// } -// else if (majorVersion == MAX_MAJOR_VERSION) -// { -// // Force 1.0 -// minorVersion = 0; -// } -// else if (majorVersion == MIN_MAJOR_VERSION) -// { -// // Force 0.4 -// minorVersion = 4; -// } - - m_log.InfoFormat("[ARCHIVER]: Creating version {0}.{1} OAR", majorVersion, minorVersion); - if (majorVersion == 1) - { - m_log.WarnFormat("[ARCHIVER]: Please be aware that version 1.0 OARs are not compatible with OpenSim versions prior to 0.7.4. Do not use the --all option if you want to produce a compatible OAR"); - } - - String s; - - using (StringWriter sw = new StringWriter()) - { - using (XmlTextWriter xtw = new XmlTextWriter(sw)) - { - xtw.Formatting = Formatting.Indented; - xtw.WriteStartDocument(); - xtw.WriteStartElement("archive"); - xtw.WriteAttributeString("major_version", majorVersion.ToString()); - xtw.WriteAttributeString("minor_version", minorVersion.ToString()); - - xtw.WriteStartElement("creation_info"); - DateTime now = DateTime.UtcNow; - TimeSpan t = now - new DateTime(1970, 1, 1); - xtw.WriteElementString("datetime", ((int)t.TotalSeconds).ToString()); - if (!MultiRegionFormat) - xtw.WriteElementString("id", m_rootScene.RegionInfo.RegionID.ToString()); - xtw.WriteEndElement(); - - xtw.WriteElementString("assets_included", SaveAssets.ToString()); - - if (MultiRegionFormat) - { - WriteRegionsManifest(scenesGroup, xtw); - } - else - { - xtw.WriteStartElement("region_info"); - WriteRegionInfo(m_rootScene, xtw); - xtw.WriteEndElement(); - } - - xtw.WriteEndElement(); - - xtw.Flush(); - } - - s = sw.ToString(); - } - - return s; - } - - /// - /// Writes the list of regions included in a multi-region OAR. - /// - private static void WriteRegionsManifest(ArchiveScenesGroup scenesGroup, XmlTextWriter xtw) - { - xtw.WriteStartElement("regions"); - - // Write the regions in order: rows from South to North, then regions from West to East. - // The list of regions can have "holes"; we write empty elements in their position. - - for (uint y = (uint)scenesGroup.Rect.Top; y < scenesGroup.Rect.Bottom; ++y) - { - SortedDictionary row; - if (scenesGroup.Regions.TryGetValue(y, out row)) - { - xtw.WriteStartElement("row"); - - for (uint x = (uint)scenesGroup.Rect.Left; x < scenesGroup.Rect.Right; ++x) - { - Scene scene; - if (row.TryGetValue(x, out scene)) - { - xtw.WriteStartElement("region"); - xtw.WriteElementString("id", scene.RegionInfo.RegionID.ToString()); - xtw.WriteElementString("dir", scenesGroup.GetRegionDir(scene.RegionInfo.RegionID)); - WriteRegionInfo(scene, xtw); - xtw.WriteEndElement(); - } - else - { - // Write a placeholder for a missing region - xtw.WriteElementString("region", ""); - } - } - - xtw.WriteEndElement(); - } - else - { - // Write a placeholder for a missing row - xtw.WriteElementString("row", ""); - } - } - - xtw.WriteEndElement(); // "regions" - } - - protected static void WriteRegionInfo(Scene scene, XmlTextWriter xtw) - { - bool isMegaregion; - Vector2 size; - - IRegionCombinerModule rcMod = scene.RequestModuleInterface(); - - if (rcMod != null) - isMegaregion = rcMod.IsRootForMegaregion(scene.RegionInfo.RegionID); - else - isMegaregion = false; - - if (isMegaregion) - size = rcMod.GetSizeOfMegaregion(scene.RegionInfo.RegionID); - else - size = new Vector2((float)Constants.RegionSize, (float)Constants.RegionSize); - - xtw.WriteElementString("is_megaregion", isMegaregion.ToString()); - xtw.WriteElementString("size_in_meters", string.Format("{0},{1}", size.X, size.Y)); - } - - - protected void Save(Scene scene, List sceneObjects, string regionDir) - { - if (regionDir != string.Empty) - regionDir = ArchiveConstants.REGIONS_PATH + regionDir + "/"; - - m_log.InfoFormat("[ARCHIVER]: Adding region settings to archive."); - - // Write out region settings - string settingsPath = String.Format("{0}{1}{2}.xml", - regionDir, ArchiveConstants.SETTINGS_PATH, scene.RegionInfo.RegionName); - m_archiveWriter.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(scene.RegionInfo.RegionSettings)); - - m_log.InfoFormat("[ARCHIVER]: Adding parcel settings to archive."); - - // Write out land data (aka parcel) settings - List landObjects = scene.LandChannel.AllParcels(); - foreach (ILandObject lo in landObjects) - { - LandData landData = lo.LandData; - string landDataPath = String.Format("{0}{1}{2}.xml", - regionDir, ArchiveConstants.LANDDATA_PATH, landData.GlobalID.ToString()); - m_archiveWriter.WriteFile(landDataPath, LandDataSerializer.Serialize(landData, m_options)); - } - - m_log.InfoFormat("[ARCHIVER]: Adding terrain information to archive."); - - // Write out terrain - string terrainPath = String.Format("{0}{1}{2}.r32", - regionDir, ArchiveConstants.TERRAINS_PATH, scene.RegionInfo.RegionName); - - MemoryStream ms = new MemoryStream(); - scene.RequestModuleInterface().SaveToStream(terrainPath, ms); - m_archiveWriter.WriteFile(terrainPath, ms.ToArray()); - ms.Close(); - - m_log.InfoFormat("[ARCHIVER]: Adding scene objects to archive."); - - // Write out scene object metadata - IRegionSerialiserModule serializer = scene.RequestModuleInterface(); - foreach (SceneObjectGroup sceneObject in sceneObjects) - { - //m_log.DebugFormat("[ARCHIVER]: Saving {0} {1}, {2}", entity.Name, entity.UUID, entity.GetType()); - - string serializedObject = serializer.SerializeGroupToXml2(sceneObject, m_options); - string objectPath = string.Format("{0}{1}", regionDir, ArchiveHelpers.CreateObjectPath(sceneObject)); - m_archiveWriter.WriteFile(objectPath, serializedObject); - } - } - - protected void ReceivedAllAssets( - ICollection assetsFoundUuids, ICollection assetsNotFoundUuids) - { - foreach (UUID uuid in assetsNotFoundUuids) - { - m_log.DebugFormat("[ARCHIVER]: Could not find asset {0}", uuid); - } - - // m_log.InfoFormat( - // "[ARCHIVER]: Received {0} of {1} assets requested", - // assetsFoundUuids.Count, assetsFoundUuids.Count + assetsNotFoundUuids.Count); - - CloseArchive(String.Empty); - } - - - /// - /// Closes the archive and notifies that we're done. - /// - /// The error that occurred, or empty for success - protected void CloseArchive(string errorMessage) - { - try - { - if (m_archiveWriter != null) - m_archiveWriter.Close(); - m_saveStream.Close(); - } - catch (Exception e) - { - m_log.Error(string.Format("[ARCHIVER]: Error closing archive: {0} ", e.Message), e); - if (errorMessage == string.Empty) - errorMessage = e.Message; - } - - m_log.InfoFormat("[ARCHIVER]: Finished writing out OAR for {0}", m_rootScene.RegionInfo.RegionName); - - m_rootScene.EventManager.TriggerOarFileSaved(m_requestId, errorMessage); - } - - } -} diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiverModule.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiverModule.cs index 26535a9..2a87dc2 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiverModule.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiverModule.cs @@ -170,7 +170,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver m_log.InfoFormat( "[ARCHIVER]: Writing archive for region {0} to {1}", Scene.RegionInfo.RegionName, savePath); - new ArchiveWriteRequestPreparation(Scene, savePath, requestId).ArchiveRegion(options); + new ArchiveWriteRequest(Scene, savePath, requestId).ArchiveRegion(options); } public void ArchiveRegion(Stream saveStream) @@ -185,7 +185,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver public void ArchiveRegion(Stream saveStream, Guid requestId, Dictionary options) { - new ArchiveWriteRequestPreparation(Scene, saveStream, requestId).ArchiveRegion(options); + new ArchiveWriteRequest(Scene, saveStream, requestId).ArchiveRegion(options); } public void DearchiveRegion(string loadPath) diff --git a/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs b/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs index cfdfd8c..abbaf41 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs @@ -311,7 +311,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests tar.WriteFile( ArchiveConstants.CONTROL_FILE_PATH, - new ArchiveWriteRequestPreparation(m_scene, (Stream)null, Guid.Empty).CreateControlFile(new ArchiveScenesGroup())); + new ArchiveWriteRequest(m_scene, (Stream)null, Guid.Empty).CreateControlFile(new ArchiveScenesGroup())); SceneObjectGroup sog1 = SceneHelpers.CreateSceneObject(1, ownerId, "obj1-", 0x11); SceneObjectPart sop2 @@ -369,7 +369,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests tar.WriteFile( ArchiveConstants.CONTROL_FILE_PATH, - new ArchiveWriteRequestPreparation(m_scene, (Stream)null, Guid.Empty).CreateControlFile(new ArchiveScenesGroup())); + new ArchiveWriteRequest(m_scene, (Stream)null, Guid.Empty).CreateControlFile(new ArchiveScenesGroup())); SceneObjectPart part1 = CreateSceneObjectPart1(); part1.SitTargetOrientation = new Quaternion(0.2f, 0.3f, 0.4f, 0.5f); @@ -559,7 +559,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests tar.WriteDir(ArchiveConstants.TERRAINS_PATH); tar.WriteFile( ArchiveConstants.CONTROL_FILE_PATH, - new ArchiveWriteRequestPreparation(m_scene, (Stream)null, Guid.Empty).CreateControlFile(new ArchiveScenesGroup())); + new ArchiveWriteRequest(m_scene, (Stream)null, Guid.Empty).CreateControlFile(new ArchiveScenesGroup())); RegionSettings rs = new RegionSettings(); rs.AgentLimit = 17; -- cgit v1.1 From 5dd2569bf7d50217d1786c3268a537745b2e9fd3 Mon Sep 17 00:00:00 2001 From: Oren Hurvitz Date: Mon, 27 Aug 2012 12:47:04 +0300 Subject: Added unit tests for multi-region OARs --- .../World/Archiver/Tests/ArchiverTests.cs | 384 +++++++++++++++++---- 1 file changed, 326 insertions(+), 58 deletions(-) (limited to 'OpenSim/Region/CoreModules/World') diff --git a/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs b/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs index abbaf41..0a30905 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs @@ -57,23 +57,25 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests private Guid m_lastRequestId; private string m_lastErrorMessage; + protected SceneHelpers m_sceneHelpers; protected TestScene m_scene; protected ArchiverModule m_archiverModule; + protected SerialiserModule m_serialiserModule; protected TaskInventoryItem m_soundItem; [SetUp] public void SetUp() { + new SceneManager(); + m_archiverModule = new ArchiverModule(); - SerialiserModule serialiserModule = new SerialiserModule(); + m_serialiserModule = new SerialiserModule(); TerrainModule terrainModule = new TerrainModule(); - m_scene = new SceneHelpers().SetupScene(); - SceneHelpers.SetupSceneModules(m_scene, m_archiverModule, serialiserModule, terrainModule); - - new SceneManager(); - SceneManager.Instance.Add(m_scene); + m_sceneHelpers = new SceneHelpers(); + m_scene = m_sceneHelpers.SetupScene(); + SceneHelpers.SetupSceneModules(m_scene, m_archiverModule, m_serialiserModule, terrainModule); } private void LoadCompleted(Guid requestId, List loadedScenes, string errorMessage) @@ -132,26 +134,10 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests TestHelpers.InMethod(); // log4net.Config.XmlConfigurator.Configure(); - SceneObjectPart part1 = CreateSceneObjectPart1(); - SceneObjectGroup sog1 = new SceneObjectGroup(part1); - m_scene.AddNewSceneObject(sog1, false); - - SceneObjectPart part2 = CreateSceneObjectPart2(); - - AssetNotecard nc = new AssetNotecard(); - nc.BodyText = "Hello World!"; - nc.Encode(); - UUID ncAssetUuid = new UUID("00000000-0000-0000-1000-000000000000"); - UUID ncItemUuid = new UUID("00000000-0000-0000-1100-000000000000"); - AssetBase ncAsset - = AssetHelpers.CreateAsset(ncAssetUuid, AssetType.Notecard, nc.AssetData, UUID.Zero); - m_scene.AssetService.Store(ncAsset); - SceneObjectGroup sog2 = new SceneObjectGroup(part2); - TaskInventoryItem ncItem - = new TaskInventoryItem { Name = "ncItem", AssetID = ncAssetUuid, ItemID = ncItemUuid }; - part2.Inventory.AddInventoryItem(ncItem, true); - - m_scene.AddNewSceneObject(sog2, false); + SceneObjectGroup sog1; + SceneObjectGroup sog2; + UUID ncAssetUuid; + CreateTestObjects(m_scene, out sog1, out sog2, out ncAssetUuid); MemoryStream archiveWriteStream = new MemoryStream(); m_scene.EventManager.OnOarFileSaved += SaveCompleted; @@ -215,6 +201,30 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests // TODO: Test presence of more files and contents of files. } + private void CreateTestObjects(Scene scene, out SceneObjectGroup sog1, out SceneObjectGroup sog2, out UUID ncAssetUuid) + { + SceneObjectPart part1 = CreateSceneObjectPart1(); + sog1 = new SceneObjectGroup(part1); + scene.AddNewSceneObject(sog1, false); + + AssetNotecard nc = new AssetNotecard(); + nc.BodyText = "Hello World!"; + nc.Encode(); + ncAssetUuid = UUID.Random(); + UUID ncItemUuid = UUID.Random(); + AssetBase ncAsset + = AssetHelpers.CreateAsset(ncAssetUuid, AssetType.Notecard, nc.AssetData, UUID.Zero); + m_scene.AssetService.Store(ncAsset); + + TaskInventoryItem ncItem + = new TaskInventoryItem { Name = "ncItem", AssetID = ncAssetUuid, ItemID = ncItemUuid }; + SceneObjectPart part2 = CreateSceneObjectPart2(); + sog2 = new SceneObjectGroup(part2); + part2.Inventory.AddInventoryItem(ncItem, true); + + scene.AddNewSceneObject(sog2, false); + } + /// /// Test saving an OpenSim Region Archive with the no assets option /// @@ -392,31 +402,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests Assert.That(soundDataResourceName, Is.Not.Null); byte[] soundData; - Console.WriteLine("Loading " + soundDataResourceName); - using (Stream resource = assembly.GetManifestResourceStream(soundDataResourceName)) - { - using (BinaryReader br = new BinaryReader(resource)) - { - // FIXME: Use the inspector instead - soundData = br.ReadBytes(99999999); - UUID soundUuid = UUID.Parse("00000000-0000-0000-0000-000000000001"); - string soundAssetFileName - = ArchiveConstants.ASSETS_PATH + soundUuid - + ArchiveConstants.ASSET_TYPE_TO_EXTENSION[(sbyte)AssetType.SoundWAV]; - tar.WriteFile(soundAssetFileName, soundData); - - /* - AssetBase soundAsset = AssetHelpers.CreateAsset(soundUuid, soundData); - scene.AssetService.Store(soundAsset); - asset1FileName = ArchiveConstants.ASSETS_PATH + soundUuid + ".wav"; - */ - - TaskInventoryItem item1 - = new TaskInventoryItem { AssetID = soundUuid, ItemID = soundItemUuid, Name = soundItemName }; - part1.Inventory.AddInventoryItem(item1, true); - } - } - + UUID soundUuid; + CreateSoundAsset(tar, assembly, soundDataResourceName, out soundData, out soundUuid); + + TaskInventoryItem item1 + = new TaskInventoryItem { AssetID = soundUuid, ItemID = soundItemUuid, Name = soundItemName }; + part1.Inventory.AddInventoryItem(item1, true); m_scene.AddNewSceneObject(object1, false); string object1FileName = string.Format( @@ -438,6 +429,34 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests Assert.That(m_lastErrorMessage, Is.Null); + TestLoadedRegion(part1, soundItemName, soundData); + } + + private static void CreateSoundAsset(TarArchiveWriter tar, Assembly assembly, string soundDataResourceName, out byte[] soundData, out UUID soundUuid) + { + using (Stream resource = assembly.GetManifestResourceStream(soundDataResourceName)) + { + using (BinaryReader br = new BinaryReader(resource)) + { + // FIXME: Use the inspector instead + soundData = br.ReadBytes(99999999); + soundUuid = UUID.Parse("00000000-0000-0000-0000-000000000001"); + string soundAssetFileName + = ArchiveConstants.ASSETS_PATH + soundUuid + + ArchiveConstants.ASSET_TYPE_TO_EXTENSION[(sbyte)AssetType.SoundWAV]; + tar.WriteFile(soundAssetFileName, soundData); + + /* + AssetBase soundAsset = AssetHelpers.CreateAsset(soundUuid, soundData); + scene.AssetService.Store(soundAsset); + asset1FileName = ArchiveConstants.ASSETS_PATH + soundUuid + ".wav"; + */ + } + } + } + + private void TestLoadedRegion(SceneObjectPart part1, string soundItemName, byte[] soundData) + { SceneObjectPart object1PartLoaded = m_scene.GetSceneObjectPart(part1.Name); Assert.That(object1PartLoaded, Is.Not.Null, "object1 was not loaded"); @@ -457,9 +476,6 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests Assert.That(loadedSoundAsset.Data, Is.EqualTo(soundData), "saved and loaded sound data do not match"); Assert.Greater(m_scene.LandChannel.AllParcels().Count, 0, "incorrect number of parcels"); - - // Temporary - Console.WriteLine("Successfully completed {0}", MethodBase.GetCurrentMethod()); } /// @@ -519,11 +535,10 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests SerialiserModule serialiserModule = new SerialiserModule(); TerrainModule terrainModule = new TerrainModule(); - TestScene scene2 = new SceneHelpers().SetupScene(); + m_sceneHelpers = new SceneHelpers(); + TestScene scene2 = m_sceneHelpers.SetupScene(); SceneHelpers.SetupSceneModules(scene2, archiverModule, serialiserModule, terrainModule); - SceneManager.Instance.Add(scene2); - // Make sure there's a valid owner for the owner we saved (this should have been wiped if the code is // behaving correctly UserAccountHelpers.CreateUserWithInventory(scene2, objectOwner); @@ -669,7 +684,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests SerialiserModule serialiserModule = new SerialiserModule(); TerrainModule terrainModule = new TerrainModule(); - Scene scene = new SceneHelpers().SetupScene(); + Scene scene = m_sceneHelpers.SetupScene(); SceneHelpers.SetupSceneModules(scene, archiverModule, serialiserModule, terrainModule); m_scene.AddNewSceneObject(new SceneObjectGroup(part2), false); @@ -705,5 +720,258 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests Assert.That(object2PartMerged.GroupPosition, Is.EqualTo(part2.GroupPosition), "object2 group position not equal after merge"); } } + + /// + /// Test saving a multi-region OAR. + /// + [Test] + public void TestSaveMultiRegionOar() + { + TestHelpers.InMethod(); + + // Create test regions + + int WIDTH = 2; + int HEIGHT = 2; + + List scenes = new List(); + + // Maps (Directory in OAR file -> scene) + Dictionary regionPaths = new Dictionary(); + + // Maps (Scene -> expected object paths) + Dictionary> expectedPaths = new Dictionary>(); + + // List of expected assets + List expectedAssets = new List(); + + for (uint y = 0; y < HEIGHT; y++) + { + for (uint x = 0; x < WIDTH; x++) + { + Scene scene; + if (x == 0 && y == 0) + { + scene = m_scene; // this scene was already created in SetUp() + } + else + { + scene = m_sceneHelpers.SetupScene(string.Format("Unit test region {0}", (y * WIDTH) + x + 1), UUID.Random(), 1000 + x, 1000 + y); + SceneHelpers.SetupSceneModules(scene, new ArchiverModule(), m_serialiserModule, new TerrainModule()); + } + scenes.Add(scene); + + string dir = String.Format("{0}_{1}_{2}", x + 1, y + 1, scene.RegionInfo.RegionName.Replace(" ", "_")); + regionPaths[dir] = scene; + + SceneObjectGroup sog1; + SceneObjectGroup sog2; + UUID ncAssetUuid; + + CreateTestObjects(scene, out sog1, out sog2, out ncAssetUuid); + + expectedPaths[scene.RegionInfo.RegionID] = new List(); + expectedPaths[scene.RegionInfo.RegionID].Add(ArchiveHelpers.CreateObjectPath(sog1)); + expectedPaths[scene.RegionInfo.RegionID].Add(ArchiveHelpers.CreateObjectPath(sog2)); + + expectedAssets.Add(ncAssetUuid); + } + } + + + // Save OAR + + MemoryStream archiveWriteStream = new MemoryStream(); + m_scene.EventManager.OnOarFileSaved += SaveCompleted; + + Guid requestId = new Guid("00000000-0000-0000-0000-808080808080"); + + Dictionary options = new Dictionary(); + options.Add("all", true); + + lock (this) + { + m_archiverModule.ArchiveRegion(archiveWriteStream, requestId, options); + Monitor.Wait(this, 60000); + } + + + // Check that the OAR contains the expected data + + Assert.That(m_lastRequestId, Is.EqualTo(requestId)); + + byte[] archive = archiveWriteStream.ToArray(); + MemoryStream archiveReadStream = new MemoryStream(archive); + TarArchiveReader tar = new TarArchiveReader(archiveReadStream); + + Dictionary> foundPaths = new Dictionary>(); + List foundAssets = new List(); + + foreach (Scene scene in scenes) + { + foundPaths[scene.RegionInfo.RegionID] = new List(); + } + + string filePath; + TarArchiveReader.TarEntryType tarEntryType; + + byte[] data = tar.ReadEntry(out filePath, out tarEntryType); + Assert.That(filePath, Is.EqualTo(ArchiveConstants.CONTROL_FILE_PATH)); + + ArchiveReadRequest arr = new ArchiveReadRequest(m_scene, (Stream)null, false, false, Guid.Empty); + arr.LoadControlFile(filePath, data, new DearchiveScenesInfo()); + + Assert.That(arr.ControlFileLoaded, Is.True); + + while (tar.ReadEntry(out filePath, out tarEntryType) != null) + { + if (filePath.StartsWith(ArchiveConstants.ASSETS_PATH)) + { + // Assets are shared, so this file doesn't belong to any specific region. + string fileName = filePath.Remove(0, ArchiveConstants.ASSETS_PATH.Length); + if (fileName.EndsWith("_notecard.txt")) + foundAssets.Add(UUID.Parse(fileName.Substring(0, fileName.Length - "_notecard.txt".Length))); + } + else + { + // This file belongs to one of the regions. Find out which one. + Assert.IsTrue(filePath.StartsWith(ArchiveConstants.REGIONS_PATH)); + string[] parts = filePath.Split(new Char[] { '/' }, 3); + Assert.AreEqual(3, parts.Length); + string regionDirectory = parts[1]; + string relativePath = parts[2]; + Scene scene = regionPaths[regionDirectory]; + + if (relativePath.StartsWith(ArchiveConstants.OBJECTS_PATH)) + { + foundPaths[scene.RegionInfo.RegionID].Add(relativePath); + } + } + } + + Assert.AreEqual(scenes.Count, foundPaths.Count); + foreach (Scene scene in scenes) + { + Assert.That(foundPaths[scene.RegionInfo.RegionID], Is.EquivalentTo(expectedPaths[scene.RegionInfo.RegionID])); + } + + Assert.That(foundAssets, Is.EquivalentTo(expectedAssets)); + } + + /// + /// Test loading a multi-region OAR. + /// + [Test] + public void TestLoadMultiRegionOar() + { + TestHelpers.InMethod(); + + // Create an ArchiveScenesGroup with the regions in the OAR. This is needed to generate the control file. + + int WIDTH = 2; + int HEIGHT = 2; + + for (uint y = 0; y < HEIGHT; y++) + { + for (uint x = 0; x < WIDTH; x++) + { + Scene scene; + if (x == 0 && y == 0) + { + scene = m_scene; // this scene was already created in SetUp() + } + else + { + scene = m_sceneHelpers.SetupScene(string.Format("Unit test region {0}", (y * WIDTH) + x + 1), UUID.Random(), 1000 + x, 1000 + y); + SceneHelpers.SetupSceneModules(scene, new ArchiverModule(), m_serialiserModule, new TerrainModule()); + } + } + } + + ArchiveScenesGroup scenesGroup = new ArchiveScenesGroup(); + SceneManager.Instance.ForEachScene(delegate(Scene scene) + { + scenesGroup.AddScene(scene); + }); + scenesGroup.CalcSceneLocations(); + + // Generate the OAR file + + MemoryStream archiveWriteStream = new MemoryStream(); + TarArchiveWriter tar = new TarArchiveWriter(archiveWriteStream); + + ArchiveWriteRequest writeRequest = new ArchiveWriteRequest(m_scene, (Stream)null, Guid.Empty); + writeRequest.MultiRegionFormat = true; + tar.WriteFile( + ArchiveConstants.CONTROL_FILE_PATH, writeRequest.CreateControlFile(scenesGroup)); + + SceneObjectPart part1 = CreateSceneObjectPart1(); + part1.SitTargetOrientation = new Quaternion(0.2f, 0.3f, 0.4f, 0.5f); + part1.SitTargetPosition = new Vector3(1, 2, 3); + + SceneObjectGroup object1 = new SceneObjectGroup(part1); + + // Let's put some inventory items into our object + string soundItemName = "sound-item1"; + UUID soundItemUuid = UUID.Parse("00000000-0000-0000-0000-000000000002"); + Type type = GetType(); + Assembly assembly = type.Assembly; + string soundDataResourceName = null; + string[] names = assembly.GetManifestResourceNames(); + foreach (string name in names) + { + if (name.EndsWith(".Resources.test-sound.wav")) + soundDataResourceName = name; + } + Assert.That(soundDataResourceName, Is.Not.Null); + + byte[] soundData; + UUID soundUuid; + CreateSoundAsset(tar, assembly, soundDataResourceName, out soundData, out soundUuid); + + TaskInventoryItem item1 + = new TaskInventoryItem { AssetID = soundUuid, ItemID = soundItemUuid, Name = soundItemName }; + part1.Inventory.AddInventoryItem(item1, true); + m_scene.AddNewSceneObject(object1, false); + + string object1FileName = string.Format( + "{0}_{1:000}-{2:000}-{3:000}__{4}.xml", + part1.Name, + Math.Round(part1.GroupPosition.X), Math.Round(part1.GroupPosition.Y), Math.Round(part1.GroupPosition.Z), + part1.UUID); + string path = "regions/1_1_Unit_test_region/" + ArchiveConstants.OBJECTS_PATH + object1FileName; + tar.WriteFile(path, SceneObjectSerializer.ToXml2Format(object1)); + + tar.Close(); + + + // Delete the current objects, to test that they're loaded from the OAR and didn't + // just remain in the scene. + SceneManager.Instance.ForEachScene(delegate(Scene scene) + { + scene.DeleteAllSceneObjects(); + }); + + // Create a "hole", to test that that the corresponding region isn't loaded from the OAR + SceneManager.Instance.CloseScene(SceneManager.Instance.Scenes[1]); + + + // Check thay the OAR file contains the expected data + + MemoryStream archiveReadStream = new MemoryStream(archiveWriteStream.ToArray()); + + lock (this) + { + m_scene.EventManager.OnOarFileLoaded += LoadCompleted; + m_archiverModule.DearchiveRegion(archiveReadStream); + } + + Assert.That(m_lastErrorMessage, Is.Null); + + Assert.AreEqual(3, SceneManager.Instance.Scenes.Count); + + TestLoadedRegion(part1, soundItemName, soundData); + } + } } -- cgit v1.1