/* * Copyright (c) Contributors, http://opensimulator.org/ * See CONTRIBUTORS.TXT for a full list of copyright holders. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * Neither the name of the OpenSimulator Project nor the * names of its contributors may be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ using System; using System.Collections.Generic; using System.IO; using System.IO.Compression; using System.Reflection; using System.Threading; using System.Text; using System.Xml; using System.Xml.Linq; using log4net; using OpenMetaverse; using OpenSim.Framework; using OpenSim.Framework.Serialization; using OpenSim.Framework.Serialization.External; using OpenSim.Region.CoreModules.World.Archiver; using OpenSim.Region.Framework.Scenes; using OpenSim.Region.Framework.Scenes.Serialization; using OpenSim.Region.Framework.Interfaces; using OpenSim.Services.Interfaces; namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver { public class InventoryArchiveReadRequest { private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); /// /// The maximum major version of archive that we can read. Minor versions shouldn't need a max number since version /// bumps here should be compatible. /// public static int MAX_MAJOR_VERSION = 1; protected TarArchiveReader archive; private UserAccount m_userInfo; private string m_invPath; /// /// Do we want to merge this load with existing inventory? /// protected bool m_merge; /// /// We only use this to request modules /// protected Scene m_scene; /// /// The stream from which the inventory archive will be loaded. /// private Stream m_loadStream; /// /// Has the control file been loaded for this archive? /// public bool ControlFileLoaded { get; private set; } /// /// Do we want to enforce the check. IAR versions before 0.2 and 1.1 do not guarantee this order, so we can't /// enforce. /// public bool EnforceControlFileCheck { get; private set; } protected bool m_assetsLoaded; protected bool m_inventoryNodesLoaded; protected int m_successfulAssetRestores; protected int m_failedAssetRestores; protected int m_successfulItemRestores; /// /// Root destination folder for the IAR load. /// protected InventoryFolderBase m_rootDestinationFolder; /// /// Inventory nodes loaded from the iar. /// protected HashSet m_loadedNodes = new HashSet(); /// /// In order to load identically named folders, we need to keep track of the folders that we have already /// resolved. /// Dictionary m_resolvedFolders = new Dictionary(); /// /// Record the creator id that should be associated with an asset. This is used to adjust asset creator ids /// after OSP resolution (since OSP creators are only stored in the item /// protected Dictionary m_creatorIdForAssetId = new Dictionary(); public InventoryArchiveReadRequest( Scene scene, UserAccount userInfo, string invPath, string loadPath, bool merge) : this( scene, userInfo, invPath, new GZipStream(ArchiveHelpers.GetStream(loadPath), CompressionMode.Decompress), merge) { } public InventoryArchiveReadRequest( Scene scene, UserAccount userInfo, string invPath, Stream loadStream, bool merge) { m_scene = scene; m_merge = merge; m_userInfo = userInfo; m_invPath = invPath; m_loadStream = loadStream; // FIXME: Do not perform this check since older versions of OpenSim do save the control file after other things // (I thought they weren't). We will need to bump the version number and perform this check on all // subsequent IAR versions only ControlFileLoaded = true; } /// /// Execute the request /// /// /// Only call this once. To load another IAR, construct another request object. /// /// /// A list of the inventory nodes loaded. If folders were loaded then only the root folders are /// returned /// /// Thrown if load fails. public HashSet Execute() { try { string filePath = "ERROR"; List folderCandidates = InventoryArchiveUtils.FindFolderByPath( m_scene.InventoryService, m_userInfo.PrincipalID, m_invPath); if (folderCandidates.Count == 0) { // Possibly provide an option later on to automatically create this folder if it does not exist m_log.ErrorFormat("[INVENTORY ARCHIVER]: Inventory path {0} does not exist", m_invPath); return m_loadedNodes; } m_rootDestinationFolder = folderCandidates[0]; archive = new TarArchiveReader(m_loadStream); byte[] data; TarArchiveReader.TarEntryType entryType; while ((data = archive.ReadEntry(out filePath, out entryType)) != null) { if (filePath == ArchiveConstants.CONTROL_FILE_PATH) { LoadControlFile(filePath, data); } else if (filePath.StartsWith(ArchiveConstants.ASSETS_PATH)) { LoadAssetFile(filePath, data); } else if (filePath.StartsWith(ArchiveConstants.INVENTORY_PATH)) { LoadInventoryFile(filePath, entryType, data); } } archive.Close(); m_log.DebugFormat( "[INVENTORY ARCHIVER]: Successfully loaded {0} assets with {1} failures", m_successfulAssetRestores, m_failedAssetRestores); m_log.InfoFormat("[INVENTORY ARCHIVER]: Successfully loaded {0} items", m_successfulItemRestores); return m_loadedNodes; } finally { m_loadStream.Close(); } } public void Close() { if (m_loadStream != null) m_loadStream.Close(); } /// /// Replicate the inventory paths in the archive to the user's inventory as necessary. /// /// The item archive path to replicate /// The root folder for the inventory load /// /// The folders that we have resolved so far for a given archive path. /// This method will add more folders if necessary /// /// /// Track the inventory nodes created. /// /// The last user inventory folder created or found for the archive path public InventoryFolderBase ReplicateArchivePathToUserInventory( string iarPath, InventoryFolderBase rootDestFolder, Dictionary resolvedFolders, HashSet loadedNodes) { string iarPathExisting = iarPath; // m_log.DebugFormat( // "[INVENTORY ARCHIVER]: Loading folder {0} {1}", rootDestFolder.Name, rootDestFolder.ID); InventoryFolderBase destFolder = ResolveDestinationFolder(rootDestFolder, ref iarPathExisting, resolvedFolders); // m_log.DebugFormat( // "[INVENTORY ARCHIVER]: originalArchivePath [{0}], section already loaded [{1}]", // iarPath, iarPathExisting); string iarPathToCreate = iarPath.Substring(iarPathExisting.Length); CreateFoldersForPath(destFolder, iarPathExisting, iarPathToCreate, resolvedFolders, loadedNodes); return destFolder; } /// /// Resolve a destination folder /// /// /// We require here a root destination folder (usually the root of the user's inventory) and the archive /// path. We also pass in a list of previously resolved folders in case we've found this one previously. /// /// /// The item archive path to resolve. The portion of the path passed back is that /// which corresponds to the resolved desintation folder. /// /// The root folder for the inventory load /// /// /// The folders that we have resolved so far for a given archive path. /// /// /// The folder in the user's inventory that matches best the archive path given. If no such folder was found /// then the passed in root destination folder is returned. /// protected InventoryFolderBase ResolveDestinationFolder( InventoryFolderBase rootDestFolder, ref string archivePath, Dictionary resolvedFolders) { // string originalArchivePath = archivePath; while (archivePath.Length > 0) { // m_log.DebugFormat("[INVENTORY ARCHIVER]: Trying to resolve destination folder {0}", archivePath); if (resolvedFolders.ContainsKey(archivePath)) { // m_log.DebugFormat( // "[INVENTORY ARCHIVER]: Found previously created folder from archive path {0}", archivePath); return resolvedFolders[archivePath]; } else { if (m_merge) { // TODO: Using m_invPath is totally wrong - what we need to do is strip the uuid from the // iar name and try to find that instead. string plainPath = ArchiveConstants.ExtractPlainPathFromIarPath(archivePath); List folderCandidates = InventoryArchiveUtils.FindFolderByPath( m_scene.InventoryService, m_userInfo.PrincipalID, plainPath); if (folderCandidates.Count != 0) { InventoryFolderBase destFolder = folderCandidates[0]; resolvedFolders[archivePath] = destFolder; return destFolder; } } // Don't include the last slash so find the penultimate one int penultimateSlashIndex = archivePath.LastIndexOf("/", archivePath.Length - 2); if (penultimateSlashIndex >= 0) { // Remove the last section of path so that we can see if we've already resolved the parent archivePath = archivePath.Remove(penultimateSlashIndex + 1); } else { // m_log.DebugFormat( // "[INVENTORY ARCHIVER]: Found no previously created folder for archive path {0}", // originalArchivePath); archivePath = string.Empty; return rootDestFolder; } } } return rootDestFolder; } /// /// Create a set of folders for the given path. /// /// /// The root folder from which the creation will take place. /// /// /// the part of the iar path that already exists /// /// /// The path to replicate in the user's inventory from iar /// /// /// The folders that we have resolved so far for a given archive path. /// /// /// Track the inventory nodes created. /// protected void CreateFoldersForPath( InventoryFolderBase destFolder, string iarPathExisting, string iarPathToReplicate, Dictionary resolvedFolders, HashSet loadedNodes) { string[] rawDirsToCreate = iarPathToReplicate.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries); for (int i = 0; i < rawDirsToCreate.Length; i++) { // m_log.DebugFormat("[INVENTORY ARCHIVER]: Creating folder {0} from IAR", rawDirsToCreate[i]); if (!rawDirsToCreate[i].Contains(ArchiveConstants.INVENTORY_NODE_NAME_COMPONENT_SEPARATOR)) continue; int identicalNameIdentifierIndex = rawDirsToCreate[i].LastIndexOf( ArchiveConstants.INVENTORY_NODE_NAME_COMPONENT_SEPARATOR); string newFolderName = rawDirsToCreate[i].Remove(identicalNameIdentifierIndex); newFolderName = InventoryArchiveUtils.UnescapeArchivePath(newFolderName); UUID newFolderId = UUID.Random(); // Asset type has to be Unknown here rather than Folder, otherwise the created folder can't be // deleted once the client has relogged. // The root folder appears to be labelled AssetType.Folder (shows up as "Category" in the client) // even though there is a AssetType.RootCategory destFolder = new InventoryFolderBase( newFolderId, newFolderName, m_userInfo.PrincipalID, (short)AssetType.Unknown, destFolder.ID, 1); m_scene.InventoryService.AddFolder(destFolder); // Record that we have now created this folder iarPathExisting += rawDirsToCreate[i] + "/"; m_log.DebugFormat("[INVENTORY ARCHIVER]: Created folder {0} from IAR", iarPathExisting); resolvedFolders[iarPathExisting] = destFolder; if (0 == i) loadedNodes.Add(destFolder); } } /// /// Load an item from the archive /// /// The archive path for the item /// The raw item data /// The root destination folder for loaded items /// All the inventory nodes (items and folders) loaded so far protected InventoryItemBase LoadItem(byte[] data, InventoryFolderBase loadFolder) { InventoryItemBase item = UserInventoryItemSerializer.Deserialize(data); // Don't use the item ID that's in the file item.ID = UUID.Random(); UUID ospResolvedId = OspResolver.ResolveOspa(item.CreatorId, m_scene.UserAccountService); if (UUID.Zero != ospResolvedId) // The user exists in this grid { // m_log.DebugFormat("[INVENTORY ARCHIVER]: Found creator {0} via OSPA resolution", ospResolvedId); // item.CreatorIdAsUuid = ospResolvedId; // Don't preserve the OSPA in the creator id (which actually gets persisted to the // database). Instead, replace with the UUID that we found. item.CreatorId = ospResolvedId.ToString(); item.CreatorData = string.Empty; } else if (item.CreatorData == null || item.CreatorData == String.Empty) { item.CreatorId = m_userInfo.PrincipalID.ToString(); // item.CreatorIdAsUuid = new UUID(item.CreatorId); } item.Owner = m_userInfo.PrincipalID; // Reset folder ID to the one in which we want to load it item.Folder = loadFolder.ID; // Record the creator id for the item's asset so that we can use it later, if necessary, when the asset // is loaded. // FIXME: This relies on the items coming before the assets in the TAR file. Need to create stronger // checks for this, and maybe even an external tool for creating OARs which enforces this, rather than // relying on native tar tools. m_creatorIdForAssetId[item.AssetID] = item.CreatorIdAsUuid; m_scene.AddInventoryItem(item); return item; } /// /// Load an asset /// /// /// /// true if asset was successfully loaded, false otherwise private bool LoadAsset(string assetPath, byte[] data) { //IRegionSerialiser serialiser = scene.RequestModuleInterface(); // Right now we're nastily obtaining the UUID from the filename string filename = assetPath.Remove(0, ArchiveConstants.ASSETS_PATH.Length); int i = filename.LastIndexOf(ArchiveConstants.ASSET_EXTENSION_SEPARATOR); if (i == -1) { m_log.ErrorFormat( "[INVENTORY ARCHIVER]: Could not find extension information in asset path {0} since it's missing the separator {1}. Skipping", assetPath, ArchiveConstants.ASSET_EXTENSION_SEPARATOR); return false; } string extension = filename.Substring(i); string rawUuid = filename.Remove(filename.Length - extension.Length); UUID assetId = new UUID(rawUuid); if (ArchiveConstants.EXTENSION_TO_ASSET_TYPE.ContainsKey(extension)) { sbyte assetType = ArchiveConstants.EXTENSION_TO_ASSET_TYPE[extension]; if (assetType == (sbyte)AssetType.Unknown) { m_log.WarnFormat("[INVENTORY ARCHIVER]: Importing {0} byte asset {1} with unknown type", data.Length, assetId); } else if (assetType == (sbyte)AssetType.Object) { if (m_creatorIdForAssetId.ContainsKey(assetId)) { string xmlData = Utils.BytesToString(data); List sceneObjects = new List(); CoalescedSceneObjects coa = null; if (CoalescedSceneObjectsSerializer.TryFromXml(xmlData, out coa)) { // m_log.DebugFormat( // "[INVENTORY ARCHIVER]: Loaded coalescence {0} has {1} objects", assetId, coa.Count); if (coa.Objects.Count == 0) { m_log.WarnFormat( "[INVENTORY ARCHIVE READ REQUEST]: Aborting load of coalesced object from asset {0} as it has zero loaded components", assetId); return false; } sceneObjects.AddRange(coa.Objects); } else { SceneObjectGroup deserializedObject = SceneObjectSerializer.FromOriginalXmlFormat(xmlData); if (deserializedObject != null) { sceneObjects.Add(deserializedObject); } else { m_log.WarnFormat( "[INVENTORY ARCHIVE READ REQUEST]: Aborting load of object from asset {0} as deserialization failed", assetId); return false; } } foreach (SceneObjectGroup sog in sceneObjects) foreach (SceneObjectPart sop in sog.Parts) if (sop.CreatorData == null || sop.CreatorData == "") sop.CreatorID = m_creatorIdForAssetId[assetId]; if (coa != null) data = Utils.StringToBytes(CoalescedSceneObjectsSerializer.ToXml(coa)); else data = Utils.StringToBytes(SceneObjectSerializer.ToOriginalXmlFormat(sceneObjects[0])); } } //m_log.DebugFormat("[INVENTORY ARCHIVER]: Importing asset {0}, type {1}", uuid, assetType); AssetBase asset = new AssetBase(assetId, "From IAR", assetType, UUID.Zero.ToString()); asset.Data = data; m_scene.AssetService.Store(asset); return true; } else { m_log.ErrorFormat( "[INVENTORY ARCHIVER]: Tried to dearchive data with path {0} with an unknown type extension {1}", assetPath, extension); return false; } } /// /// Load control file /// /// /// public void LoadControlFile(string path, byte[] data) { XDocument doc = XDocument.Parse(Encoding.ASCII.GetString(data)); XElement archiveElement = doc.Element("archive"); int majorVersion = int.Parse(archiveElement.Attribute("major_version").Value); int minorVersion = int.Parse(archiveElement.Attribute("minor_version").Value); string version = string.Format("{0}.{1}", majorVersion, minorVersion); if (majorVersion > MAX_MAJOR_VERSION) { throw new Exception( string.Format( "The IAR you are trying to load has major version number of {0} but this version of OpenSim can only load IARs with major version number {1} and below", majorVersion, MAX_MAJOR_VERSION)); } ControlFileLoaded = true; m_log.InfoFormat("[INVENTORY ARCHIVER]: Loading IAR with version {0}", version); } /// /// Load inventory file /// /// /// /// protected void LoadInventoryFile(string path, TarArchiveReader.TarEntryType entryType, byte[] data) { if (!ControlFileLoaded) throw new Exception( string.Format( "The IAR you are trying to load does not list {0} before {1}. Aborting load", ArchiveConstants.CONTROL_FILE_PATH, ArchiveConstants.INVENTORY_PATH)); if (m_assetsLoaded) throw new Exception( string.Format( "The IAR you are trying to load does not list all {0} before {1}. Aborting load", ArchiveConstants.INVENTORY_PATH, ArchiveConstants.ASSETS_PATH)); path = path.Substring(ArchiveConstants.INVENTORY_PATH.Length); // Trim off the file portion if we aren't already dealing with a directory path if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY != entryType) path = path.Remove(path.LastIndexOf("/") + 1); InventoryFolderBase foundFolder = ReplicateArchivePathToUserInventory( path, m_rootDestinationFolder, m_resolvedFolders, m_loadedNodes); if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY != entryType) { InventoryItemBase item = LoadItem(data, foundFolder); if (item != null) { m_successfulItemRestores++; // If we aren't loading the folder containing the item then well need to update the // viewer separately for that item. if (!m_loadedNodes.Contains(foundFolder)) m_loadedNodes.Add(item); } } m_inventoryNodesLoaded = true; } /// /// Load asset file /// /// /// protected void LoadAssetFile(string path, byte[] data) { if (!ControlFileLoaded) throw new Exception( string.Format( "The IAR you are trying to load does not list {0} before {1}. Aborting load", ArchiveConstants.CONTROL_FILE_PATH, ArchiveConstants.ASSETS_PATH)); if (!m_inventoryNodesLoaded) throw new Exception( string.Format( "The IAR you are trying to load does not list all {0} before {1}. Aborting load", ArchiveConstants.INVENTORY_PATH, ArchiveConstants.ASSETS_PATH)); if (LoadAsset(path, data)) m_successfulAssetRestores++; else m_failedAssetRestores++; if ((m_successfulAssetRestores) % 50 == 0) m_log.DebugFormat( "[INVENTORY ARCHIVER]: Loaded {0} assets...", m_successfulAssetRestores); m_assetsLoaded = true; } } }