From d7e6fe488d5258ec92230b6c652969f2d0420376 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 7 Sep 2012 23:30:54 +0100 Subject: Rename ArchiveWriteRequestPreparatio nto ArchiveWriteRequest since after the multi-OAR patch there is now only one class that handles this operation. Adapation of 0004-Renamed-ArchiveWriteRequestPreparation-to-ArchiveWri.patch in http://opensimulator.org/mantis/view.php?id=6105 since that did not directly apply --- .../World/Archiver/ArchiveWriteRequest.cs | 634 +++++++++++++++++++++ 1 file changed, 634 insertions(+) create mode 100644 OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs (limited to 'OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs') diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs new file mode 100644 index 0000000..d751b1c --- /dev/null +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs @@ -0,0 +1,634 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Reflection; +using System.Text.RegularExpressions; +using System.Threading; +using System.Xml; +using log4net; +using OpenMetaverse; +using OpenSim.Framework; +using OpenSim.Framework.Serialization; +using OpenSim.Region.CoreModules.World.Terrain; +using OpenSim.Region.Framework.Interfaces; +using OpenSim.Region.Framework.Scenes; +using Ionic.Zlib; +using GZipStream = Ionic.Zlib.GZipStream; +using CompressionMode = Ionic.Zlib.CompressionMode; +using OpenSim.Framework.Serialization.External; + +namespace OpenSim.Region.CoreModules.World.Archiver +{ + /// + /// Prepare to write out an archive. + /// + public class ArchiveWriteRequest + { + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + /// + /// The minimum major version of OAR that we can write. + /// + public static int MIN_MAJOR_VERSION = 0; + + /// + /// The maximum major version of OAR that we can write. + /// + public static int MAX_MAJOR_VERSION = 1; + + /// + /// Whether we're saving a multi-region archive. + /// + public bool MultiRegionFormat { get; set; } + + /// + /// Determine whether this archive will save assets. Default is true. + /// + public bool SaveAssets { get; set; } + + /// + /// Determines which objects will be included in the archive, according to their permissions. + /// Default is null, meaning no permission checks. + /// + public string CheckPermissions { get; set; } + + protected Scene m_rootScene; + protected Stream m_saveStream; + protected TarArchiveWriter m_archiveWriter; + protected Guid m_requestId; + protected Dictionary m_options; + + /// + /// Constructor + /// + /// Calling module + /// The path to which to save data. + /// The id associated with this request + /// + /// If there was a problem opening a stream for the file specified by the savePath + /// + public ArchiveWriteRequest(Scene scene, string savePath, Guid requestId) : this(scene, requestId) + { + try + { + m_saveStream = new GZipStream(new FileStream(savePath, FileMode.Create), CompressionMode.Compress, CompressionLevel.BestCompression); + } + catch (EntryPointNotFoundException e) + { + m_log.ErrorFormat( + "[ARCHIVER]: Mismatch between Mono and zlib1g library version when trying to create compression stream." + + "If you've manually installed Mono, have you appropriately updated zlib1g as well?"); + m_log.ErrorFormat("{0} {1}", e.Message, e.StackTrace); + } + } + + /// + /// Constructor. + /// + /// The root scene to archive + /// The stream to which to save data. + /// The id associated with this request + public ArchiveWriteRequest(Scene scene, Stream saveStream, Guid requestId) : this(scene, requestId) + { + m_saveStream = saveStream; + } + + protected ArchiveWriteRequest(Scene scene, Guid requestId) + { + m_rootScene = scene; + m_requestId = requestId; + m_archiveWriter = null; + + MultiRegionFormat = false; + SaveAssets = true; + CheckPermissions = null; + } + + /// + /// Archive the region requested. + /// + /// if there was an io problem with creating the file + public void ArchiveRegion(Dictionary options) + { + m_options = options; + + if (options.ContainsKey("all") && (bool)options["all"]) + MultiRegionFormat = true; + + if (options.ContainsKey("noassets") && (bool)options["noassets"]) + SaveAssets = false; + + Object temp; + if (options.TryGetValue("checkPermissions", out temp)) + CheckPermissions = (string)temp; + + + // Find the regions to archive + ArchiveScenesGroup scenesGroup = new ArchiveScenesGroup(); + if (MultiRegionFormat) + { + m_log.InfoFormat("[ARCHIVER]: Saving {0} regions", SceneManager.Instance.Scenes.Count); + SceneManager.Instance.ForEachScene(delegate(Scene scene) + { + scenesGroup.AddScene(scene); + }); + } + else + { + scenesGroup.AddScene(m_rootScene); + } + scenesGroup.CalcSceneLocations(); + + + m_archiveWriter = new TarArchiveWriter(m_saveStream); + + try + { + // Write out control file. It should be first so that it will be found ASAP when loading the file. + m_archiveWriter.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, CreateControlFile(scenesGroup)); + m_log.InfoFormat("[ARCHIVER]: Added control file to archive."); + + // Archive the regions + + Dictionary assetUuids = new Dictionary(); + + scenesGroup.ForEachScene(delegate(Scene scene) + { + string regionDir = MultiRegionFormat ? scenesGroup.GetRegionDir(scene.RegionInfo.RegionID) : ""; + ArchiveOneRegion(scene, regionDir, assetUuids); + }); + + // Archive the assets + + if (SaveAssets) + { + m_log.DebugFormat("[ARCHIVER]: Saving {0} assets", assetUuids.Count); + + // Asynchronously request all the assets required to perform this archive operation + AssetsRequest ar + = new AssetsRequest( + new AssetsArchiver(m_archiveWriter), assetUuids, + m_rootScene.AssetService, m_rootScene.UserAccountService, + m_rootScene.RegionInfo.ScopeID, options, ReceivedAllAssets); + + Util.FireAndForget(o => ar.Execute()); + + // CloseArchive() will be called from ReceivedAllAssets() + } + else + { + m_log.DebugFormat("[ARCHIVER]: Not saving assets since --noassets was specified"); + CloseArchive(string.Empty); + } + } + catch (Exception e) + { + CloseArchive(e.Message); + throw; + } + } + + + private void ArchiveOneRegion(Scene scene, string regionDir, Dictionary assetUuids) + { + m_log.InfoFormat("[ARCHIVER]: Writing region {0}", scene.RegionInfo.RegionName); + + EntityBase[] entities = scene.GetEntities(); + List sceneObjects = new List(); + + int numObjectsSkippedPermissions = 0; + + // Filter entities so that we only have scene objects. + // FIXME: Would be nicer to have this as a proper list in SceneGraph, since lots of methods + // end up having to do this + IPermissionsModule permissionsModule = scene.RequestModuleInterface(); + foreach (EntityBase entity in entities) + { + if (entity is SceneObjectGroup) + { + SceneObjectGroup sceneObject = (SceneObjectGroup)entity; + + if (!sceneObject.IsDeleted && !sceneObject.IsAttachment) + { + if (!CanUserArchiveObject(scene.RegionInfo.EstateSettings.EstateOwner, sceneObject, CheckPermissions, permissionsModule)) + { + // The user isn't allowed to copy/transfer this object, so it will not be included in the OAR. + ++numObjectsSkippedPermissions; + } + else + { + sceneObjects.Add(sceneObject); + } + } + } + } + + if (SaveAssets) + { + UuidGatherer assetGatherer = new UuidGatherer(scene.AssetService); + int prevAssets = assetUuids.Count; + + foreach (SceneObjectGroup sceneObject in sceneObjects) + { + assetGatherer.GatherAssetUuids(sceneObject, assetUuids); + } + + m_log.DebugFormat( + "[ARCHIVER]: {0} scene objects to serialize requiring save of {1} assets", + sceneObjects.Count, assetUuids.Count - prevAssets); + } + + if (numObjectsSkippedPermissions > 0) + { + m_log.DebugFormat( + "[ARCHIVER]: {0} scene objects skipped due to lack of permissions", + numObjectsSkippedPermissions); + } + + // Make sure that we also request terrain texture assets + RegionSettings regionSettings = scene.RegionInfo.RegionSettings; + + if (regionSettings.TerrainTexture1 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_1) + assetUuids[regionSettings.TerrainTexture1] = AssetType.Texture; + + if (regionSettings.TerrainTexture2 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_2) + assetUuids[regionSettings.TerrainTexture2] = AssetType.Texture; + + if (regionSettings.TerrainTexture3 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_3) + assetUuids[regionSettings.TerrainTexture3] = AssetType.Texture; + + if (regionSettings.TerrainTexture4 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_4) + assetUuids[regionSettings.TerrainTexture4] = AssetType.Texture; + + Save(scene, sceneObjects, regionDir); + } + + /// + /// Checks whether the user has permission to export an object group to an OAR. + /// + /// The user + /// The object group + /// Which permissions to check: "C" = Copy, "T" = Transfer + /// The scene's permissions module + /// Whether the user is allowed to export the object to an OAR + private bool CanUserArchiveObject(UUID user, SceneObjectGroup objGroup, string checkPermissions, IPermissionsModule permissionsModule) + { + if (checkPermissions == null) + return true; + + if (permissionsModule == null) + return true; // this shouldn't happen + + // Check whether the user is permitted to export all of the parts in the SOG. If any + // part can't be exported then the entire SOG can't be exported. + + bool permitted = true; + //int primNumber = 1; + + foreach (SceneObjectPart obj in objGroup.Parts) + { + uint perm; + PermissionClass permissionClass = permissionsModule.GetPermissionClass(user, obj); + switch (permissionClass) + { + case PermissionClass.Owner: + perm = obj.BaseMask; + break; + case PermissionClass.Group: + perm = obj.GroupMask | obj.EveryoneMask; + break; + case PermissionClass.Everyone: + default: + perm = obj.EveryoneMask; + break; + } + + bool canCopy = (perm & (uint)PermissionMask.Copy) != 0; + bool canTransfer = (perm & (uint)PermissionMask.Transfer) != 0; + + // Special case: if Everyone can copy the object then this implies it can also be + // Transferred. + // However, if the user is the Owner then we don't check EveryoneMask, because it seems that the mask + // always (incorrectly) includes the Copy bit set in this case. But that's a mistake: the viewer + // does NOT show that the object has Everyone-Copy permissions, and doesn't allow it to be copied. + if (permissionClass != PermissionClass.Owner) + canTransfer |= (obj.EveryoneMask & (uint)PermissionMask.Copy) != 0; + + bool partPermitted = true; + if (checkPermissions.Contains("C") && !canCopy) + partPermitted = false; + if (checkPermissions.Contains("T") && !canTransfer) + partPermitted = false; + + // If the user is the Creator of the object then it can always be included in the OAR + bool creator = (obj.CreatorID.Guid == user.Guid); + if (creator) + partPermitted = true; + + //string name = (objGroup.PrimCount == 1) ? objGroup.Name : string.Format("{0} ({1}/{2})", obj.Name, primNumber, objGroup.PrimCount); + //m_log.DebugFormat("[ARCHIVER]: Object permissions: {0}: Base={1:X4}, Owner={2:X4}, Everyone={3:X4}, permissionClass={4}, checkPermissions={5}, canCopy={6}, canTransfer={7}, creator={8}, permitted={9}", + // name, obj.BaseMask, obj.OwnerMask, obj.EveryoneMask, + // permissionClass, checkPermissions, canCopy, canTransfer, creator, partPermitted); + + if (!partPermitted) + { + permitted = false; + break; + } + + //++primNumber; + } + + return permitted; + } + + /// + /// Create the control file. + /// + /// + public string CreateControlFile(ArchiveScenesGroup scenesGroup) + { + int majorVersion; + int minorVersion; + + if (MultiRegionFormat) + { + majorVersion = MAX_MAJOR_VERSION; + minorVersion = 0; + } + else + { + // To support older versions of OpenSim, we continue to create single-region OARs + // using the old file format. In the future this format will be discontinued. + majorVersion = 0; + minorVersion = 8; + } +// +// if (m_options.ContainsKey("version")) +// { +// string[] parts = m_options["version"].ToString().Split('.'); +// if (parts.Length >= 1) +// { +// majorVersion = Int32.Parse(parts[0]); +// +// if (parts.Length >= 2) +// minorVersion = Int32.Parse(parts[1]); +// } +// } +// +// if (majorVersion < MIN_MAJOR_VERSION || majorVersion > MAX_MAJOR_VERSION) +// { +// throw new Exception( +// string.Format( +// "OAR version number for save must be between {0} and {1}", +// MIN_MAJOR_VERSION, MAX_MAJOR_VERSION)); +// } +// else if (majorVersion == MAX_MAJOR_VERSION) +// { +// // Force 1.0 +// minorVersion = 0; +// } +// else if (majorVersion == MIN_MAJOR_VERSION) +// { +// // Force 0.4 +// minorVersion = 4; +// } + + m_log.InfoFormat("[ARCHIVER]: Creating version {0}.{1} OAR", majorVersion, minorVersion); + if (majorVersion == 1) + { + m_log.WarnFormat("[ARCHIVER]: Please be aware that version 1.0 OARs are not compatible with OpenSim versions prior to 0.7.4. Do not use the --all option if you want to produce a compatible OAR"); + } + + String s; + + using (StringWriter sw = new StringWriter()) + { + using (XmlTextWriter xtw = new XmlTextWriter(sw)) + { + xtw.Formatting = Formatting.Indented; + xtw.WriteStartDocument(); + xtw.WriteStartElement("archive"); + xtw.WriteAttributeString("major_version", majorVersion.ToString()); + xtw.WriteAttributeString("minor_version", minorVersion.ToString()); + + xtw.WriteStartElement("creation_info"); + DateTime now = DateTime.UtcNow; + TimeSpan t = now - new DateTime(1970, 1, 1); + xtw.WriteElementString("datetime", ((int)t.TotalSeconds).ToString()); + if (!MultiRegionFormat) + xtw.WriteElementString("id", m_rootScene.RegionInfo.RegionID.ToString()); + xtw.WriteEndElement(); + + xtw.WriteElementString("assets_included", SaveAssets.ToString()); + + if (MultiRegionFormat) + { + WriteRegionsManifest(scenesGroup, xtw); + } + else + { + xtw.WriteStartElement("region_info"); + WriteRegionInfo(m_rootScene, xtw); + xtw.WriteEndElement(); + } + + xtw.WriteEndElement(); + + xtw.Flush(); + } + + s = sw.ToString(); + } + + return s; + } + + /// + /// Writes the list of regions included in a multi-region OAR. + /// + private static void WriteRegionsManifest(ArchiveScenesGroup scenesGroup, XmlTextWriter xtw) + { + xtw.WriteStartElement("regions"); + + // Write the regions in order: rows from South to North, then regions from West to East. + // The list of regions can have "holes"; we write empty elements in their position. + + for (uint y = (uint)scenesGroup.Rect.Top; y < scenesGroup.Rect.Bottom; ++y) + { + SortedDictionary row; + if (scenesGroup.Regions.TryGetValue(y, out row)) + { + xtw.WriteStartElement("row"); + + for (uint x = (uint)scenesGroup.Rect.Left; x < scenesGroup.Rect.Right; ++x) + { + Scene scene; + if (row.TryGetValue(x, out scene)) + { + xtw.WriteStartElement("region"); + xtw.WriteElementString("id", scene.RegionInfo.RegionID.ToString()); + xtw.WriteElementString("dir", scenesGroup.GetRegionDir(scene.RegionInfo.RegionID)); + WriteRegionInfo(scene, xtw); + xtw.WriteEndElement(); + } + else + { + // Write a placeholder for a missing region + xtw.WriteElementString("region", ""); + } + } + + xtw.WriteEndElement(); + } + else + { + // Write a placeholder for a missing row + xtw.WriteElementString("row", ""); + } + } + + xtw.WriteEndElement(); // "regions" + } + + protected static void WriteRegionInfo(Scene scene, XmlTextWriter xtw) + { + bool isMegaregion; + Vector2 size; + + IRegionCombinerModule rcMod = scene.RequestModuleInterface(); + + if (rcMod != null) + isMegaregion = rcMod.IsRootForMegaregion(scene.RegionInfo.RegionID); + else + isMegaregion = false; + + if (isMegaregion) + size = rcMod.GetSizeOfMegaregion(scene.RegionInfo.RegionID); + else + size = new Vector2((float)Constants.RegionSize, (float)Constants.RegionSize); + + xtw.WriteElementString("is_megaregion", isMegaregion.ToString()); + xtw.WriteElementString("size_in_meters", string.Format("{0},{1}", size.X, size.Y)); + } + + + protected void Save(Scene scene, List sceneObjects, string regionDir) + { + if (regionDir != string.Empty) + regionDir = ArchiveConstants.REGIONS_PATH + regionDir + "/"; + + m_log.InfoFormat("[ARCHIVER]: Adding region settings to archive."); + + // Write out region settings + string settingsPath = String.Format("{0}{1}{2}.xml", + regionDir, ArchiveConstants.SETTINGS_PATH, scene.RegionInfo.RegionName); + m_archiveWriter.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(scene.RegionInfo.RegionSettings)); + + m_log.InfoFormat("[ARCHIVER]: Adding parcel settings to archive."); + + // Write out land data (aka parcel) settings + List landObjects = scene.LandChannel.AllParcels(); + foreach (ILandObject lo in landObjects) + { + LandData landData = lo.LandData; + string landDataPath = String.Format("{0}{1}{2}.xml", + regionDir, ArchiveConstants.LANDDATA_PATH, landData.GlobalID.ToString()); + m_archiveWriter.WriteFile(landDataPath, LandDataSerializer.Serialize(landData, m_options)); + } + + m_log.InfoFormat("[ARCHIVER]: Adding terrain information to archive."); + + // Write out terrain + string terrainPath = String.Format("{0}{1}{2}.r32", + regionDir, ArchiveConstants.TERRAINS_PATH, scene.RegionInfo.RegionName); + + MemoryStream ms = new MemoryStream(); + scene.RequestModuleInterface().SaveToStream(terrainPath, ms); + m_archiveWriter.WriteFile(terrainPath, ms.ToArray()); + ms.Close(); + + m_log.InfoFormat("[ARCHIVER]: Adding scene objects to archive."); + + // Write out scene object metadata + IRegionSerialiserModule serializer = scene.RequestModuleInterface(); + foreach (SceneObjectGroup sceneObject in sceneObjects) + { + //m_log.DebugFormat("[ARCHIVER]: Saving {0} {1}, {2}", entity.Name, entity.UUID, entity.GetType()); + + string serializedObject = serializer.SerializeGroupToXml2(sceneObject, m_options); + string objectPath = string.Format("{0}{1}", regionDir, ArchiveHelpers.CreateObjectPath(sceneObject)); + m_archiveWriter.WriteFile(objectPath, serializedObject); + } + } + + protected void ReceivedAllAssets( + ICollection assetsFoundUuids, ICollection assetsNotFoundUuids) + { + foreach (UUID uuid in assetsNotFoundUuids) + { + m_log.DebugFormat("[ARCHIVER]: Could not find asset {0}", uuid); + } + + // m_log.InfoFormat( + // "[ARCHIVER]: Received {0} of {1} assets requested", + // assetsFoundUuids.Count, assetsFoundUuids.Count + assetsNotFoundUuids.Count); + + CloseArchive(String.Empty); + } + + + /// + /// Closes the archive and notifies that we're done. + /// + /// The error that occurred, or empty for success + protected void CloseArchive(string errorMessage) + { + try + { + if (m_archiveWriter != null) + m_archiveWriter.Close(); + m_saveStream.Close(); + } + catch (Exception e) + { + m_log.Error(string.Format("[ARCHIVER]: Error closing archive: {0} ", e.Message), e); + if (errorMessage == string.Empty) + errorMessage = e.Message; + } + + m_log.InfoFormat("[ARCHIVER]: Finished writing out OAR for {0}", m_rootScene.RegionInfo.RegionName); + + m_rootScene.EventManager.TriggerOarFileSaved(m_requestId, errorMessage); + } + + } +} -- cgit v1.1