/* * Copyright (c) Contributors, http://opensimulator.org/ * See CONTRIBUTORS.TXT for a full list of copyright holders. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * Neither the name of the OpenSimulator Project nor the * names of its contributors may be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ using System; using System.Collections.Generic; using System.IO; using System.Reflection; using System.Text.RegularExpressions; using System.Threading; using log4net; using OpenMetaverse; using OpenMetaverse.Assets; using OpenMetaverse.StructuredData; using OpenSim.Framework; using OpenSim.Region.Framework.Scenes.Serialization; using OpenSim.Services.Interfaces; using OpenSimAssetType = OpenSim.Framework.SLUtil.OpenSimAssetType; namespace OpenSim.Region.Framework.Scenes { /// /// Gather uuids for a given entity. /// /// /// This does a deep inspection of the entity to retrieve all the assets it uses (whether as textures, as scripts /// contained in inventory, as scripts contained in objects contained in another object's inventory, etc. Assets /// are only retrieved when they are necessary to carry out the inspection (i.e. a serialized object needs to be /// retrieved to work out which assets it references). /// public class UuidGatherer { private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); /// /// Is gathering complete? /// public bool Complete { get { return m_assetUuidsToInspect.Count <= 0; } } /// /// The dictionary of UUIDs gathered so far. If Complete == true then this is all the reachable UUIDs. /// /// The gathered uuids. public IDictionary GatheredUuids { get; private set; } public HashSet FailedUUIDs { get; private set; } public HashSet UncertainAssetsUUIDs { get; private set; } public int possibleNotAssetCount { get; set; } public int ErrorCount { get; private set; } /// /// Gets the next UUID to inspect. /// /// If there is no next UUID then returns null public UUID? NextUuidToInspect { get { if (Complete) return null; else return m_assetUuidsToInspect.Peek(); } } protected IAssetService m_assetService; protected Queue m_assetUuidsToInspect; /// /// Initializes a new instance of the class. /// /// In this case the collection of gathered assets will start out blank. /// /// Asset service. /// public UuidGatherer(IAssetService assetService) : this(assetService, new Dictionary(), new HashSet (),new HashSet ()) {} public UuidGatherer(IAssetService assetService, IDictionary collector) : this(assetService, collector, new HashSet (), new HashSet ()) {} /// /// Initializes a new instance of the class. /// /// /// Asset service. /// /// /// Gathered UUIDs will be collected in this dictionary. /// It can be pre-populated if you want to stop the gatherer from analyzing assets that have already been fetched and inspected. /// public UuidGatherer(IAssetService assetService, IDictionary collector, HashSet failedIDs, HashSet uncertainAssetsUUIDs) { m_assetService = assetService; GatheredUuids = collector; // FIXME: Not efficient for searching, can improve. m_assetUuidsToInspect = new Queue(); FailedUUIDs = failedIDs; UncertainAssetsUUIDs = uncertainAssetsUUIDs; ErrorCount = 0; possibleNotAssetCount = 0; } /// /// Adds the asset uuid for inspection during the gathering process. /// /// true, if for inspection was added, false otherwise. /// UUID. public bool AddForInspection(UUID uuid) { if(uuid == UUID.Zero) return false; if(FailedUUIDs.Contains(uuid)) { if(UncertainAssetsUUIDs.Contains(uuid)) possibleNotAssetCount++; else ErrorCount++; return false; } if(GatheredUuids.ContainsKey(uuid)) return false; if (m_assetUuidsToInspect.Contains(uuid)) return false; // m_log.DebugFormat("[UUID GATHERER]: Adding asset {0} for inspection", uuid); m_assetUuidsToInspect.Enqueue(uuid); return true; } /// /// Gather all the asset uuids associated with a given object. /// /// /// This includes both those directly associated with /// it (e.g. face textures) and recursively, those of items within it's inventory (e.g. objects contained /// within this object). /// /// The scene object for which to gather assets public void AddForInspection(SceneObjectGroup sceneObject) { // m_log.DebugFormat( // "[UUID GATHERER]: Getting assets for object {0}, {1}", sceneObject.Name, sceneObject.UUID); if(sceneObject.IsDeleted) return; SceneObjectPart[] parts = sceneObject.Parts; for (int i = 0; i < parts.Length; i++) { SceneObjectPart part = parts[i]; // m_log.DebugFormat( // "[UUID GATHERER]: Getting part {0}, {1} for object {2}", part.Name, part.UUID, sceneObject.UUID); try { Primitive.TextureEntry textureEntry = part.Shape.Textures; if (textureEntry != null) { // Get the prim's default texture. This will be used for faces which don't have their own texture if (textureEntry.DefaultTexture != null) RecordTextureEntryAssetUuids(textureEntry.DefaultTexture); if (textureEntry.FaceTextures != null) { // Loop through the rest of the texture faces (a non-null face means the face is different from DefaultTexture) foreach (Primitive.TextureEntryFace texture in textureEntry.FaceTextures) { if (texture != null) RecordTextureEntryAssetUuids(texture); } } } // If the prim is a sculpt then preserve this information too if (part.Shape.SculptTexture != UUID.Zero) GatheredUuids[part.Shape.SculptTexture] = (sbyte)AssetType.Texture; if (part.Shape.ProjectionTextureUUID != UUID.Zero) GatheredUuids[part.Shape.ProjectionTextureUUID] = (sbyte)AssetType.Texture; UUID collisionSound = part.CollisionSound; if ( collisionSound != UUID.Zero && collisionSound != part.invalidCollisionSoundUUID) GatheredUuids[collisionSound] = (sbyte)AssetType.Sound; if (part.ParticleSystem.Length > 0) { try { Primitive.ParticleSystem ps = new Primitive.ParticleSystem(part.ParticleSystem, 0); if (ps.Texture != UUID.Zero) GatheredUuids[ps.Texture] = (sbyte)AssetType.Texture; } catch (Exception) { m_log.WarnFormat( "[UUID GATHERER]: Could not check particle system for part {0} {1} in object {2} {3} since it is corrupt. Continuing.", part.Name, part.UUID, sceneObject.Name, sceneObject.UUID); } } TaskInventoryDictionary taskDictionary = (TaskInventoryDictionary)part.TaskInventory.Clone(); // Now analyze this prim's inventory items to preserve all the uuids that they reference foreach (TaskInventoryItem tii in taskDictionary.Values) { // m_log.DebugFormat( // "[ARCHIVER]: Analysing item {0} asset type {1} in {2} {3}", // tii.Name, tii.Type, part.Name, part.UUID); AddForInspection(tii.AssetID, (sbyte)tii.Type); } // FIXME: We need to make gathering modular but we cannot yet, since gatherers are not guaranteed // to be called with scene objects that are in a scene (e.g. in the case of hg asset mapping and // inventory transfer. There needs to be a way for a module to register a method without assuming a // Scene.EventManager is present. // part.ParentGroup.Scene.EventManager.TriggerGatherUuids(part, assetUuids); // still needed to retrieve textures used as materials for any parts containing legacy materials stored in DynAttrs RecordMaterialsUuids(part); } catch (Exception e) { m_log.ErrorFormat("[UUID GATHERER]: Failed to get part - {0}", e); } } } /// /// Gathers the next set of assets returned by the next uuid to get from the asset service. /// /// false if gathering is already complete, true otherwise public bool GatherNext() { if (Complete) return false; UUID nextToInspect = m_assetUuidsToInspect.Dequeue(); // m_log.DebugFormat("[UUID GATHERER]: Inspecting asset {0}", nextToInspect); GetAssetUuids(nextToInspect); return true; } /// /// Gathers all remaining asset UUIDS no matter how many calls are required to the asset service. /// /// false if gathering is already complete, true otherwise public bool GatherAll() { if (Complete) return false; while (GatherNext()); return true; } /// /// Gather all the asset uuids associated with the asset referenced by a given uuid /// /// /// This includes both those directly associated with /// it (e.g. face textures) and recursively, those of items within it's inventory (e.g. objects contained /// within this object). /// This method assumes that the asset type associated with this asset in persistent storage is correct (which /// should always be the case). So with this method we always need to retrieve asset data even if the asset /// is of a type which is known not to reference any other assets /// /// The uuid of the asset for which to gather referenced assets private void GetAssetUuids(UUID assetUuid) { if(assetUuid == UUID.Zero) return; if(FailedUUIDs.Contains(assetUuid)) { if(UncertainAssetsUUIDs.Contains(assetUuid)) possibleNotAssetCount++; else ErrorCount++; return; } // avoid infinite loops if (GatheredUuids.ContainsKey(assetUuid)) return; AssetBase assetBase; try { assetBase = GetAsset(assetUuid); } catch (Exception e) { m_log.ErrorFormat("[UUID GATHERER]: Failed to get asset {0} : {1}", assetUuid, e.Message); ErrorCount++; FailedUUIDs.Add(assetUuid); return; } if(assetBase == null) { // m_log.ErrorFormat("[UUID GATHERER]: asset {0} not found", assetUuid); FailedUUIDs.Add(assetUuid); if(UncertainAssetsUUIDs.Contains(assetUuid)) possibleNotAssetCount++; else ErrorCount++; return; } if(UncertainAssetsUUIDs.Contains(assetUuid)) UncertainAssetsUUIDs.Remove(assetUuid); sbyte assetType = assetBase.Type; if(assetBase.Data == null || assetBase.Data.Length == 0) { // m_log.ErrorFormat("[UUID GATHERER]: asset {0}, type {1} has no data", assetUuid, assetType); ErrorCount++; FailedUUIDs.Add(assetUuid); return; } GatheredUuids[assetUuid] = assetType; try { if ((sbyte)AssetType.Bodypart == assetType || (sbyte)AssetType.Clothing == assetType) { RecordWearableAssetUuids(assetBase); } else if ((sbyte)AssetType.Gesture == assetType) { RecordGestureAssetUuids(assetBase); } else if ((sbyte)AssetType.Notecard == assetType) { RecordTextEmbeddedAssetUuids(assetBase); } else if ((sbyte)AssetType.LSLText == assetType) { RecordTextEmbeddedAssetUuids(assetBase); } else if ((sbyte)OpenSimAssetType.Material == assetType) { RecordMaterialAssetUuids(assetBase); } else if ((sbyte)AssetType.Object == assetType) { RecordSceneObjectAssetUuids(assetBase); } } catch (Exception e) { m_log.ErrorFormat("[UUID GATHERER]: Failed to gather uuids for asset with id {0} type {1}: {2}", assetUuid, assetType, e.Message); GatheredUuids.Remove(assetUuid); ErrorCount++; FailedUUIDs.Add(assetUuid); } } private void AddForInspection(UUID assetUuid, sbyte assetType) { if(assetUuid == UUID.Zero) return; // Here, we want to collect uuids which require further asset fetches but mark the others as gathered if(FailedUUIDs.Contains(assetUuid)) { if(UncertainAssetsUUIDs.Contains(assetUuid)) possibleNotAssetCount++; else ErrorCount++; return; } if(GatheredUuids.ContainsKey(assetUuid)) return; try { if ((sbyte)AssetType.Bodypart == assetType || (sbyte)AssetType.Clothing == assetType || (sbyte)AssetType.Gesture == assetType || (sbyte)AssetType.Notecard == assetType || (sbyte)AssetType.LSLText == assetType || (sbyte)OpenSimAssetType.Material == assetType || (sbyte)AssetType.Object == assetType) { AddForInspection(assetUuid); } else { GatheredUuids[assetUuid] = assetType; } } catch (Exception) { m_log.ErrorFormat( "[UUID GATHERER]: Failed to gather uuids for asset id {0}, type {1}", assetUuid, assetType); throw; } } /// /// Collect all the asset uuids found in one face of a Texture Entry. /// private void RecordTextureEntryAssetUuids(Primitive.TextureEntryFace texture) { GatheredUuids[texture.TextureID] = (sbyte)AssetType.Texture; if (texture.MaterialID != UUID.Zero) AddForInspection(texture.MaterialID); } /// /// Gather all of the texture asset UUIDs used to reference "Materials" such as normal and specular maps /// stored in legacy format in part.DynAttrs /// /// private void RecordMaterialsUuids(SceneObjectPart part) { // scan thru the dynAttrs map of this part for any textures used as materials OSD osdMaterials = null; lock (part.DynAttrs) { if (part.DynAttrs.ContainsStore("OpenSim", "Materials")) { OSDMap materialsStore = part.DynAttrs.GetStore("OpenSim", "Materials"); if (materialsStore == null) return; materialsStore.TryGetValue("Materials", out osdMaterials); } if (osdMaterials != null) { //m_log.Info("[UUID Gatherer]: found Materials: " + OSDParser.SerializeJsonString(osd)); if (osdMaterials is OSDArray) { OSDArray matsArr = osdMaterials as OSDArray; foreach (OSDMap matMap in matsArr) { try { if (matMap.ContainsKey("Material")) { OSDMap mat = matMap["Material"] as OSDMap; if (mat.ContainsKey("NormMap")) { UUID normalMapId = mat["NormMap"].AsUUID(); if (normalMapId != UUID.Zero) { GatheredUuids[normalMapId] = (sbyte)AssetType.Texture; //m_log.Info("[UUID Gatherer]: found normal map ID: " + normalMapId.ToString()); } } if (mat.ContainsKey("SpecMap")) { UUID specularMapId = mat["SpecMap"].AsUUID(); if (specularMapId != UUID.Zero) { GatheredUuids[specularMapId] = (sbyte)AssetType.Texture; //m_log.Info("[UUID Gatherer]: found specular map ID: " + specularMapId.ToString()); } } } } catch (Exception e) { m_log.Warn("[UUID Gatherer]: exception getting materials: " + e.Message); } } } } } } /// /// Get an asset synchronously, potentially using an asynchronous callback. If the /// asynchronous callback is used, we will wait for it to complete. /// /// /// protected virtual AssetBase GetAsset(UUID uuid) { return m_assetService.Get(uuid.ToString()); } /// /// Record the asset uuids embedded within the given text (e.g. a script). /// /// private void RecordTextEmbeddedAssetUuids(AssetBase textAsset) { // m_log.DebugFormat("[ASSET GATHERER]: Getting assets for uuid references in asset {0}", embeddingAssetId); string text = Utils.BytesToString(textAsset.Data); // m_log.DebugFormat("[UUID GATHERER]: Text {0}", text); MatchCollection uuidMatches = Util.PermissiveUUIDPattern.Matches(text); // m_log.DebugFormat("[UUID GATHERER]: Found {0} matches in text", uuidMatches.Count); foreach (Match uuidMatch in uuidMatches) { UUID uuid = new UUID(uuidMatch.Value); if(uuid == UUID.Zero) continue; // m_log.DebugFormat("[UUID GATHERER]: Recording {0} in text", uuid); if(!UncertainAssetsUUIDs.Contains(uuid)) UncertainAssetsUUIDs.Add(uuid); AddForInspection(uuid); } } /// /// Record the uuids referenced by the given wearable asset /// /// private void RecordWearableAssetUuids(AssetBase assetBase) { //m_log.Debug(new System.Text.ASCIIEncoding().GetString(bodypartAsset.Data)); AssetWearable wearableAsset = new AssetBodypart(assetBase.FullID, assetBase.Data); wearableAsset.Decode(); //m_log.DebugFormat( // "[ARCHIVER]: Wearable asset {0} references {1} assets", wearableAssetUuid, wearableAsset.Textures.Count); foreach (UUID uuid in wearableAsset.Textures.Values) GatheredUuids[uuid] = (sbyte)AssetType.Texture; } /// /// Get all the asset uuids associated with a given object. This includes both those directly associated with /// it (e.g. face textures) and recursively, those of items within it's inventory (e.g. objects contained /// within this object). /// /// private void RecordSceneObjectAssetUuids(AssetBase sceneObjectAsset) { string xml = Utils.BytesToString(sceneObjectAsset.Data); if (String.IsNullOrEmpty(xml)) m_log.ErrorFormat("[UUIDGatherer]: Asset {0} - {1} has a zero length XML blob!", sceneObjectAsset.Name, sceneObjectAsset.ID); else { CoalescedSceneObjects coa; if (CoalescedSceneObjectsSerializer.TryFromXml(xml, out coa)) { foreach (SceneObjectGroup sog in coa.Objects) AddForInspection(sog); } else { SceneObjectGroup sog = SceneObjectSerializer.FromOriginalXmlFormat(xml); if (null != sog) AddForInspection(sog); } } } /// /// Get the asset uuid associated with a gesture /// /// private void RecordGestureAssetUuids(AssetBase gestureAsset) { using (MemoryStream ms = new MemoryStream(gestureAsset.Data)) using (StreamReader sr = new StreamReader(ms)) { sr.ReadLine(); // Unknown (Version?) sr.ReadLine(); // Unknown sr.ReadLine(); // Unknown sr.ReadLine(); // Name sr.ReadLine(); // Comment ? int count = Convert.ToInt32(sr.ReadLine()); // Item count for (int i = 0 ; i < count ; i++) { string type = sr.ReadLine(); if (type == null) break; string name = sr.ReadLine(); if (name == null) break; string id = sr.ReadLine(); if (id == null) break; string unknown = sr.ReadLine(); if (unknown == null) break; // If it can be parsed as a UUID, it is an asset ID UUID uuid; if (UUID.TryParse(id, out uuid)) GatheredUuids[uuid] = (sbyte)AssetType.Animation; // the asset is either an Animation or a Sound, but this distinction isn't important } } } /// /// Get the asset uuid's referenced in a material. /// private void RecordMaterialAssetUuids(AssetBase materialAsset) { OSDMap mat; try { mat = (OSDMap)OSDParser.DeserializeLLSDXml(materialAsset.Data); } catch (Exception e) { m_log.WarnFormat("[Materials]: cannot decode material asset {0}: {1}", materialAsset.ID, e.Message); return; } UUID normMap = mat["NormMap"].AsUUID(); if (normMap != UUID.Zero) GatheredUuids[normMap] = (sbyte)AssetType.Texture; UUID specMap = mat["SpecMap"].AsUUID(); if (specMap != UUID.Zero) GatheredUuids[specMap] = (sbyte)AssetType.Texture; } } public class HGUuidGatherer : UuidGatherer { private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); protected string m_assetServerURL; public HGUuidGatherer(IAssetService assetService, string assetServerURL) : this(assetService, assetServerURL, new Dictionary()) {} public HGUuidGatherer(IAssetService assetService, string assetServerURL, IDictionary collector) : base(assetService, collector) { m_assetServerURL = assetServerURL; if (!m_assetServerURL.EndsWith("/") && !m_assetServerURL.EndsWith("=")) m_assetServerURL = m_assetServerURL + "/"; } protected override AssetBase GetAsset(UUID uuid) { if (string.Empty == m_assetServerURL) return base.GetAsset(uuid); else return FetchAsset(uuid); } public AssetBase FetchAsset(UUID assetID) { // Test if it's already here AssetBase asset = m_assetService.Get(assetID.ToString()); if (asset == null) { // It's not, so fetch it from abroad asset = m_assetService.Get(m_assetServerURL + assetID.ToString()); if (asset != null) m_log.DebugFormat("[HGUUIDGatherer]: Copied asset {0} from {1} to local asset server", assetID, m_assetServerURL); else m_log.DebugFormat("[HGUUIDGatherer]: Failed to fetch asset {0} from {1}", assetID, m_assetServerURL); } //else // m_log.DebugFormat("[HGUUIDGatherer]: Asset {0} from {1} was already here", assetID, m_assetServerURL); return asset; } } }