From 9822bb664b58693790201606126e3fd26f7a4d52 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Sat, 2 Feb 2013 02:57:38 +0000 Subject: Log missing assets on "fcache assets" found. This ignores references found by scanning LSL/notecard files since these are the source of false positives. This also changes UuidGatherer to reutn an AssetType.Unknown for embedded script/lsl references instead of Texture, since these are often not textures. This is added to help people in determining when they have missing assets such as textures, etc. In this case, one wants to run "fcache clear" first. --- .../Region/CoreModules/Asset/FlotsamAssetCache.cs | 41 ++++++++++++++-------- 1 file changed, 26 insertions(+), 15 deletions(-) (limited to 'OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs') diff --git a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs index 8e800cb..00af175 100644 --- a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs +++ b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs @@ -707,32 +707,43 @@ namespace OpenSim.Region.CoreModules.Asset { UuidGatherer gatherer = new UuidGatherer(m_AssetService); + HashSet uniqueUuids = new HashSet(); Dictionary assets = new Dictionary(); + foreach (Scene s in m_Scenes) { StampRegionStatusFile(s.RegionInfo.RegionID); s.ForEachSOG(delegate(SceneObjectGroup e) - { + { gatherer.GatherAssetUuids(e, assets); - }); - } - foreach (UUID assetID in assets.Keys) - { - string filename = GetFileName(assetID.ToString()); + foreach (UUID assetID in assets.Keys) + { + uniqueUuids.Add(assetID); - if (File.Exists(filename)) - { - File.SetLastAccessTime(filename, DateTime.Now); - } - else if (storeUncached) - { - m_AssetService.Get(assetID.ToString()); - } + string filename = GetFileName(assetID.ToString()); + + if (File.Exists(filename)) + { + File.SetLastAccessTime(filename, DateTime.Now); + } + else if (storeUncached) + { + AssetBase cachedAsset = m_AssetService.Get(assetID.ToString()); + if (cachedAsset == null && assets[assetID] != AssetType.Unknown) + m_log.DebugFormat( + "[FLOTSAM ASSET CACHE]: Could not find asset {0}, type {1} referenced by object {2} at {3} in scene {4} when pre-caching all scene assets", + assetID, assets[assetID], e.Name, e.AbsolutePosition, s.Name); + } + } + + assets.Clear(); + }); } - return assets.Keys.Count; + + return uniqueUuids.Count; } /// -- cgit v1.1 From b08977ea7d9e35066e3cbf367fad58c2d8bc227e Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 8 Feb 2013 21:21:20 +0000 Subject: Don't allow exceptions to propogate from FlotsamAssetCache which may occur when deleting expired files or stamping the region status file. Changes various error level log lines to warn since these are not fatal to the operation of OpenSimulator --- .../Region/CoreModules/Asset/FlotsamAssetCache.cs | 89 ++++++++++++++-------- 1 file changed, 57 insertions(+), 32 deletions(-) (limited to 'OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs') diff --git a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs index 00af175..3cba9b4 100644 --- a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs +++ b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs @@ -299,7 +299,7 @@ namespace OpenSim.Region.CoreModules.Asset } catch (Exception e) { - m_log.ErrorFormat( + m_log.WarnFormat( "[FLOTSAM ASSET CACHE]: Failed to update cache for asset {0}. Exception {1} {2}", asset.ID, e.Message, e.StackTrace); } @@ -339,12 +339,13 @@ namespace OpenSim.Region.CoreModules.Asset /// Try to get an asset from the file cache. /// /// - /// + /// An asset retrieved from the file cache. null if there was a problem retrieving an asset. private AssetBase GetFromFileCache(string id) { AssetBase asset = null; - + string filename = GetFileName(id); + if (File.Exists(filename)) { FileStream stream = null; @@ -359,7 +360,7 @@ namespace OpenSim.Region.CoreModules.Asset } catch (System.Runtime.Serialization.SerializationException e) { - m_log.ErrorFormat( + m_log.WarnFormat( "[FLOTSAM ASSET CACHE]: Failed to get file {0} for asset {1}. Exception {2} {3}", filename, id, e.Message, e.StackTrace); @@ -371,7 +372,7 @@ namespace OpenSim.Region.CoreModules.Asset } catch (Exception e) { - m_log.ErrorFormat( + m_log.WarnFormat( "[FLOTSAM ASSET CACHE]: Failed to get file {0} for asset {1}. Exception {2} {3}", filename, id, e.Message, e.StackTrace); } @@ -469,7 +470,7 @@ namespace OpenSim.Region.CoreModules.Asset } catch (Exception e) { - m_log.ErrorFormat( + m_log.WarnFormat( "[FLOTSAM ASSET CACHE]: Failed to expire cached file {0}. Exception {1} {2}", id, e.Message, e.StackTrace); } @@ -520,29 +521,39 @@ namespace OpenSim.Region.CoreModules.Asset /// private void CleanExpiredFiles(string dir, DateTime purgeLine) { - foreach (string file in Directory.GetFiles(dir)) + try { - if (File.GetLastAccessTime(file) < purgeLine) + foreach (string file in Directory.GetFiles(dir)) { - File.Delete(file); + if (File.GetLastAccessTime(file) < purgeLine) + { + File.Delete(file); + } } - } - // Recurse into lower tiers - foreach (string subdir in Directory.GetDirectories(dir)) - { - CleanExpiredFiles(subdir, purgeLine); - } + // Recurse into lower tiers + foreach (string subdir in Directory.GetDirectories(dir)) + { + CleanExpiredFiles(subdir, purgeLine); + } - // Check if a tier directory is empty, if so, delete it - int dirSize = Directory.GetFiles(dir).Length + Directory.GetDirectories(dir).Length; - if (dirSize == 0) - { - Directory.Delete(dir); + // Check if a tier directory is empty, if so, delete it + int dirSize = Directory.GetFiles(dir).Length + Directory.GetDirectories(dir).Length; + if (dirSize == 0) + { + Directory.Delete(dir); + } + else if (dirSize >= m_CacheWarnAt) + { + m_log.WarnFormat( + "[FLOTSAM ASSET CACHE]: Cache folder exceeded CacheWarnAt limit {0} {1}. Suggest increasing tiers, tier length, or reducing cache expiration", + dir, dirSize); + } } - else if (dirSize >= m_CacheWarnAt) + catch (Exception e) { - m_log.WarnFormat("[FLOTSAM ASSET CACHE]: Cache folder exceeded CacheWarnAt limit {0} {1}. Suggest increasing tiers, tier length, or reducing cache expiration", dir, dirSize); + m_log.Warn( + string.Format("[FLOTSAM ASSET CACHE]: Could not complete clean of expired files in {0}, exception ", dir), e); } } @@ -601,7 +612,7 @@ namespace OpenSim.Region.CoreModules.Asset } catch (IOException e) { - m_log.ErrorFormat( + m_log.WarnFormat( "[FLOTSAM ASSET CACHE]: Failed to write asset {0} to temporary location {1} (final {2}) on cache in {3}. Exception {4} {5}.", asset.ID, tempname, filename, directory, e.Message, e.StackTrace); @@ -680,17 +691,31 @@ namespace OpenSim.Region.CoreModules.Asset /// /// This notes the last time the Region had a deep asset scan performed on it. /// - /// - private void StampRegionStatusFile(UUID RegionID) + /// + private void StampRegionStatusFile(UUID regionID) { - string RegionCacheStatusFile = Path.Combine(m_CacheDirectory, "RegionStatus_" + RegionID.ToString() + ".fac"); - if (File.Exists(RegionCacheStatusFile)) + string RegionCacheStatusFile = Path.Combine(m_CacheDirectory, "RegionStatus_" + regionID.ToString() + ".fac"); + + try { - File.SetLastWriteTime(RegionCacheStatusFile, DateTime.Now); + if (File.Exists(RegionCacheStatusFile)) + { + File.SetLastWriteTime(RegionCacheStatusFile, DateTime.Now); + } + else + { + File.WriteAllText( + RegionCacheStatusFile, + "Please do not delete this file unless you are manually clearing your Flotsam Asset Cache."); + } } - else + catch (Exception e) { - File.WriteAllText(RegionCacheStatusFile, "Please do not delete this file unless you are manually clearing your Flotsam Asset Cache."); + m_log.Warn( + string.Format( + "[FLOTSAM ASSET CACHE]: Could not stamp region status file for region {0}. Exception ", + regionID), + e); } } @@ -759,7 +784,7 @@ namespace OpenSim.Region.CoreModules.Asset } catch (Exception e) { - m_log.ErrorFormat( + m_log.WarnFormat( "[FLOTSAM ASSET CACHE]: Couldn't clear asset cache directory {0} from {1}. Exception {2} {3}", dir, m_CacheDirectory, e.Message, e.StackTrace); } @@ -773,7 +798,7 @@ namespace OpenSim.Region.CoreModules.Asset } catch (Exception e) { - m_log.ErrorFormat( + m_log.WarnFormat( "[FLOTSAM ASSET CACHE]: Couldn't clear asset cache file {0} from {1}. Exception {1} {2}", file, m_CacheDirectory, e.Message, e.StackTrace); } -- cgit v1.1 From aba66c71122a2ca9a3c2fb807b360e9fff8b9026 Mon Sep 17 00:00:00 2001 From: Melanie Thielker Date: Sat, 23 Mar 2013 13:12:41 +0100 Subject: Make sharing errors not spew and let the cache retry the files --- OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs') diff --git a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs index a0f1e8c..ec388f4 100644 --- a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs +++ b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs @@ -369,7 +369,7 @@ namespace OpenSim.Region.CoreModules.Asset AssetBase asset = null; string filename = GetFileName(id); - if (File.Exists(filename)) + while (File.Exists(filename)) { FileStream stream = null; try @@ -380,6 +380,8 @@ namespace OpenSim.Region.CoreModules.Asset asset = (AssetBase)bformatter.Deserialize(stream); m_DiskHits++; + + break; } catch (System.Runtime.Serialization.SerializationException e) { @@ -392,12 +394,24 @@ namespace OpenSim.Region.CoreModules.Asset // {different version of AssetBase} -- we should attempt to // delete it and re-cache File.Delete(filename); + + break; + } + catch (IOException e) + { + // This is a sharing violation: File exists but can't be opened because it's + // being written + Thread.Sleep(100); + + continue; } catch (Exception e) { m_log.ErrorFormat( "[FLOTSAM ASSET CACHE]: Failed to get file {0} for asset {1}. Exception {2} {3}", filename, id, e.Message, e.StackTrace); + + break; } finally { -- cgit v1.1 From 9fee431cc8a054dd4d6b20392cbefd0a0f8343f1 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Sat, 30 Mar 2013 01:21:16 +0000 Subject: In the flotasm asset cache, if we get a request for a file that we're actively writing, simply return null instead of first logging an exception. --- .../Region/CoreModules/Asset/FlotsamAssetCache.cs | 52 +++++++++++----------- 1 file changed, 27 insertions(+), 25 deletions(-) (limited to 'OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs') diff --git a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs index 3cba9b4..2afe065 100644 --- a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs +++ b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs @@ -341,11 +341,35 @@ namespace OpenSim.Region.CoreModules.Asset /// /// An asset retrieved from the file cache. null if there was a problem retrieving an asset. private AssetBase GetFromFileCache(string id) - { - AssetBase asset = null; - + { string filename = GetFileName(id); +#if WAIT_ON_INPROGRESS_REQUESTS + // Check if we're already downloading this asset. If so, try to wait for it to + // download. + if (m_WaitOnInprogressTimeout > 0) + { + m_RequestsForInprogress++; + + ManualResetEvent waitEvent; + if (m_CurrentlyWriting.TryGetValue(filename, out waitEvent)) + { + waitEvent.WaitOne(m_WaitOnInprogressTimeout); + return Get(id); + } + } +#else + // Track how often we have the problem that an asset is requested while + // it is still being downloaded by a previous request. + if (m_CurrentlyWriting.Contains(filename)) + { + m_RequestsForInprogress++; + return null; + } +#endif + + AssetBase asset = null; + if (File.Exists(filename)) { FileStream stream = null; @@ -383,28 +407,6 @@ namespace OpenSim.Region.CoreModules.Asset } } -#if WAIT_ON_INPROGRESS_REQUESTS - // Check if we're already downloading this asset. If so, try to wait for it to - // download. - if (m_WaitOnInprogressTimeout > 0) - { - m_RequestsForInprogress++; - - ManualResetEvent waitEvent; - if (m_CurrentlyWriting.TryGetValue(filename, out waitEvent)) - { - waitEvent.WaitOne(m_WaitOnInprogressTimeout); - return Get(id); - } - } -#else - // Track how often we have the problem that an asset is requested while - // it is still being downloaded by a previous request. - if (m_CurrentlyWriting.Contains(filename)) - { - m_RequestsForInprogress++; - } -#endif return asset; } -- cgit v1.1