From 5e4d6cab00cb29cd088ab7b62ab13aff103b64cb Mon Sep 17 00:00:00 2001 From: onefang Date: Sun, 19 May 2019 21:24:15 +1000 Subject: Dump OpenSim 0.9.0.1 into it's own branch. --- .../Region/CoreModules/Asset/CenomeAssetCache.cs | 38 ++- OpenSim/Region/CoreModules/Asset/CoreAssetCache.cs | 23 +- .../Region/CoreModules/Asset/FlotsamAssetCache.cs | 371 ++++++++++++++++----- .../CoreModules/Asset/GlynnTuckerAssetCache.cs | 24 +- .../Asset/Tests/FlotsamAssetCacheTests.cs | 2 +- 5 files changed, 347 insertions(+), 111 deletions(-) (limited to 'OpenSim/Region/CoreModules/Asset') diff --git a/OpenSim/Region/CoreModules/Asset/CenomeAssetCache.cs b/OpenSim/Region/CoreModules/Asset/CenomeAssetCache.cs index ebec9d2..403236c 100644 --- a/OpenSim/Region/CoreModules/Asset/CenomeAssetCache.cs +++ b/OpenSim/Region/CoreModules/Asset/CenomeAssetCache.cs @@ -41,16 +41,16 @@ namespace OpenSim.Region.CoreModules.Asset /// /// /// - /// Cache is enabled by setting "AssetCaching" configuration to value "CenomeMemoryAssetCache". - /// When cache is successfully enable log should have message + /// Cache is enabled by setting "AssetCaching" configuration to value "CenomeMemoryAssetCache". + /// When cache is successfully enable log should have message /// "[ASSET CACHE]: Cenome asset cache enabled (MaxSize = XXX bytes, MaxCount = XXX, ExpirationTime = XXX)". /// /// /// Cache's size is limited by two parameters: - /// maximal allowed size in bytes and maximal allowed asset count. When new asset + /// maximal allowed size in bytes and maximal allowed asset count. When new asset /// is added to cache that have achieved either size or count limitation, cache /// will automatically remove less recently used assets from cache. Additionally - /// asset's lifetime is controlled by expiration time. + /// asset's lifetime is controlled by expiration time. /// /// /// @@ -91,10 +91,10 @@ namespace OpenSim.Region.CoreModules.Asset /// /// [Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "CenomeMemoryAssetCache")] - public class CenomeMemoryAssetCache : IImprovedAssetCache, ISharedRegionModule + public class CenomeMemoryAssetCache : IAssetCache, ISharedRegionModule { private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - + /// /// Cache's default maximal asset count. /// @@ -119,7 +119,7 @@ namespace OpenSim.Region.CoreModules.Asset /// Asset's default expiration time in the cache. /// public static readonly TimeSpan DefaultExpirationTime = TimeSpan.FromMinutes(30.0); - + /// /// Cache object. /// @@ -192,7 +192,7 @@ namespace OpenSim.Region.CoreModules.Asset expirationTime); } - #region IImprovedAssetCache Members + #region IAssetCache Members public bool Check(string id) { @@ -213,7 +213,7 @@ namespace OpenSim.Region.CoreModules.Asset if (asset != null) { // m_log.DebugFormat("[CENOME ASSET CACHE]: Caching asset {0}", asset.ID); - + long size = asset.Data != null ? asset.Data.Length : 1; m_cache.Set(asset.ID, asset, size); m_cachedCount++; @@ -221,6 +221,11 @@ namespace OpenSim.Region.CoreModules.Asset } + public void CacheNegative(string id) + { + // We don't do negative caching + } + /// /// Clear asset cache. /// @@ -241,7 +246,7 @@ namespace OpenSim.Region.CoreModules.Asset } /// - /// Get asset stored + /// Get asset stored /// /// /// The asset's id. @@ -255,10 +260,9 @@ namespace OpenSim.Region.CoreModules.Asset /// Cache doesn't guarantee in any situation that asset is stored to it. /// /// - public AssetBase Get(string id) + public bool Get(string id, out AssetBase assetBase) { m_getCount++; - AssetBase assetBase; if (m_cache.TryGetValue(id, out assetBase)) m_hitCount++; @@ -278,8 +282,8 @@ namespace OpenSim.Region.CoreModules.Asset // if (null == assetBase) // m_log.DebugFormat("[CENOME ASSET CACHE]: Asset {0} not in cache", id); - - return assetBase; + + return true; } #endregion @@ -294,7 +298,7 @@ namespace OpenSim.Region.CoreModules.Asset get { return "CenomeMemoryAssetCache"; } } - public Type ReplaceableInterface + public Type ReplaceableInterface { get { return null; } } @@ -308,7 +312,7 @@ namespace OpenSim.Region.CoreModules.Asset public void AddRegion(Scene scene) { if (m_enabled) - scene.RegisterModuleInterface(this); + scene.RegisterModuleInterface(this); } /// @@ -344,7 +348,7 @@ namespace OpenSim.Region.CoreModules.Asset if (name != Name) return; - + long maxSize = DefaultMaxSize; int maxCount = DefaultMaxCount; TimeSpan expirationTime = DefaultExpirationTime; diff --git a/OpenSim/Region/CoreModules/Asset/CoreAssetCache.cs b/OpenSim/Region/CoreModules/Asset/CoreAssetCache.cs index f720748..10c0e85 100644 --- a/OpenSim/Region/CoreModules/Asset/CoreAssetCache.cs +++ b/OpenSim/Region/CoreModules/Asset/CoreAssetCache.cs @@ -40,7 +40,7 @@ using OpenSim.Services.Interfaces; namespace OpenSim.Region.CoreModules.Asset { [Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "CoreAssetCache")] - public class CoreAssetCache : ISharedRegionModule, IImprovedAssetCache + public class CoreAssetCache : ISharedRegionModule, IAssetCache { private static readonly ILog m_log = LogManager.GetLogger( @@ -54,7 +54,7 @@ namespace OpenSim.Region.CoreModules.Asset get { return "CoreAssetCache"; } } - public Type ReplaceableInterface + public Type ReplaceableInterface { get { return null; } } @@ -98,7 +98,7 @@ namespace OpenSim.Region.CoreModules.Asset public void AddRegion(Scene scene) { if (m_Enabled) - scene.RegisterModuleInterface(this); + scene.RegisterModuleInterface(this); } public void RemoveRegion(Scene scene) @@ -110,12 +110,15 @@ namespace OpenSim.Region.CoreModules.Asset } //////////////////////////////////////////////////////////// - // IImprovedAssetCache + // IAssetCache // public bool Check(string id) { // XXX This is probably not an efficient implementation. - return Get(id) != null; + AssetBase asset; + if (!Get(id, out asset)) + return false; + return asset != null; } public void Cache(AssetBase asset) @@ -124,9 +127,15 @@ namespace OpenSim.Region.CoreModules.Asset m_Cache.Store(asset.ID, asset); } - public AssetBase Get(string id) + public void CacheNegative(string id) { - return (AssetBase)m_Cache.Get(id); + // We don't do negative caching + } + + public bool Get(string id, out AssetBase asset) + { + asset = (AssetBase)m_Cache.Get(id); + return true; } public void Expire(string id) diff --git a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs index 17646fb..c1bf544 100644 --- a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs +++ b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs @@ -25,7 +25,7 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -// Uncomment to make asset Get requests for existing +// Uncomment to make asset Get requests for existing // #define WAIT_ON_INPROGRESS_REQUESTS using System; @@ -55,16 +55,18 @@ using OpenSim.Services.Interfaces; namespace OpenSim.Region.CoreModules.Asset { [Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "FlotsamAssetCache")] - public class FlotsamAssetCache : ISharedRegionModule, IImprovedAssetCache, IAssetService + public class FlotsamAssetCache : ISharedRegionModule, IAssetCache, IAssetService { private static readonly ILog m_log = LogManager.GetLogger( MethodBase.GetCurrentMethod().DeclaringType); private bool m_Enabled; + private bool m_timerRunning; + private bool m_cleanupRunning; private const string m_ModuleName = "FlotsamAssetCache"; - private const string m_DefaultCacheDirectory = "../caches/assetcache"; + private const string m_DefaultCacheDirectory = "./assetcache"; private string m_CacheDirectory = m_DefaultCacheDirectory; private readonly List m_InvalidChars = new List(); @@ -76,6 +78,7 @@ namespace OpenSim.Region.CoreModules.Asset private static ulong m_RequestsForInprogress; private static ulong m_DiskHits; private static ulong m_MemoryHits; + private static ulong m_weakRefHits; #if WAIT_ON_INPROGRESS_REQUESTS private Dictionary m_CurrentlyWriting = new Dictionary(); @@ -89,12 +92,17 @@ namespace OpenSim.Region.CoreModules.Asset private ExpiringCache m_MemoryCache; private bool m_MemoryCacheEnabled = false; + private ExpiringCache m_negativeCache; + private bool m_negativeCacheEnabled = true; + private bool m_negativeCacheSliding = false; + // Expiration is expressed in hours. - private const double m_DefaultMemoryExpiration = 2; + private double m_MemoryExpiration = 0.016; private const double m_DefaultFileExpiration = 48; - private TimeSpan m_MemoryExpiration = TimeSpan.FromHours(m_DefaultMemoryExpiration); + // Negative cache is in seconds + private int m_negativeExpiration = 120; private TimeSpan m_FileExpiration = TimeSpan.FromHours(m_DefaultFileExpiration); - private TimeSpan m_FileExpirationCleanupTimer = TimeSpan.FromHours(0.166); + private TimeSpan m_FileExpirationCleanupTimer = TimeSpan.FromHours(1.0); private static int m_CacheDirectoryTiers = 1; private static int m_CacheDirectoryTierLen = 3; @@ -104,6 +112,11 @@ namespace OpenSim.Region.CoreModules.Asset private IAssetService m_AssetService; private List m_Scenes = new List(); + private object timerLock = new object(); + + private Dictionary weakAssetReferences = new Dictionary(); + private object weakAssetReferencesLock = new object(); + private bool m_updateFileTimeOnCacheHit = false; public FlotsamAssetCache() { @@ -111,7 +124,7 @@ namespace OpenSim.Region.CoreModules.Asset m_InvalidChars.AddRange(Path.GetInvalidFileNameChars()); } - public Type ReplaceableInterface + public Type ReplaceableInterface { get { return null; } } @@ -124,7 +137,7 @@ namespace OpenSim.Region.CoreModules.Asset public void Initialise(IConfigSource source) { IConfig moduleConfig = source.Configs["Modules"]; - + if (moduleConfig != null) { string name = moduleConfig.GetString("AssetCaching", String.Empty); @@ -132,6 +145,7 @@ namespace OpenSim.Region.CoreModules.Asset if (name == Name) { m_MemoryCache = new ExpiringCache(); + m_negativeCache = new ExpiringCache(); m_Enabled = true; m_log.InfoFormat("[FLOTSAM ASSET CACHE]: {0} enabled", this.Name); @@ -148,12 +162,18 @@ namespace OpenSim.Region.CoreModules.Asset m_CacheDirectory = assetConfig.GetString("CacheDirectory", m_DefaultCacheDirectory); m_MemoryCacheEnabled = assetConfig.GetBoolean("MemoryCacheEnabled", m_MemoryCacheEnabled); - m_MemoryExpiration = TimeSpan.FromHours(assetConfig.GetDouble("MemoryCacheTimeout", m_DefaultMemoryExpiration)); - + m_MemoryExpiration = assetConfig.GetDouble("MemoryCacheTimeout", m_MemoryExpiration); + m_MemoryExpiration *= 3600.0; // config in hours to seconds + + m_negativeCacheEnabled = assetConfig.GetBoolean("NegativeCacheEnabled", m_negativeCacheEnabled); + m_negativeExpiration = assetConfig.GetInt("NegativeCacheTimeout", m_negativeExpiration); + m_negativeCacheSliding = assetConfig.GetBoolean("NegativeCacheSliding", m_negativeCacheSliding); + m_updateFileTimeOnCacheHit = assetConfig.GetBoolean("UpdateFileTimeOnCacheHit", m_updateFileTimeOnCacheHit); + #if WAIT_ON_INPROGRESS_REQUESTS m_WaitOnInprogressTimeout = assetConfig.GetInt("WaitOnInprogressTimeout", 3000); #endif - + m_LogLevel = assetConfig.GetInt("LogLevel", m_LogLevel); m_HitRateDisplay = (ulong)assetConfig.GetLong("HitRateDisplay", (long)m_HitRateDisplay); @@ -170,14 +190,6 @@ namespace OpenSim.Region.CoreModules.Asset m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Cache Directory {0}", m_CacheDirectory); - if (m_FileCacheEnabled && (m_FileExpiration > TimeSpan.Zero) && (m_FileExpirationCleanupTimer > TimeSpan.Zero)) - { - m_CacheCleanTimer = new System.Timers.Timer(m_FileExpirationCleanupTimer.TotalMilliseconds); - m_CacheCleanTimer.AutoReset = true; - m_CacheCleanTimer.Elapsed += CleanupExpiredFiles; - lock (m_CacheCleanTimer) - m_CacheCleanTimer.Start(); - } if (m_CacheDirectoryTiers < 1) { @@ -217,9 +229,8 @@ namespace OpenSim.Region.CoreModules.Asset { if (m_Enabled) { - scene.RegisterModuleInterface(this); + scene.RegisterModuleInterface(this); m_Scenes.Add(scene); - } } @@ -227,23 +238,61 @@ namespace OpenSim.Region.CoreModules.Asset { if (m_Enabled) { - scene.UnregisterModuleInterface(this); + scene.UnregisterModuleInterface(this); m_Scenes.Remove(scene); + lock(timerLock) + { + if(m_timerRunning && m_Scenes.Count <= 0) + { + m_timerRunning = false; + m_CacheCleanTimer.Stop(); + m_CacheCleanTimer.Close(); + } + } } } public void RegionLoaded(Scene scene) { - if (m_Enabled && m_AssetService == null) - m_AssetService = scene.RequestModuleInterface(); + if (m_Enabled) + { + if(m_AssetService == null) + m_AssetService = scene.RequestModuleInterface(); + lock(timerLock) + { + if(!m_timerRunning) + { + if (m_FileCacheEnabled && (m_FileExpiration > TimeSpan.Zero) && (m_FileExpirationCleanupTimer > TimeSpan.Zero)) + { + m_CacheCleanTimer = new System.Timers.Timer(m_FileExpirationCleanupTimer.TotalMilliseconds); + m_CacheCleanTimer.AutoReset = false; + m_CacheCleanTimer.Elapsed += CleanupExpiredFiles; + m_CacheCleanTimer.Start(); + m_timerRunning = true; + } + } + } + if (m_MemoryCacheEnabled) + m_MemoryCache = new ExpiringCache(); + + lock(weakAssetReferencesLock) + weakAssetReferences = new Dictionary(); + } } //////////////////////////////////////////////////////////// - // IImprovedAssetCache + // IAssetCache // + private void UpdateWeakReference(string key, AssetBase asset) + { + WeakReference aref = new WeakReference(asset); + lock(weakAssetReferencesLock) + weakAssetReferences[key] = aref; + } private void UpdateMemoryCache(string key, AssetBase asset) { + // NOTE DO NOT USE SLIDEEXPIRE option on current libomv m_MemoryCache.AddOrUpdate(key, asset, m_MemoryExpiration); } @@ -257,11 +306,11 @@ namespace OpenSim.Region.CoreModules.Asset if (File.Exists(filename)) { UpdateFileLastAccessTime(filename); - } - else + } + else { // Once we start writing, make sure we flag that we're writing - // that object to the cache so that we don't try to write the + // that object to the cache so that we don't try to write the // same file multiple times. lock (m_CurrentlyWriting) { @@ -306,6 +355,7 @@ namespace OpenSim.Region.CoreModules.Asset if (asset != null) { //m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Caching asset with id {0}", asset.ID); + UpdateWeakReference(asset.ID, asset); if (m_MemoryCacheEnabled) UpdateMemoryCache(asset.ID, asset); @@ -315,6 +365,17 @@ namespace OpenSim.Region.CoreModules.Asset } } + public void CacheNegative(string id) + { + if (m_negativeCacheEnabled) + { + if (m_negativeCacheSliding) + m_negativeCache.AddOrUpdate(id, null, TimeSpan.FromSeconds(m_negativeExpiration)); + else + m_negativeCache.AddOrUpdate(id, null, m_negativeExpiration); + } + } + /// /// Updates the cached file with the current time. /// @@ -333,6 +394,25 @@ namespace OpenSim.Region.CoreModules.Asset } } + private AssetBase GetFromWeakReference(string id) + { + AssetBase asset = null; + WeakReference aref; + + lock(weakAssetReferencesLock) + { + if (weakAssetReferences.TryGetValue(id, out aref)) + { + asset = aref.Target as AssetBase; + if(asset == null) + weakAssetReferences.Remove(id); + else + m_weakRefHits++; + } + } + return asset; + } + /// /// Try to get an asset from the in-memory cache. /// @@ -359,7 +439,7 @@ namespace OpenSim.Region.CoreModules.Asset /// /// An asset retrieved from the file cache. null if there was a problem retrieving an asset. private AssetBase GetFromFileCache(string id) - { + { string filename = GetFileName(id); #if WAIT_ON_INPROGRESS_REQUESTS @@ -394,6 +474,8 @@ namespace OpenSim.Region.CoreModules.Asset { using (FileStream stream = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read)) { + if (stream.Length == 0) // Empty file will trigger exception below + return null; BinaryFormatter bformatter = new BinaryFormatter(); asset = (AssetBase)bformatter.Deserialize(stream); @@ -451,23 +533,57 @@ namespace OpenSim.Region.CoreModules.Asset return found; } + // For IAssetService public AssetBase Get(string id) { + AssetBase asset; + Get(id, out asset); + return asset; + } + + public bool Get(string id, out AssetBase asset) + { + asset = null; + m_Requests++; - AssetBase asset = null; + object dummy; + if (m_negativeCache.TryGetValue(id, out dummy)) + { + return false; + } - if (m_MemoryCacheEnabled) + asset = GetFromWeakReference(id); + if (asset != null && m_updateFileTimeOnCacheHit) + { + string filename = GetFileName(id); + UpdateFileLastAccessTime(filename); + } + + if (m_MemoryCacheEnabled && asset == null) + { asset = GetFromMemoryCache(id); + if(asset != null) + { + UpdateWeakReference(id,asset); + if (m_updateFileTimeOnCacheHit) + { + string filename = GetFileName(id); + UpdateFileLastAccessTime(filename); + } + } + } if (asset == null && m_FileCacheEnabled) { asset = GetFromFileCache(id); - - if (m_MemoryCacheEnabled && asset != null) - UpdateMemoryCache(id, asset); + if(asset != null) + UpdateWeakReference(id,asset); } + if (m_MemoryCacheEnabled && asset != null) + UpdateMemoryCache(id, asset); + if (((m_LogLevel >= 1)) && (m_HitRateDisplay != 0) && (m_Requests % m_HitRateDisplay == 0)) { m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Cache Get :: {0} :: {1}", id, asset == null ? "Miss" : "Hit"); @@ -475,7 +591,7 @@ namespace OpenSim.Region.CoreModules.Asset GenerateCacheHitReport().ForEach(l => m_log.InfoFormat("[FLOTSAM ASSET CACHE]: {0}", l)); } - return asset; + return true; } public bool Check(string id) @@ -490,7 +606,9 @@ namespace OpenSim.Region.CoreModules.Asset public AssetBase GetCached(string id) { - return Get(id); + AssetBase asset; + Get(id, out asset); + return asset; } public void Expire(string id) @@ -511,6 +629,9 @@ namespace OpenSim.Region.CoreModules.Asset if (m_MemoryCacheEnabled) m_MemoryCache.Remove(id); + + lock(weakAssetReferencesLock) + weakAssetReferences.Remove(id); } catch (Exception e) { @@ -525,7 +646,7 @@ namespace OpenSim.Region.CoreModules.Asset if (m_LogLevel >= 2) m_log.Debug("[FLOTSAM ASSET CACHE]: Clearing caches."); - if (m_FileCacheEnabled) + if (m_FileCacheEnabled && Directory.Exists(m_CacheDirectory)) { foreach (string dir in Directory.GetDirectories(m_CacheDirectory)) { @@ -534,7 +655,12 @@ namespace OpenSim.Region.CoreModules.Asset } if (m_MemoryCacheEnabled) - m_MemoryCache.Clear(); + m_MemoryCache = new ExpiringCache(); + if (m_negativeCacheEnabled) + m_negativeCache = new ExpiringCache(); + + lock(weakAssetReferencesLock) + weakAssetReferences = new Dictionary(); } private void CleanupExpiredFiles(object source, ElapsedEventArgs e) @@ -542,6 +668,12 @@ namespace OpenSim.Region.CoreModules.Asset if (m_LogLevel >= 2) m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Checking for expired files older then {0}.", m_FileExpiration); + lock(timerLock) + { + if(!m_timerRunning || m_cleanupRunning) + return; + m_cleanupRunning = true; + } // Purge all files last accessed prior to this point DateTime purgeLine = DateTime.Now - m_FileExpiration; @@ -549,27 +681,34 @@ namespace OpenSim.Region.CoreModules.Asset // before cleaning up expired files we must scan the objects in the scene to make sure that we retain // such local assets if they have not been recently accessed. TouchAllSceneAssets(false); + if(Directory.Exists(m_CacheDirectory)) + { + foreach (string dir in Directory.GetDirectories(m_CacheDirectory)) + CleanExpiredFiles(dir, purgeLine); + } - foreach (string dir in Directory.GetDirectories(m_CacheDirectory)) + lock(timerLock) { - CleanExpiredFiles(dir, purgeLine); + if(m_timerRunning) + m_CacheCleanTimer.Start(); + m_cleanupRunning = false; } } /// - /// Recurses through specified directory checking for asset files last - /// accessed prior to the specified purge line and deletes them. Also + /// Recurses through specified directory checking for asset files last + /// accessed prior to the specified purge line and deletes them. Also /// removes empty tier directories. /// /// /// private void CleanExpiredFiles(string dir, DateTime purgeLine) { - // Yet another "directory we are trying to remove doesn't exist, so don't complain" idiocy fix. - if (!Directory.Exists(dir)) - return; try { + if(!Directory.Exists(dir)) + return; + foreach (string file in Directory.GetFiles(dir)) { if (File.GetLastAccessTime(file) < purgeLine) @@ -593,14 +732,19 @@ namespace OpenSim.Region.CoreModules.Asset else if (dirSize >= m_CacheWarnAt) { m_log.WarnFormat( - "[FLOTSAM ASSET CACHE]: Cache folder exceeded CacheWarnAt limit {0} {1}. Suggest increasing tiers, tier length, or reducing cache expiration", + "[FLOTSAM ASSET CACHE]: Cache folder exceeded CacheWarnAt limit {0} {1}. Suggest increasing tiers, tier length, or reducing cache expiration", dir, dirSize); } } + catch (DirectoryNotFoundException) + { + // If we get here, another node on the same box has + // already removed the directory. Continue with next. + } catch (Exception e) { -//// TODO - I'm almost sure this is another case of failing to delete something that doesn't actually exist. -//// m_log.Warn(string.Format("[FLOTSAM ASSET CACHE]: Could not complete clean of expired files in {0}", dir)); + m_log.Warn( + string.Format("[FLOTSAM ASSET CACHE]: Could not complete clean of expired files in {0}, exception ", dir), e); } } @@ -628,7 +772,7 @@ namespace OpenSim.Region.CoreModules.Asset } /// - /// Writes a file to the file cache, creating any nessesary + /// Writes a file to the file cache, creating any nessesary /// tier directories along the way /// /// @@ -640,7 +784,7 @@ namespace OpenSim.Region.CoreModules.Asset // Make sure the target cache directory exists string directory = Path.GetDirectoryName(filename); - // Write file first to a temp name, so that it doesn't look + // Write file first to a temp name, so that it doesn't look // like it's already cached while it's still writing. string tempname = Path.Combine(directory, Path.GetRandomFileName()); @@ -652,7 +796,7 @@ namespace OpenSim.Region.CoreModules.Asset { Directory.CreateDirectory(directory); } - + stream = File.Open(tempname, FileMode.Create); BinaryFormatter bformatter = new BinaryFormatter(); bformatter.Serialize(stream, asset); @@ -665,6 +809,9 @@ namespace OpenSim.Region.CoreModules.Asset return; } + catch (UnauthorizedAccessException) + { + } finally { if (stream != null) @@ -686,7 +833,7 @@ namespace OpenSim.Region.CoreModules.Asset // This situation occurs fairly rarely anyway. We assume in this that moves are atomic on the // filesystem. File.Move(tempname, filename); - + if (m_LogLevel >= 2) m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Cache Stored :: {0}", asset.ID); } @@ -725,6 +872,9 @@ namespace OpenSim.Region.CoreModules.Asset /// private int GetFileCacheCount(string dir) { + if(!Directory.Exists(dir)) + return 0; + int count = Directory.GetFiles(dir).Length; foreach (string subdir in Directory.GetDirectories(dir)) @@ -743,7 +893,7 @@ namespace OpenSim.Region.CoreModules.Asset { string RegionCacheStatusFile = Path.Combine(m_CacheDirectory, "RegionStatus_" + regionID.ToString() + ".fac"); - try + try { if (File.Exists(RegionCacheStatusFile)) { @@ -752,7 +902,7 @@ namespace OpenSim.Region.CoreModules.Asset else { File.WriteAllText( - RegionCacheStatusFile, + RegionCacheStatusFile, "Please do not delete this file unless you are manually clearing your Flotsam Asset Cache."); } } @@ -760,14 +910,14 @@ namespace OpenSim.Region.CoreModules.Asset { m_log.Warn( string.Format( - "[FLOTSAM ASSET CACHE]: Could not stamp region status file for region {0}. Exception ", - regionID), + "[FLOTSAM ASSET CACHE]: Could not stamp region status file for region {0}. Exception ", + regionID), e); } } /// - /// Iterates through all Scenes, doing a deep scan through assets + /// Iterates through all Scenes, doing a deep scan through assets /// to update the access time of all assets present in the scene or referenced by assets /// in the scene. /// @@ -786,10 +936,16 @@ namespace OpenSim.Region.CoreModules.Asset StampRegionStatusFile(s.RegionInfo.RegionID); s.ForEachSOG(delegate(SceneObjectGroup e) - { + { + if(!m_timerRunning && !storeUncached) + return; + gatherer.AddForInspection(e); gatherer.GatherAll(); + if(!m_timerRunning && !storeUncached) + return; + foreach (UUID assetID in gatherer.GatheredUuids.Keys) { if (!assetsFound.ContainsKey(assetID)) @@ -799,6 +955,7 @@ namespace OpenSim.Region.CoreModules.Asset if (File.Exists(filename)) { UpdateFileLastAccessTime(filename); + assetsFound[assetID] = true; } else if (storeUncached) { @@ -818,7 +975,14 @@ namespace OpenSim.Region.CoreModules.Asset } gatherer.GatheredUuids.Clear(); + if(!m_timerRunning && !storeUncached) + return; + + if(!storeUncached) + Thread.Sleep(50); }); + if(!m_timerRunning && !storeUncached) + break; } return assetsFound.Count; @@ -829,6 +993,9 @@ namespace OpenSim.Region.CoreModules.Asset /// private void ClearFileCache() { + if(!Directory.Exists(m_CacheDirectory)) + return; + foreach (string dir in Directory.GetDirectories(m_CacheDirectory)) { try @@ -862,22 +1029,35 @@ namespace OpenSim.Region.CoreModules.Asset { List outputLines = new List(); - double fileHitRate = (double)m_DiskHits / m_Requests * 100.0; + double invReq = 100.0 / m_Requests; + + double weakHitRate = m_weakRefHits * invReq; + int weakEntries = weakAssetReferences.Count; + + double fileHitRate = m_DiskHits * invReq; + double TotalHitRate = weakHitRate + fileHitRate; + + outputLines.Add( + string.Format("Total requests: {0}", m_Requests)); outputLines.Add( - string.Format("File Hit Rate: {0}% for {1} requests", fileHitRate.ToString("0.00"), m_Requests)); + string.Format("unCollected Hit Rate: {0}% ({1} entries)", weakHitRate.ToString("0.00"),weakEntries)); + outputLines.Add( + string.Format("File Hit Rate: {0}%", fileHitRate.ToString("0.00"))); if (m_MemoryCacheEnabled) { - double memHitRate = (double)m_MemoryHits / m_Requests * 100.0; - + double HitRate = m_MemoryHits * invReq; outputLines.Add( - string.Format("Memory Hit Rate: {0}% for {1} requests", memHitRate.ToString("0.00"), m_Requests)); + string.Format("Memory Hit Rate: {0}%", HitRate.ToString("0.00"))); + + TotalHitRate += HitRate; } + outputLines.Add( + string.Format("Total Hit Rate: {0}%", TotalHitRate.ToString("0.00"))); outputLines.Add( string.Format( - "Unnecessary requests due to requests for assets that are currently downloading: {0}", - m_RequestsForInprogress)); + "Requests overlap during file writing: {0}", m_RequestsForInprogress)); return outputLines; } @@ -914,9 +1094,9 @@ namespace OpenSim.Region.CoreModules.Asset if (m_FileCacheEnabled) { con.Output("Deep scans have previously been performed on the following regions:"); - + foreach (string s in Directory.GetFiles(m_CacheDirectory, "*.fac")) - { + { string RegionID = s.Remove(0,s.IndexOf("_")).Replace(".fac",""); DateTime RegionDeepScanTMStamp = File.GetLastWriteTime(s); con.OutputFormat("Region: {0}, {1}", RegionID, RegionDeepScanTMStamp.ToString("MM/dd/yyyy hh:mm:ss")); @@ -959,7 +1139,7 @@ namespace OpenSim.Region.CoreModules.Asset con.Output("Memory cache not enabled."); } } - + if (clearFile) { if (m_FileCacheEnabled) @@ -976,13 +1156,44 @@ namespace OpenSim.Region.CoreModules.Asset break; case "assets": - con.Output("Ensuring assets are cached for all scenes."); + lock(timerLock) + { + if(m_cleanupRunning) + { + con.OutputFormat("FloatSam assets check already running"); + return; + } + m_cleanupRunning = true; + } - WorkManager.RunInThread(delegate + con.Output("FloatSam Ensuring assets are cached for all scenes."); + + WorkManager.RunInThreadPool(delegate { + bool wasRunning= false; + lock(timerLock) + { + if(m_timerRunning) + { + m_CacheCleanTimer.Stop(); + m_timerRunning = false; + wasRunning = true; + Thread.Sleep(100); + } + } int assetReferenceTotal = TouchAllSceneAssets(true); + GC.Collect(); + lock(timerLock) + { + if(wasRunning) + { + m_CacheCleanTimer.Start(); + m_timerRunning = true; + } + m_cleanupRunning = false; + } con.OutputFormat("Completed check with {0} assets.", assetReferenceTotal); - }, null, "TouchAllSceneAssets"); + }, null, "TouchAllSceneAssets", false); break; @@ -1037,19 +1248,23 @@ namespace OpenSim.Region.CoreModules.Asset public AssetMetadata GetMetadata(string id) { - AssetBase asset = Get(id); + AssetBase asset; + Get(id, out asset); return asset.Metadata; } public byte[] GetData(string id) { - AssetBase asset = Get(id); + AssetBase asset; + Get(id, out asset); return asset.Data; } public bool Get(string id, object sender, AssetRetrieved handler) { - AssetBase asset = Get(id); + AssetBase asset; + if (!Get(id, out asset)) + return false; handler(id, sender, asset); return true; } @@ -1057,12 +1272,12 @@ namespace OpenSim.Region.CoreModules.Asset public bool[] AssetsExist(string[] ids) { bool[] exist = new bool[ids.Length]; - + for (int i = 0; i < ids.Length; i++) { exist[i] = Check(ids[i]); } - + return exist; } @@ -1080,7 +1295,9 @@ namespace OpenSim.Region.CoreModules.Asset public bool UpdateContent(string id, byte[] data) { - AssetBase asset = Get(id); + AssetBase asset; + if (!Get(id, out asset)) + return false; asset.Data = data; Cache(asset); return true; diff --git a/OpenSim/Region/CoreModules/Asset/GlynnTuckerAssetCache.cs b/OpenSim/Region/CoreModules/Asset/GlynnTuckerAssetCache.cs index 5f76ac2..abe9b23 100644 --- a/OpenSim/Region/CoreModules/Asset/GlynnTuckerAssetCache.cs +++ b/OpenSim/Region/CoreModules/Asset/GlynnTuckerAssetCache.cs @@ -41,7 +41,7 @@ using OpenSim.Services.Interfaces; namespace OpenSim.Region.CoreModules.Asset { [Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "GlynnTuckerAssetCache")] - public class GlynnTuckerAssetCache : ISharedRegionModule, IImprovedAssetCache + public class GlynnTuckerAssetCache : ISharedRegionModule, IAssetCache { private static readonly ILog m_log = LogManager.GetLogger( @@ -55,7 +55,7 @@ namespace OpenSim.Region.CoreModules.Asset // Instrumentation private uint m_DebugRate; - public Type ReplaceableInterface + public Type ReplaceableInterface { get { return null; } } @@ -100,7 +100,7 @@ namespace OpenSim.Region.CoreModules.Asset public void AddRegion(Scene scene) { if (m_Enabled) - scene.RegisterModuleInterface(this); + scene.RegisterModuleInterface(this); } public void RemoveRegion(Scene scene) @@ -112,7 +112,7 @@ namespace OpenSim.Region.CoreModules.Asset } //////////////////////////////////////////////////////////// - // IImprovedAssetCache + // IAssetCache // public bool Check(string id) @@ -126,14 +126,20 @@ namespace OpenSim.Region.CoreModules.Asset m_Cache.AddOrUpdate(asset.ID, asset); } - public AssetBase Get(string id) + public void CacheNegative(string id) { - Object asset = null; - m_Cache.TryGet(id, out asset); + // We don't do negative caching + } + + public bool Get(string id, out AssetBase asset) + { + Object a = null; + m_Cache.TryGet(id, out a); - Debug(asset); + Debug(a); - return (AssetBase)asset; + asset = (AssetBase)a; + return true; } public void Expire(string id) diff --git a/OpenSim/Region/CoreModules/Asset/Tests/FlotsamAssetCacheTests.cs b/OpenSim/Region/CoreModules/Asset/Tests/FlotsamAssetCacheTests.cs index 73e4431..dbb7941 100644 --- a/OpenSim/Region/CoreModules/Asset/Tests/FlotsamAssetCacheTests.cs +++ b/OpenSim/Region/CoreModules/Asset/Tests/FlotsamAssetCacheTests.cs @@ -112,7 +112,7 @@ namespace OpenSim.Region.CoreModules.Asset.Tests { TestHelpers.InMethod(); // log4net.Config.XmlConfigurator.Configure(); - + AssetBase asset = AssetHelpers.CreateNotecardAsset(); asset.ID = TestHelpers.ParseTail(0x2).ToString(); -- cgit v1.1