aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
authorMelanie2009-10-12 23:21:32 +0100
committerMelanie2009-10-12 23:21:32 +0100
commitbf68dad64398f7bd4ae2992d345071d261c32e80 (patch)
treebe0135fbc5c7bd4f457b7e5a1f19d4ee40e3bab8
parentStop null values from being returned on database queries (diff)
downloadopensim-SC_OLD-bf68dad64398f7bd4ae2992d345071d261c32e80.zip
opensim-SC_OLD-bf68dad64398f7bd4ae2992d345071d261c32e80.tar.gz
opensim-SC_OLD-bf68dad64398f7bd4ae2992d345071d261c32e80.tar.bz2
opensim-SC_OLD-bf68dad64398f7bd4ae2992d345071d261c32e80.tar.xz
0004246: [Patch] FlotsamAssetCache deep scan & cache
Thank you, mcortez.
Diffstat (limited to '')
-rw-r--r--OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs307
-rw-r--r--bin/config-include/FlotsamCache.ini.example7
2 files changed, 277 insertions, 37 deletions
diff --git a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs
index b81ab41..4041b63 100644
--- a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs
+++ b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs
@@ -43,6 +43,7 @@ using Mono.Addins;
43using OpenMetaverse; 43using OpenMetaverse;
44 44
45using OpenSim.Framework; 45using OpenSim.Framework;
46using OpenSim.Framework.Console;
46using OpenSim.Region.Framework.Interfaces; 47using OpenSim.Region.Framework.Interfaces;
47using OpenSim.Region.Framework.Scenes; 48using OpenSim.Region.Framework.Scenes;
48using OpenSim.Services.Interfaces; 49using OpenSim.Services.Interfaces;
@@ -54,7 +55,7 @@ using OpenSim.Services.Interfaces;
54namespace Flotsam.RegionModules.AssetCache 55namespace Flotsam.RegionModules.AssetCache
55{ 56{
56 [Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule")] 57 [Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule")]
57 public class FlotsamAssetCache : ISharedRegionModule, IImprovedAssetCache 58 public class FlotsamAssetCache : ISharedRegionModule, IImprovedAssetCache, IAssetService
58 { 59 {
59 private static readonly ILog m_log = 60 private static readonly ILog m_log =
60 LogManager.GetLogger( 61 LogManager.GetLogger(
@@ -102,6 +103,11 @@ namespace Flotsam.RegionModules.AssetCache
102 103
103 private System.Timers.Timer m_CachCleanTimer = new System.Timers.Timer(); 104 private System.Timers.Timer m_CachCleanTimer = new System.Timers.Timer();
104 105
106 private IAssetService m_AssetService = null;
107 private List<Scene> m_Scenes = new List<Scene>();
108
109 private bool m_DeepScanBeforePurge = false;
110
105 public FlotsamAssetCache() 111 public FlotsamAssetCache()
106 { 112 {
107 m_InvalidChars.AddRange(Path.GetInvalidPathChars()); 113 m_InvalidChars.AddRange(Path.GetInvalidPathChars());
@@ -121,6 +127,7 @@ namespace Flotsam.RegionModules.AssetCache
121 public void Initialise(IConfigSource source) 127 public void Initialise(IConfigSource source)
122 { 128 {
123 IConfig moduleConfig = source.Configs["Modules"]; 129 IConfig moduleConfig = source.Configs["Modules"];
130
124 131
125 if (moduleConfig != null) 132 if (moduleConfig != null)
126 { 133 {
@@ -195,7 +202,13 @@ namespace Flotsam.RegionModules.AssetCache
195 202
196 m_CacheWarnAt = assetConfig.GetInt("CacheWarnAt", 30000); 203 m_CacheWarnAt = assetConfig.GetInt("CacheWarnAt", 30000);
197 204
198 205 m_DeepScanBeforePurge = assetConfig.GetBoolean("DeepScanBeforePurge", false);
206
207 MainConsole.Instance.Commands.AddCommand(this.Name, true, "fcache status", "fcache status", "Display cache status", HandleConsoleCommand);
208 MainConsole.Instance.Commands.AddCommand(this.Name, true, "fcache clear", "fcache clear [file] [memory]", "Remove all assets in the file and/or memory cache", HandleConsoleCommand);
209 MainConsole.Instance.Commands.AddCommand(this.Name, true, "fcache assets", "fcache assets", "Attempt a deep scan and cache of all assets in all scenes", HandleConsoleCommand);
210 MainConsole.Instance.Commands.AddCommand(this.Name, true, "fcache expire", "fcache expire <datetime>", "Purge cached assets older then the specified date/time", HandleConsoleCommand);
211
199 } 212 }
200 } 213 }
201 } 214 }
@@ -213,16 +226,23 @@ namespace Flotsam.RegionModules.AssetCache
213 if (m_Enabled) 226 if (m_Enabled)
214 { 227 {
215 scene.RegisterModuleInterface<IImprovedAssetCache>(this); 228 scene.RegisterModuleInterface<IImprovedAssetCache>(this);
229 m_Scenes.Add(scene);
216 230
217 //scene.AddCommand(this, "flotsamcache", "", "Display a list of console commands for the Flotsam Asset Cache", HandleConsoleCommand); 231 if (m_AssetService == null)
218 scene.AddCommand(this, "flotsamcache counts", "flotsamcache counts", "Display the number of cached assets", HandleConsoleCommand); 232 {
219 scene.AddCommand(this, "flotsamcache clearmem", "flotsamcache clearmem", "Remove all assets cached in memory", HandleConsoleCommand); 233 m_AssetService = scene.RequestModuleInterface<IAssetService>();
220 scene.AddCommand(this, "flotsamcache clearfile", "flotsamcache clearfile", "Remove all assets cached on disk", HandleConsoleCommand); 234
235 }
221 } 236 }
222 } 237 }
223 238
224 public void RemoveRegion(Scene scene) 239 public void RemoveRegion(Scene scene)
225 { 240 {
241 if (m_Enabled)
242 {
243 scene.UnregisterModuleInterface<IImprovedAssetCache>(this);
244 m_Scenes.Remove(scene);
245 }
226 } 246 }
227 247
228 public void RegionLoaded(Scene scene) 248 public void RegionLoaded(Scene scene)
@@ -442,31 +462,47 @@ namespace Flotsam.RegionModules.AssetCache
442 if (m_LogLevel >= 2) 462 if (m_LogLevel >= 2)
443 m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Checking for expired files older then {0}.", m_FileExpiration.ToString()); 463 m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Checking for expired files older then {0}.", m_FileExpiration.ToString());
444 464
465 // Purge all files last accessed prior to this point
466 DateTime purgeLine = DateTime.Now - m_FileExpiration;
467
468 // An optional deep scan at this point will ensure assets present in scenes,
469 // or referenced by objects in the scene, but not recently accessed
470 // are not purged.
471 if (m_DeepScanBeforePurge)
472 {
473 CacheScenes();
474 }
475
445 foreach (string dir in Directory.GetDirectories(m_CacheDirectory)) 476 foreach (string dir in Directory.GetDirectories(m_CacheDirectory))
446 { 477 {
447 CleanExpiredFiles(dir); 478 CleanExpiredFiles(dir, purgeLine);
448 } 479 }
449 } 480 }
450 481
451 /// <summary> 482 /// <summary>
452 /// Recurses through specified directory checking for expired asset files and deletes them. Also removes empty directories. 483 /// Recurses through specified directory checking for asset files last
484 /// accessed prior to the specified purge line and deletes them. Also
485 /// removes empty tier directories.
453 /// </summary> 486 /// </summary>
454 /// <param name="dir"></param> 487 /// <param name="dir"></param>
455 private void CleanExpiredFiles(string dir) 488 private void CleanExpiredFiles(string dir, DateTime purgeLine)
456 { 489 {
490
457 foreach (string file in Directory.GetFiles(dir)) 491 foreach (string file in Directory.GetFiles(dir))
458 { 492 {
459 if (DateTime.Now - File.GetLastAccessTime(file) > m_FileExpiration) 493 if (File.GetLastAccessTime(file) < purgeLine)
460 { 494 {
461 File.Delete(file); 495 File.Delete(file);
462 } 496 }
463 } 497 }
464 498
499 // Recurse into lower tiers
465 foreach (string subdir in Directory.GetDirectories(dir)) 500 foreach (string subdir in Directory.GetDirectories(dir))
466 { 501 {
467 CleanExpiredFiles(subdir); 502 CleanExpiredFiles(subdir, purgeLine);
468 } 503 }
469 504
505 // Check if a tier directory is empty, if so, delete it
470 int dirSize = Directory.GetFiles(dir).Length + Directory.GetDirectories(dir).Length; 506 int dirSize = Directory.GetFiles(dir).Length + Directory.GetDirectories(dir).Length;
471 if (dirSize == 0) 507 if (dirSize == 0)
472 { 508 {
@@ -478,6 +514,11 @@ namespace Flotsam.RegionModules.AssetCache
478 } 514 }
479 } 515 }
480 516
517 /// <summary>
518 /// Determines the filename for an AssetID stored in the file cache
519 /// </summary>
520 /// <param name="id"></param>
521 /// <returns></returns>
481 private string GetFileName(string id) 522 private string GetFileName(string id)
482 { 523 {
483 // Would it be faster to just hash the darn thing? 524 // Would it be faster to just hash the darn thing?
@@ -496,14 +537,23 @@ namespace Flotsam.RegionModules.AssetCache
496 return Path.Combine(path, id); 537 return Path.Combine(path, id);
497 } 538 }
498 539
540 /// <summary>
541 /// Writes a file to the file cache, creating any nessesary
542 /// tier directories along the way
543 /// </summary>
544 /// <param name="filename"></param>
545 /// <param name="asset"></param>
499 private void WriteFileCache(string filename, AssetBase asset) 546 private void WriteFileCache(string filename, AssetBase asset)
500 { 547 {
501 Stream stream = null; 548 Stream stream = null;
549
502 // Make sure the target cache directory exists 550 // Make sure the target cache directory exists
503 string directory = Path.GetDirectoryName(filename); 551 string directory = Path.GetDirectoryName(filename);
552
504 // Write file first to a temp name, so that it doesn't look 553 // Write file first to a temp name, so that it doesn't look
505 // like it's already cached while it's still writing. 554 // like it's already cached while it's still writing.
506 string tempname = Path.Combine(directory, Path.GetRandomFileName()); 555 string tempname = Path.Combine(directory, Path.GetRandomFileName());
556
507 try 557 try
508 { 558 {
509 if (!Directory.Exists(directory)) 559 if (!Directory.Exists(directory))
@@ -563,6 +613,11 @@ namespace Flotsam.RegionModules.AssetCache
563 } 613 }
564 } 614 }
565 615
616 /// <summary>
617 /// Scan through the file cache, and return number of assets currently cached.
618 /// </summary>
619 /// <param name="dir"></param>
620 /// <returns></returns>
566 private int GetFileCacheCount(string dir) 621 private int GetFileCacheCount(string dir)
567 { 622 {
568 int count = Directory.GetFiles(dir).Length; 623 int count = Directory.GetFiles(dir).Length;
@@ -575,55 +630,180 @@ namespace Flotsam.RegionModules.AssetCache
575 return count; 630 return count;
576 } 631 }
577 632
633 /// <summary>
634 /// This notes the last time the Region had a deep asset scan performed on it.
635 /// </summary>
636 /// <param name="RegionID"></param>
637 private void StampRegionStatusFile(UUID RegionID)
638 {
639 string RegionCacheStatusFile = Path.Combine(m_CacheDirectory, "RegionStatus_" + RegionID.ToString() + ".fac");
640 if (File.Exists(RegionCacheStatusFile))
641 {
642 File.SetLastWriteTime(RegionCacheStatusFile, DateTime.Now);
643 }
644 else
645 {
646 File.WriteAllText(RegionCacheStatusFile, "Please do not delete this file unless you are manually clearing your Flotsam Asset Cache.");
647 }
648 }
649
650 /// <summary>
651 /// Iterates through all Scenes, doing a deep scan through assets
652 /// to cache all assets present in the scene or referenced by assets
653 /// in the scene
654 /// </summary>
655 /// <returns></returns>
656 private int CacheScenes()
657 {
658 UuidGatherer gatherer = new UuidGatherer(m_AssetService);
659
660 Dictionary<UUID, int> assets = new Dictionary<UUID, int>();
661 foreach (Scene s in m_Scenes)
662 {
663 StampRegionStatusFile(s.RegionInfo.RegionID);
664
665 s.ForEachSOG(delegate(SceneObjectGroup e)
666 {
667 gatherer.GatherAssetUuids(e, assets);
668 }
669 );
670 }
671
672 foreach (UUID assetID in assets.Keys)
673 {
674 string filename = GetFileName(assetID.ToString());
675
676 if (File.Exists(filename))
677 {
678 File.SetLastAccessTime(filename, DateTime.Now);
679 }
680 else
681 {
682 m_AssetService.Get(assetID.ToString());
683 }
684 }
685
686 return assets.Keys.Count;
687 }
688
689 /// <summary>
690 /// Deletes all cache contents
691 /// </summary>
692 private void ClearFileCache()
693 {
694 foreach (string dir in Directory.GetDirectories(m_CacheDirectory))
695 {
696 try
697 {
698 Directory.Delete(dir, true);
699 }
700 catch (Exception e)
701 {
702 LogException(e);
703 }
704 }
705
706 foreach (string file in Directory.GetFiles(m_CacheDirectory))
707 {
708 try
709 {
710 File.Delete(file);
711 }
712 catch (Exception e)
713 {
714 LogException(e);
715 }
716 }
717 }
718
578 #region Console Commands 719 #region Console Commands
579 private void HandleConsoleCommand(string module, string[] cmdparams) 720 private void HandleConsoleCommand(string module, string[] cmdparams)
580 { 721 {
581 if (cmdparams.Length == 2) 722 if (cmdparams.Length >= 2)
582 { 723 {
583 string cmd = cmdparams[1]; 724 string cmd = cmdparams[1];
584 switch (cmd) 725 switch (cmd)
585 { 726 {
586 case "count": 727 case "status":
587 case "counts": 728 m_log.InfoFormat("[FLOTSAM ASSET CACHE] Memory Cache : {0} assets", m_MemoryCache.Count);
588 m_log.InfoFormat("[FLOTSAM ASSET CACHE] Memory Cache : {0}", m_MemoryCache.Count);
589 729
590 int fileCount = GetFileCacheCount(m_CacheDirectory); 730 int fileCount = GetFileCacheCount(m_CacheDirectory);
591 m_log.InfoFormat("[FLOTSAM ASSET CACHE] File Cache : {0}", fileCount); 731 m_log.InfoFormat("[FLOTSAM ASSET CACHE] File Cache : {0} assets", fileCount);
592 732
593 break; 733 foreach ( string s in Directory.GetFiles(m_CacheDirectory, "*.fac" ) )
734 {
735 m_log.Info("[FLOTSAM ASSET CACHE] Deep Scans were performed on the following regions:");
736
737 string RegionID = s.Remove(0,s.IndexOf("_")).Replace(".fac","");
738 DateTime RegionDeepScanTMStamp = File.GetLastWriteTime(s);
739 m_log.InfoFormat("[FLOTSAM ASSET CACHE] Region: {0}, {1}", RegionID, RegionDeepScanTMStamp.ToString("MM/dd/yyyy hh:mm:ss"));
740 }
594 741
595 case "clearmem":
596 m_MemoryCache.Clear();
597 m_log.InfoFormat("[FLOTSAM ASSET CACHE] Memory Cache Cleared, there are now {0} items in the memory cache", m_MemoryCache.Count);
598 break; 742 break;
599 743
600 case "clearfile": 744 case "clear":
601 foreach (string dir in Directory.GetDirectories(m_CacheDirectory)) 745 if (cmdparams.Length < 3)
602 { 746 {
603 try 747 m_log.Warn("[FLOTSAM ASSET CACHE] Please specify memory and/or file cache.");
604 { 748 break;
605 Directory.Delete(dir, true);
606 }
607 catch (Exception e)
608 {
609 LogException(e);
610 }
611 } 749 }
612 750 foreach (string s in cmdparams)
613 foreach (string file in Directory.GetFiles(m_CacheDirectory))
614 { 751 {
615 try 752 if (s.ToLower() == "memory")
616 { 753 {
617 File.Delete(file); 754 m_MemoryCache.Clear();
755 m_log.Info("[FLOTSAM ASSET CACHE] Memory cache cleared.");
618 } 756 }
619 catch (Exception e) 757 else if (s.ToLower() == "file")
620 { 758 {
621 LogException(e); 759 ClearFileCache();
760 m_log.Info("[FLOTSAM ASSET CACHE] File cache cleared.");
622 } 761 }
623 } 762 }
763 break;
764
765
766 case "assets":
767 m_log.Info("[FLOTSAM ASSET CACHE] Caching all assets, in all scenes.");
768
769 Util.FireAndForget(delegate {
770 int assetsCached = CacheScenes();
771 m_log.InfoFormat("[FLOTSAM ASSET CACHE] Completed Scene Caching, {0} assets found.", assetsCached);
772
773 });
624 774
625 break; 775 break;
626 776
777 case "expire":
778
779
780 if (cmdparams.Length >= 3)
781 {
782 m_log.InfoFormat("[FLOTSAM ASSET CACHE] Invalid parameters for Expire, please specify a valid date & time", cmd);
783 break;
784 }
785
786 string s_expirationDate = "";
787 DateTime expirationDate;
788
789 if (cmdparams.Length > 3)
790 {
791 s_expirationDate = string.Join(" ", cmdparams, 2, cmdparams.Length - 2);
792 }
793 else
794 {
795 s_expirationDate = cmdparams[2];
796 }
797
798 if (!DateTime.TryParse(s_expirationDate, out expirationDate))
799 {
800 m_log.InfoFormat("[FLOTSAM ASSET CACHE] {0} is not a valid date & time", cmd);
801 break;
802 }
803
804 CleanExpiredFiles(m_CacheDirectory, expirationDate);
805
806 break;
627 default: 807 default:
628 m_log.InfoFormat("[FLOTSAM ASSET CACHE] Unknown command {0}", cmd); 808 m_log.InfoFormat("[FLOTSAM ASSET CACHE] Unknown command {0}", cmd);
629 break; 809 break;
@@ -631,13 +811,66 @@ namespace Flotsam.RegionModules.AssetCache
631 } 811 }
632 else if (cmdparams.Length == 1) 812 else if (cmdparams.Length == 1)
633 { 813 {
634 m_log.InfoFormat("[FLOTSAM ASSET CACHE] flotsamcache counts - Display the number of cached assets"); 814 m_log.InfoFormat("[FLOTSAM ASSET CACHE] flotsamcache status - Display cache status");
635 m_log.InfoFormat("[FLOTSAM ASSET CACHE] flotsamcache clearmem - Remove all assets cached in memory"); 815 m_log.InfoFormat("[FLOTSAM ASSET CACHE] flotsamcache clearmem - Remove all assets cached in memory");
636 m_log.InfoFormat("[FLOTSAM ASSET CACHE] flotsamcache clearfile - Remove all assets cached on disk"); 816 m_log.InfoFormat("[FLOTSAM ASSET CACHE] flotsamcache clearfile - Remove all assets cached on disk");
817 m_log.InfoFormat("[FLOTSAM ASSET CACHE] flotsamcache cachescenes - Attempt a deep cache of all assets in all scenes");
818 m_log.InfoFormat("[FLOTSAM ASSET CACHE] flotsamcache <datetime> - Purge assets older then the specified date & time");
637 819
638 } 820 }
639 } 821 }
640 822
641 #endregion 823 #endregion
824
825 #region IAssetService Members
826
827
828 public AssetMetadata GetMetadata(string id)
829 {
830 AssetBase asset = Get(id);
831 return asset.Metadata;
832 }
833
834 public byte[] GetData(string id)
835 {
836 AssetBase asset = Get(id);
837 return asset.Data;
838 }
839
840 public bool Get(string id, object sender, AssetRetrieved handler)
841 {
842 AssetBase asset = Get(id);
843 handler(id, sender, asset);
844 return true;
845 }
846
847 public string Store(AssetBase asset)
848 {
849 if ((asset.FullID == null) || (asset.FullID == UUID.Zero))
850 {
851 asset.FullID = UUID.Random();
852 }
853
854 Cache(asset);
855
856 return asset.ID;
857
858 }
859
860 public bool UpdateContent(string id, byte[] data)
861 {
862 AssetBase asset = Get(id);
863 asset.Data = data;
864 Cache(asset);
865 return true;
866 }
867
868 public bool Delete(string id)
869 {
870 Expire(id);
871 return true;
872 }
873
874 #endregion
642 } 875 }
643} \ No newline at end of file 876} \ No newline at end of file
diff --git a/bin/config-include/FlotsamCache.ini.example b/bin/config-include/FlotsamCache.ini.example
index abb3b9a..b50d7ec 100644
--- a/bin/config-include/FlotsamCache.ini.example
+++ b/bin/config-include/FlotsamCache.ini.example
@@ -50,3 +50,10 @@
50 50
51 ; Warning level for cache directory size 51 ; Warning level for cache directory size
52 ;CacheWarnAt = 30000 52 ;CacheWarnAt = 30000
53
54 ; Perform a deep scan of all assets within all regions, looking for all assets
55 ; present or referenced. Mark all assets found that are already present in the
56 ; cache, and request all assets that are found that are not already cached (this
57 ; will cause those assets to be cached)
58 ;
59 ; DeepScanBeforePurge = false