From 1f2472d0fcd86a7ae09c01ecb3508eab001ce033 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 11 Oct 2012 23:28:53 +0100 Subject: Extend "show stats" command to "show stats [list|all|]" This allows different categories of stats to be shown, with options to list categories or show all stats. Currently categories are scene and simulator and only a very few stats are currently registered via this mechanism. This commit also adds percentage stats for packets and blocks reused from the packet pool. --- .../Framework/Monitoring/SimExtraStatsCollector.cs | 16 +- OpenSim/Framework/Monitoring/StatsManager.cs | 193 +++++++++++++++++++-- 2 files changed, 185 insertions(+), 24 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs index 8ac9090..aa86202 100644 --- a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs @@ -359,13 +359,19 @@ Asset service request failures: {3}" + Environment.NewLine, inPacketsPerSecond, outPacketsPerSecond, pendingDownloads, pendingUploads, unackedBytes, totalFrameTime, netFrameTime, physicsFrameTime, otherFrameTime, agentFrameTime, imageFrameTime)); - foreach (KeyValuePair kvp in StatsManager.RegisteredStats) - { - Stat stat = kvp.Value; + Dictionary> sceneStats; - if (stat.Category == "scene" && stat.Verbosity == StatVerbosity.Info) + if (StatsManager.TryGetStats("scene", out sceneStats)) + { + foreach (KeyValuePair> kvp in sceneStats) { - sb.AppendFormat("Slow frames ({0}): {1}\n", stat.Container, stat.Value); + foreach (Stat stat in kvp.Value.Values) + { + if (stat.Verbosity == StatVerbosity.Info) + { + sb.AppendFormat("{0} ({1}): {2}{3}\n", stat.Name, stat.Container, stat.Value, stat.UnitName); + } + } } } diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index b5dc24f..a67c5f8 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -27,6 +27,7 @@ using System; using System.Collections.Generic; +using OpenSim.Framework.Console; namespace OpenSim.Framework.Monitoring { @@ -35,13 +36,23 @@ namespace OpenSim.Framework.Monitoring /// public class StatsManager { + // Subcommand used to list other stats. + public const string AllSubCommand = "all"; + + // Subcommand used to list other stats. + public const string ListSubCommand = "list"; + + // All subcommands + public static HashSet SubCommands = new HashSet { AllSubCommand, ListSubCommand }; + /// - /// Registered stats. + /// Registered stats categorized by category/container/shortname /// /// - /// Do not add or remove from this dictionary. + /// Do not add or remove directly from this dictionary. /// - public static Dictionary RegisteredStats = new Dictionary(); + public static Dictionary>> RegisteredStats + = new Dictionary>>(); private static AssetStatsCollector assetStats; private static UserStatsCollector userStats; @@ -51,6 +62,76 @@ namespace OpenSim.Framework.Monitoring public static UserStatsCollector UserStats { get { return userStats; } } public static SimExtraStatsCollector SimExtraStats { get { return simExtraStats; } } + public static void RegisterConsoleCommands(CommandConsole console) + { + console.Commands.AddCommand( + "General", + false, + "show stats", + "show stats [list|all|]", + "Show statistical information for this server", + "If no final argument is specified then legacy statistics information is currently shown.\n" + + "If list is specified then statistic categories are shown.\n" + + "If all is specified then all registered statistics are shown.\n" + + "If a category name is specified then only statistics from that category are shown.\n" + + "THIS STATS FACILITY IS EXPERIMENTAL AND DOES NOT YET CONTAIN ALL STATS", + HandleShowStatsCommand); + } + + public static void HandleShowStatsCommand(string module, string[] cmd) + { + ICommandConsole con = MainConsole.Instance; + + if (cmd.Length > 2) + { + var categoryName = cmd[2]; + + if (categoryName == AllSubCommand) + { + foreach (var category in RegisteredStats.Values) + { + OutputCategoryStatsToConsole(con, category); + } + } + else if (categoryName == ListSubCommand) + { + con.Output("Statistic categories available are:"); + foreach (string category in RegisteredStats.Keys) + con.OutputFormat(" {0}", category); + } + else + { + Dictionary> category; + if (!RegisteredStats.TryGetValue(categoryName, out category)) + { + con.OutputFormat("No such category as {0}", categoryName); + } + else + { + OutputCategoryStatsToConsole(con, category); + } + } + } + else + { + // Legacy + con.Output(SimExtraStats.Report()); + } + } + + private static void OutputCategoryStatsToConsole( + ICommandConsole con, Dictionary> category) + { + foreach (var container in category.Values) + { + foreach (Stat stat in container.Values) + { + con.OutputFormat( + "{0}.{1}.{2} : {3}{4}", stat.Category, stat.Container, stat.ShortName, stat.Value, stat.UnitName); + } + } + } + /// /// Start collecting statistics related to assets. /// Should only be called once. @@ -73,43 +154,100 @@ namespace OpenSim.Framework.Monitoring return userStats; } + /// + /// Registers a statistic. + /// + /// + /// public static bool RegisterStat(Stat stat) { + Dictionary> category = null, newCategory; + Dictionary container = null, newContainer; + lock (RegisteredStats) { - if (RegisteredStats.ContainsKey(stat.UniqueName)) - { - // XXX: For now just return false. This is to avoid problems in regression tests where all tests - // in a class are run in the same instance of the VM. + // Stat name is not unique across category/container/shortname key. + // XXX: For now just return false. This is to avoid problems in regression tests where all tests + // in a class are run in the same instance of the VM. + if (TryGetStat(stat, out category, out container)) return false; -// throw new Exception( -// "StatsManager already contains stat with ShortName {0} in Category {1}", stat.ShortName, stat.Category); - } + // We take a copy-on-write approach here of replacing dictionaries when keys are added or removed. + // This means that we don't need to lock or copy them on iteration, which will be a much more + // common operation after startup. + if (container != null) + newContainer = new Dictionary(container); + else + newContainer = new Dictionary(); + + if (category != null) + newCategory = new Dictionary>(category); + else + newCategory = new Dictionary>(); - // We take a replace-on-write approach here so that we don't need to generate a new Dictionary - Dictionary newRegisteredStats = new Dictionary(RegisteredStats); - newRegisteredStats[stat.UniqueName] = stat; - RegisteredStats = newRegisteredStats; + newContainer[stat.ShortName] = stat; + newCategory[stat.Container] = newContainer; + RegisteredStats[stat.Category] = newCategory; } return true; } + /// + /// Deregister a statistic + /// > + /// + /// > category = null, newCategory; + Dictionary container = null, newContainer; + lock (RegisteredStats) { - if (!RegisteredStats.ContainsKey(stat.UniqueName)) + if (!TryGetStat(stat, out category, out container)) return false; - Dictionary newRegisteredStats = new Dictionary(RegisteredStats); - newRegisteredStats.Remove(stat.UniqueName); - RegisteredStats = newRegisteredStats; + newContainer = new Dictionary(container); + newContainer.Remove(stat.UniqueName); + + newCategory = new Dictionary>(category); + newCategory.Remove(stat.Container); + + newCategory[stat.Container] = newContainer; + RegisteredStats[stat.Category] = newCategory; return true; } } + + public static bool TryGetStats(string category, out Dictionary> stats) + { + return RegisteredStats.TryGetValue(category, out stats); + } + + public static bool TryGetStat( + Stat stat, + out Dictionary> category, + out Dictionary container) + { + category = null; + container = null; + + lock (RegisteredStats) + { + if (RegisteredStats.TryGetValue(stat.Category, out category)) + { + if (category.TryGetValue(stat.Container, out container)) + { + if (container.ContainsKey(stat.ShortName)) + return true; + } + } + } + + return false; + } } /// @@ -157,9 +295,26 @@ namespace OpenSim.Framework.Monitoring public virtual double Value { get; set; } + /// + /// Constructor + /// + /// Short name for the stat. Must not contain spaces. e.g. "LongFrames" + /// Human readable name for the stat. e.g. "Long frames" + /// + /// Unit name for the stat. Should be preceeded by a space if the unit name isn't normally appeneded immediately to the value. + /// e.g. " frames" + /// + /// Category under which this stat should appear, e.g. "scene". Do not capitalize. + /// Entity to which this stat relates. e.g. scene name if this is a per scene stat. + /// Verbosity of stat. Controls whether it will appear in short stat display or only full display. + /// Description of stat public Stat( string shortName, string name, string unitName, string category, string container, StatVerbosity verbosity, string description) { + if (StatsManager.SubCommands.Contains(category)) + throw new Exception( + string.Format("Stat cannot be in category '{0}' since this is reserved for a subcommand", category)); + ShortName = shortName; Name = name; UnitName = unitName; @@ -203,7 +358,7 @@ namespace OpenSim.Framework.Monitoring public PercentageStat( string shortName, string name, string category, string container, StatVerbosity verbosity, string description) - : base(shortName, name, " %", category, container, verbosity, description) + : base(shortName, name, "%", category, container, verbosity, description) { } } -- cgit v1.1 From 2e9ef015f7b73a3942011a36a9f94ce59d848dc0 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 11 Oct 2012 23:58:37 +0100 Subject: Fix packetpool for ImprovedTerseObjectUpdate packets. These were neither being returned or in many places reused. Getting packets from a pool rather than deallocating and reallocating reduces memory churn which in turn reduces garbage collection time and frequency. --- OpenSim/Framework/Monitoring/StatsManager.cs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index a67c5f8..d365190 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -126,8 +126,7 @@ namespace OpenSim.Framework.Monitoring { foreach (Stat stat in container.Values) { - con.OutputFormat( - "{0}.{1}.{2} : {3}{4}", stat.Category, stat.Container, stat.ShortName, stat.Value, stat.UnitName); + con.Output(stat.ToConsoleString()); } } } @@ -330,6 +329,12 @@ namespace OpenSim.Framework.Monitoring { return string.Format("{0}+{1}+{2}", container, category, shortName); } + + public virtual string ToConsoleString() + { + return string.Format( + "{0}.{1}.{2} : {3}{4}", Category, Container, ShortName, Value, UnitName); + } } public class PercentageStat : Stat @@ -358,8 +363,13 @@ namespace OpenSim.Framework.Monitoring public PercentageStat( string shortName, string name, string category, string container, StatVerbosity verbosity, string description) - : base(shortName, name, "%", category, container, verbosity, description) + : base(shortName, name, "%", category, container, verbosity, description) {} + + public override string ToConsoleString() { + return string.Format( + "{0}.{1}.{2} : {3:0.###}{4} ({5}/{6})", + Category, Container, ShortName, Value, UnitName, Antecedent, Consequent); } } } \ No newline at end of file -- cgit v1.1 From 387ce8ef35e7084895524507d6bba987b8c4a5d0 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 12 Oct 2012 00:10:51 +0100 Subject: Fix build break by moving OpenSim.Framework.Console back below HttpServer in the build order. Luckily, it turns out Framework.Monitoring doesn't need to reference Console directly. --- OpenSim/Framework/Monitoring/StatsManager.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index d365190..d7aff03 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -27,7 +27,6 @@ using System; using System.Collections.Generic; -using OpenSim.Framework.Console; namespace OpenSim.Framework.Monitoring { @@ -62,7 +61,7 @@ namespace OpenSim.Framework.Monitoring public static UserStatsCollector UserStats { get { return userStats; } } public static SimExtraStatsCollector SimExtraStats { get { return simExtraStats; } } - public static void RegisterConsoleCommands(CommandConsole console) + public static void RegisterConsoleCommands(ICommandConsole console) { console.Commands.AddCommand( "General", -- cgit v1.1 From 59a17ad676326d5affc2e221ef9c02166a85c6fd Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 12 Oct 2012 00:26:15 +0100 Subject: Fix percentage stats to multiply by 100. Adjust container name for packetpool stats. --- OpenSim/Framework/Monitoring/StatsManager.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index d7aff03..31989e5 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -351,7 +351,7 @@ namespace OpenSim.Framework.Monitoring if (c == 0) return 0; - return (double)Antecedent / c; + return (double)Antecedent / c * 100; } set @@ -367,7 +367,7 @@ namespace OpenSim.Framework.Monitoring public override string ToConsoleString() { return string.Format( - "{0}.{1}.{2} : {3:0.###}{4} ({5}/{6})", + "{0}.{1}.{2} : {3:0.##}{4} ({5}/{6})", Category, Container, ShortName, Value, UnitName, Antecedent, Consequent); } } -- cgit v1.1 From ab7b7c5d3df03decbcaa3b8bf7683f1268f2be92 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 12 Oct 2012 02:59:28 +0100 Subject: Get Watchdog to log thread removal --- OpenSim/Framework/Monitoring/Watchdog.cs | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index 7964f28..a20326d 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -231,7 +231,25 @@ namespace OpenSim.Framework.Monitoring private static bool RemoveThread(int threadID) { lock (m_threads) - return m_threads.Remove(threadID); + { + ThreadWatchdogInfo twi; + if (m_threads.TryGetValue(threadID, out twi)) + { + m_log.DebugFormat( + "[WATCHDOG]: Removing thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId); + + m_threads.Remove(threadID); + + return true; + } + else + { + m_log.WarnFormat( + "[WATCHDOG]: Requested to remove thread with ID {0} but this is not being monitored", threadID); + + return false; + } + } } public static bool AbortThread(int threadID) -- cgit v1.1 From 4e5b2346a5700b14687a33175ba54a93960a9d33 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 16 Oct 2012 23:44:52 +0100 Subject: Add LastMemoryChurn stat using existing data so we can more quickly tell how memory churn changes rather than waiting for the average to move. --- OpenSim/Framework/Monitoring/BaseStatsCollector.cs | 6 +++++- OpenSim/Framework/Monitoring/MemoryWatchdog.cs | 10 +++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs index 57a63ef..2903b6e 100644 --- a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs @@ -49,7 +49,11 @@ namespace OpenSim.Framework.Monitoring Math.Round(GC.GetTotalMemory(false) / 1024.0 / 1024.0)); sb.AppendFormat( - "OpenSim object memory churn : {0} MB/s\n", + "OpenSim last object memory churn : {0} MB/s\n", + Math.Round((MemoryWatchdog.LastMemoryChurn * 1000) / 1024.0 / 1024, 3)); + + sb.AppendFormat( + "OpenSim average object memory churn : {0} MB/s\n", Math.Round((MemoryWatchdog.AverageMemoryChurn * 1000) / 1024.0 / 1024, 3)); sb.AppendFormat( diff --git a/OpenSim/Framework/Monitoring/MemoryWatchdog.cs b/OpenSim/Framework/Monitoring/MemoryWatchdog.cs index a23cf1f..c6010cd 100644 --- a/OpenSim/Framework/Monitoring/MemoryWatchdog.cs +++ b/OpenSim/Framework/Monitoring/MemoryWatchdog.cs @@ -60,7 +60,7 @@ namespace OpenSim.Framework.Monitoring private static bool m_enabled; /// - /// Average memory churn in bytes per millisecond. + /// Last memory churn in bytes per millisecond. /// public static double AverageMemoryChurn { @@ -68,6 +68,14 @@ namespace OpenSim.Framework.Monitoring } /// + /// Average memory churn in bytes per millisecond. + /// + public static double LastMemoryChurn + { + get { if (m_samples.Count > 0) return m_samples.Last(); else return 0; } + } + + /// /// Maximum number of statistical samples. /// /// -- cgit v1.1