5 files changed, 433 insertions, 14 deletions

diff --git a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs
index 9ee0876..446e3c0 100644
--- a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs
+++ b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs
@@ -43,27 +43,32 @@ namespace OpenSim.Framework.Monitoring
             StringBuilder sb = new StringBuilder(Environment.NewLine);
             sb.Append("MEMORY STATISTICS");
             sb.Append(Environment.NewLine);
-            sb.Append(
-                string.Format(
+            sb.AppendFormat(
                     "Allocated to OpenSim objects: {0} MB\n",
-                    Math.Round(GC.GetTotalMemory(false) / 1024.0 / 1024.0)));
+                    Math.Round(GC.GetTotalMemory(false) / 1024.0 / 1024.0));
+
+            sb.AppendFormat(
+                "OpenSim last object memory churn    : {0} MB/s\n",
+                Math.Round((MemoryWatchdog.LastMemoryChurn * 1000) / 1024.0 / 1024, 3));
+
+            sb.AppendFormat(
+                "OpenSim average object memory churn : {0} MB/s\n",
+                Math.Round((MemoryWatchdog.AverageMemoryChurn * 1000) / 1024.0 / 1024, 3));
 
             Process myprocess = Process.GetCurrentProcess();
             if (!myprocess.HasExited)
             {
                 myprocess.Refresh();
-                sb.Append(
-                    string.Format(
+                sb.AppendFormat(
                         "Process memory:      Physical {0} MB \t Paged {1} MB \t Virtual {2} MB\n",
                         Math.Round(Process.GetCurrentProcess().WorkingSet64 / 1024.0 / 1024.0),
                         Math.Round(Process.GetCurrentProcess().PagedMemorySize64 / 1024.0 / 1024.0),
-                        Math.Round(Process.GetCurrentProcess().VirtualMemorySize64 / 1024.0 / 1024.0)));
-                sb.Append(
-                    string.Format(
+                        Math.Round(Process.GetCurrentProcess().VirtualMemorySize64 / 1024.0 / 1024.0));
+                sb.AppendFormat(
                         "Peak process memory: Physical {0} MB \t Paged {1} MB \t Virtual {2} MB\n",
                         Math.Round(Process.GetCurrentProcess().PeakWorkingSet64 / 1024.0 / 1024.0),
                         Math.Round(Process.GetCurrentProcess().PeakPagedMemorySize64 / 1024.0 / 1024.0),
-                        Math.Round(Process.GetCurrentProcess().PeakVirtualMemorySize64 / 1024.0 / 1024.0)));
+                        Math.Round(Process.GetCurrentProcess().PeakVirtualMemorySize64 / 1024.0 / 1024.0));
             }
             else
                 sb.Append("Process reported as Exited \n");
diff --git a/OpenSim/Framework/Monitoring/MemoryWatchdog.cs b/OpenSim/Framework/Monitoring/MemoryWatchdog.cs
index a23cf1f..c6010cd 100644
--- a/OpenSim/Framework/Monitoring/MemoryWatchdog.cs
+++ b/OpenSim/Framework/Monitoring/MemoryWatchdog.cs
@@ -60,7 +60,7 @@ namespace OpenSim.Framework.Monitoring
         private static bool m_enabled;
 
         /// <summary>
-        /// Average memory churn in bytes per millisecond.
+        /// Last memory churn in bytes per millisecond.
         /// </summary>
         public static double AverageMemoryChurn
         {
@@ -68,6 +68,14 @@ namespace OpenSim.Framework.Monitoring
         }
 
         /// <summary>
+        /// Average memory churn in bytes per millisecond.
+        /// </summary>
+        public static double LastMemoryChurn
+        {
+            get { if (m_samples.Count > 0) return m_samples.Last(); else return 0; }
+        }
+
+        /// <summary>
         /// Maximum number of statistical samples.
         /// </summary>
         /// <remarks>
diff --git a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs
index cdd7cc7..aa86202 100644
--- a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs
+++ b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs
@@ -355,10 +355,25 @@ Asset service request failures: {3}" + Environment.NewLine,
             sb.Append(Environment.NewLine);
             sb.Append(
                 string.Format(
-                    "{0,6:0}  {1,6:0}  {2,6:0}  {3,6:0}  {4,6:0}  {5,6:0.0}  {6,6:0.0}  {7,6:0.0}  {8,6:0.0}  {9,6:0.0}  {10,6:0.0}",
+                    "{0,6:0}  {1,6:0}  {2,6:0}  {3,6:0}  {4,6:0}  {5,6:0.0}  {6,6:0.0}  {7,6:0.0}  {8,6:0.0}  {9,6:0.0}  {10,6:0.0}\n\n",
                     inPacketsPerSecond, outPacketsPerSecond, pendingDownloads, pendingUploads, unackedBytes, totalFrameTime,
                     netFrameTime, physicsFrameTime, otherFrameTime, agentFrameTime, imageFrameTime));
-            sb.Append(Environment.NewLine);
+
+            Dictionary<string, Dictionary<string, Stat>> sceneStats;
+
+            if (StatsManager.TryGetStats("scene", out sceneStats))
+            {
+                foreach (KeyValuePair<string, Dictionary<string, Stat>> kvp in sceneStats)
+                {
+                    foreach (Stat stat in kvp.Value.Values)
+                    {
+                        if (stat.Verbosity == StatVerbosity.Info)
+                        {
+                            sb.AppendFormat("{0} ({1}): {2}{3}\n", stat.Name, stat.Container, stat.Value, stat.UnitName);
+                        }
+                    }
+                }
+            }
 
             /*
             sb.Append(Environment.NewLine);
diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs
index d78fa6a..4844336 100644
--- a/OpenSim/Framework/Monitoring/StatsManager.cs
+++ b/OpenSim/Framework/Monitoring/StatsManager.cs
@@ -25,6 +25,9 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+using System;
+using System.Collections.Generic;
+
 namespace OpenSim.Framework.Monitoring
 {
     /// <summary>
@@ -32,6 +35,24 @@ namespace OpenSim.Framework.Monitoring
     /// </summary>
     public class StatsManager
     {
+        // Subcommand used to list other stats.
+        public const string AllSubCommand = "all";
+
+        // Subcommand used to list other stats.
+        public const string ListSubCommand = "list";
+
+        // All subcommands
+        public static HashSet<string> SubCommands = new HashSet<string> { AllSubCommand, ListSubCommand };
+
+        /// <summary>
+        /// Registered stats categorized by category/container/shortname
+        /// </summary>
+        /// <remarks>
+        /// Do not add or remove directly from this dictionary.
+        /// </remarks>
+        public static Dictionary<string, Dictionary<string, Dictionary<string, Stat>>> RegisteredStats
+            = new Dictionary<string, Dictionary<string, Dictionary<string, Stat>>>();
+
         private static AssetStatsCollector assetStats;
         private static UserStatsCollector userStats;
         private static SimExtraStatsCollector simExtraStats = new SimExtraStatsCollector();
@@ -40,6 +61,75 @@ namespace OpenSim.Framework.Monitoring
         public static UserStatsCollector UserStats { get { return userStats; } }
         public static SimExtraStatsCollector SimExtraStats { get { return simExtraStats; } }
 
+        public static void RegisterConsoleCommands(ICommandConsole console)
+        {
+            console.Commands.AddCommand(
+                "General",
+                false,
+                "show stats",
+                "show stats [list|all|<category>]",
+                "Show statistical information for this server",
+                "If no final argument is specified then legacy statistics information is currently shown.\n"
+                    + "If list is specified then statistic categories are shown.\n"
+                    + "If all is specified then all registered statistics are shown.\n"
+                    + "If a category name is specified then only statistics from that category are shown.\n"
+                    + "THIS STATS FACILITY IS EXPERIMENTAL AND DOES NOT YET CONTAIN ALL STATS",
+                HandleShowStatsCommand);
+        }
+
+        public static void HandleShowStatsCommand(string module, string[] cmd)
+        {
+            ICommandConsole con = MainConsole.Instance;
+
+            if (cmd.Length > 2)
+            {
+                var categoryName = cmd[2];
+
+                if (categoryName == AllSubCommand)
+                {
+                    foreach (var category in RegisteredStats.Values)
+                    {
+                        OutputCategoryStatsToConsole(con, category);
+                    }
+                }
+                else if (categoryName == ListSubCommand)
+                {
+                    con.Output("Statistic categories available are:");
+                    foreach (string category in RegisteredStats.Keys)
+                        con.OutputFormat("  {0}", category);
+                }
+                else
+                {
+                    Dictionary<string, Dictionary<string, Stat>> category;
+                    if (!RegisteredStats.TryGetValue(categoryName, out category))
+                    {
+                        con.OutputFormat("No such category as {0}", categoryName);
+                    }
+                    else
+                    {
+                        OutputCategoryStatsToConsole(con, category);
+                    }
+                }
+            }
+            else
+            {
+                // Legacy
+                con.Output(SimExtraStats.Report());
+            }
+        }
+
+        private static void OutputCategoryStatsToConsole(
+            ICommandConsole con, Dictionary<string, Dictionary<string, Stat>> category)
+        {
+            foreach (var container in category.Values)
+            {
+                foreach (Stat stat in container.Values)
+                {
+                    con.Output(stat.ToConsoleString());
+                }
+            }
+        }
+
         /// <summary>
         /// Start collecting statistics related to assets.
         /// Should only be called once.
@@ -61,5 +151,275 @@ namespace OpenSim.Framework.Monitoring
 
             return userStats;
         }
+
+        /// <summary>
+        /// Registers a statistic.
+        /// </summary>
+        /// <param name='stat'></param>
+        /// <returns></returns>
+        public static bool RegisterStat(Stat stat)
+        {
+            Dictionary<string, Dictionary<string, Stat>> category = null, newCategory;
+            Dictionary<string, Stat> container = null, newContainer;
+
+            lock (RegisteredStats)
+            {
+                // Stat name is not unique across category/container/shortname key.
+                // XXX: For now just return false.  This is to avoid problems in regression tests where all tests
+                // in a class are run in the same instance of the VM.
+                if (TryGetStat(stat, out category, out container))
+                    return false;
+
+                // We take a copy-on-write approach here of replacing dictionaries when keys are added or removed.
+                // This means that we don't need to lock or copy them on iteration, which will be a much more
+                // common operation after startup.
+                if (container != null)
+                    newContainer = new Dictionary<string, Stat>(container);
+                else
+                    newContainer = new Dictionary<string, Stat>();
+
+                if (category != null)
+                    newCategory = new Dictionary<string, Dictionary<string, Stat>>(category);
+                else
+                    newCategory = new Dictionary<string, Dictionary<string, Stat>>();
+
+                newContainer[stat.ShortName] = stat;
+                newCategory[stat.Container] = newContainer;
+                RegisteredStats[stat.Category] = newCategory;
+            }
+
+            return true;
+        }
+
+        /// <summary>
+        /// Deregister a statistic
+        /// </summary>>
+        /// <param name='stat'></param>
+        /// <returns></returns
+        public static bool DeregisterStat(Stat stat)
+        {
+            Dictionary<string, Dictionary<string, Stat>> category = null, newCategory;
+            Dictionary<string, Stat> container = null, newContainer;
+
+            lock (RegisteredStats)
+            {
+                if (!TryGetStat(stat, out category, out container))
+                    return false;
+
+                newContainer = new Dictionary<string, Stat>(container);
+                newContainer.Remove(stat.ShortName);
+
+                newCategory = new Dictionary<string, Dictionary<string, Stat>>(category);
+                newCategory.Remove(stat.Container);
+
+                newCategory[stat.Container] = newContainer;
+                RegisteredStats[stat.Category] = newCategory;
+
+                return true;
+            }
+        }
+
+        public static bool TryGetStats(string category, out Dictionary<string, Dictionary<string, Stat>> stats)
+        {
+            return RegisteredStats.TryGetValue(category, out stats);
+        }
+
+        public static bool TryGetStat(
+            Stat stat,
+            out Dictionary<string, Dictionary<string, Stat>> category,
+            out Dictionary<string, Stat> container)
+        {
+            category = null;
+            container = null;
+
+            lock (RegisteredStats)
+            {
+                if (RegisteredStats.TryGetValue(stat.Category, out category))
+                {
+                    if (category.TryGetValue(stat.Container, out container))
+                    {
+                        if (container.ContainsKey(stat.ShortName))
+                            return true;
+                    }
+                }
+            }
+
+            return false;
+        }
+    }
+
+    /// <summary>
+    /// Stat type.
+    /// </summary>
+    /// <remarks>
+    /// A push stat is one which is continually updated and so it's value can simply by read.
+    /// A pull stat is one where reading the value triggers a collection method - the stat is not continually updated.
+    /// </remarks>
+    public enum StatType
+    {
+        Push,
+        Pull
+    }
+
+    /// <summary>
+    /// Verbosity of stat.
+    /// </summary>
+    /// <remarks>
+    /// Info will always be displayed.
+    /// </remarks>
+    public enum StatVerbosity
+    {
+        Debug,
+        Info
+    }
+
+    /// <summary>
+    /// Holds individual static details
+    /// </summary>
+    public class Stat
+    {
+        /// <summary>
+        /// Category of this stat (e.g. cache, scene, etc).
+        /// </summary>
+        public string Category { get; private set; }
+
+        /// <summary>
+        /// Containing name for this stat.
+        /// FIXME: In the case of a scene, this is currently the scene name (though this leaves
+        /// us with a to-be-resolved problem of non-unique region names).
+        /// </summary>
+        /// <value>
+        /// The container.
+        /// </value>
+        public string Container { get; private set; }
+
+        public StatType StatType { get; private set; }
+
+        /// <summary>
+        /// Action used to update this stat when the value is requested if it's a pull type.
+        /// </summary>
+        public Action<Stat> PullAction { get; private set; }
+
+        public StatVerbosity Verbosity { get; private set; }
+        public string ShortName { get; private set; }
+        public string Name { get; private set; }
+        public string Description { get; private set; }
+        public virtual string UnitName { get; private set; }
+
+        public virtual double Value
+        {
+            get
+            {
+                // Asking for an update here means that the updater cannot access this value without infinite recursion.
+                // XXX: A slightly messy but simple solution may be to flick a flag so we can tell if this is being
+                // called by the pull action and just return the value.
+                if (StatType == StatType.Pull)
+                    PullAction(this);
+
+                return m_value;
+            }
+
+            set
+            {
+                m_value = value;
+            }
+        }
+
+        private double m_value;
+
+        /// <summary>
+        /// Constructor
+        /// </summary>
+        /// <param name='shortName'>Short name for the stat.  Must not contain spaces.  e.g. "LongFrames"</param>
+        /// <param name='name'>Human readable name for the stat.  e.g. "Long frames"</param>
+        /// <param name='description'>Description of stat</param>
+        /// <param name='unitName'>
+        /// Unit name for the stat.  Should be preceeded by a space if the unit name isn't normally appeneded immediately to the value.
+        /// e.g. " frames"
+        /// </param>
+        /// <param name='category'>Category under which this stat should appear, e.g. "scene".  Do not capitalize.</param>
+        /// <param name='container'>Entity to which this stat relates.  e.g. scene name if this is a per scene stat.</param>
+        /// <param name='type'>Push or pull</param>
+        /// <param name='pullAction'>Pull stats need an action to update the stat on request.  Push stats should set null here.</param>
+        /// <param name='verbosity'>Verbosity of stat.  Controls whether it will appear in short stat display or only full display.</param>
+        public Stat(
+            string shortName,
+            string name,
+            string description,
+            string unitName,
+            string category,
+            string container,
+            StatType type,
+            Action<Stat> pullAction,
+            StatVerbosity verbosity)
+        {
+            if (StatsManager.SubCommands.Contains(category))
+                throw new Exception(
+                    string.Format("Stat cannot be in category '{0}' since this is reserved for a subcommand", category));
+
+            ShortName = shortName;
+            Name = name;
+            Description = description;
+            UnitName = unitName;
+            Category = category;
+            Container = container;
+            StatType = type;
+
+            if (StatType == StatType.Push && pullAction != null)
+                throw new Exception("A push stat cannot have a pull action");
+            else
+                PullAction = pullAction;
+
+            Verbosity = verbosity;
+        }
+
+        public virtual string ToConsoleString()
+        {
+            return string.Format(
+                "{0}.{1}.{2} : {3}{4}", Category, Container, ShortName, Value, UnitName);
+        }
+    }
+
+    public class PercentageStat : Stat
+    {
+        public int Antecedent { get; set; }
+        public int Consequent { get; set; }
+
+        public override double Value
+        {
+            get
+            {
+                int c = Consequent;
+
+                // Avoid any chance of a multi-threaded divide-by-zero
+                if (c == 0)
+                    return 0;
+
+                return (double)Antecedent / c * 100;
+            }
+
+            set
+            {
+                throw new Exception("Cannot set value on a PercentageStat");
+            }
+        }
+
+        public PercentageStat(
+            string shortName,
+            string name,
+            string description,
+            string category,
+            string container,
+            StatType type,
+            Action<Stat> pullAction,
+            StatVerbosity verbosity)
+            : base(shortName, name, description, "%", category, container, type, pullAction, verbosity) {}
+
+        public override string ToConsoleString()
+        {
+            return string.Format(
+                "{0}.{1}.{2} : {3:0.##}{4} ({5}/{6})",
+                Category, Container, ShortName, Value, UnitName, Antecedent, Consequent);
+        }
     }
 }
\ No newline at end of file
diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs
index b709baa..28d6d5c 100644
--- a/OpenSim/Framework/Monitoring/Watchdog.cs
+++ b/OpenSim/Framework/Monitoring/Watchdog.cs
@@ -89,6 +89,17 @@ namespace OpenSim.Framework.Monitoring
                 FirstTick = Environment.TickCount & Int32.MaxValue;
                 LastTick = FirstTick;
             }
+
+            public ThreadWatchdogInfo(ThreadWatchdogInfo previousTwi)
+            {
+                Thread = previousTwi.Thread;
+                FirstTick = previousTwi.FirstTick;
+                LastTick = previousTwi.LastTick;
+                Timeout = previousTwi.Timeout;
+                IsTimedOut = previousTwi.IsTimedOut;
+                AlarmIfTimeout = previousTwi.AlarmIfTimeout;
+                AlarmMethod = previousTwi.AlarmMethod;
+            }
         }
 
         /// <summary>
@@ -220,7 +231,25 @@ namespace OpenSim.Framework.Monitoring
         private static bool RemoveThread(int threadID)
         {
             lock (m_threads)
-                return m_threads.Remove(threadID);
+            {
+                ThreadWatchdogInfo twi;
+                if (m_threads.TryGetValue(threadID, out twi))
+                {
+                    m_log.DebugFormat(
+                        "[WATCHDOG]: Removing thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
+
+                    m_threads.Remove(threadID);
+
+                    return true;
+                }
+                else
+                {
+                    m_log.WarnFormat(
+                        "[WATCHDOG]: Requested to remove thread with ID {0} but this is not being monitored", threadID);
+
+                    return false;
+                }
+            }
         }
 
         public static bool AbortThread(int threadID)
@@ -335,7 +364,9 @@ namespace OpenSim.Framework.Monitoring
                                 if (callbackInfos == null)
                                     callbackInfos = new List<ThreadWatchdogInfo>();
 
-                                callbackInfos.Add(threadInfo);
+                                // Send a copy of the watchdog info to prevent race conditions where the watchdog
+                                // thread updates the monitoring info after an alarm has been sent out.
+                                callbackInfos.Add(new ThreadWatchdogInfo(threadInfo));
                             }
                         }
                     }