8 files changed, 674 insertions, 15 deletions

diff --git a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs
index 9ee0876..446e3c0 100644
--- a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs
+++ b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs
@@ -43,27 +43,32 @@ namespace OpenSim.Framework.Monitoring
             StringBuilder sb = new StringBuilder(Environment.NewLine);
             sb.Append("MEMORY STATISTICS");
             sb.Append(Environment.NewLine);
-            sb.Append(
-                string.Format(
+            sb.AppendFormat(
                     "Allocated to OpenSim objects: {0} MB\n",
-                    Math.Round(GC.GetTotalMemory(false) / 1024.0 / 1024.0)));
+                    Math.Round(GC.GetTotalMemory(false) / 1024.0 / 1024.0));
+
+            sb.AppendFormat(
+                "OpenSim last object memory churn    : {0} MB/s\n",
+                Math.Round((MemoryWatchdog.LastMemoryChurn * 1000) / 1024.0 / 1024, 3));
+
+            sb.AppendFormat(
+                "OpenSim average object memory churn : {0} MB/s\n",
+                Math.Round((MemoryWatchdog.AverageMemoryChurn * 1000) / 1024.0 / 1024, 3));
 
             Process myprocess = Process.GetCurrentProcess();
             if (!myprocess.HasExited)
             {
                 myprocess.Refresh();
-                sb.Append(
-                    string.Format(
+                sb.AppendFormat(
                         "Process memory:      Physical {0} MB \t Paged {1} MB \t Virtual {2} MB\n",
                         Math.Round(Process.GetCurrentProcess().WorkingSet64 / 1024.0 / 1024.0),
                         Math.Round(Process.GetCurrentProcess().PagedMemorySize64 / 1024.0 / 1024.0),
-                        Math.Round(Process.GetCurrentProcess().VirtualMemorySize64 / 1024.0 / 1024.0)));
-                sb.Append(
-                    string.Format(
+                        Math.Round(Process.GetCurrentProcess().VirtualMemorySize64 / 1024.0 / 1024.0));
+                sb.AppendFormat(
                         "Peak process memory: Physical {0} MB \t Paged {1} MB \t Virtual {2} MB\n",
                         Math.Round(Process.GetCurrentProcess().PeakWorkingSet64 / 1024.0 / 1024.0),
                         Math.Round(Process.GetCurrentProcess().PeakPagedMemorySize64 / 1024.0 / 1024.0),
-                        Math.Round(Process.GetCurrentProcess().PeakVirtualMemorySize64 / 1024.0 / 1024.0)));
+                        Math.Round(Process.GetCurrentProcess().PeakVirtualMemorySize64 / 1024.0 / 1024.0));
             }
             else
                 sb.Append("Process reported as Exited \n");
diff --git a/OpenSim/Framework/Monitoring/MemoryWatchdog.cs b/OpenSim/Framework/Monitoring/MemoryWatchdog.cs
index a23cf1f..c6010cd 100644
--- a/OpenSim/Framework/Monitoring/MemoryWatchdog.cs
+++ b/OpenSim/Framework/Monitoring/MemoryWatchdog.cs
@@ -60,7 +60,7 @@ namespace OpenSim.Framework.Monitoring
         private static bool m_enabled;
 
         /// <summary>
-        /// Average memory churn in bytes per millisecond.
+        /// Last memory churn in bytes per millisecond.
         /// </summary>
         public static double AverageMemoryChurn
         {
@@ -68,6 +68,14 @@ namespace OpenSim.Framework.Monitoring
         }
 
         /// <summary>
+        /// Average memory churn in bytes per millisecond.
+        /// </summary>
+        public static double LastMemoryChurn
+        {
+            get { if (m_samples.Count > 0) return m_samples.Last(); else return 0; }
+        }
+
+        /// <summary>
         /// Maximum number of statistical samples.
         /// </summary>
         /// <remarks>
diff --git a/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs
new file mode 100644
index 0000000..1f2bb40
--- /dev/null
+++ b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs
@@ -0,0 +1,33 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("OpenSim.Framework.Monitoring")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("http://opensimulator.org")]
+[assembly: AssemblyProduct("OpenSim")]
+[assembly: AssemblyCopyright("OpenSimulator developers")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("74506fe3-2f9d-44c1-94c9-a30f79d9e0cb")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+[assembly: AssemblyVersion("0.7.5.*")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
diff --git a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs
index cdd7cc7..aa86202 100644
--- a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs
+++ b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs
@@ -355,10 +355,25 @@ Asset service request failures: {3}" + Environment.NewLine,
             sb.Append(Environment.NewLine);
             sb.Append(
                 string.Format(
-                    "{0,6:0}  {1,6:0}  {2,6:0}  {3,6:0}  {4,6:0}  {5,6:0.0}  {6,6:0.0}  {7,6:0.0}  {8,6:0.0}  {9,6:0.0}  {10,6:0.0}",
+                    "{0,6:0}  {1,6:0}  {2,6:0}  {3,6:0}  {4,6:0}  {5,6:0.0}  {6,6:0.0}  {7,6:0.0}  {8,6:0.0}  {9,6:0.0}  {10,6:0.0}\n\n",
                     inPacketsPerSecond, outPacketsPerSecond, pendingDownloads, pendingUploads, unackedBytes, totalFrameTime,
                     netFrameTime, physicsFrameTime, otherFrameTime, agentFrameTime, imageFrameTime));
-            sb.Append(Environment.NewLine);
+
+            Dictionary<string, Dictionary<string, Stat>> sceneStats;
+
+            if (StatsManager.TryGetStats("scene", out sceneStats))
+            {
+                foreach (KeyValuePair<string, Dictionary<string, Stat>> kvp in sceneStats)
+                {
+                    foreach (Stat stat in kvp.Value.Values)
+                    {
+                        if (stat.Verbosity == StatVerbosity.Info)
+                        {
+                            sb.AppendFormat("{0} ({1}): {2}{3}\n", stat.Name, stat.Container, stat.Value, stat.UnitName);
+                        }
+                    }
+                }
+            }
 
             /*
             sb.Append(Environment.NewLine);
diff --git a/OpenSim/Framework/Monitoring/Stats/PercentageStat.cs b/OpenSim/Framework/Monitoring/Stats/PercentageStat.cs
new file mode 100644
index 0000000..60bed55
--- /dev/null
+++ b/OpenSim/Framework/Monitoring/Stats/PercentageStat.cs
@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) Contributors, http://opensimulator.org/
+ * See CONTRIBUTORS.TXT for a full list of copyright holders.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the OpenSimulator Project nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace OpenSim.Framework.Monitoring
+{
+    public class PercentageStat : Stat
+    {
+        public long Antecedent { get; set; }
+        public long Consequent { get; set; }
+
+        public override double Value
+        {
+            get
+            {
+                // Asking for an update here means that the updater cannot access this value without infinite recursion.
+                // XXX: A slightly messy but simple solution may be to flick a flag so we can tell if this is being
+                // called by the pull action and just return the value.
+                if (StatType == StatType.Pull)
+                    PullAction(this);
+
+                long c = Consequent;
+
+                // Avoid any chance of a multi-threaded divide-by-zero
+                if (c == 0)
+                    return 0;
+
+                return (double)Antecedent / c * 100;
+            }
+
+            set
+            {
+                throw new InvalidOperationException("Cannot set value on a PercentageStat");
+            }
+        }
+
+        public PercentageStat(
+            string shortName,
+            string name,
+            string description,
+            string category,
+            string container,
+            StatType type,
+            Action<Stat> pullAction,
+            StatVerbosity verbosity)
+            : base(shortName, name, description, "%", category, container, type, pullAction, verbosity) {}
+
+        public override string ToConsoleString()
+        {
+            StringBuilder sb = new StringBuilder();
+
+            sb.AppendFormat(
+                "{0}.{1}.{2} : {3:0.##}{4} ({5}/{6})",
+                Category, Container, ShortName, Value, UnitName, Antecedent, Consequent);
+
+            AppendMeasuresOfInterest(sb);
+
+            return sb.ToString();
+        }
+    }
+}
\ No newline at end of file
diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs
new file mode 100644
index 0000000..f91251b
--- /dev/null
+++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs
@@ -0,0 +1,238 @@
+/*
+ * Copyright (c) Contributors, http://opensimulator.org/
+ * See CONTRIBUTORS.TXT for a full list of copyright holders.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the OpenSimulator Project nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace OpenSim.Framework.Monitoring
+{
+    /// <summary>
+    /// Holds individual statistic details
+    /// </summary>
+    public class Stat
+    {
+        /// <summary>
+        /// Category of this stat (e.g. cache, scene, etc).
+        /// </summary>
+        public string Category { get; private set; }
+
+        /// <summary>
+        /// Containing name for this stat.
+        /// FIXME: In the case of a scene, this is currently the scene name (though this leaves
+        /// us with a to-be-resolved problem of non-unique region names).
+        /// </summary>
+        /// <value>
+        /// The container.
+        /// </value>
+        public string Container { get; private set; }
+
+        public StatType StatType { get; private set; }
+
+        public MeasuresOfInterest MeasuresOfInterest { get; private set; }
+
+        /// <summary>
+        /// Action used to update this stat when the value is requested if it's a pull type.
+        /// </summary>
+        public Action<Stat> PullAction { get; private set; }
+
+        public StatVerbosity Verbosity { get; private set; }
+        public string ShortName { get; private set; }
+        public string Name { get; private set; }
+        public string Description { get; private set; }
+        public virtual string UnitName { get; private set; }
+
+        public virtual double Value
+        {
+            get
+            {
+                // Asking for an update here means that the updater cannot access this value without infinite recursion.
+                // XXX: A slightly messy but simple solution may be to flick a flag so we can tell if this is being
+                // called by the pull action and just return the value.
+                if (StatType == StatType.Pull)
+                    PullAction(this);
+
+                return m_value;
+            }
+
+            set
+            {
+                m_value = value;
+            }
+        }
+
+        private double m_value;
+
+        /// <summary>
+        /// Historical samples for calculating measures of interest average.
+        /// </summary>
+        /// <remarks>
+        /// Will be null if no measures of interest require samples.
+        /// </remarks>
+        private static Queue<double> m_samples;
+
+        /// <summary>
+        /// Maximum number of statistical samples.
+        /// </summary>
+        /// <remarks>
+        /// At the moment this corresponds to 1 minute since the sampling rate is every 2.5 seconds as triggered from
+        /// the main Watchdog.
+        /// </remarks>
+        private static int m_maxSamples = 24;
+
+        public Stat(
+            string shortName,
+            string name,
+            string description,
+            string unitName,
+            string category,
+            string container,
+            StatType type,
+            Action<Stat> pullAction,
+            StatVerbosity verbosity) 
+                : this(
+                    shortName, 
+                    name, 
+                    description, 
+                    unitName, 
+                    category, 
+                    container, 
+                    type, 
+                    MeasuresOfInterest.None, 
+                    pullAction, 
+                    verbosity)
+        {
+        }
+
+        /// <summary>
+        /// Constructor
+        /// </summary>
+        /// <param name='shortName'>Short name for the stat.  Must not contain spaces.  e.g. "LongFrames"</param>
+        /// <param name='name'>Human readable name for the stat.  e.g. "Long frames"</param>
+        /// <param name='description'>Description of stat</param>
+        /// <param name='unitName'>
+        /// Unit name for the stat.  Should be preceeded by a space if the unit name isn't normally appeneded immediately to the value.
+        /// e.g. " frames"
+        /// </param>
+        /// <param name='category'>Category under which this stat should appear, e.g. "scene".  Do not capitalize.</param>
+        /// <param name='container'>Entity to which this stat relates.  e.g. scene name if this is a per scene stat.</param>
+        /// <param name='type'>Push or pull</param>
+        /// <param name='pullAction'>Pull stats need an action to update the stat on request.  Push stats should set null here.</param>
+        /// <param name='moi'>Measures of interest</param>
+        /// <param name='verbosity'>Verbosity of stat.  Controls whether it will appear in short stat display or only full display.</param>
+        public Stat(
+            string shortName,
+            string name,
+            string description,
+            string unitName,
+            string category,
+            string container,
+            StatType type,
+            MeasuresOfInterest moi,
+            Action<Stat> pullAction,
+            StatVerbosity verbosity)
+        {
+            if (StatsManager.SubCommands.Contains(category))
+                throw new Exception(
+                    string.Format("Stat cannot be in category '{0}' since this is reserved for a subcommand", category));
+
+            ShortName = shortName;
+            Name = name;
+            Description = description;
+            UnitName = unitName;
+            Category = category;
+            Container = container;
+            StatType = type;
+
+            if (StatType == StatType.Push && pullAction != null)
+                throw new Exception("A push stat cannot have a pull action");
+            else
+                PullAction = pullAction;
+
+            MeasuresOfInterest = moi;
+
+            if ((moi & MeasuresOfInterest.AverageChangeOverTime) == MeasuresOfInterest.AverageChangeOverTime)
+                m_samples = new Queue<double>(m_maxSamples);
+
+            Verbosity = verbosity;
+        }
+
+        /// <summary>
+        /// Record a value in the sample set.
+        /// </summary>
+        /// <remarks>
+        /// Do not call this if MeasuresOfInterest.None
+        /// </remarks>
+        public void RecordValue()
+        {
+            double newValue = Value;
+
+            lock (m_samples)
+            {
+                if (m_samples.Count >= m_maxSamples)
+                    m_samples.Dequeue();
+
+                m_samples.Enqueue(newValue);
+            }
+        }
+
+        public virtual string ToConsoleString()
+        {
+            StringBuilder sb = new StringBuilder();
+            sb.AppendFormat("{0}.{1}.{2} : {3}{4}", Category, Container, ShortName, Value, UnitName);
+
+            AppendMeasuresOfInterest(sb);
+
+            return sb.ToString();
+        }
+
+        protected void AppendMeasuresOfInterest(StringBuilder sb)
+        {
+            if ((MeasuresOfInterest & MeasuresOfInterest.AverageChangeOverTime) 
+                == MeasuresOfInterest.AverageChangeOverTime)
+            {
+                double totalChange = 0;
+                double? lastSample = null;
+
+                lock (m_samples)
+                {
+                    foreach (double s in m_samples)
+                    {
+                        if (lastSample != null)
+                            totalChange += s - (double)lastSample;
+
+                        lastSample = s;
+                    }
+                }
+
+                int divisor = m_samples.Count <= 1 ? 1 : m_samples.Count - 1;
+
+                sb.AppendFormat(", {0:0.##}{1}/s", totalChange / divisor / (Watchdog.WATCHDOG_INTERVAL_MS / 1000), UnitName);
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs
index d78fa6a..0762b01 100644
--- a/OpenSim/Framework/Monitoring/StatsManager.cs
+++ b/OpenSim/Framework/Monitoring/StatsManager.cs
@@ -25,6 +25,10 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+using System;
+using System.Collections.Generic;
+using System.Text;
+
 namespace OpenSim.Framework.Monitoring
 {
     /// <summary>
@@ -32,6 +36,24 @@ namespace OpenSim.Framework.Monitoring
     /// </summary>
     public class StatsManager
     {
+        // Subcommand used to list other stats.
+        public const string AllSubCommand = "all";
+
+        // Subcommand used to list other stats.
+        public const string ListSubCommand = "list";
+
+        // All subcommands
+        public static HashSet<string> SubCommands = new HashSet<string> { AllSubCommand, ListSubCommand };
+
+        /// <summary>
+        /// Registered stats categorized by category/container/shortname
+        /// </summary>
+        /// <remarks>
+        /// Do not add or remove directly from this dictionary.
+        /// </remarks>
+        public static Dictionary<string, Dictionary<string, Dictionary<string, Stat>>> RegisteredStats
+            = new Dictionary<string, Dictionary<string, Dictionary<string, Stat>>>();
+
         private static AssetStatsCollector assetStats;
         private static UserStatsCollector userStats;
         private static SimExtraStatsCollector simExtraStats = new SimExtraStatsCollector();
@@ -40,6 +62,75 @@ namespace OpenSim.Framework.Monitoring
         public static UserStatsCollector UserStats { get { return userStats; } }
         public static SimExtraStatsCollector SimExtraStats { get { return simExtraStats; } }
 
+        public static void RegisterConsoleCommands(ICommandConsole console)
+        {
+            console.Commands.AddCommand(
+                "General",
+                false,
+                "show stats",
+                "show stats [list|all|<category>]",
+                "Show statistical information for this server",
+                "If no final argument is specified then legacy statistics information is currently shown.\n"
+                    + "If list is specified then statistic categories are shown.\n"
+                    + "If all is specified then all registered statistics are shown.\n"
+                    + "If a category name is specified then only statistics from that category are shown.\n"
+                    + "THIS STATS FACILITY IS EXPERIMENTAL AND DOES NOT YET CONTAIN ALL STATS",
+                HandleShowStatsCommand);
+        }
+
+        public static void HandleShowStatsCommand(string module, string[] cmd)
+        {
+            ICommandConsole con = MainConsole.Instance;
+
+            if (cmd.Length > 2)
+            {
+                var categoryName = cmd[2];
+
+                if (categoryName == AllSubCommand)
+                {
+                    foreach (var category in RegisteredStats.Values)
+                    {
+                        OutputCategoryStatsToConsole(con, category);
+                    }
+                }
+                else if (categoryName == ListSubCommand)
+                {
+                    con.Output("Statistic categories available are:");
+                    foreach (string category in RegisteredStats.Keys)
+                        con.OutputFormat("  {0}", category);
+                }
+                else
+                {
+                    Dictionary<string, Dictionary<string, Stat>> category;
+                    if (!RegisteredStats.TryGetValue(categoryName, out category))
+                    {
+                        con.OutputFormat("No such category as {0}", categoryName);
+                    }
+                    else
+                    {
+                        OutputCategoryStatsToConsole(con, category);
+                    }
+                }
+            }
+            else
+            {
+                // Legacy
+                con.Output(SimExtraStats.Report());
+            }
+        }
+
+        private static void OutputCategoryStatsToConsole(
+            ICommandConsole con, Dictionary<string, Dictionary<string, Stat>> category)
+        {
+            foreach (var container in category.Values)
+            {
+                foreach (Stat stat in container.Values)
+                {
+                    con.Output(stat.ToConsoleString());
+                }
+            }
+        }
+
         /// <summary>
         /// Start collecting statistics related to assets.
         /// Should only be called once.
@@ -61,5 +152,153 @@ namespace OpenSim.Framework.Monitoring
 
             return userStats;
         }
+
+        /// <summary>
+        /// Registers a statistic.
+        /// </summary>
+        /// <param name='stat'></param>
+        /// <returns></returns>
+        public static bool RegisterStat(Stat stat)
+        {
+            Dictionary<string, Dictionary<string, Stat>> category = null, newCategory;
+            Dictionary<string, Stat> container = null, newContainer;
+
+            lock (RegisteredStats)
+            {
+                // Stat name is not unique across category/container/shortname key.
+                // XXX: For now just return false.  This is to avoid problems in regression tests where all tests
+                // in a class are run in the same instance of the VM.
+                if (TryGetStat(stat, out category, out container))
+                    return false;
+
+                // We take a copy-on-write approach here of replacing dictionaries when keys are added or removed.
+                // This means that we don't need to lock or copy them on iteration, which will be a much more
+                // common operation after startup.
+                if (container != null)
+                    newContainer = new Dictionary<string, Stat>(container);
+                else
+                    newContainer = new Dictionary<string, Stat>();
+
+                if (category != null)
+                    newCategory = new Dictionary<string, Dictionary<string, Stat>>(category);
+                else
+                    newCategory = new Dictionary<string, Dictionary<string, Stat>>();
+
+                newContainer[stat.ShortName] = stat;
+                newCategory[stat.Container] = newContainer;
+                RegisteredStats[stat.Category] = newCategory;
+            }
+
+            return true;
+        }
+
+        /// <summary>
+        /// Deregister a statistic
+        /// </summary>>
+        /// <param name='stat'></param>
+        /// <returns></returns
+        public static bool DeregisterStat(Stat stat)
+        {
+            Dictionary<string, Dictionary<string, Stat>> category = null, newCategory;
+            Dictionary<string, Stat> container = null, newContainer;
+
+            lock (RegisteredStats)
+            {
+                if (!TryGetStat(stat, out category, out container))
+                    return false;
+
+                newContainer = new Dictionary<string, Stat>(container);
+                newContainer.Remove(stat.ShortName);
+
+                newCategory = new Dictionary<string, Dictionary<string, Stat>>(category);
+                newCategory.Remove(stat.Container);
+
+                newCategory[stat.Container] = newContainer;
+                RegisteredStats[stat.Category] = newCategory;
+
+                return true;
+            }
+        }
+
+        public static bool TryGetStats(string category, out Dictionary<string, Dictionary<string, Stat>> stats)
+        {
+            return RegisteredStats.TryGetValue(category, out stats);
+        }
+
+        public static bool TryGetStat(
+            Stat stat,
+            out Dictionary<string, Dictionary<string, Stat>> category,
+            out Dictionary<string, Stat> container)
+        {
+            category = null;
+            container = null;
+
+            lock (RegisteredStats)
+            {
+                if (RegisteredStats.TryGetValue(stat.Category, out category))
+                {
+                    if (category.TryGetValue(stat.Container, out container))
+                    {
+                        if (container.ContainsKey(stat.ShortName))
+                            return true;
+                    }
+                }
+            }
+
+            return false;
+        }
+
+        public static void RecordStats()
+        {
+            lock (RegisteredStats)
+            {
+                foreach (Dictionary<string, Dictionary<string, Stat>> category in RegisteredStats.Values)
+                {
+                    foreach (Dictionary<string, Stat> container in category.Values)
+                    {
+                        foreach (Stat stat in container.Values)
+                        {
+                            if (stat.MeasuresOfInterest != MeasuresOfInterest.None)
+                                stat.RecordValue();
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    /// <summary>
+    /// Stat type.
+    /// </summary>
+    /// <remarks>
+    /// A push stat is one which is continually updated and so it's value can simply by read.
+    /// A pull stat is one where reading the value triggers a collection method - the stat is not continually updated.
+    /// </remarks>
+    public enum StatType
+    {
+        Push,
+        Pull
+    }
+
+    /// <summary>
+    /// Measures of interest for this stat.
+    /// </summary>
+    [Flags]
+    public enum MeasuresOfInterest
+    {
+        None,
+        AverageChangeOverTime
+    }
+
+    /// <summary>
+    /// Verbosity of stat.
+    /// </summary>
+    /// <remarks>
+    /// Info will always be displayed.
+    /// </remarks>
+    public enum StatVerbosity
+    {
+        Debug,
+        Info
     }
 }
\ No newline at end of file
diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs
index b709baa..69d2db5 100644
--- a/OpenSim/Framework/Monitoring/Watchdog.cs
+++ b/OpenSim/Framework/Monitoring/Watchdog.cs
@@ -39,7 +39,7 @@ namespace OpenSim.Framework.Monitoring
     public static class Watchdog
     {
         /// <summary>Timer interval in milliseconds for the watchdog timer</summary>
-        const double WATCHDOG_INTERVAL_MS = 2500.0d;
+        public const double WATCHDOG_INTERVAL_MS = 2500.0d;
 
         /// <summary>Default timeout in milliseconds before a thread is considered dead</summary>
         public const int DEFAULT_WATCHDOG_TIMEOUT_MS = 5000;
@@ -89,6 +89,17 @@ namespace OpenSim.Framework.Monitoring
                 FirstTick = Environment.TickCount & Int32.MaxValue;
                 LastTick = FirstTick;
             }
+
+            public ThreadWatchdogInfo(ThreadWatchdogInfo previousTwi)
+            {
+                Thread = previousTwi.Thread;
+                FirstTick = previousTwi.FirstTick;
+                LastTick = previousTwi.LastTick;
+                Timeout = previousTwi.Timeout;
+                IsTimedOut = previousTwi.IsTimedOut;
+                AlarmIfTimeout = previousTwi.AlarmIfTimeout;
+                AlarmMethod = previousTwi.AlarmMethod;
+            }
         }
 
         /// <summary>
@@ -220,7 +231,25 @@ namespace OpenSim.Framework.Monitoring
         private static bool RemoveThread(int threadID)
         {
             lock (m_threads)
-                return m_threads.Remove(threadID);
+            {
+                ThreadWatchdogInfo twi;
+                if (m_threads.TryGetValue(threadID, out twi))
+                {
+                    m_log.DebugFormat(
+                        "[WATCHDOG]: Removing thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
+
+                    m_threads.Remove(threadID);
+
+                    return true;
+                }
+                else
+                {
+                    m_log.WarnFormat(
+                        "[WATCHDOG]: Requested to remove thread with ID {0} but this is not being monitored", threadID);
+
+                    return false;
+                }
+            }
         }
 
         public static bool AbortThread(int threadID)
@@ -335,7 +364,9 @@ namespace OpenSim.Framework.Monitoring
                                 if (callbackInfos == null)
                                     callbackInfos = new List<ThreadWatchdogInfo>();
 
-                                callbackInfos.Add(threadInfo);
+                                // Send a copy of the watchdog info to prevent race conditions where the watchdog
+                                // thread updates the monitoring info after an alarm has been sent out.
+                                callbackInfos.Add(new ThreadWatchdogInfo(threadInfo));
                             }
                         }
                     }
@@ -349,6 +380,8 @@ namespace OpenSim.Framework.Monitoring
             if (MemoryWatchdog.Enabled)
                 MemoryWatchdog.Update();
 
+            StatsManager.RecordStats();
+
             m_watchdogTimer.Start();
         }
     }