* Implements new 'Monitoring' system for reporting performance.

* Mostly the same set as the StatsMonitor used for Viewer notification, but exposes some new frametimes - including EventMS, PhysicsUpdateMS, LandUpdateMS; new memory monitoring - both GC.TotalMemory and Process.PrivateWorkingMemory64; also exposes ThreadCount (using System.Diagnostics.Process)
* Type 'monitor report' on the console to see output.
* SNMP Implementation forthcoming.

14 files changed, 471 insertions, 8 deletions

diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/IMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/IMonitor.cs
new file mode 100644
index 0000000..a51dccd
--- /dev/null
+++ b/OpenSim/Region/CoreModules/Framework/Monitoring/IMonitor.cs
@@ -0,0 +1,9 @@
+namespace OpenSim.Region.CoreModules.Framework.Monitoring
+{
+    interface IMonitor
+    {
+        double GetValue();
+        string GetName();
+        string GetFriendlyValue(); // Convert to readable numbers
+    }
+}
diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/MonitorModule.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/MonitorModule.cs
new file mode 100644
index 0000000..d4a7692
--- /dev/null
+++ b/OpenSim/Region/CoreModules/Framework/Monitoring/MonitorModule.cs
@@ -0,0 +1,70 @@
+using System.Collections.Generic;
+using System.Reflection;
+using log4net;
+using Nini.Config;
+using OpenSim.Region.CoreModules.Framework.Monitoring.Monitors;
+using OpenSim.Region.Framework.Interfaces;
+using OpenSim.Region.Framework.Scenes;
+
+namespace OpenSim.Region.CoreModules.Framework.Monitoring
+{
+    public class MonitorModule : IRegionModule 
+    {
+        private Scene m_scene;
+        private readonly List<IMonitor> m_monitors = new List<IMonitor>();
+        private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
+
+        public void DebugMonitors(string module, string[] args)
+        {
+            foreach (IMonitor monitor in m_monitors)
+            {
+                m_log.Info("[MonitorModule] " + m_scene.RegionInfo.RegionName + " reports " + monitor.GetName() + " = " + monitor.GetValue());
+            }
+        }
+
+        #region Implementation of IRegionModule
+
+        public void Initialise(Scene scene, IConfigSource source)
+        {
+            m_scene = scene;
+
+
+            m_scene.AddCommand(this, "monitor report",
+                               "monitor report",
+                               "Returns a variety of statistics about the current region and/or simulator",
+                               DebugMonitors);
+        }
+
+        public void PostInitialise()
+        {
+            m_monitors.Add(new AgentCountMonitor(m_scene));
+            m_monitors.Add(new ChildAgentCountMonitor(m_scene));
+            m_monitors.Add(new GCMemoryMonitor());
+            m_monitors.Add(new ObjectCountMonitor(m_scene));
+            m_monitors.Add(new PhysicsFrameMonitor(m_scene));
+            m_monitors.Add(new PhysicsUpdateFrameMonitor(m_scene));
+            m_monitors.Add(new PWSMemoryMonitor());
+            m_monitors.Add(new ThreadCountMonitor());
+            m_monitors.Add(new TotalFrameMonitor(m_scene));
+            m_monitors.Add(new EventFrameMonitor(m_scene));
+            m_monitors.Add(new LandFrameMonitor(m_scene));
+        }
+
+        public void Close()
+        {
+            
+        }
+
+        public string Name
+        {
+            get { return "Region Health Monitoring Module"; }
+        }
+
+        public bool IsSharedModule
+        {
+            get { return false; }
+        }
+
+        #endregion
+    }
+}
diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/AgentCountMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/AgentCountMonitor.cs
new file mode 100644
index 0000000..edc6e6b
--- /dev/null
+++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/AgentCountMonitor.cs
@@ -0,0 +1,33 @@
+using OpenSim.Region.Framework.Scenes;
+
+namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors
+{
+    class AgentCountMonitor : IMonitor
+    {
+        private readonly Scene m_scene;
+
+        public AgentCountMonitor(Scene scene)
+        {
+            m_scene = scene;
+        }
+
+        #region Implementation of IMonitor
+
+        public double GetValue()
+        {
+            return m_scene.SceneGraph.GetRootAgentCount();
+        }
+
+        public string GetName()
+        {
+            return "Root Agent Count";
+        }
+
+        public string GetFriendlyValue()
+        {
+            return (int)GetValue() + " agent(s)";
+        }
+
+        #endregion
+    }
+}
diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/ChildAgentCountMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/ChildAgentCountMonitor.cs
new file mode 100644
index 0000000..afe6b79
--- /dev/null
+++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/ChildAgentCountMonitor.cs
@@ -0,0 +1,33 @@
+using OpenSim.Region.Framework.Scenes;
+
+namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors
+{
+    class ChildAgentCountMonitor : IMonitor
+    {
+        private readonly Scene m_scene;
+
+        public ChildAgentCountMonitor(Scene scene)
+        {
+            m_scene = scene;
+        }
+
+        #region Implementation of IMonitor
+
+        public double GetValue()
+        {
+            return m_scene.SceneGraph.GetChildAgentCount();
+        }
+
+        public string GetName()
+        {
+            return "Child Agent Count";
+        }
+
+        public string GetFriendlyValue()
+        {
+            return (int)GetValue() + " child agent(s)";
+        }
+
+        #endregion
+    }
+}
diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/EventFrameMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/EventFrameMonitor.cs
new file mode 100644
index 0000000..dec5a9e
--- /dev/null
+++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/EventFrameMonitor.cs
@@ -0,0 +1,33 @@
+using OpenSim.Region.Framework.Scenes;
+
+namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors
+{
+    class EventFrameMonitor : IMonitor
+    {
+        private readonly Scene m_scene;
+
+        public EventFrameMonitor(Scene scene)
+        {
+            m_scene = scene;
+        }
+
+        #region Implementation of IMonitor
+
+        public double GetValue()
+        {
+            return m_scene.MonitorEventTime;
+        }
+
+        public string GetName()
+        {
+            return "Total Event Frame Time";
+        }
+
+        public string GetFriendlyValue()
+        {
+            return (int)GetValue() + "ms";
+        }
+
+        #endregion
+    }
+}
diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/GCMemoryMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/GCMemoryMonitor.cs
new file mode 100644
index 0000000..cd67fea
--- /dev/null
+++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/GCMemoryMonitor.cs
@@ -0,0 +1,26 @@
+using System;
+
+namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors
+{
+    class GCMemoryMonitor : IMonitor
+    {
+        #region Implementation of IMonitor
+
+        public double GetValue()
+        {
+            return GC.GetTotalMemory(false);
+        }
+
+        public string GetName()
+        {
+            return "GC Reported Memory";
+        }
+
+        public string GetFriendlyValue()
+        {
+            return (int)(GetValue() / (1024*1024)) + "MB (Global)";
+        }
+
+        #endregion
+    }
+}
diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/LandFrameMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/LandFrameMonitor.cs
new file mode 100644
index 0000000..d883fc7
--- /dev/null
+++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/LandFrameMonitor.cs
@@ -0,0 +1,33 @@
+using OpenSim.Region.Framework.Scenes;
+
+namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors
+{
+    class LandFrameMonitor : IMonitor
+    {
+        private readonly Scene m_scene;
+
+        public LandFrameMonitor(Scene scene)
+        {
+            m_scene = scene;
+        }
+
+        #region Implementation of IMonitor
+
+        public double GetValue()
+        {
+            return m_scene.MonitorLandTime;
+        }
+
+        public string GetName()
+        {
+            return "Land Frame Time";
+        }
+
+        public string GetFriendlyValue()
+        {
+            return (int)GetValue() + "ms";
+        }
+
+        #endregion
+    }
+}
diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/ObjectCountMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/ObjectCountMonitor.cs
new file mode 100644
index 0000000..dd9b19d
--- /dev/null
+++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/ObjectCountMonitor.cs
@@ -0,0 +1,33 @@
+using OpenSim.Region.Framework.Scenes;
+
+namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors
+{
+    class ObjectCountMonitor : IMonitor
+    {
+        private readonly Scene m_scene;
+
+        public ObjectCountMonitor(Scene scene)
+        {
+            m_scene = scene;
+        }
+
+        #region Implementation of IMonitor
+
+        public double GetValue()
+        {
+            return m_scene.SceneGraph.GetTotalObjectsCount();
+        }
+
+        public string GetName()
+        {
+            return "Total Objects Count";
+        }
+
+        public string GetFriendlyValue()
+        {
+            return (int)GetValue() + " Object(s)";
+        }
+
+        #endregion
+    }
+}
diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/PWSMemoryMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/PWSMemoryMonitor.cs
new file mode 100644
index 0000000..88f2938
--- /dev/null
+++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/PWSMemoryMonitor.cs
@@ -0,0 +1,26 @@
+using System;
+
+namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors
+{
+    class PWSMemoryMonitor : IMonitor
+    {
+        #region Implementation of IMonitor
+
+        public double GetValue()
+        {
+            return System.Diagnostics.Process.GetCurrentProcess().PrivateMemorySize64;
+        }
+
+        public string GetName()
+        {
+            return "Private Working Set Memory";
+        }
+
+        public string GetFriendlyValue()
+        {
+            return (int)(GetValue() / (1024 * 1024)) + "MB (Global)";
+        }
+
+        #endregion
+    }
+}
diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/PhysicsFrameMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/PhysicsFrameMonitor.cs
new file mode 100644
index 0000000..4d62e4f
--- /dev/null
+++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/PhysicsFrameMonitor.cs
@@ -0,0 +1,33 @@
+using OpenSim.Region.Framework.Scenes;
+
+namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors
+{
+    class PhysicsFrameMonitor : IMonitor
+    {
+        private readonly Scene m_scene;
+
+        public PhysicsFrameMonitor(Scene scene)
+        {
+            m_scene = scene;
+        }
+
+        #region Implementation of IMonitor
+
+        public double GetValue()
+        {
+            return m_scene.MonitorPhysicsSyncTime + m_scene.MonitorPhysicsUpdateTime;
+        }
+
+        public string GetName()
+        {
+            return "Total Physics Frame Time";
+        }
+
+        public string GetFriendlyValue()
+        {
+            return (int)GetValue() + "ms";
+        }
+
+        #endregion
+    }
+}
diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/PhysicsUpdateFrameMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/PhysicsUpdateFrameMonitor.cs
new file mode 100644
index 0000000..91ac282
--- /dev/null
+++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/PhysicsUpdateFrameMonitor.cs
@@ -0,0 +1,33 @@
+using OpenSim.Region.Framework.Scenes;
+
+namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors
+{
+    class PhysicsUpdateFrameMonitor : IMonitor
+    {
+        private readonly Scene m_scene;
+
+        public PhysicsUpdateFrameMonitor(Scene scene)
+        {
+            m_scene = scene;
+        }
+
+        #region Implementation of IMonitor
+
+        public double GetValue()
+        {
+            return m_scene.MonitorPhysicsUpdateTime;
+        }
+
+        public string GetName()
+        {
+            return "Physics Update Frame Time";
+        }
+
+        public string GetFriendlyValue()
+        {
+            return (int)GetValue() + "ms";
+        }
+
+        #endregion
+    }
+}
diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/ThreadCountMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/ThreadCountMonitor.cs
new file mode 100644
index 0000000..9300a93
--- /dev/null
+++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/ThreadCountMonitor.cs
@@ -0,0 +1,25 @@
+
+namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors
+{
+    class ThreadCountMonitor : IMonitor
+    {
+        #region Implementation of IMonitor
+
+        public double GetValue()
+        {
+            return System.Diagnostics.Process.GetCurrentProcess().Threads.Count;
+        }
+
+        public string GetName()
+        {
+            return "Total Threads";
+        }
+
+        public string GetFriendlyValue()
+        {
+            return (int)GetValue() + " Thread(s) (Global)";
+        }
+
+        #endregion
+    }
+}
diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/TotalFrameMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/TotalFrameMonitor.cs
new file mode 100644
index 0000000..dea1f94
--- /dev/null
+++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/TotalFrameMonitor.cs
@@ -0,0 +1,33 @@
+using OpenSim.Region.Framework.Scenes;
+
+namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors
+{
+    class TotalFrameMonitor : IMonitor
+    {
+        private readonly Scene m_scene;
+
+        public TotalFrameMonitor(Scene scene)
+        {
+            m_scene = scene;
+        }
+
+        #region Implementation of IMonitor
+
+        public double GetValue()
+        {
+            return m_scene.MonitorFrameTime;
+        }
+
+        public string GetName()
+        {
+            return "Total Frame Time";
+        }
+
+        public string GetFriendlyValue()
+        {
+            return (int)GetValue() + "ms";
+        }
+
+        #endregion
+    }
+}
diff --git a/OpenSim/Region/Framework/Scenes/Scene.cs b/OpenSim/Region/Framework/Scenes/Scene.cs
index 78ccb55..07fdc9f 100644
--- a/OpenSim/Region/Framework/Scenes/Scene.cs
+++ b/OpenSim/Region/Framework/Scenes/Scene.cs
@@ -274,6 +274,21 @@ namespace OpenSim.Region.Framework.Scenes
         private int physicsMS2;
         private int physicsMS;
         private int otherMS;
+        private int tempOnRezMS;
+        private int eventMS;
+        private int backupMS;
+        private int terrainMS;
+        private int landMS;
+
+        public int MonitorFrameTime { get { return frameMS; } }
+        public int MonitorPhysicsUpdateTime { get { return physicsMS; } }
+        public int MonitorPhysicsSyncTime { get { return physicsMS2; } }
+        public int MonitorOtherTime { get { return otherMS; } }
+        public int MonitorTempOnRezTime { get { return tempOnRezMS; } }
+        public int MonitorEventTime { get { return eventMS; } } // This may need to be divided into each event?
+        public int MonitorBackupTime { get { return backupMS; } }
+        public int MonitorTerrainTime { get { return terrainMS; } }
+        public int MonitorLandTime { get { return landMS; } }
 
         private bool m_physics_enabled = true;
         private bool m_scripts_enabled = true;
@@ -1026,7 +1041,8 @@ namespace OpenSim.Region.Framework.Scenes
                 TimeSpan SinceLastFrame = DateTime.UtcNow - m_lastupdate;
                 physicsFPS = 0f;
 
-                maintc = maintc = frameMS = otherMS = Environment.TickCount;
+                maintc = maintc = otherMS = Environment.TickCount;
+                int tmpFrameMS = maintc;
 
                 // Increment the frame counter
                 ++m_frame;
@@ -1046,15 +1062,16 @@ namespace OpenSim.Region.Framework.Scenes
                     if (m_frame % m_update_presences == 0)
                         m_sceneGraph.UpdatePresences();
 
-                    physicsMS2 = Environment.TickCount;
+                    int TempPhysicsMS2 = Environment.TickCount;
                     if ((m_frame % m_update_physics == 0) && m_physics_enabled)
                         m_sceneGraph.UpdatePreparePhysics();
-                    physicsMS2 = Environment.TickCount - physicsMS2;
+                    TempPhysicsMS2 = Environment.TickCount - TempPhysicsMS2;
+                    physicsMS2 = TempPhysicsMS2;
 
                     if (m_frame % m_update_entitymovement == 0)
                         m_sceneGraph.UpdateScenePresenceMovement();
 
-                    physicsMS = Environment.TickCount;
+                    int TempPhysicsMS = Environment.TickCount;
                     if (m_frame % m_update_physics == 0)
                     {
                         if (m_physics_enabled)
@@ -1062,30 +1079,56 @@ namespace OpenSim.Region.Framework.Scenes
                         if (SynchronizeScene != null)
                             SynchronizeScene(this);
                     }
-                    physicsMS = Environment.TickCount - physicsMS;
-                    physicsMS += physicsMS2;
+                    TempPhysicsMS = Environment.TickCount - TempPhysicsMS;
+                    physicsMS = TempPhysicsMS;
 
                     // Delete temp-on-rez stuff
                     if (m_frame % m_update_backup == 0)
+                    {
+                        int tozMS = Environment.TickCount;
                         CleanTempObjects();
+                        tozMS -= Environment.TickCount;
+                        tempOnRezMS = tozMS;
+                    }
 
                     if (RegionStatus != RegionStatus.SlaveScene)
                     {
                         if (m_frame % m_update_events == 0)
+                        {
+                            int evMS = Environment.TickCount;
                             UpdateEvents();
+                            evMS -= Environment.TickCount;
+                            eventMS = evMS;
+                        }
 
                         if (m_frame % m_update_backup == 0)
+                        {
+                            int backMS = Environment.TickCount;
                             UpdateStorageBackup();
+                            backMS -= Environment.TickCount;
+                            backupMS = backMS;
+                        }
 
                         if (m_frame % m_update_terrain == 0)
+                        {
+                            int terMS = Environment.TickCount;
                             UpdateTerrain();
+                            terMS -= Environment.TickCount;
+                            terrainMS = terMS;
+                        }
 
                         if (m_frame % m_update_land == 0)
+                        {
+                            int ldMS = Environment.TickCount;
                             UpdateLand();
+                            ldMS -= Environment.TickCount;
+                            landMS = ldMS;
+                        }
 
                         int tickCount = Environment.TickCount;
                         otherMS = tickCount - otherMS;
-                        frameMS = tickCount - frameMS;
+                        tmpFrameMS -= tickCount;
+                        frameMS = tmpFrameMS;
 
                         // if (m_frame%m_update_avatars == 0)
                         //   UpdateInWorldTime();
@@ -1097,7 +1140,7 @@ namespace OpenSim.Region.Framework.Scenes
                         StatsReporter.SetObjects(m_sceneGraph.GetTotalObjectsCount());
                         StatsReporter.SetActiveObjects(m_sceneGraph.GetActiveObjectsCount());
                         StatsReporter.addFrameMS(frameMS);
-                        StatsReporter.addPhysicsMS(physicsMS);
+                        StatsReporter.addPhysicsMS(physicsMS + physicsMS2);
                         StatsReporter.addOtherMS(otherMS);
                         StatsReporter.SetActiveScripts(m_sceneGraph.GetActiveScriptsCount());
                         StatsReporter.addScriptLines(m_sceneGraph.GetScriptLPS());