From 711dde34e4e5da954a58393e1a177e8c6969b8b5 Mon Sep 17 00:00:00 2001 From: Adam Frisby Date: Sun, 1 Nov 2009 19:37:40 +1100 Subject: * Implements new 'Monitoring' system for reporting performance. * Mostly the same set as the StatsMonitor used for Viewer notification, but exposes some new frametimes - including EventMS, PhysicsUpdateMS, LandUpdateMS; new memory monitoring - both GC.TotalMemory and Process.PrivateWorkingMemory64; also exposes ThreadCount (using System.Diagnostics.Process) * Type 'monitor report' on the console to see output. * SNMP Implementation forthcoming. --- .../CoreModules/Framework/Monitoring/IMonitor.cs | 9 +++ .../Framework/Monitoring/MonitorModule.cs | 70 ++++++++++++++++++++++ .../Monitoring/Monitors/AgentCountMonitor.cs | 33 ++++++++++ .../Monitoring/Monitors/ChildAgentCountMonitor.cs | 33 ++++++++++ .../Monitoring/Monitors/EventFrameMonitor.cs | 33 ++++++++++ .../Monitoring/Monitors/GCMemoryMonitor.cs | 26 ++++++++ .../Monitoring/Monitors/LandFrameMonitor.cs | 33 ++++++++++ .../Monitoring/Monitors/ObjectCountMonitor.cs | 33 ++++++++++ .../Monitoring/Monitors/PWSMemoryMonitor.cs | 26 ++++++++ .../Monitoring/Monitors/PhysicsFrameMonitor.cs | 33 ++++++++++ .../Monitors/PhysicsUpdateFrameMonitor.cs | 33 ++++++++++ .../Monitoring/Monitors/ThreadCountMonitor.cs | 25 ++++++++ .../Monitoring/Monitors/TotalFrameMonitor.cs | 33 ++++++++++ 13 files changed, 420 insertions(+) create mode 100644 OpenSim/Region/CoreModules/Framework/Monitoring/IMonitor.cs create mode 100644 OpenSim/Region/CoreModules/Framework/Monitoring/MonitorModule.cs create mode 100644 OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/AgentCountMonitor.cs create mode 100644 OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/ChildAgentCountMonitor.cs create mode 100644 OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/EventFrameMonitor.cs create mode 100644 OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/GCMemoryMonitor.cs create mode 100644 OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/LandFrameMonitor.cs create mode 100644 OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/ObjectCountMonitor.cs create mode 100644 OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/PWSMemoryMonitor.cs create mode 100644 OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/PhysicsFrameMonitor.cs create mode 100644 OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/PhysicsUpdateFrameMonitor.cs create mode 100644 OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/ThreadCountMonitor.cs create mode 100644 OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/TotalFrameMonitor.cs (limited to 'OpenSim/Region/CoreModules/Framework') diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/IMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/IMonitor.cs new file mode 100644 index 0000000..a51dccd --- /dev/null +++ b/OpenSim/Region/CoreModules/Framework/Monitoring/IMonitor.cs @@ -0,0 +1,9 @@ +namespace OpenSim.Region.CoreModules.Framework.Monitoring +{ + interface IMonitor + { + double GetValue(); + string GetName(); + string GetFriendlyValue(); // Convert to readable numbers + } +} diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/MonitorModule.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/MonitorModule.cs new file mode 100644 index 0000000..d4a7692 --- /dev/null +++ b/OpenSim/Region/CoreModules/Framework/Monitoring/MonitorModule.cs @@ -0,0 +1,70 @@ +using System.Collections.Generic; +using System.Reflection; +using log4net; +using Nini.Config; +using OpenSim.Region.CoreModules.Framework.Monitoring.Monitors; +using OpenSim.Region.Framework.Interfaces; +using OpenSim.Region.Framework.Scenes; + +namespace OpenSim.Region.CoreModules.Framework.Monitoring +{ + public class MonitorModule : IRegionModule + { + private Scene m_scene; + private readonly List m_monitors = new List(); + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + public void DebugMonitors(string module, string[] args) + { + foreach (IMonitor monitor in m_monitors) + { + m_log.Info("[MonitorModule] " + m_scene.RegionInfo.RegionName + " reports " + monitor.GetName() + " = " + monitor.GetValue()); + } + } + + #region Implementation of IRegionModule + + public void Initialise(Scene scene, IConfigSource source) + { + m_scene = scene; + + + m_scene.AddCommand(this, "monitor report", + "monitor report", + "Returns a variety of statistics about the current region and/or simulator", + DebugMonitors); + } + + public void PostInitialise() + { + m_monitors.Add(new AgentCountMonitor(m_scene)); + m_monitors.Add(new ChildAgentCountMonitor(m_scene)); + m_monitors.Add(new GCMemoryMonitor()); + m_monitors.Add(new ObjectCountMonitor(m_scene)); + m_monitors.Add(new PhysicsFrameMonitor(m_scene)); + m_monitors.Add(new PhysicsUpdateFrameMonitor(m_scene)); + m_monitors.Add(new PWSMemoryMonitor()); + m_monitors.Add(new ThreadCountMonitor()); + m_monitors.Add(new TotalFrameMonitor(m_scene)); + m_monitors.Add(new EventFrameMonitor(m_scene)); + m_monitors.Add(new LandFrameMonitor(m_scene)); + } + + public void Close() + { + + } + + public string Name + { + get { return "Region Health Monitoring Module"; } + } + + public bool IsSharedModule + { + get { return false; } + } + + #endregion + } +} diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/AgentCountMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/AgentCountMonitor.cs new file mode 100644 index 0000000..edc6e6b --- /dev/null +++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/AgentCountMonitor.cs @@ -0,0 +1,33 @@ +using OpenSim.Region.Framework.Scenes; + +namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors +{ + class AgentCountMonitor : IMonitor + { + private readonly Scene m_scene; + + public AgentCountMonitor(Scene scene) + { + m_scene = scene; + } + + #region Implementation of IMonitor + + public double GetValue() + { + return m_scene.SceneGraph.GetRootAgentCount(); + } + + public string GetName() + { + return "Root Agent Count"; + } + + public string GetFriendlyValue() + { + return (int)GetValue() + " agent(s)"; + } + + #endregion + } +} diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/ChildAgentCountMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/ChildAgentCountMonitor.cs new file mode 100644 index 0000000..afe6b79 --- /dev/null +++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/ChildAgentCountMonitor.cs @@ -0,0 +1,33 @@ +using OpenSim.Region.Framework.Scenes; + +namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors +{ + class ChildAgentCountMonitor : IMonitor + { + private readonly Scene m_scene; + + public ChildAgentCountMonitor(Scene scene) + { + m_scene = scene; + } + + #region Implementation of IMonitor + + public double GetValue() + { + return m_scene.SceneGraph.GetChildAgentCount(); + } + + public string GetName() + { + return "Child Agent Count"; + } + + public string GetFriendlyValue() + { + return (int)GetValue() + " child agent(s)"; + } + + #endregion + } +} diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/EventFrameMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/EventFrameMonitor.cs new file mode 100644 index 0000000..dec5a9e --- /dev/null +++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/EventFrameMonitor.cs @@ -0,0 +1,33 @@ +using OpenSim.Region.Framework.Scenes; + +namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors +{ + class EventFrameMonitor : IMonitor + { + private readonly Scene m_scene; + + public EventFrameMonitor(Scene scene) + { + m_scene = scene; + } + + #region Implementation of IMonitor + + public double GetValue() + { + return m_scene.MonitorEventTime; + } + + public string GetName() + { + return "Total Event Frame Time"; + } + + public string GetFriendlyValue() + { + return (int)GetValue() + "ms"; + } + + #endregion + } +} diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/GCMemoryMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/GCMemoryMonitor.cs new file mode 100644 index 0000000..cd67fea --- /dev/null +++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/GCMemoryMonitor.cs @@ -0,0 +1,26 @@ +using System; + +namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors +{ + class GCMemoryMonitor : IMonitor + { + #region Implementation of IMonitor + + public double GetValue() + { + return GC.GetTotalMemory(false); + } + + public string GetName() + { + return "GC Reported Memory"; + } + + public string GetFriendlyValue() + { + return (int)(GetValue() / (1024*1024)) + "MB (Global)"; + } + + #endregion + } +} diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/LandFrameMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/LandFrameMonitor.cs new file mode 100644 index 0000000..d883fc7 --- /dev/null +++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/LandFrameMonitor.cs @@ -0,0 +1,33 @@ +using OpenSim.Region.Framework.Scenes; + +namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors +{ + class LandFrameMonitor : IMonitor + { + private readonly Scene m_scene; + + public LandFrameMonitor(Scene scene) + { + m_scene = scene; + } + + #region Implementation of IMonitor + + public double GetValue() + { + return m_scene.MonitorLandTime; + } + + public string GetName() + { + return "Land Frame Time"; + } + + public string GetFriendlyValue() + { + return (int)GetValue() + "ms"; + } + + #endregion + } +} diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/ObjectCountMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/ObjectCountMonitor.cs new file mode 100644 index 0000000..dd9b19d --- /dev/null +++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/ObjectCountMonitor.cs @@ -0,0 +1,33 @@ +using OpenSim.Region.Framework.Scenes; + +namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors +{ + class ObjectCountMonitor : IMonitor + { + private readonly Scene m_scene; + + public ObjectCountMonitor(Scene scene) + { + m_scene = scene; + } + + #region Implementation of IMonitor + + public double GetValue() + { + return m_scene.SceneGraph.GetTotalObjectsCount(); + } + + public string GetName() + { + return "Total Objects Count"; + } + + public string GetFriendlyValue() + { + return (int)GetValue() + " Object(s)"; + } + + #endregion + } +} diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/PWSMemoryMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/PWSMemoryMonitor.cs new file mode 100644 index 0000000..88f2938 --- /dev/null +++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/PWSMemoryMonitor.cs @@ -0,0 +1,26 @@ +using System; + +namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors +{ + class PWSMemoryMonitor : IMonitor + { + #region Implementation of IMonitor + + public double GetValue() + { + return System.Diagnostics.Process.GetCurrentProcess().PrivateMemorySize64; + } + + public string GetName() + { + return "Private Working Set Memory"; + } + + public string GetFriendlyValue() + { + return (int)(GetValue() / (1024 * 1024)) + "MB (Global)"; + } + + #endregion + } +} diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/PhysicsFrameMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/PhysicsFrameMonitor.cs new file mode 100644 index 0000000..4d62e4f --- /dev/null +++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/PhysicsFrameMonitor.cs @@ -0,0 +1,33 @@ +using OpenSim.Region.Framework.Scenes; + +namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors +{ + class PhysicsFrameMonitor : IMonitor + { + private readonly Scene m_scene; + + public PhysicsFrameMonitor(Scene scene) + { + m_scene = scene; + } + + #region Implementation of IMonitor + + public double GetValue() + { + return m_scene.MonitorPhysicsSyncTime + m_scene.MonitorPhysicsUpdateTime; + } + + public string GetName() + { + return "Total Physics Frame Time"; + } + + public string GetFriendlyValue() + { + return (int)GetValue() + "ms"; + } + + #endregion + } +} diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/PhysicsUpdateFrameMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/PhysicsUpdateFrameMonitor.cs new file mode 100644 index 0000000..91ac282 --- /dev/null +++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/PhysicsUpdateFrameMonitor.cs @@ -0,0 +1,33 @@ +using OpenSim.Region.Framework.Scenes; + +namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors +{ + class PhysicsUpdateFrameMonitor : IMonitor + { + private readonly Scene m_scene; + + public PhysicsUpdateFrameMonitor(Scene scene) + { + m_scene = scene; + } + + #region Implementation of IMonitor + + public double GetValue() + { + return m_scene.MonitorPhysicsUpdateTime; + } + + public string GetName() + { + return "Physics Update Frame Time"; + } + + public string GetFriendlyValue() + { + return (int)GetValue() + "ms"; + } + + #endregion + } +} diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/ThreadCountMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/ThreadCountMonitor.cs new file mode 100644 index 0000000..9300a93 --- /dev/null +++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/ThreadCountMonitor.cs @@ -0,0 +1,25 @@ + +namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors +{ + class ThreadCountMonitor : IMonitor + { + #region Implementation of IMonitor + + public double GetValue() + { + return System.Diagnostics.Process.GetCurrentProcess().Threads.Count; + } + + public string GetName() + { + return "Total Threads"; + } + + public string GetFriendlyValue() + { + return (int)GetValue() + " Thread(s) (Global)"; + } + + #endregion + } +} diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/TotalFrameMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/TotalFrameMonitor.cs new file mode 100644 index 0000000..dea1f94 --- /dev/null +++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/TotalFrameMonitor.cs @@ -0,0 +1,33 @@ +using OpenSim.Region.Framework.Scenes; + +namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors +{ + class TotalFrameMonitor : IMonitor + { + private readonly Scene m_scene; + + public TotalFrameMonitor(Scene scene) + { + m_scene = scene; + } + + #region Implementation of IMonitor + + public double GetValue() + { + return m_scene.MonitorFrameTime; + } + + public string GetName() + { + return "Total Frame Time"; + } + + public string GetFriendlyValue() + { + return (int)GetValue() + "ms"; + } + + #endregion + } +} -- cgit v1.1 From 838bc80ab9273c2834794535886a86c7574bb0d3 Mon Sep 17 00:00:00 2001 From: Adam Frisby Date: Mon, 2 Nov 2009 00:05:49 +1100 Subject: * Implemented some tweaks to monitoring module. * Output is prettier & more useful. * Added 'Alerts' to allow rules to be constructed using Monitors to detect for events such as deadlocks. This will be translated to SNMP Traps when I get SNMP implemented. --- .../Framework/Monitoring/Alerts/DeadlockAlert.cs | 37 ++++++++++++++++++++++ .../Framework/Monitoring/MonitorModule.cs | 25 ++++++++++++++- .../Monitoring/Monitors/LastFrameTimeMonitor.cs | 34 ++++++++++++++++++++ 3 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 OpenSim/Region/CoreModules/Framework/Monitoring/Alerts/DeadlockAlert.cs create mode 100644 OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/LastFrameTimeMonitor.cs (limited to 'OpenSim/Region/CoreModules/Framework') diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Alerts/DeadlockAlert.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Alerts/DeadlockAlert.cs new file mode 100644 index 0000000..b546ccb --- /dev/null +++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Alerts/DeadlockAlert.cs @@ -0,0 +1,37 @@ +using OpenSim.Region.CoreModules.Framework.Monitoring.Monitors; + +namespace OpenSim.Region.CoreModules.Framework.Monitoring.Alerts +{ + class DeadlockAlert : IAlert + { + private LastFrameTimeMonitor m_monitor; + + public DeadlockAlert(LastFrameTimeMonitor m_monitor) + { + this.m_monitor = m_monitor; + } + + #region Implementation of IAlert + + public string GetName() + { + return "Potential Deadlock Alert"; + } + + public void Test() + { + if (m_monitor.GetValue() > 60 * 1000) + { + if(OnTriggerAlert != null) + { + OnTriggerAlert(typeof (DeadlockAlert), + (int) (m_monitor.GetValue()/1000) + " second(s) since last frame processed.", true); + } + } + } + + public event Alert OnTriggerAlert; + + #endregion + } +} diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/MonitorModule.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/MonitorModule.cs index d4a7692..769af8d 100644 --- a/OpenSim/Region/CoreModules/Framework/Monitoring/MonitorModule.cs +++ b/OpenSim/Region/CoreModules/Framework/Monitoring/MonitorModule.cs @@ -2,6 +2,7 @@ using System.Reflection; using log4net; using Nini.Config; +using OpenSim.Region.CoreModules.Framework.Monitoring.Alerts; using OpenSim.Region.CoreModules.Framework.Monitoring.Monitors; using OpenSim.Region.Framework.Interfaces; using OpenSim.Region.Framework.Scenes; @@ -12,13 +13,22 @@ namespace OpenSim.Region.CoreModules.Framework.Monitoring { private Scene m_scene; private readonly List m_monitors = new List(); + private readonly List m_alerts = new List(); private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); public void DebugMonitors(string module, string[] args) { foreach (IMonitor monitor in m_monitors) { - m_log.Info("[MonitorModule] " + m_scene.RegionInfo.RegionName + " reports " + monitor.GetName() + " = " + monitor.GetValue()); + m_log.Info("[MonitorModule] " + m_scene.RegionInfo.RegionName + " reports " + monitor.GetName() + " = " + monitor.GetFriendlyValue()); + } + } + + public void TestAlerts() + { + foreach (IAlert alert in m_alerts) + { + alert.Test(); } } @@ -48,6 +58,19 @@ namespace OpenSim.Region.CoreModules.Framework.Monitoring m_monitors.Add(new TotalFrameMonitor(m_scene)); m_monitors.Add(new EventFrameMonitor(m_scene)); m_monitors.Add(new LandFrameMonitor(m_scene)); + m_monitors.Add(new LastFrameTimeMonitor(m_scene)); + + m_alerts.Add(new DeadlockAlert(m_monitors.Find(x => x is LastFrameTimeMonitor) as LastFrameTimeMonitor)); + + foreach (IAlert alert in m_alerts) + { + alert.OnTriggerAlert += OnTriggerAlert; + } + } + + void OnTriggerAlert(System.Type reporter, string reason, bool fatal) + { + m_log.Error("[Monitor] " + reporter.Name + " for " + m_scene.RegionInfo.RegionName + " reports " + reason + " (Fatal: " + fatal + ")"); } public void Close() diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/LastFrameTimeMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/LastFrameTimeMonitor.cs new file mode 100644 index 0000000..36363f8 --- /dev/null +++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/LastFrameTimeMonitor.cs @@ -0,0 +1,34 @@ +using System; +using OpenSim.Region.Framework.Scenes; + +namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors +{ + class LastFrameTimeMonitor : IMonitor + { + private readonly Scene m_scene; + + public LastFrameTimeMonitor(Scene scene) + { + m_scene = scene; + } + + #region Implementation of IMonitor + + public double GetValue() + { + return Environment.TickCount - m_scene.MonitorLastFrameTick; + } + + public string GetName() + { + return "Last Completed Frame At"; + } + + public string GetFriendlyValue() + { + return (int)GetValue() + "ms ago"; + } + + #endregion + } +} -- cgit v1.1 From 3c0881c83c18489ca1887658adfa8a61c5fdd806 Mon Sep 17 00:00:00 2001 From: Adam Frisby Date: Mon, 2 Nov 2009 00:20:41 +1100 Subject: * Committing missing file from last commit. --- OpenSim/Region/CoreModules/Framework/Monitoring/IAlert.cs | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 OpenSim/Region/CoreModules/Framework/Monitoring/IAlert.cs (limited to 'OpenSim/Region/CoreModules/Framework') diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/IAlert.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/IAlert.cs new file mode 100644 index 0000000..b533df9 --- /dev/null +++ b/OpenSim/Region/CoreModules/Framework/Monitoring/IAlert.cs @@ -0,0 +1,13 @@ +using System; + +namespace OpenSim.Region.CoreModules.Framework.Monitoring +{ + internal delegate void Alert(Type reporter, string reason, bool fatal); + + interface IAlert + { + string GetName(); + void Test(); + event Alert OnTriggerAlert; + } +} -- cgit v1.1