From 838bc80ab9273c2834794535886a86c7574bb0d3 Mon Sep 17 00:00:00 2001 From: Adam Frisby Date: Mon, 2 Nov 2009 00:05:49 +1100 Subject: * Implemented some tweaks to monitoring module. * Output is prettier & more useful. * Added 'Alerts' to allow rules to be constructed using Monitors to detect for events such as deadlocks. This will be translated to SNMP Traps when I get SNMP implemented. --- .../Framework/Monitoring/Alerts/DeadlockAlert.cs | 37 ++++++++++++++++++++++ .../Framework/Monitoring/MonitorModule.cs | 25 ++++++++++++++- .../Monitoring/Monitors/LastFrameTimeMonitor.cs | 34 ++++++++++++++++++++ OpenSim/Region/Framework/Scenes/Scene.cs | 3 ++ 4 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 OpenSim/Region/CoreModules/Framework/Monitoring/Alerts/DeadlockAlert.cs create mode 100644 OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/LastFrameTimeMonitor.cs (limited to 'OpenSim') diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Alerts/DeadlockAlert.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Alerts/DeadlockAlert.cs new file mode 100644 index 0000000..b546ccb --- /dev/null +++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Alerts/DeadlockAlert.cs @@ -0,0 +1,37 @@ +using OpenSim.Region.CoreModules.Framework.Monitoring.Monitors; + +namespace OpenSim.Region.CoreModules.Framework.Monitoring.Alerts +{ + class DeadlockAlert : IAlert + { + private LastFrameTimeMonitor m_monitor; + + public DeadlockAlert(LastFrameTimeMonitor m_monitor) + { + this.m_monitor = m_monitor; + } + + #region Implementation of IAlert + + public string GetName() + { + return "Potential Deadlock Alert"; + } + + public void Test() + { + if (m_monitor.GetValue() > 60 * 1000) + { + if(OnTriggerAlert != null) + { + OnTriggerAlert(typeof (DeadlockAlert), + (int) (m_monitor.GetValue()/1000) + " second(s) since last frame processed.", true); + } + } + } + + public event Alert OnTriggerAlert; + + #endregion + } +} diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/MonitorModule.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/MonitorModule.cs index d4a7692..769af8d 100644 --- a/OpenSim/Region/CoreModules/Framework/Monitoring/MonitorModule.cs +++ b/OpenSim/Region/CoreModules/Framework/Monitoring/MonitorModule.cs @@ -2,6 +2,7 @@ using System.Reflection; using log4net; using Nini.Config; +using OpenSim.Region.CoreModules.Framework.Monitoring.Alerts; using OpenSim.Region.CoreModules.Framework.Monitoring.Monitors; using OpenSim.Region.Framework.Interfaces; using OpenSim.Region.Framework.Scenes; @@ -12,13 +13,22 @@ namespace OpenSim.Region.CoreModules.Framework.Monitoring { private Scene m_scene; private readonly List m_monitors = new List(); + private readonly List m_alerts = new List(); private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); public void DebugMonitors(string module, string[] args) { foreach (IMonitor monitor in m_monitors) { - m_log.Info("[MonitorModule] " + m_scene.RegionInfo.RegionName + " reports " + monitor.GetName() + " = " + monitor.GetValue()); + m_log.Info("[MonitorModule] " + m_scene.RegionInfo.RegionName + " reports " + monitor.GetName() + " = " + monitor.GetFriendlyValue()); + } + } + + public void TestAlerts() + { + foreach (IAlert alert in m_alerts) + { + alert.Test(); } } @@ -48,6 +58,19 @@ namespace OpenSim.Region.CoreModules.Framework.Monitoring m_monitors.Add(new TotalFrameMonitor(m_scene)); m_monitors.Add(new EventFrameMonitor(m_scene)); m_monitors.Add(new LandFrameMonitor(m_scene)); + m_monitors.Add(new LastFrameTimeMonitor(m_scene)); + + m_alerts.Add(new DeadlockAlert(m_monitors.Find(x => x is LastFrameTimeMonitor) as LastFrameTimeMonitor)); + + foreach (IAlert alert in m_alerts) + { + alert.OnTriggerAlert += OnTriggerAlert; + } + } + + void OnTriggerAlert(System.Type reporter, string reason, bool fatal) + { + m_log.Error("[Monitor] " + reporter.Name + " for " + m_scene.RegionInfo.RegionName + " reports " + reason + " (Fatal: " + fatal + ")"); } public void Close() diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/LastFrameTimeMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/LastFrameTimeMonitor.cs new file mode 100644 index 0000000..36363f8 --- /dev/null +++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/LastFrameTimeMonitor.cs @@ -0,0 +1,34 @@ +using System; +using OpenSim.Region.Framework.Scenes; + +namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors +{ + class LastFrameTimeMonitor : IMonitor + { + private readonly Scene m_scene; + + public LastFrameTimeMonitor(Scene scene) + { + m_scene = scene; + } + + #region Implementation of IMonitor + + public double GetValue() + { + return Environment.TickCount - m_scene.MonitorLastFrameTick; + } + + public string GetName() + { + return "Last Completed Frame At"; + } + + public string GetFriendlyValue() + { + return (int)GetValue() + "ms ago"; + } + + #endregion + } +} diff --git a/OpenSim/Region/Framework/Scenes/Scene.cs b/OpenSim/Region/Framework/Scenes/Scene.cs index 07fdc9f..1e7803f 100644 --- a/OpenSim/Region/Framework/Scenes/Scene.cs +++ b/OpenSim/Region/Framework/Scenes/Scene.cs @@ -279,6 +279,7 @@ namespace OpenSim.Region.Framework.Scenes private int backupMS; private int terrainMS; private int landMS; + private int lastCompletedFrame; public int MonitorFrameTime { get { return frameMS; } } public int MonitorPhysicsUpdateTime { get { return physicsMS; } } @@ -289,6 +290,7 @@ namespace OpenSim.Region.Framework.Scenes public int MonitorBackupTime { get { return backupMS; } } public int MonitorTerrainTime { get { return terrainMS; } } public int MonitorLandTime { get { return landMS; } } + public int MonitorLastFrameTick { get { return lastCompletedFrame; } } private bool m_physics_enabled = true; private bool m_scripts_enabled = true; @@ -1129,6 +1131,7 @@ namespace OpenSim.Region.Framework.Scenes otherMS = tickCount - otherMS; tmpFrameMS -= tickCount; frameMS = tmpFrameMS; + lastCompletedFrame = tickCount; // if (m_frame%m_update_avatars == 0) // UpdateInWorldTime(); -- cgit v1.1