From 838bc80ab9273c2834794535886a86c7574bb0d3 Mon Sep 17 00:00:00 2001
From: Adam Frisby
Date: Mon, 2 Nov 2009 00:05:49 +1100
Subject: * Implemented some tweaks to monitoring module. * Output is prettier
& more useful. * Added 'Alerts' to allow rules to be constructed using
Monitors to detect for events such as deadlocks. This will be translated to
SNMP Traps when I get SNMP implemented.
---
.../Framework/Monitoring/Alerts/DeadlockAlert.cs | 37 ++++++++++++++++++++++
.../Framework/Monitoring/MonitorModule.cs | 25 ++++++++++++++-
.../Monitoring/Monitors/LastFrameTimeMonitor.cs | 34 ++++++++++++++++++++
OpenSim/Region/Framework/Scenes/Scene.cs | 3 ++
4 files changed, 98 insertions(+), 1 deletion(-)
create mode 100644 OpenSim/Region/CoreModules/Framework/Monitoring/Alerts/DeadlockAlert.cs
create mode 100644 OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/LastFrameTimeMonitor.cs
diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Alerts/DeadlockAlert.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Alerts/DeadlockAlert.cs
new file mode 100644
index 0000000..b546ccb
--- /dev/null
+++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Alerts/DeadlockAlert.cs
@@ -0,0 +1,37 @@
+using OpenSim.Region.CoreModules.Framework.Monitoring.Monitors;
+
+namespace OpenSim.Region.CoreModules.Framework.Monitoring.Alerts
+{
+ class DeadlockAlert : IAlert
+ {
+ private LastFrameTimeMonitor m_monitor;
+
+ public DeadlockAlert(LastFrameTimeMonitor m_monitor)
+ {
+ this.m_monitor = m_monitor;
+ }
+
+ #region Implementation of IAlert
+
+ public string GetName()
+ {
+ return "Potential Deadlock Alert";
+ }
+
+ public void Test()
+ {
+ if (m_monitor.GetValue() > 60 * 1000)
+ {
+ if(OnTriggerAlert != null)
+ {
+ OnTriggerAlert(typeof (DeadlockAlert),
+ (int) (m_monitor.GetValue()/1000) + " second(s) since last frame processed.", true);
+ }
+ }
+ }
+
+ public event Alert OnTriggerAlert;
+
+ #endregion
+ }
+}
diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/MonitorModule.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/MonitorModule.cs
index d4a7692..769af8d 100644
--- a/OpenSim/Region/CoreModules/Framework/Monitoring/MonitorModule.cs
+++ b/OpenSim/Region/CoreModules/Framework/Monitoring/MonitorModule.cs
@@ -2,6 +2,7 @@
using System.Reflection;
using log4net;
using Nini.Config;
+using OpenSim.Region.CoreModules.Framework.Monitoring.Alerts;
using OpenSim.Region.CoreModules.Framework.Monitoring.Monitors;
using OpenSim.Region.Framework.Interfaces;
using OpenSim.Region.Framework.Scenes;
@@ -12,13 +13,22 @@ namespace OpenSim.Region.CoreModules.Framework.Monitoring
{
private Scene m_scene;
private readonly List<IMonitor> m_monitors = new List<IMonitor>();
+ private readonly List<IAlert> m_alerts = new List<IAlert>();
private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
public void DebugMonitors(string module, string[] args)
{
foreach (IMonitor monitor in m_monitors)
{
- m_log.Info("[MonitorModule] " + m_scene.RegionInfo.RegionName + " reports " + monitor.GetName() + " = " + monitor.GetValue());
+ m_log.Info("[MonitorModule] " + m_scene.RegionInfo.RegionName + " reports " + monitor.GetName() + " = " + monitor.GetFriendlyValue());
+ }
+ }
+
+ public void TestAlerts()
+ {
+ foreach (IAlert alert in m_alerts)
+ {
+ alert.Test();
}
}
@@ -48,6 +58,19 @@ namespace OpenSim.Region.CoreModules.Framework.Monitoring
m_monitors.Add(new TotalFrameMonitor(m_scene));
m_monitors.Add(new EventFrameMonitor(m_scene));
m_monitors.Add(new LandFrameMonitor(m_scene));
+ m_monitors.Add(new LastFrameTimeMonitor(m_scene));
+
+ m_alerts.Add(new DeadlockAlert(m_monitors.Find(x => x is LastFrameTimeMonitor) as LastFrameTimeMonitor));
+
+ foreach (IAlert alert in m_alerts)
+ {
+ alert.OnTriggerAlert += OnTriggerAlert;
+ }
+ }
+
+ void OnTriggerAlert(System.Type reporter, string reason, bool fatal)
+ {
+ m_log.Error("[Monitor] " + reporter.Name + " for " + m_scene.RegionInfo.RegionName + " reports " + reason + " (Fatal: " + fatal + ")");
}
public void Close()
diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/LastFrameTimeMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/LastFrameTimeMonitor.cs
new file mode 100644
index 0000000..36363f8
--- /dev/null
+++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/LastFrameTimeMonitor.cs
@@ -0,0 +1,34 @@
+using System;
+using OpenSim.Region.Framework.Scenes;
+
+namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors
+{
+ class LastFrameTimeMonitor : IMonitor
+ {
+ private readonly Scene m_scene;
+
+ public LastFrameTimeMonitor(Scene scene)
+ {
+ m_scene = scene;
+ }
+
+ #region Implementation of IMonitor
+
+ public double GetValue()
+ {
+ return Environment.TickCount - m_scene.MonitorLastFrameTick;
+ }
+
+ public string GetName()
+ {
+ return "Last Completed Frame At";
+ }
+
+ public string GetFriendlyValue()
+ {
+ return (int)GetValue() + "ms ago";
+ }
+
+ #endregion
+ }
+}
diff --git a/OpenSim/Region/Framework/Scenes/Scene.cs b/OpenSim/Region/Framework/Scenes/Scene.cs
index 07fdc9f..1e7803f 100644
--- a/OpenSim/Region/Framework/Scenes/Scene.cs
+++ b/OpenSim/Region/Framework/Scenes/Scene.cs
@@ -279,6 +279,7 @@ namespace OpenSim.Region.Framework.Scenes
private int backupMS;
private int terrainMS;
private int landMS;
+ private int lastCompletedFrame;
public int MonitorFrameTime { get { return frameMS; } }
public int MonitorPhysicsUpdateTime { get { return physicsMS; } }
@@ -289,6 +290,7 @@ namespace OpenSim.Region.Framework.Scenes
public int MonitorBackupTime { get { return backupMS; } }
public int MonitorTerrainTime { get { return terrainMS; } }
public int MonitorLandTime { get { return landMS; } }
+ public int MonitorLastFrameTick { get { return lastCompletedFrame; } }
private bool m_physics_enabled = true;
private bool m_scripts_enabled = true;
@@ -1129,6 +1131,7 @@ namespace OpenSim.Region.Framework.Scenes
otherMS = tickCount - otherMS;
tmpFrameMS -= tickCount;
frameMS = tmpFrameMS;
+ lastCompletedFrame = tickCount;
// if (m_frame%m_update_avatars == 0)
// UpdateInWorldTime();
--
cgit v1.1