* Implemented some tweaks to monitoring module.

* Output is prettier & more useful.
* Added 'Alerts' to allow rules to be constructed using Monitors to detect for events such as deadlocks. This will be translated to SNMP Traps when I get SNMP implemented.

4 files changed, 98 insertions, 1 deletions

diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Alerts/DeadlockAlert.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Alerts/DeadlockAlert.cs
new file mode 100644
index 0000000..b546ccb
--- /dev/null
+++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Alerts/DeadlockAlert.cs
@@ -0,0 +1,37 @@
+using OpenSim.Region.CoreModules.Framework.Monitoring.Monitors;
+
+namespace OpenSim.Region.CoreModules.Framework.Monitoring.Alerts
+{
+    class DeadlockAlert : IAlert
+    {
+        private LastFrameTimeMonitor m_monitor;
+
+        public DeadlockAlert(LastFrameTimeMonitor m_monitor)
+        {
+            this.m_monitor = m_monitor;
+        }
+
+        #region Implementation of IAlert
+
+        public string GetName()
+        {
+            return "Potential Deadlock Alert";
+        }
+
+        public void Test()
+        {
+            if (m_monitor.GetValue() > 60 * 1000)
+            {
+                if(OnTriggerAlert != null)
+                {
+                    OnTriggerAlert(typeof (DeadlockAlert),
+                                   (int) (m_monitor.GetValue()/1000) + " second(s) since last frame processed.", true);
+                }
+            }
+        }
+
+        public event Alert OnTriggerAlert;
+
+        #endregion
+    }
+}
diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/MonitorModule.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/MonitorModule.cs
index d4a7692..769af8d 100644
--- a/OpenSim/Region/CoreModules/Framework/Monitoring/MonitorModule.cs
+++ b/OpenSim/Region/CoreModules/Framework/Monitoring/MonitorModule.cs
@@ -2,6 +2,7 @@
 using System.Reflection;
 using log4net;
 using Nini.Config;
+using OpenSim.Region.CoreModules.Framework.Monitoring.Alerts;
 using OpenSim.Region.CoreModules.Framework.Monitoring.Monitors;
 using OpenSim.Region.Framework.Interfaces;
 using OpenSim.Region.Framework.Scenes;
@@ -12,13 +13,22 @@ namespace OpenSim.Region.CoreModules.Framework.Monitoring
     {
         private Scene m_scene;
         private readonly List<IMonitor> m_monitors = new List<IMonitor>();
+        private readonly List<IAlert> m_alerts = new List<IAlert>();
         private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
 
         public void DebugMonitors(string module, string[] args)
         {
             foreach (IMonitor monitor in m_monitors)
             {
-                m_log.Info("[MonitorModule] " + m_scene.RegionInfo.RegionName + " reports " + monitor.GetName() + " = " + monitor.GetValue());
+                m_log.Info("[MonitorModule] " + m_scene.RegionInfo.RegionName + " reports " + monitor.GetName() + " = " + monitor.GetFriendlyValue());
+            }
+        }
+
+        public void TestAlerts()
+        {
+            foreach (IAlert alert in m_alerts)
+            {
+                alert.Test();
             }
         }
 
@@ -48,6 +58,19 @@ namespace OpenSim.Region.CoreModules.Framework.Monitoring
             m_monitors.Add(new TotalFrameMonitor(m_scene));
             m_monitors.Add(new EventFrameMonitor(m_scene));
             m_monitors.Add(new LandFrameMonitor(m_scene));
+            m_monitors.Add(new LastFrameTimeMonitor(m_scene));
+
+            m_alerts.Add(new DeadlockAlert(m_monitors.Find(x => x is LastFrameTimeMonitor) as LastFrameTimeMonitor));
+
+            foreach (IAlert alert in m_alerts)
+            {
+                alert.OnTriggerAlert += OnTriggerAlert;
+            }
+        }
+
+        void OnTriggerAlert(System.Type reporter, string reason, bool fatal)
+        {
+            m_log.Error("[Monitor] " + reporter.Name + " for " + m_scene.RegionInfo.RegionName + " reports " + reason + " (Fatal: " + fatal + ")");
         }
 
         public void Close()
diff --git a/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/LastFrameTimeMonitor.cs b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/LastFrameTimeMonitor.cs
new file mode 100644
index 0000000..36363f8
--- /dev/null
+++ b/OpenSim/Region/CoreModules/Framework/Monitoring/Monitors/LastFrameTimeMonitor.cs
@@ -0,0 +1,34 @@
+using System;
+using OpenSim.Region.Framework.Scenes;
+
+namespace OpenSim.Region.CoreModules.Framework.Monitoring.Monitors
+{
+    class LastFrameTimeMonitor : IMonitor
+    {
+        private readonly Scene m_scene;
+
+        public LastFrameTimeMonitor(Scene scene)
+        {
+            m_scene = scene;
+        }
+
+        #region Implementation of IMonitor
+
+        public double GetValue()
+        {
+            return Environment.TickCount - m_scene.MonitorLastFrameTick;
+        }
+
+        public string GetName()
+        {
+            return "Last Completed Frame At";
+        }
+
+        public string GetFriendlyValue()
+        {
+            return (int)GetValue() + "ms ago";
+        }
+
+        #endregion
+    }
+}
diff --git a/OpenSim/Region/Framework/Scenes/Scene.cs b/OpenSim/Region/Framework/Scenes/Scene.cs
index 07fdc9f..1e7803f 100644
--- a/OpenSim/Region/Framework/Scenes/Scene.cs
+++ b/OpenSim/Region/Framework/Scenes/Scene.cs
@@ -279,6 +279,7 @@ namespace OpenSim.Region.Framework.Scenes
         private int backupMS;
         private int terrainMS;
         private int landMS;
+        private int lastCompletedFrame;
 
         public int MonitorFrameTime { get { return frameMS; } }
         public int MonitorPhysicsUpdateTime { get { return physicsMS; } }
@@ -289,6 +290,7 @@ namespace OpenSim.Region.Framework.Scenes
         public int MonitorBackupTime { get { return backupMS; } }
         public int MonitorTerrainTime { get { return terrainMS; } }
         public int MonitorLandTime { get { return landMS; } }
+        public int MonitorLastFrameTick { get { return lastCompletedFrame; } }
 
         private bool m_physics_enabled = true;
         private bool m_scripts_enabled = true;
@@ -1129,6 +1131,7 @@ namespace OpenSim.Region.Framework.Scenes
                         otherMS = tickCount - otherMS;
                         tmpFrameMS -= tickCount;
                         frameMS = tmpFrameMS;
+                        lastCompletedFrame = tickCount;
 
                         // if (m_frame%m_update_avatars == 0)
                         //   UpdateInWorldTime();