From 76bd3de2fd243d0c910404af8a9998de746b04c4 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Mon, 5 Aug 2013 19:22:47 +0100 Subject: Add checks monitoring framework to provide alerts if certain conditions do not hold. Not yet in use. --- OpenSim/Framework/Monitoring/Checks/Check.cs | 118 ++++++++++++ OpenSim/Framework/Monitoring/ChecksManager.cs | 262 ++++++++++++++++++++++++++ OpenSim/Framework/Monitoring/StatsManager.cs | 6 +- OpenSim/Framework/Monitoring/Watchdog.cs | 1 + OpenSim/Framework/Servers/ServerBase.cs | 1 + 5 files changed, 385 insertions(+), 3 deletions(-) create mode 100644 OpenSim/Framework/Monitoring/Checks/Check.cs create mode 100644 OpenSim/Framework/Monitoring/ChecksManager.cs (limited to 'OpenSim/Framework') diff --git a/OpenSim/Framework/Monitoring/Checks/Check.cs b/OpenSim/Framework/Monitoring/Checks/Check.cs new file mode 100644 index 0000000..594386a --- /dev/null +++ b/OpenSim/Framework/Monitoring/Checks/Check.cs @@ -0,0 +1,118 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Text; + +namespace OpenSim.Framework.Monitoring +{ + public class Check + { +// private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + public static readonly char[] DisallowedShortNameCharacters = { '.' }; + + /// + /// Category of this stat (e.g. cache, scene, etc). + /// + public string Category { get; private set; } + + /// + /// Containing name for this stat. + /// FIXME: In the case of a scene, this is currently the scene name (though this leaves + /// us with a to-be-resolved problem of non-unique region names). + /// + /// + /// The container. + /// + public string Container { get; private set; } + + /// + /// Action used to check whether alert should go off. + /// + /// + /// Should return true if check passes. False otherwise. + /// + public Func CheckFunc { get; private set; } + + /// + /// Message from the last failure, if any. If there is no message or no failure then will be null. + /// + /// + /// Should be set by the CheckFunc when applicable. + /// + public string LastFailureMessage { get; set; } + + public StatVerbosity Verbosity { get; private set; } + public string ShortName { get; private set; } + public string Name { get; private set; } + public string Description { get; private set; } + + public Check( + string shortName, + string name, + string description, + string category, + string container, + Func checkFunc, + StatVerbosity verbosity) + { + if (ChecksManager.SubCommands.Contains(category)) + throw new Exception( + string.Format("Alert cannot be in category '{0}' since this is reserved for a subcommand", category)); + + foreach (char c in DisallowedShortNameCharacters) + { + if (shortName.IndexOf(c) != -1) + throw new Exception(string.Format("Alert name {0} cannot contain character {1}", shortName, c)); + } + + ShortName = shortName; + Name = name; + Description = description; + Category = category; + Container = container; + CheckFunc = checkFunc; + Verbosity = verbosity; + } + + public bool CheckIt() + { + return CheckFunc(this); + } + + public virtual string ToConsoleString() + { + return string.Format( + "{0}.{1}.{2} - {3}", + Category, + Container, + ShortName, + Description); + } + } +} \ No newline at end of file diff --git a/OpenSim/Framework/Monitoring/ChecksManager.cs b/OpenSim/Framework/Monitoring/ChecksManager.cs new file mode 100644 index 0000000..e4a7f8c --- /dev/null +++ b/OpenSim/Framework/Monitoring/ChecksManager.cs @@ -0,0 +1,262 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; +using log4net; + +namespace OpenSim.Framework.Monitoring +{ + /// + /// Static class used to register/deregister checks on runtime conditions. + /// + public static class ChecksManager + { + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + // Subcommand used to list other stats. + public const string ListSubCommand = "list"; + + // All subcommands + public static HashSet SubCommands = new HashSet { ListSubCommand }; + + /// + /// Checks categorized by category/container/shortname + /// + /// + /// Do not add or remove directly from this dictionary. + /// + public static SortedDictionary>> RegisteredChecks + = new SortedDictionary>>(); + + public static void RegisterConsoleCommands(ICommandConsole console) + { + console.Commands.AddCommand( + "General", + false, + "show checks", + "show checks", + "Show checks configured for this server", + "If no argument is specified then info on all checks will be shown.\n" + + "'list' argument will show check categories.\n" + + "THIS FACILITY IS EXPERIMENTAL", + HandleShowchecksCommand); + } + + public static void HandleShowchecksCommand(string module, string[] cmd) + { + ICommandConsole con = MainConsole.Instance; + + if (cmd.Length > 2) + { + foreach (string name in cmd.Skip(2)) + { + string[] components = name.Split('.'); + + string categoryName = components[0]; +// string containerName = components.Length > 1 ? components[1] : null; + + if (categoryName == ListSubCommand) + { + con.Output("check categories available are:"); + + foreach (string category in RegisteredChecks.Keys) + con.OutputFormat(" {0}", category); + } +// else +// { +// SortedDictionary> category; +// if (!Registeredchecks.TryGetValue(categoryName, out category)) +// { +// con.OutputFormat("No such category as {0}", categoryName); +// } +// else +// { +// if (String.IsNullOrEmpty(containerName)) +// { +// OutputConfiguredToConsole(con, category); +// } +// else +// { +// SortedDictionary container; +// if (category.TryGetValue(containerName, out container)) +// { +// OutputContainerChecksToConsole(con, container); +// } +// else +// { +// con.OutputFormat("No such container {0} in category {1}", containerName, categoryName); +// } +// } +// } +// } + } + } + else + { + OutputAllChecksToConsole(con); + } + } + + /// + /// Registers a statistic. + /// + /// + /// + public static bool RegisterCheck(Check check) + { + SortedDictionary> category = null, newCategory; + SortedDictionary container = null, newContainer; + + lock (RegisteredChecks) + { + // Check name is not unique across category/container/shortname key. + // XXX: For now just return false. This is to avoid problems in regression tests where all tests + // in a class are run in the same instance of the VM. + if (TryGetCheckParents(check, out category, out container)) + return false; + + // We take a copy-on-write approach here of replacing dictionaries when keys are added or removed. + // This means that we don't need to lock or copy them on iteration, which will be a much more + // common operation after startup. + if (container != null) + newContainer = new SortedDictionary(container); + else + newContainer = new SortedDictionary(); + + if (category != null) + newCategory = new SortedDictionary>(category); + else + newCategory = new SortedDictionary>(); + + newContainer[check.ShortName] = check; + newCategory[check.Container] = newContainer; + RegisteredChecks[check.Category] = newCategory; + } + + return true; + } + + /// + /// Deregister an check + /// > + /// + /// + public static bool DeregisterCheck(Check check) + { + SortedDictionary> category = null, newCategory; + SortedDictionary container = null, newContainer; + + lock (RegisteredChecks) + { + if (!TryGetCheckParents(check, out category, out container)) + return false; + + newContainer = new SortedDictionary(container); + newContainer.Remove(check.ShortName); + + newCategory = new SortedDictionary>(category); + newCategory.Remove(check.Container); + + newCategory[check.Container] = newContainer; + RegisteredChecks[check.Category] = newCategory; + + return true; + } + } + + public static bool TryGetCheckParents( + Check check, + out SortedDictionary> category, + out SortedDictionary container) + { + category = null; + container = null; + + lock (RegisteredChecks) + { + if (RegisteredChecks.TryGetValue(check.Category, out category)) + { + if (category.TryGetValue(check.Container, out container)) + { + if (container.ContainsKey(check.ShortName)) + return true; + } + } + } + + return false; + } + + public static void CheckChecks() + { + lock (RegisteredChecks) + { + foreach (SortedDictionary> category in RegisteredChecks.Values) + { + foreach (SortedDictionary container in category.Values) + { + foreach (Check check in container.Values) + { + if (!check.CheckIt()) + m_log.WarnFormat( + "[CHECKS MANAGER]: Check {0}.{1}.{2} failed with message {3}", check.Category, check.Container, check.ShortName, check.LastFailureMessage); + } + } + } + } + } + + private static void OutputAllChecksToConsole(ICommandConsole con) + { + foreach (var category in RegisteredChecks.Values) + { + OutputCategoryChecksToConsole(con, category); + } + } + + private static void OutputCategoryChecksToConsole( + ICommandConsole con, SortedDictionary> category) + { + foreach (var container in category.Values) + { + OutputContainerChecksToConsole(con, container); + } + } + + private static void OutputContainerChecksToConsole(ICommandConsole con, SortedDictionary container) + { + foreach (Check check in container.Values) + { + con.Output(check.ToConsoleString()); + } + } + } +} \ No newline at end of file diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index e6a2304..87197f4 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -35,9 +35,9 @@ using OpenMetaverse.StructuredData; namespace OpenSim.Framework.Monitoring { /// - /// Singleton used to provide access to statistics reporters + /// Static class used to register/deregister/fetch statistics /// - public class StatsManager + public static class StatsManager { // Subcommand used to list other stats. public const string AllSubCommand = "all"; @@ -257,7 +257,7 @@ namespace OpenSim.Framework.Monitoring // } /// - /// Registers a statistic. + /// Register a statistic. /// /// /// diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index 3f992b1..45762a6 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -380,6 +380,7 @@ namespace OpenSim.Framework.Monitoring if (MemoryWatchdog.Enabled) MemoryWatchdog.Update(); + ChecksManager.CheckChecks(); StatsManager.RecordStats(); m_watchdogTimer.Start(); diff --git a/OpenSim/Framework/Servers/ServerBase.cs b/OpenSim/Framework/Servers/ServerBase.cs index 029b848..a8e0f81 100644 --- a/OpenSim/Framework/Servers/ServerBase.cs +++ b/OpenSim/Framework/Servers/ServerBase.cs @@ -272,6 +272,7 @@ namespace OpenSim.Framework.Servers "shutdown", "Quit the application", (mod, args) => Shutdown()); + ChecksManager.RegisterConsoleCommands(m_console); StatsManager.RegisterConsoleCommands(m_console); } -- cgit v1.1