From 134f86e8d5c414409631b25b8c6f0ee45fbd8631 Mon Sep 17 00:00:00 2001 From: David Walter Seikel Date: Thu, 3 Nov 2016 21:44:39 +1000 Subject: Initial update to OpenSim 0.8.2.1 source code. --- .../Framework/Monitoring/AssetStatsCollector.cs | 26 ++ OpenSim/Framework/Monitoring/BaseStatsCollector.cs | 19 +- OpenSim/Framework/Monitoring/Checks/Check.cs | 118 ++++++ OpenSim/Framework/Monitoring/ChecksManager.cs | 262 ++++++++++++++ .../Monitoring/Interfaces/IStatsCollector.cs | 9 + OpenSim/Framework/Monitoring/JobEngine.cs | 341 +++++++++++++++++ OpenSim/Framework/Monitoring/MemoryWatchdog.cs | 8 +- .../Monitoring/Properties/AssemblyInfo.cs | 4 +- .../Framework/Monitoring/ServerStatsCollector.cs | 346 ++++++++++++++++++ .../Framework/Monitoring/SimExtraStatsCollector.cs | 106 ++++-- OpenSim/Framework/Monitoring/Stats/CounterStat.cs | 119 ++++++ .../Framework/Monitoring/Stats/EventHistogram.cs | 173 +++++++++ .../Framework/Monitoring/Stats/PercentageStat.cs | 16 + OpenSim/Framework/Monitoring/Stats/Stat.cs | 102 +++++- OpenSim/Framework/Monitoring/StatsLogger.cs | 151 ++++++++ OpenSim/Framework/Monitoring/StatsManager.cs | 403 +++++++++++++++++---- OpenSim/Framework/Monitoring/UserStatsCollector.cs | 18 + OpenSim/Framework/Monitoring/Watchdog.cs | 106 +++--- OpenSim/Framework/Monitoring/WorkManager.cs | 290 +++++++++++++++ 19 files changed, 2442 insertions(+), 175 deletions(-) create mode 100644 OpenSim/Framework/Monitoring/Checks/Check.cs create mode 100644 OpenSim/Framework/Monitoring/ChecksManager.cs create mode 100644 OpenSim/Framework/Monitoring/JobEngine.cs create mode 100644 OpenSim/Framework/Monitoring/ServerStatsCollector.cs mode change 100644 => 100755 OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs create mode 100755 OpenSim/Framework/Monitoring/Stats/CounterStat.cs create mode 100755 OpenSim/Framework/Monitoring/Stats/EventHistogram.cs create mode 100644 OpenSim/Framework/Monitoring/StatsLogger.cs create mode 100644 OpenSim/Framework/Monitoring/WorkManager.cs (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/AssetStatsCollector.cs b/OpenSim/Framework/Monitoring/AssetStatsCollector.cs index 2a4d45b..6a0f676 100644 --- a/OpenSim/Framework/Monitoring/AssetStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/AssetStatsCollector.cs @@ -28,6 +28,8 @@ using System; using System.Timers; +using OpenMetaverse.StructuredData; + namespace OpenSim.Framework.Monitoring { /// @@ -100,5 +102,29 @@ Asset requests yesterday : {3} ({4} per hour) of which {5} were not found", AssetRequestsToday, assetRequestsTodayPerHour, AssetRequestsNotFoundToday, AssetRequestsYesterday, assetRequestsYesterdayPerHour, AssetRequestsNotFoundYesterday); } + + public override string XReport(string uptime, string version) + { + return OSDParser.SerializeJsonString(OReport(uptime, version)); + } + + public override OSDMap OReport(string uptime, string version) + { + double elapsedHours = (DateTime.Now - startTime).TotalHours; + if (elapsedHours <= 0) { elapsedHours = 1; } // prevent divide by zero + + long assetRequestsTodayPerHour = (long)Math.Round(AssetRequestsToday / elapsedHours); + long assetRequestsYesterdayPerHour = (long)Math.Round(AssetRequestsYesterday / 24.0); + + OSDMap ret = new OSDMap(); + ret.Add("AssetRequestsToday", OSD.FromLong(AssetRequestsToday)); + ret.Add("AssetRequestsTodayPerHour", OSD.FromLong(assetRequestsTodayPerHour)); + ret.Add("AssetRequestsNotFoundToday", OSD.FromLong(AssetRequestsNotFoundToday)); + ret.Add("AssetRequestsYesterday", OSD.FromLong(AssetRequestsYesterday)); + ret.Add("AssetRequestsYesterdayPerHour", OSD.FromLong(assetRequestsYesterdayPerHour)); + ret.Add("AssetRequestsNotFoundYesterday", OSD.FromLong(assetRequestsNotFoundYesterday)); + + return ret; + } } } diff --git a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs index 2903b6e..20495f6 100644 --- a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs @@ -1,4 +1,4 @@ -/* +/* * Copyright (c) Contributors, http://opensimulator.org/ * See CONTRIBUTORS.TXT for a full list of copyright holders. * @@ -45,16 +45,16 @@ namespace OpenSim.Framework.Monitoring sb.Append(Environment.NewLine); sb.AppendFormat( - "Allocated to OpenSim objects: {0} MB\n", + "Heap allocated to OpenSim : {0} MB\n", Math.Round(GC.GetTotalMemory(false) / 1024.0 / 1024.0)); sb.AppendFormat( - "OpenSim last object memory churn : {0} MB/s\n", - Math.Round((MemoryWatchdog.LastMemoryChurn * 1000) / 1024.0 / 1024, 3)); + "Last heap allocation rate : {0} MB/s\n", + Math.Round((MemoryWatchdog.LastHeapAllocationRate * 1000) / 1024.0 / 1024, 3)); sb.AppendFormat( - "OpenSim average object memory churn : {0} MB/s\n", - Math.Round((MemoryWatchdog.AverageMemoryChurn * 1000) / 1024.0 / 1024, 3)); + "Average heap allocation rate: {0} MB/s\n", + Math.Round((MemoryWatchdog.AverageHeapAllocationRate * 1000) / 1024.0 / 1024, 3)); sb.AppendFormat( "Process memory : {0} MB\n", @@ -67,5 +67,12 @@ namespace OpenSim.Framework.Monitoring { return (string) Math.Round(GC.GetTotalMemory(false) / 1024.0 / 1024.0).ToString() ; } + + public virtual OSDMap OReport(string uptime, string version) + { + OSDMap ret = new OSDMap(); + ret.Add("TotalMemory", new OSDReal(Math.Round(GC.GetTotalMemory(false) / 1024.0 / 1024.0))); + return ret; + } } } diff --git a/OpenSim/Framework/Monitoring/Checks/Check.cs b/OpenSim/Framework/Monitoring/Checks/Check.cs new file mode 100644 index 0000000..594386a --- /dev/null +++ b/OpenSim/Framework/Monitoring/Checks/Check.cs @@ -0,0 +1,118 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Text; + +namespace OpenSim.Framework.Monitoring +{ + public class Check + { +// private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + public static readonly char[] DisallowedShortNameCharacters = { '.' }; + + /// + /// Category of this stat (e.g. cache, scene, etc). + /// + public string Category { get; private set; } + + /// + /// Containing name for this stat. + /// FIXME: In the case of a scene, this is currently the scene name (though this leaves + /// us with a to-be-resolved problem of non-unique region names). + /// + /// + /// The container. + /// + public string Container { get; private set; } + + /// + /// Action used to check whether alert should go off. + /// + /// + /// Should return true if check passes. False otherwise. + /// + public Func CheckFunc { get; private set; } + + /// + /// Message from the last failure, if any. If there is no message or no failure then will be null. + /// + /// + /// Should be set by the CheckFunc when applicable. + /// + public string LastFailureMessage { get; set; } + + public StatVerbosity Verbosity { get; private set; } + public string ShortName { get; private set; } + public string Name { get; private set; } + public string Description { get; private set; } + + public Check( + string shortName, + string name, + string description, + string category, + string container, + Func checkFunc, + StatVerbosity verbosity) + { + if (ChecksManager.SubCommands.Contains(category)) + throw new Exception( + string.Format("Alert cannot be in category '{0}' since this is reserved for a subcommand", category)); + + foreach (char c in DisallowedShortNameCharacters) + { + if (shortName.IndexOf(c) != -1) + throw new Exception(string.Format("Alert name {0} cannot contain character {1}", shortName, c)); + } + + ShortName = shortName; + Name = name; + Description = description; + Category = category; + Container = container; + CheckFunc = checkFunc; + Verbosity = verbosity; + } + + public bool CheckIt() + { + return CheckFunc(this); + } + + public virtual string ToConsoleString() + { + return string.Format( + "{0}.{1}.{2} - {3}", + Category, + Container, + ShortName, + Description); + } + } +} \ No newline at end of file diff --git a/OpenSim/Framework/Monitoring/ChecksManager.cs b/OpenSim/Framework/Monitoring/ChecksManager.cs new file mode 100644 index 0000000..e4a7f8c --- /dev/null +++ b/OpenSim/Framework/Monitoring/ChecksManager.cs @@ -0,0 +1,262 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; +using log4net; + +namespace OpenSim.Framework.Monitoring +{ + /// + /// Static class used to register/deregister checks on runtime conditions. + /// + public static class ChecksManager + { + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + // Subcommand used to list other stats. + public const string ListSubCommand = "list"; + + // All subcommands + public static HashSet SubCommands = new HashSet { ListSubCommand }; + + /// + /// Checks categorized by category/container/shortname + /// + /// + /// Do not add or remove directly from this dictionary. + /// + public static SortedDictionary>> RegisteredChecks + = new SortedDictionary>>(); + + public static void RegisterConsoleCommands(ICommandConsole console) + { + console.Commands.AddCommand( + "General", + false, + "show checks", + "show checks", + "Show checks configured for this server", + "If no argument is specified then info on all checks will be shown.\n" + + "'list' argument will show check categories.\n" + + "THIS FACILITY IS EXPERIMENTAL", + HandleShowchecksCommand); + } + + public static void HandleShowchecksCommand(string module, string[] cmd) + { + ICommandConsole con = MainConsole.Instance; + + if (cmd.Length > 2) + { + foreach (string name in cmd.Skip(2)) + { + string[] components = name.Split('.'); + + string categoryName = components[0]; +// string containerName = components.Length > 1 ? components[1] : null; + + if (categoryName == ListSubCommand) + { + con.Output("check categories available are:"); + + foreach (string category in RegisteredChecks.Keys) + con.OutputFormat(" {0}", category); + } +// else +// { +// SortedDictionary> category; +// if (!Registeredchecks.TryGetValue(categoryName, out category)) +// { +// con.OutputFormat("No such category as {0}", categoryName); +// } +// else +// { +// if (String.IsNullOrEmpty(containerName)) +// { +// OutputConfiguredToConsole(con, category); +// } +// else +// { +// SortedDictionary container; +// if (category.TryGetValue(containerName, out container)) +// { +// OutputContainerChecksToConsole(con, container); +// } +// else +// { +// con.OutputFormat("No such container {0} in category {1}", containerName, categoryName); +// } +// } +// } +// } + } + } + else + { + OutputAllChecksToConsole(con); + } + } + + /// + /// Registers a statistic. + /// + /// + /// + public static bool RegisterCheck(Check check) + { + SortedDictionary> category = null, newCategory; + SortedDictionary container = null, newContainer; + + lock (RegisteredChecks) + { + // Check name is not unique across category/container/shortname key. + // XXX: For now just return false. This is to avoid problems in regression tests where all tests + // in a class are run in the same instance of the VM. + if (TryGetCheckParents(check, out category, out container)) + return false; + + // We take a copy-on-write approach here of replacing dictionaries when keys are added or removed. + // This means that we don't need to lock or copy them on iteration, which will be a much more + // common operation after startup. + if (container != null) + newContainer = new SortedDictionary(container); + else + newContainer = new SortedDictionary(); + + if (category != null) + newCategory = new SortedDictionary>(category); + else + newCategory = new SortedDictionary>(); + + newContainer[check.ShortName] = check; + newCategory[check.Container] = newContainer; + RegisteredChecks[check.Category] = newCategory; + } + + return true; + } + + /// + /// Deregister an check + /// > + /// + /// + public static bool DeregisterCheck(Check check) + { + SortedDictionary> category = null, newCategory; + SortedDictionary container = null, newContainer; + + lock (RegisteredChecks) + { + if (!TryGetCheckParents(check, out category, out container)) + return false; + + newContainer = new SortedDictionary(container); + newContainer.Remove(check.ShortName); + + newCategory = new SortedDictionary>(category); + newCategory.Remove(check.Container); + + newCategory[check.Container] = newContainer; + RegisteredChecks[check.Category] = newCategory; + + return true; + } + } + + public static bool TryGetCheckParents( + Check check, + out SortedDictionary> category, + out SortedDictionary container) + { + category = null; + container = null; + + lock (RegisteredChecks) + { + if (RegisteredChecks.TryGetValue(check.Category, out category)) + { + if (category.TryGetValue(check.Container, out container)) + { + if (container.ContainsKey(check.ShortName)) + return true; + } + } + } + + return false; + } + + public static void CheckChecks() + { + lock (RegisteredChecks) + { + foreach (SortedDictionary> category in RegisteredChecks.Values) + { + foreach (SortedDictionary container in category.Values) + { + foreach (Check check in container.Values) + { + if (!check.CheckIt()) + m_log.WarnFormat( + "[CHECKS MANAGER]: Check {0}.{1}.{2} failed with message {3}", check.Category, check.Container, check.ShortName, check.LastFailureMessage); + } + } + } + } + } + + private static void OutputAllChecksToConsole(ICommandConsole con) + { + foreach (var category in RegisteredChecks.Values) + { + OutputCategoryChecksToConsole(con, category); + } + } + + private static void OutputCategoryChecksToConsole( + ICommandConsole con, SortedDictionary> category) + { + foreach (var container in category.Values) + { + OutputContainerChecksToConsole(con, container); + } + } + + private static void OutputContainerChecksToConsole(ICommandConsole con, SortedDictionary container) + { + foreach (Check check in container.Values) + { + con.Output(check.ToConsoleString()); + } + } + } +} \ No newline at end of file diff --git a/OpenSim/Framework/Monitoring/Interfaces/IStatsCollector.cs b/OpenSim/Framework/Monitoring/Interfaces/IStatsCollector.cs index 99f75e3..40df562 100644 --- a/OpenSim/Framework/Monitoring/Interfaces/IStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/Interfaces/IStatsCollector.cs @@ -25,6 +25,8 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +using OpenMetaverse.StructuredData; + namespace OpenSim.Framework.Monitoring { /// @@ -45,5 +47,12 @@ namespace OpenSim.Framework.Monitoring /// A /// string XReport(string uptime, string version); + + /// + /// Report back collected statistical information as an OSDMap of key/values + /// + /// + /// + OSDMap OReport(string uptime, string version); } } diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs new file mode 100644 index 0000000..6db9a67 --- /dev/null +++ b/OpenSim/Framework/Monitoring/JobEngine.cs @@ -0,0 +1,341 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Concurrent; +using System.Reflection; +using System.Threading; +using log4net; +using OpenSim.Framework; + +namespace OpenSim.Framework.Monitoring +{ + public class JobEngine + { + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + public int LogLevel { get; set; } + + public string Name { get; private set; } + + public string LoggingName { get; private set; } + + /// + /// Is this engine running? + /// + public bool IsRunning { get; private set; } + + /// + /// The current job that the engine is running. + /// + /// + /// Will be null if no job is currently running. + /// + public Job CurrentJob { get; private set; } + + /// + /// Number of jobs waiting to be processed. + /// + public int JobsWaiting { get { return m_jobQueue.Count; } } + + /// + /// The timeout in milliseconds to wait for at least one event to be written when the recorder is stopping. + /// + public int RequestProcessTimeoutOnStop { get; set; } + + /// + /// Controls whether we need to warn in the log about exceeding the max queue size. + /// + /// + /// This is flipped to false once queue max has been exceeded and back to true when it falls below max, in + /// order to avoid spamming the log with lots of warnings. + /// + private bool m_warnOverMaxQueue = true; + + private BlockingCollection m_jobQueue; + + private CancellationTokenSource m_cancelSource; + + /// + /// Used to signal that we are ready to complete stop. + /// + private ManualResetEvent m_finishedProcessingAfterStop = new ManualResetEvent(false); + + public JobEngine(string name, string loggingName) + { + Name = name; + LoggingName = loggingName; + + RequestProcessTimeoutOnStop = 5000; + } + + public void Start() + { + lock (this) + { + if (IsRunning) + return; + + IsRunning = true; + + m_finishedProcessingAfterStop.Reset(); + + m_jobQueue = new BlockingCollection(new ConcurrentQueue(), 5000); + m_cancelSource = new CancellationTokenSource(); + + WorkManager.StartThread( + ProcessRequests, + Name, + ThreadPriority.Normal, + false, + true, + null, + int.MaxValue); + } + } + + public void Stop() + { + lock (this) + { + try + { + if (!IsRunning) + return; + + IsRunning = false; + + int requestsLeft = m_jobQueue.Count; + + if (requestsLeft <= 0) + { + m_cancelSource.Cancel(); + } + else + { + m_log.InfoFormat("[{0}]: Waiting to write {1} events after stop.", LoggingName, requestsLeft); + + while (requestsLeft > 0) + { + if (!m_finishedProcessingAfterStop.WaitOne(RequestProcessTimeoutOnStop)) + { + // After timeout no events have been written + if (requestsLeft == m_jobQueue.Count) + { + m_log.WarnFormat( + "[{0}]: No requests processed after {1} ms wait. Discarding remaining {2} requests", + LoggingName, RequestProcessTimeoutOnStop, requestsLeft); + + break; + } + } + + requestsLeft = m_jobQueue.Count; + } + } + } + finally + { + m_cancelSource.Dispose(); + } + } + } + + /// + /// Make a job. + /// + /// + /// We provide this method to replace the constructor so that we can later pool job objects if necessary to + /// reduce memory churn. Normally one would directly call QueueJob() with parameters anyway. + /// + /// + /// Name. + /// Action. + /// Common identifier. + public static Job MakeJob(string name, Action action, string commonId = null) + { + return Job.MakeJob(name, action, commonId); + } + + /// + /// Remove the next job queued for processing. + /// + /// + /// Returns null if there is no next job. + /// Will not remove a job currently being performed. + /// + public Job RemoveNextJob() + { + Job nextJob; + m_jobQueue.TryTake(out nextJob); + + return nextJob; + } + + /// + /// Queue the job for processing. + /// + /// true, if job was queued, false otherwise. + /// Name of job. This appears on the console and in logging. + /// Action to perform. + /// + /// Common identifier for a set of jobs. This is allows a set of jobs to be removed + /// if required (e.g. all jobs for a given agent. Optional. + /// + public bool QueueJob(string name, Action action, string commonId = null) + { + return QueueJob(MakeJob(name, action, commonId)); + } + + /// + /// Queue the job for processing. + /// + /// true, if job was queued, false otherwise. + /// The job + /// + public bool QueueJob(Job job) + { + if (m_jobQueue.Count < m_jobQueue.BoundedCapacity) + { + m_jobQueue.Add(job); + + if (!m_warnOverMaxQueue) + m_warnOverMaxQueue = true; + + return true; + } + else + { + if (m_warnOverMaxQueue) + { + m_log.WarnFormat( + "[{0}]: Job queue at maximum capacity, not recording job from {1} in {2}", + LoggingName, job.Name, Name); + + m_warnOverMaxQueue = false; + } + + return false; + } + } + + private void ProcessRequests() + { + try + { + while (IsRunning || m_jobQueue.Count > 0) + { + try + { + CurrentJob = m_jobQueue.Take(m_cancelSource.Token); + } + catch (ObjectDisposedException e) + { + // If we see this whilst not running then it may be due to a race where this thread checks + // IsRunning after the stopping thread sets it to false and disposes of the cancellation source. + if (IsRunning) + throw e; + else + break; + } + + if (LogLevel >= 1) + m_log.DebugFormat("[{0}]: Processing job {1}", LoggingName, CurrentJob.Name); + + try + { + CurrentJob.Action(); + } + catch (Exception e) + { + m_log.Error( + string.Format( + "[{0}]: Job {1} failed, continuing. Exception ", LoggingName, CurrentJob.Name), e); + } + + if (LogLevel >= 1) + m_log.DebugFormat("[{0}]: Processed job {1}", LoggingName, CurrentJob.Name); + + CurrentJob = null; + } + } + catch (OperationCanceledException) + { + } + + m_finishedProcessingAfterStop.Set(); + } + + public class Job + { + /// + /// Name of the job. + /// + /// + /// This appears on console and debug output. + /// + public string Name { get; private set; } + + /// + /// Common ID for this job. + /// + /// + /// This allows all jobs with a certain common ID (e.g. a client UUID) to be removed en-masse if required. + /// Can be null if this is not required. + /// + public string CommonId { get; private set; } + + /// + /// Action to perform when this job is processed. + /// + public Action Action { get; private set; } + + private Job(string name, string commonId, Action action) + { + Name = name; + CommonId = commonId; + Action = action; + } + + /// + /// Make a job. It needs to be separately queued. + /// + /// + /// We provide this method to replace the constructor so that we can pool job objects if necessary to + /// to reduce memory churn. Normally one would directly call JobEngine.QueueJob() with parameters anyway. + /// + /// + /// Name. + /// Action. + /// Common identifier. + public static Job MakeJob(string name, Action action, string commonId = null) + { + return new Job(name, commonId, action); + } + } + } +} \ No newline at end of file diff --git a/OpenSim/Framework/Monitoring/MemoryWatchdog.cs b/OpenSim/Framework/Monitoring/MemoryWatchdog.cs index c6010cd..c474622 100644 --- a/OpenSim/Framework/Monitoring/MemoryWatchdog.cs +++ b/OpenSim/Framework/Monitoring/MemoryWatchdog.cs @@ -60,17 +60,17 @@ namespace OpenSim.Framework.Monitoring private static bool m_enabled; /// - /// Last memory churn in bytes per millisecond. + /// Average heap allocation rate in bytes per millisecond. /// - public static double AverageMemoryChurn + public static double AverageHeapAllocationRate { get { if (m_samples.Count > 0) return m_samples.Average(); else return 0; } } /// - /// Average memory churn in bytes per millisecond. + /// Last heap allocation in bytes /// - public static double LastMemoryChurn + public static double LastHeapAllocationRate { get { if (m_samples.Count > 0) return m_samples.Last(); else return 0; } } diff --git a/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs index 1f2bb40..a617b93 100644 --- a/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs +++ b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs @@ -29,5 +29,5 @@ using System.Runtime.InteropServices; // Build Number // Revision // -[assembly: AssemblyVersion("0.7.5.*")] -[assembly: AssemblyFileVersion("1.0.0.0")] +[assembly: AssemblyVersion("0.8.3.*")] + diff --git a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs new file mode 100644 index 0000000..77315bb --- /dev/null +++ b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs @@ -0,0 +1,346 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Net.NetworkInformation; +using System.Text; +using System.Threading; +using log4net; +using Nini.Config; +using OpenMetaverse.StructuredData; +using OpenSim.Framework; + +namespace OpenSim.Framework.Monitoring +{ + public class ServerStatsCollector + { + private readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); + private readonly string LogHeader = "[SERVER STATS]"; + + public bool Enabled = false; + private static Dictionary RegisteredStats = new Dictionary(); + + public readonly string CategoryServer = "server"; + + public readonly string ContainerThreadpool = "threadpool"; + public readonly string ContainerProcessor = "processor"; + public readonly string ContainerMemory = "memory"; + public readonly string ContainerNetwork = "network"; + public readonly string ContainerProcess = "process"; + + public string NetworkInterfaceTypes = "Ethernet"; + + readonly int performanceCounterSampleInterval = 500; +// int lastperformanceCounterSampleTime = 0; + + private class PerfCounterControl + { + public PerformanceCounter perfCounter; + public int lastFetch; + public string name; + public PerfCounterControl(PerformanceCounter pPc) + : this(pPc, String.Empty) + { + } + public PerfCounterControl(PerformanceCounter pPc, string pName) + { + perfCounter = pPc; + lastFetch = 0; + name = pName; + } + } + + PerfCounterControl processorPercentPerfCounter = null; + + // IRegionModuleBase.Initialize + public void Initialise(IConfigSource source) + { + if (source == null) + return; + + IConfig cfg = source.Configs["Monitoring"]; + + if (cfg != null) + Enabled = cfg.GetBoolean("ServerStatsEnabled", true); + + if (Enabled) + { + NetworkInterfaceTypes = cfg.GetString("NetworkInterfaceTypes", "Ethernet"); + } + } + + public void Start() + { + if (RegisteredStats.Count == 0) + RegisterServerStats(); + } + + public void Close() + { + if (RegisteredStats.Count > 0) + { + foreach (Stat stat in RegisteredStats.Values) + { + StatsManager.DeregisterStat(stat); + stat.Dispose(); + } + RegisteredStats.Clear(); + } + } + + private void MakeStat(string pName, string pDesc, string pUnit, string pContainer, Action act) + { + MakeStat(pName, pDesc, pUnit, pContainer, act, MeasuresOfInterest.None); + } + + private void MakeStat(string pName, string pDesc, string pUnit, string pContainer, Action act, MeasuresOfInterest moi) + { + string desc = pDesc; + if (desc == null) + desc = pName; + Stat stat = new Stat(pName, pName, desc, pUnit, CategoryServer, pContainer, StatType.Pull, moi, act, StatVerbosity.Debug); + StatsManager.RegisterStat(stat); + RegisteredStats.Add(pName, stat); + } + + public void RegisterServerStats() + { +// lastperformanceCounterSampleTime = Util.EnvironmentTickCount(); + PerformanceCounter tempPC; + Stat tempStat; + string tempName; + + try + { + tempName = "CPUPercent"; + tempPC = new PerformanceCounter("Processor", "% Processor Time", "_Total"); + processorPercentPerfCounter = new PerfCounterControl(tempPC); + // A long time bug in mono is that CPU percent is reported as CPU percent idle. Windows reports CPU percent busy. + tempStat = new Stat(tempName, tempName, "", "percent", CategoryServer, ContainerProcessor, + StatType.Pull, (s) => { GetNextValue(s, processorPercentPerfCounter); }, + StatVerbosity.Info); + StatsManager.RegisterStat(tempStat); + RegisteredStats.Add(tempName, tempStat); + + MakeStat("TotalProcessorTime", null, "sec", ContainerProcessor, + (s) => { s.Value = Math.Round(Process.GetCurrentProcess().TotalProcessorTime.TotalSeconds, 3); }); + + MakeStat("UserProcessorTime", null, "sec", ContainerProcessor, + (s) => { s.Value = Math.Round(Process.GetCurrentProcess().UserProcessorTime.TotalSeconds, 3); }); + + MakeStat("PrivilegedProcessorTime", null, "sec", ContainerProcessor, + (s) => { s.Value = Math.Round(Process.GetCurrentProcess().PrivilegedProcessorTime.TotalSeconds, 3); }); + + MakeStat("Threads", null, "threads", ContainerProcessor, + (s) => { s.Value = Process.GetCurrentProcess().Threads.Count; }); + } + catch (Exception e) + { + m_log.ErrorFormat("{0} Exception creating 'Process': {1}", LogHeader, e); + } + + MakeStat("BuiltinThreadpoolWorkerThreadsAvailable", null, "threads", ContainerThreadpool, + s => + { + int workerThreads, iocpThreads; + ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads); + s.Value = workerThreads; + }); + + MakeStat("BuiltinThreadpoolIOCPThreadsAvailable", null, "threads", ContainerThreadpool, + s => + { + int workerThreads, iocpThreads; + ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads); + s.Value = iocpThreads; + }); + + if (Util.FireAndForgetMethod == FireAndForgetMethod.SmartThreadPool && Util.GetSmartThreadPoolInfo() != null) + { + MakeStat("STPMaxThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().MaxThreads); + MakeStat("STPMinThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().MinThreads); + MakeStat("STPConcurrency", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().MaxConcurrentWorkItems); + MakeStat("STPActiveThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().ActiveThreads); + MakeStat("STPInUseThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().InUseThreads); + MakeStat("STPWorkItemsWaiting", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().WaitingCallbacks); + } + + MakeStat( + "HTTPRequestsMade", + "Number of outbound HTTP requests made", + "requests", + ContainerNetwork, + s => s.Value = WebUtil.RequestNumber, + MeasuresOfInterest.AverageChangeOverTime); + + try + { + List okInterfaceTypes = new List(NetworkInterfaceTypes.Split(',')); + + IEnumerable nics = NetworkInterface.GetAllNetworkInterfaces(); + foreach (NetworkInterface nic in nics) + { + if (nic.OperationalStatus != OperationalStatus.Up) + continue; + + string nicInterfaceType = nic.NetworkInterfaceType.ToString(); + if (!okInterfaceTypes.Contains(nicInterfaceType)) + { + m_log.DebugFormat("{0} Not including stats for network interface '{1}' of type '{2}'.", + LogHeader, nic.Name, nicInterfaceType); + m_log.DebugFormat("{0} To include, add to comma separated list in [Monitoring]NetworkInterfaceTypes={1}", + LogHeader, NetworkInterfaceTypes); + continue; + } + + if (nic.Supports(NetworkInterfaceComponent.IPv4)) + { + IPv4InterfaceStatistics nicStats = nic.GetIPv4Statistics(); + if (nicStats != null) + { + MakeStat("BytesRcvd/" + nic.Name, nic.Name, "KB", ContainerNetwork, + (s) => { LookupNic(s, (ns) => { return ns.BytesReceived; }, 1024.0); }); + MakeStat("BytesSent/" + nic.Name, nic.Name, "KB", ContainerNetwork, + (s) => { LookupNic(s, (ns) => { return ns.BytesSent; }, 1024.0); }); + MakeStat("TotalBytes/" + nic.Name, nic.Name, "KB", ContainerNetwork, + (s) => { LookupNic(s, (ns) => { return ns.BytesSent + ns.BytesReceived; }, 1024.0); }); + } + } + // TODO: add IPv6 (it may actually happen someday) + } + } + catch (Exception e) + { + m_log.ErrorFormat("{0} Exception creating 'Network Interface': {1}", LogHeader, e); + } + + MakeStat("ProcessMemory", null, "MB", ContainerMemory, + (s) => { s.Value = Math.Round(Process.GetCurrentProcess().WorkingSet64 / 1024d / 1024d, 3); }); + MakeStat("HeapMemory", null, "MB", ContainerMemory, + (s) => { s.Value = Math.Round(GC.GetTotalMemory(false) / 1024d / 1024d, 3); }); + MakeStat("LastHeapAllocationRate", null, "MB/sec", ContainerMemory, + (s) => { s.Value = Math.Round(MemoryWatchdog.LastHeapAllocationRate * 1000d / 1024d / 1024d, 3); }); + MakeStat("AverageHeapAllocationRate", null, "MB/sec", ContainerMemory, + (s) => { s.Value = Math.Round(MemoryWatchdog.AverageHeapAllocationRate * 1000d / 1024d / 1024d, 3); }); + } + + // Notes on performance counters: + // "How To Read Performance Counters": http://blogs.msdn.com/b/bclteam/archive/2006/06/02/618156.aspx + // "How to get the CPU Usage in C#": http://stackoverflow.com/questions/278071/how-to-get-the-cpu-usage-in-c + // "Mono Performance Counters": http://www.mono-project.com/Mono_Performance_Counters + private delegate double PerfCounterNextValue(); + + private void GetNextValue(Stat stat, PerfCounterControl perfControl) + { + if (Util.EnvironmentTickCountSubtract(perfControl.lastFetch) > performanceCounterSampleInterval) + { + if (perfControl != null && perfControl.perfCounter != null) + { + try + { + stat.Value = Math.Round(perfControl.perfCounter.NextValue(), 3); + } + catch (Exception e) + { + m_log.ErrorFormat("{0} Exception on NextValue fetching {1}: {2}", LogHeader, stat.Name, e); + } + + perfControl.lastFetch = Util.EnvironmentTickCount(); + } + } + } + + // Lookup the nic that goes with this stat and set the value by using a fetch action. + // Not sure about closure with delegates inside delegates. + private delegate double GetIPv4StatValue(IPv4InterfaceStatistics interfaceStat); + private void LookupNic(Stat stat, GetIPv4StatValue getter, double factor) + { + // Get the one nic that has the name of this stat + IEnumerable nics = NetworkInterface.GetAllNetworkInterfaces().Where( + (network) => network.Name == stat.Description); + try + { + foreach (NetworkInterface nic in nics) + { + IPv4InterfaceStatistics intrStats = nic.GetIPv4Statistics(); + if (intrStats != null) + { + double newVal = Math.Round(getter(intrStats) / factor, 3); + stat.Value = newVal; + } + break; + } + } + catch + { + // There are times interfaces go away so we just won't update the stat for this + m_log.ErrorFormat("{0} Exception fetching stat on interface '{1}'", LogHeader, stat.Description); + } + } + } + + public class ServerStatsAggregator : Stat + { + public ServerStatsAggregator( + string shortName, + string name, + string description, + string unitName, + string category, + string container + ) + : base( + shortName, + name, + description, + unitName, + category, + container, + StatType.Push, + MeasuresOfInterest.None, + null, + StatVerbosity.Info) + { + } + public override string ToConsoleString() + { + StringBuilder sb = new StringBuilder(); + + return sb.ToString(); + } + + public override OSDMap ToOSDMap() + { + OSDMap ret = new OSDMap(); + + return ret; + } + } +} diff --git a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs old mode 100644 new mode 100755 index aa86202..e4df7ee --- a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs @@ -27,6 +27,8 @@ using System; using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; using System.Text; using OpenMetaverse; using OpenMetaverse.StructuredData; @@ -39,8 +41,6 @@ namespace OpenSim.Framework.Monitoring /// public class SimExtraStatsCollector : BaseStatsCollector { - private long abnormalClientThreadTerminations; - // private long assetsInCache; // private long texturesInCache; // private long assetCacheMemoryUsage; @@ -72,11 +72,11 @@ namespace OpenSim.Framework.Monitoring private volatile float pendingUploads; private volatile float activeScripts; private volatile float scriptLinesPerSecond; - - /// - /// Number of times that a client thread terminated because of an exception - /// - public long AbnormalClientThreadTerminations { get { return abnormalClientThreadTerminations; } } + private volatile float m_frameDilation; + private volatile float m_usersLoggingIn; + private volatile float m_totalGeoPrims; + private volatile float m_totalMeshes; + private volatile float m_inUseThreads; // /// // /// These statistics are being collected by push rather than pull. Pull would be simpler, but I had the @@ -166,11 +166,6 @@ namespace OpenSim.Framework.Monitoring private IDictionary packetQueueStatsCollectors = new Dictionary(); - public void AddAbnormalClientThreadTermination() - { - abnormalClientThreadTerminations++; - } - // public void AddAsset(AssetBase asset) // { // assetsInCache++; @@ -260,6 +255,10 @@ namespace OpenSim.Framework.Monitoring { // FIXME: SimStats shouldn't allow an arbitrary stat packing order (which is inherited from the original // SimStatsPacket that was being used). + + // For an unknown reason the original designers decided not to + // include the spare MS statistic inside of this class, this is + // located inside the StatsBlock at location 21, thus it is skipped timeDilation = stats.StatsBlock[0].StatValue; simFps = stats.StatsBlock[1].StatValue; physicsFps = stats.StatsBlock[2].StatValue; @@ -281,6 +280,11 @@ namespace OpenSim.Framework.Monitoring pendingUploads = stats.StatsBlock[18].StatValue; activeScripts = stats.StatsBlock[19].StatValue; scriptLinesPerSecond = stats.StatsBlock[20].StatValue; + m_frameDilation = stats.StatsBlock[22].StatValue; + m_usersLoggingIn = stats.StatsBlock[23].StatValue; + m_totalGeoPrims = stats.StatsBlock[24].StatValue; + m_totalMeshes = stats.StatsBlock[25].StatValue; + m_inUseThreads = stats.StatsBlock[26].StatValue; } /// @@ -324,10 +328,12 @@ Asset service request failures: {3}" + Environment.NewLine, sb.Append(Environment.NewLine); sb.Append("CONNECTION STATISTICS"); sb.Append(Environment.NewLine); - sb.Append( - string.Format( - "Abnormal client thread terminations: {0}" + Environment.NewLine, - abnormalClientThreadTerminations)); + + List stats = StatsManager.GetStatsFromEachContainer("clientstack", "ClientLogoutsDueToNoReceives"); + + sb.AppendFormat( + "Client logouts due to no data receive timeout: {0}\n\n", + stats != null ? stats.Sum(s => s.Value).ToString() : "unknown"); // sb.Append(Environment.NewLine); // sb.Append("INVENTORY STATISTICS"); @@ -338,7 +344,7 @@ Asset service request failures: {3}" + Environment.NewLine, // InventoryServiceRetrievalFailures)); sb.Append(Environment.NewLine); - sb.Append("FRAME STATISTICS"); + sb.Append("SAMPLE FRAME STATISTICS"); sb.Append(Environment.NewLine); sb.Append("Dilatn SimFPS PhyFPS AgntUp RootAg ChldAg Prims AtvPrm AtvScr ScrLPS"); sb.Append(Environment.NewLine); @@ -359,11 +365,12 @@ Asset service request failures: {3}" + Environment.NewLine, inPacketsPerSecond, outPacketsPerSecond, pendingDownloads, pendingUploads, unackedBytes, totalFrameTime, netFrameTime, physicsFrameTime, otherFrameTime, agentFrameTime, imageFrameTime)); - Dictionary> sceneStats; - + /* 20130319 RA: For the moment, disable the dump of 'scene' catagory as they are mostly output by + * the two formatted printouts above. + SortedDictionary> sceneStats; if (StatsManager.TryGetStats("scene", out sceneStats)) { - foreach (KeyValuePair> kvp in sceneStats) + foreach (KeyValuePair> kvp in sceneStats) { foreach (Stat stat in kvp.Value.Values) { @@ -374,6 +381,7 @@ Asset service request failures: {3}" + Environment.NewLine, } } } + */ /* sb.Append(Environment.NewLine); @@ -405,6 +413,36 @@ Asset service request failures: {3}" + Environment.NewLine, /// public override string XReport(string uptime, string version) { + return OSDParser.SerializeJsonString(OReport(uptime, version)); + } + + /// + /// Report back collected statistical information as an OSDMap + /// + /// + public override OSDMap OReport(string uptime, string version) + { + // Get the amount of physical memory, allocated with the instance of this program, in kilobytes; + // the working set is the set of memory pages currently visible to this program in physical RAM + // memory and includes both shared (e.g. system libraries) and private data + double memUsage = Process.GetCurrentProcess().WorkingSet64 / 1024.0; + + // Get the number of threads from the system that are currently + // running + int numberThreadsRunning = 0; + foreach (ProcessThread currentThread in + Process.GetCurrentProcess().Threads) + { + // A known issue with the current process .Threads property is + // that it can return null threads, thus don't count those as + // running threads and prevent the program function from failing + if (currentThread != null && + currentThread.ThreadState == ThreadState.Running) + { + numberThreadsRunning++; + } + } + OSDMap args = new OSDMap(30); // args["AssetsInCache"] = OSD.FromString (String.Format ("{0:0.##}", AssetsInCache)); // args["TimeAfterCacheMiss"] = OSD.FromString (String.Format ("{0:0.##}", @@ -441,14 +479,28 @@ Asset service request failures: {3}" + Environment.NewLine, args["Memory"] = OSD.FromString (base.XReport (uptime, version)); args["Uptime"] = OSD.FromString (uptime); args["Version"] = OSD.FromString (version); - - string strBuffer = ""; - strBuffer = OSDParser.SerializeJsonString(args); - return strBuffer; + args["FrameDilatn"] = OSD.FromString(String.Format("{0:0.##}", m_frameDilation)); + args["Logging in Users"] = OSD.FromString(String.Format("{0:0.##}", + m_usersLoggingIn)); + args["GeoPrims"] = OSD.FromString(String.Format("{0:0.##}", + m_totalGeoPrims)); + args["Mesh Objects"] = OSD.FromString(String.Format("{0:0.##}", + m_totalMeshes)); + args["XEngine Thread Count"] = OSD.FromString(String.Format("{0:0.##}", + m_inUseThreads)); + args["Util Thread Count"] = OSD.FromString(String.Format("{0:0.##}", + Util.GetSmartThreadPoolInfo().InUseThreads)); + args["System Thread Count"] = OSD.FromString(String.Format( + "{0:0.##}", numberThreadsRunning)); + args["ProcMem"] = OSD.FromString(String.Format("{0:#,###,###.##}", + memUsage)); + + return args; } } + /// /// Pull packet queue stats from packet queues and report /// @@ -474,5 +526,11 @@ Asset service request failures: {3}" + Environment.NewLine, { return ""; } + + public OSDMap OReport(string uptime, string version) + { + OSDMap ret = new OSDMap(); + return ret; + } } } diff --git a/OpenSim/Framework/Monitoring/Stats/CounterStat.cs b/OpenSim/Framework/Monitoring/Stats/CounterStat.cs new file mode 100755 index 0000000..318cf1c --- /dev/null +++ b/OpenSim/Framework/Monitoring/Stats/CounterStat.cs @@ -0,0 +1,119 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using OpenMetaverse.StructuredData; + +namespace OpenSim.Framework.Monitoring +{ +// A statistic that wraps a counter. +// Built this way mostly so histograms and history can be created. +public class CounterStat : Stat +{ + private SortedDictionary m_histograms; + private object counterLock = new object(); + + public CounterStat( + string shortName, + string name, + string description, + string unitName, + string category, + string container, + StatVerbosity verbosity) + : base(shortName, name, description, unitName, category, container, StatType.Push, null, verbosity) + { + m_histograms = new SortedDictionary(); + } + + // Histograms are presumably added at intialization time and the list does not change thereafter. + // Thus no locking of the histogram list. + public void AddHistogram(string histoName, EventHistogram histo) + { + m_histograms.Add(histoName, histo); + } + + public delegate void ProcessHistogram(string name, EventHistogram histo); + public void ForEachHistogram(ProcessHistogram process) + { + foreach (KeyValuePair kvp in m_histograms) + { + process(kvp.Key, kvp.Value); + } + } + + public void Event() + { + this.Event(1); + } + + // Count the underlying counter. + public void Event(int cnt) + { + lock (counterLock) + { + base.Value += cnt; + + foreach (EventHistogram histo in m_histograms.Values) + { + histo.Event(cnt); + } + } + } + + // CounterStat is a basic stat plus histograms + public override OSDMap ToOSDMap() + { + // Get the foundational instance + OSDMap map = base.ToOSDMap(); + + map["StatType"] = "CounterStat"; + + // If there are any histograms, add a new field that is an array of histograms as OSDMaps + if (m_histograms.Count > 0) + { + lock (counterLock) + { + if (m_histograms.Count > 0) + { + OSDArray histos = new OSDArray(); + foreach (EventHistogram histo in m_histograms.Values) + { + histos.Add(histo.GetHistogramAsOSDMap()); + } + map.Add("Histograms", histos); + } + } + } + return map; + } +} +} diff --git a/OpenSim/Framework/Monitoring/Stats/EventHistogram.cs b/OpenSim/Framework/Monitoring/Stats/EventHistogram.cs new file mode 100755 index 0000000..f51f322 --- /dev/null +++ b/OpenSim/Framework/Monitoring/Stats/EventHistogram.cs @@ -0,0 +1,173 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using OpenMetaverse.StructuredData; + +namespace OpenSim.Framework.Monitoring +{ +// Create a time histogram of events. The histogram is built in a wrap-around +// array of equally distributed buckets. +// For instance, a minute long histogram of second sized buckets would be: +// new EventHistogram(60, 1000) +public class EventHistogram +{ + private int m_timeBase; + private int m_numBuckets; + private int m_bucketMilliseconds; + private int m_lastBucket; + private int m_totalHistogramMilliseconds; + private long[] m_histogram; + private object histoLock = new object(); + + public EventHistogram(int numberOfBuckets, int millisecondsPerBucket) + { + m_numBuckets = numberOfBuckets; + m_bucketMilliseconds = millisecondsPerBucket; + m_totalHistogramMilliseconds = m_numBuckets * m_bucketMilliseconds; + + m_histogram = new long[m_numBuckets]; + Zero(); + m_lastBucket = 0; + m_timeBase = Util.EnvironmentTickCount(); + } + + public void Event() + { + this.Event(1); + } + + // Record an event at time 'now' in the histogram. + public void Event(int cnt) + { + lock (histoLock) + { + // The time as displaced from the base of the histogram + int bucketTime = Util.EnvironmentTickCountSubtract(m_timeBase); + + // If more than the total time of the histogram, we just start over + if (bucketTime > m_totalHistogramMilliseconds) + { + Zero(); + m_lastBucket = 0; + m_timeBase = Util.EnvironmentTickCount(); + } + else + { + // To which bucket should we add this event? + int bucket = bucketTime / m_bucketMilliseconds; + + // Advance m_lastBucket to the new bucket. Zero any buckets skipped over. + while (bucket != m_lastBucket) + { + // Zero from just after the last bucket to the new bucket or the end + for (int jj = m_lastBucket + 1; jj <= Math.Min(bucket, m_numBuckets - 1); jj++) + { + m_histogram[jj] = 0; + } + m_lastBucket = bucket; + // If the new bucket is off the end, wrap around to the beginning + if (bucket > m_numBuckets) + { + bucket -= m_numBuckets; + m_lastBucket = 0; + m_histogram[m_lastBucket] = 0; + m_timeBase += m_totalHistogramMilliseconds; + } + } + } + m_histogram[m_lastBucket] += cnt; + } + } + + // Get a copy of the current histogram + public long[] GetHistogram() + { + long[] ret = new long[m_numBuckets]; + lock (histoLock) + { + int indx = m_lastBucket + 1; + for (int ii = 0; ii < m_numBuckets; ii++, indx++) + { + if (indx >= m_numBuckets) + indx = 0; + ret[ii] = m_histogram[indx]; + } + } + return ret; + } + + public OSDMap GetHistogramAsOSDMap() + { + OSDMap ret = new OSDMap(); + + ret.Add("Buckets", OSD.FromInteger(m_numBuckets)); + ret.Add("BucketMilliseconds", OSD.FromInteger(m_bucketMilliseconds)); + ret.Add("TotalMilliseconds", OSD.FromInteger(m_totalHistogramMilliseconds)); + + // Compute a number for the first bucket in the histogram. + // This will allow readers to know how this histogram relates to any previously read histogram. + int baseBucketNum = (m_timeBase / m_bucketMilliseconds) + m_lastBucket + 1; + ret.Add("BaseNumber", OSD.FromInteger(baseBucketNum)); + + ret.Add("Values", GetHistogramAsOSDArray()); + + return ret; + } + // Get a copy of the current histogram + public OSDArray GetHistogramAsOSDArray() + { + OSDArray ret = new OSDArray(m_numBuckets); + lock (histoLock) + { + int indx = m_lastBucket + 1; + for (int ii = 0; ii < m_numBuckets; ii++, indx++) + { + if (indx >= m_numBuckets) + indx = 0; + ret[ii] = OSD.FromLong(m_histogram[indx]); + } + } + return ret; + } + + // Zero out the histogram + public void Zero() + { + lock (histoLock) + { + for (int ii = 0; ii < m_numBuckets; ii++) + m_histogram[ii] = 0; + } + } +} + +} diff --git a/OpenSim/Framework/Monitoring/Stats/PercentageStat.cs b/OpenSim/Framework/Monitoring/Stats/PercentageStat.cs index 60bed55..55ddf06 100644 --- a/OpenSim/Framework/Monitoring/Stats/PercentageStat.cs +++ b/OpenSim/Framework/Monitoring/Stats/PercentageStat.cs @@ -29,6 +29,8 @@ using System; using System.Collections.Generic; using System.Text; +using OpenMetaverse.StructuredData; + namespace OpenSim.Framework.Monitoring { public class PercentageStat : Stat @@ -84,5 +86,19 @@ namespace OpenSim.Framework.Monitoring return sb.ToString(); } + + // PercentageStat is a basic stat plus percent calc + public override OSDMap ToOSDMap() + { + // Get the foundational instance + OSDMap map = base.ToOSDMap(); + + map["StatType"] = "PercentageStat"; + + map.Add("Antecedent", OSD.FromLong(Antecedent)); + map.Add("Consequent", OSD.FromLong(Consequent)); + + return map; + } } } \ No newline at end of file diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs index f91251b..a7cb2a6 100644 --- a/OpenSim/Framework/Monitoring/Stats/Stat.cs +++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs @@ -27,15 +27,23 @@ using System; using System.Collections.Generic; +using System.Linq; +using System.Reflection; using System.Text; +using log4net; +using OpenMetaverse.StructuredData; namespace OpenSim.Framework.Monitoring { /// /// Holds individual statistic details /// - public class Stat + public class Stat : IDisposable { +// private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + public static readonly char[] DisallowedShortNameCharacters = { '.' }; + /// /// Category of this stat (e.g. cache, scene, etc). /// @@ -93,7 +101,7 @@ namespace OpenSim.Framework.Monitoring /// /// Will be null if no measures of interest require samples. /// - private static Queue m_samples; + private Queue m_samples; /// /// Maximum number of statistical samples. @@ -160,6 +168,13 @@ namespace OpenSim.Framework.Monitoring throw new Exception( string.Format("Stat cannot be in category '{0}' since this is reserved for a subcommand", category)); + foreach (char c in DisallowedShortNameCharacters) + { + if (shortName.IndexOf(c) != -1) + shortName = shortName.Replace(c, '#'); +// throw new Exception(string.Format("Stat name {0} cannot contain character {1}", shortName, c)); + } + ShortName = shortName; Name = name; Description = description; @@ -181,6 +196,12 @@ namespace OpenSim.Framework.Monitoring Verbosity = verbosity; } + // IDisposable.Dispose() + public virtual void Dispose() + { + return; + } + /// /// Record a value in the sample set. /// @@ -196,6 +217,8 @@ namespace OpenSim.Framework.Monitoring if (m_samples.Count >= m_maxSamples) m_samples.Dequeue(); +// m_log.DebugFormat("[STAT]: Recording value {0} for {1}", newValue, Name); + m_samples.Enqueue(newValue); } } @@ -203,35 +226,100 @@ namespace OpenSim.Framework.Monitoring public virtual string ToConsoleString() { StringBuilder sb = new StringBuilder(); - sb.AppendFormat("{0}.{1}.{2} : {3}{4}", Category, Container, ShortName, Value, UnitName); + sb.AppendFormat( + "{0}.{1}.{2} : {3}{4}", + Category, + Container, + ShortName, + Value, + string.IsNullOrEmpty(UnitName) ? "" : string.Format(" {0}", UnitName)); AppendMeasuresOfInterest(sb); return sb.ToString(); } - protected void AppendMeasuresOfInterest(StringBuilder sb) + public virtual OSDMap ToOSDMap() { - if ((MeasuresOfInterest & MeasuresOfInterest.AverageChangeOverTime) - == MeasuresOfInterest.AverageChangeOverTime) + OSDMap ret = new OSDMap(); + ret.Add("StatType", "Stat"); // used by overloading classes to denote type of stat + + ret.Add("Category", OSD.FromString(Category)); + ret.Add("Container", OSD.FromString(Container)); + ret.Add("ShortName", OSD.FromString(ShortName)); + ret.Add("Name", OSD.FromString(Name)); + ret.Add("Description", OSD.FromString(Description)); + ret.Add("UnitName", OSD.FromString(UnitName)); + ret.Add("Value", OSD.FromReal(Value)); + + double lastChangeOverTime, averageChangeOverTime; + if (ComputeMeasuresOfInterest(out lastChangeOverTime, out averageChangeOverTime)) + { + ret.Add("LastChangeOverTime", OSD.FromReal(lastChangeOverTime)); + ret.Add("AverageChangeOverTime", OSD.FromReal(averageChangeOverTime)); + } + + return ret; + } + + // Compute the averages over time and return same. + // Return 'true' if averages were actually computed. 'false' if no average info. + public bool ComputeMeasuresOfInterest(out double lastChangeOverTime, out double averageChangeOverTime) + { + bool ret = false; + lastChangeOverTime = 0; + averageChangeOverTime = 0; + + if ((MeasuresOfInterest & MeasuresOfInterest.AverageChangeOverTime) == MeasuresOfInterest.AverageChangeOverTime) { double totalChange = 0; + double? penultimateSample = null; double? lastSample = null; lock (m_samples) { + // m_log.DebugFormat( + // "[STAT]: Samples for {0} are {1}", + // Name, string.Join(",", m_samples.Select(s => s.ToString()).ToArray())); + foreach (double s in m_samples) { if (lastSample != null) totalChange += s - (double)lastSample; + penultimateSample = lastSample; lastSample = s; } } + if (lastSample != null && penultimateSample != null) + { + lastChangeOverTime + = ((double)lastSample - (double)penultimateSample) / (Watchdog.WATCHDOG_INTERVAL_MS / 1000); + } + int divisor = m_samples.Count <= 1 ? 1 : m_samples.Count - 1; - sb.AppendFormat(", {0:0.##}{1}/s", totalChange / divisor / (Watchdog.WATCHDOG_INTERVAL_MS / 1000), UnitName); + averageChangeOverTime = totalChange / divisor / (Watchdog.WATCHDOG_INTERVAL_MS / 1000); + ret = true; + } + + return ret; + } + + protected void AppendMeasuresOfInterest(StringBuilder sb) + { + double lastChangeOverTime = 0; + double averageChangeOverTime = 0; + + if (ComputeMeasuresOfInterest(out lastChangeOverTime, out averageChangeOverTime)) + { + sb.AppendFormat( + ", {0:0.##}{1}/s, {2:0.##}{3}/s", + lastChangeOverTime, + string.IsNullOrEmpty(UnitName) ? "" : string.Format(" {0}", UnitName), + averageChangeOverTime, + string.IsNullOrEmpty(UnitName) ? "" : string.Format(" {0}", UnitName)); } } } diff --git a/OpenSim/Framework/Monitoring/StatsLogger.cs b/OpenSim/Framework/Monitoring/StatsLogger.cs new file mode 100644 index 0000000..15a37aa --- /dev/null +++ b/OpenSim/Framework/Monitoring/StatsLogger.cs @@ -0,0 +1,151 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using System.Text; +using System.Timers; +using log4net; + +namespace OpenSim.Framework.Monitoring +{ + /// + /// Provides a means to continuously log stats for debugging purposes. + /// + public static class StatsLogger + { + private static readonly ILog m_statsLog = LogManager.GetLogger("special.StatsLogger"); + + private static Timer m_loggingTimer; + private static int m_statsLogIntervalMs = 5000; + + public static void RegisterConsoleCommands(ICommandConsole console) + { + console.Commands.AddCommand( + "General", + false, + "stats record", + "stats record start|stop", + "Control whether stats are being regularly recorded to a separate file.", + "For debug purposes. Experimental.", + HandleStatsRecordCommand); + + console.Commands.AddCommand( + "General", + false, + "stats save", + "stats save ", + "Save stats snapshot to a file. If the file already exists, then the report is appended.", + "For debug purposes. Experimental.", + HandleStatsSaveCommand); + } + + public static void HandleStatsRecordCommand(string module, string[] cmd) + { + ICommandConsole con = MainConsole.Instance; + + if (cmd.Length != 3) + { + con.Output("Usage: stats record start|stop"); + return; + } + + if (cmd[2] == "start") + { + Start(); + con.OutputFormat("Now recording all stats to file every {0}ms", m_statsLogIntervalMs); + } + else if (cmd[2] == "stop") + { + Stop(); + con.Output("Stopped recording stats to file."); + } + } + + public static void HandleStatsSaveCommand(string module, string[] cmd) + { + ICommandConsole con = MainConsole.Instance; + + if (cmd.Length != 3) + { + con.Output("Usage: stats save "); + return; + } + + string path = cmd[2]; + + using (StreamWriter sw = new StreamWriter(path, true)) + { + foreach (string line in GetReport()) + sw.WriteLine(line); + } + + MainConsole.Instance.OutputFormat("Stats saved to file {0}", path); + } + + public static void Start() + { + if (m_loggingTimer != null) + Stop(); + + m_loggingTimer = new Timer(m_statsLogIntervalMs); + m_loggingTimer.AutoReset = false; + m_loggingTimer.Elapsed += Log; + m_loggingTimer.Start(); + } + + public static void Stop() + { + if (m_loggingTimer != null) + { + m_loggingTimer.Stop(); + } + } + + private static void Log(object sender, ElapsedEventArgs e) + { + foreach (string line in GetReport()) + m_statsLog.Info(line); + + m_loggingTimer.Start(); + } + + private static List GetReport() + { + List lines = new List(); + + lines.Add(string.Format("*** STATS REPORT AT {0} ***", DateTime.Now)); + + foreach (string report in StatsManager.GetAllStatsReports()) + lines.Add(report); + + return lines; + } + } +} \ No newline at end of file diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index 0762b01..3136ee8 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -26,15 +26,20 @@ */ using System; +using System.Collections; using System.Collections.Generic; +using System.Linq; using System.Text; +using OpenSim.Framework; +using OpenMetaverse.StructuredData; + namespace OpenSim.Framework.Monitoring { /// - /// Singleton used to provide access to statistics reporters + /// Static class used to register/deregister/fetch statistics /// - public class StatsManager + public static class StatsManager { // Subcommand used to list other stats. public const string AllSubCommand = "all"; @@ -51,31 +56,43 @@ namespace OpenSim.Framework.Monitoring /// /// Do not add or remove directly from this dictionary. /// - public static Dictionary>> RegisteredStats - = new Dictionary>>(); + public static SortedDictionary>> RegisteredStats + = new SortedDictionary>>(); - private static AssetStatsCollector assetStats; - private static UserStatsCollector userStats; - private static SimExtraStatsCollector simExtraStats = new SimExtraStatsCollector(); +// private static AssetStatsCollector assetStats; +// private static UserStatsCollector userStats; +// private static SimExtraStatsCollector simExtraStats = new SimExtraStatsCollector(); - public static AssetStatsCollector AssetStats { get { return assetStats; } } - public static UserStatsCollector UserStats { get { return userStats; } } - public static SimExtraStatsCollector SimExtraStats { get { return simExtraStats; } } +// public static AssetStatsCollector AssetStats { get { return assetStats; } } +// public static UserStatsCollector UserStats { get { return userStats; } } + public static SimExtraStatsCollector SimExtraStats { get; set; } public static void RegisterConsoleCommands(ICommandConsole console) { console.Commands.AddCommand( "General", false, - "show stats", - "show stats [list|all|]", + "stats show", + "stats show [list|all|([.])+", "Show statistical information for this server", "If no final argument is specified then legacy statistics information is currently shown.\n" - + "If list is specified then statistic categories are shown.\n" - + "If all is specified then all registered statistics are shown.\n" - + "If a category name is specified then only statistics from that category are shown.\n" + + "'list' argument will show statistic categories.\n" + + "'all' will show all statistics.\n" + + "A name will show statistics from that category.\n" + + "A . name will show statistics from that category in that container.\n" + + "More than one name can be given separated by spaces.\n" + "THIS STATS FACILITY IS EXPERIMENTAL AND DOES NOT YET CONTAIN ALL STATS", HandleShowStatsCommand); + + console.Commands.AddCommand( + "General", + false, + "show stats", + "show stats [list|all|([.])+", + "Alias for 'stats show' command", + HandleShowStatsCommand); + + StatsLogger.RegisterConsoleCommands(console); } public static void HandleShowStatsCommand(string module, string[] cmd) @@ -84,105 +101,286 @@ namespace OpenSim.Framework.Monitoring if (cmd.Length > 2) { - var categoryName = cmd[2]; - - if (categoryName == AllSubCommand) + foreach (string name in cmd.Skip(2)) { - foreach (var category in RegisteredStats.Values) + string[] components = name.Split('.'); + + string categoryName = components[0]; + string containerName = components.Length > 1 ? components[1] : null; + string statName = components.Length > 2 ? components[2] : null; + + if (categoryName == AllSubCommand) { - OutputCategoryStatsToConsole(con, category); + OutputAllStatsToConsole(con); } - } - else if (categoryName == ListSubCommand) - { - con.Output("Statistic categories available are:"); - foreach (string category in RegisteredStats.Keys) - con.OutputFormat(" {0}", category); - } - else - { - Dictionary> category; - if (!RegisteredStats.TryGetValue(categoryName, out category)) + else if (categoryName == ListSubCommand) { - con.OutputFormat("No such category as {0}", categoryName); + con.Output("Statistic categories available are:"); + foreach (string category in RegisteredStats.Keys) + con.OutputFormat(" {0}", category); } else { - OutputCategoryStatsToConsole(con, category); + SortedDictionary> category; + if (!RegisteredStats.TryGetValue(categoryName, out category)) + { + con.OutputFormat("No such category as {0}", categoryName); + } + else + { + if (String.IsNullOrEmpty(containerName)) + { + OutputCategoryStatsToConsole(con, category); + } + else + { + SortedDictionary container; + if (category.TryGetValue(containerName, out container)) + { + if (String.IsNullOrEmpty(statName)) + { + OutputContainerStatsToConsole(con, container); + } + else + { + Stat stat; + if (container.TryGetValue(statName, out stat)) + { + OutputStatToConsole(con, stat); + } + else + { + con.OutputFormat( + "No such stat {0} in {1}.{2}", statName, categoryName, containerName); + } + } + } + else + { + con.OutputFormat("No such container {0} in category {1}", containerName, categoryName); + } + } + } } } } else { // Legacy - con.Output(SimExtraStats.Report()); + if (SimExtraStats != null) + con.Output(SimExtraStats.Report()); + else + OutputAllStatsToConsole(con); } } - private static void OutputCategoryStatsToConsole( - ICommandConsole con, Dictionary> category) + public static List GetAllStatsReports() + { + List reports = new List(); + + foreach (var category in RegisteredStats.Values) + reports.AddRange(GetCategoryStatsReports(category)); + + return reports; + } + + private static void OutputAllStatsToConsole(ICommandConsole con) { + foreach (string report in GetAllStatsReports()) + con.Output(report); + } + + private static List GetCategoryStatsReports( + SortedDictionary> category) + { + List reports = new List(); + foreach (var container in category.Values) + reports.AddRange(GetContainerStatsReports(container)); + + return reports; + } + + private static void OutputCategoryStatsToConsole( + ICommandConsole con, SortedDictionary> category) + { + foreach (string report in GetCategoryStatsReports(category)) + con.Output(report); + } + + private static List GetContainerStatsReports(SortedDictionary container) + { + List reports = new List(); + + foreach (Stat stat in container.Values) + reports.Add(stat.ToConsoleString()); + + return reports; + } + + private static void OutputContainerStatsToConsole( + ICommandConsole con, SortedDictionary container) + { + foreach (string report in GetContainerStatsReports(container)) + con.Output(report); + } + + private static void OutputStatToConsole(ICommandConsole con, Stat stat) + { + con.Output(stat.ToConsoleString()); + } + + // Creates an OSDMap of the format: + // { categoryName: { + // containerName: { + // statName: { + // "Name": name, + // "ShortName": shortName, + // ... + // }, + // statName: { + // "Name": name, + // "ShortName": shortName, + // ... + // }, + // ... + // }, + // containerName: { + // ... + // }, + // ... + // }, + // categoryName: { + // ... + // }, + // ... + // } + // The passed in parameters will filter the categories, containers and stats returned. If any of the + // parameters are either EmptyOrNull or the AllSubCommand value, all of that type will be returned. + // Case matters. + public static OSDMap GetStatsAsOSDMap(string pCategoryName, string pContainerName, string pStatName) + { + OSDMap map = new OSDMap(); + + foreach (string catName in RegisteredStats.Keys) { - foreach (Stat stat in container.Values) + // Do this category if null spec, "all" subcommand or category name matches passed parameter. + // Skip category if none of the above. + if (!(String.IsNullOrEmpty(pCategoryName) || pCategoryName == AllSubCommand || pCategoryName == catName)) + continue; + + OSDMap contMap = new OSDMap(); + foreach (string contName in RegisteredStats[catName].Keys) { - con.Output(stat.ToConsoleString()); + if (!(string.IsNullOrEmpty(pContainerName) || pContainerName == AllSubCommand || pContainerName == contName)) + continue; + + OSDMap statMap = new OSDMap(); + + SortedDictionary theStats = RegisteredStats[catName][contName]; + foreach (string statName in theStats.Keys) + { + if (!(String.IsNullOrEmpty(pStatName) || pStatName == AllSubCommand || pStatName == statName)) + continue; + + statMap.Add(statName, theStats[statName].ToOSDMap()); + } + + contMap.Add(contName, statMap); } + map.Add(catName, contMap); } + + return map; } - /// - /// Start collecting statistics related to assets. - /// Should only be called once. - /// - public static AssetStatsCollector StartCollectingAssetStats() + public static Hashtable HandleStatsRequest(Hashtable request) { - assetStats = new AssetStatsCollector(); + Hashtable responsedata = new Hashtable(); +// string regpath = request["uri"].ToString(); + int response_code = 200; + string contenttype = "text/json"; - return assetStats; - } + string pCategoryName = StatsManager.AllSubCommand; + string pContainerName = StatsManager.AllSubCommand; + string pStatName = StatsManager.AllSubCommand; - /// - /// Start collecting statistics related to users. - /// Should only be called once. - /// - public static UserStatsCollector StartCollectingUserStats() - { - userStats = new UserStatsCollector(); + if (request.ContainsKey("cat")) pCategoryName = request["cat"].ToString(); + if (request.ContainsKey("cont")) pContainerName = request["cat"].ToString(); + if (request.ContainsKey("stat")) pStatName = request["stat"].ToString(); - return userStats; + string strOut = StatsManager.GetStatsAsOSDMap(pCategoryName, pContainerName, pStatName).ToString(); + + // If requestor wants it as a callback function, build response as a function rather than just the JSON string. + if (request.ContainsKey("callback")) + { + strOut = request["callback"].ToString() + "(" + strOut + ");"; + } + + // m_log.DebugFormat("{0} StatFetch: uri={1}, cat={2}, cont={3}, stat={4}, resp={5}", + // LogHeader, regpath, pCategoryName, pContainerName, pStatName, strOut); + + responsedata["int_response_code"] = response_code; + responsedata["content_type"] = contenttype; + responsedata["keepalive"] = false; + responsedata["str_response_string"] = strOut; + responsedata["access_control_allow_origin"] = "*"; + + return responsedata; } +// /// +// /// Start collecting statistics related to assets. +// /// Should only be called once. +// /// +// public static AssetStatsCollector StartCollectingAssetStats() +// { +// assetStats = new AssetStatsCollector(); +// +// return assetStats; +// } +// +// /// +// /// Start collecting statistics related to users. +// /// Should only be called once. +// /// +// public static UserStatsCollector StartCollectingUserStats() +// { +// userStats = new UserStatsCollector(); +// +// return userStats; +// } + /// - /// Registers a statistic. + /// Register a statistic. /// /// /// public static bool RegisterStat(Stat stat) { - Dictionary> category = null, newCategory; - Dictionary container = null, newContainer; + SortedDictionary> category = null, newCategory; + SortedDictionary container = null, newContainer; lock (RegisteredStats) { // Stat name is not unique across category/container/shortname key. // XXX: For now just return false. This is to avoid problems in regression tests where all tests // in a class are run in the same instance of the VM. - if (TryGetStat(stat, out category, out container)) + if (TryGetStatParents(stat, out category, out container)) return false; // We take a copy-on-write approach here of replacing dictionaries when keys are added or removed. // This means that we don't need to lock or copy them on iteration, which will be a much more // common operation after startup. if (container != null) - newContainer = new Dictionary(container); + newContainer = new SortedDictionary(container); else - newContainer = new Dictionary(); + newContainer = new SortedDictionary(); if (category != null) - newCategory = new Dictionary>(category); + newCategory = new SortedDictionary>(category); else - newCategory = new Dictionary>(); + newCategory = new SortedDictionary>(); newContainer[stat.ShortName] = stat; newCategory[stat.Container] = newContainer; @@ -196,21 +394,21 @@ namespace OpenSim.Framework.Monitoring /// Deregister a statistic /// > /// - /// public static bool DeregisterStat(Stat stat) { - Dictionary> category = null, newCategory; - Dictionary container = null, newContainer; + SortedDictionary> category = null, newCategory; + SortedDictionary container = null, newContainer; lock (RegisteredStats) { - if (!TryGetStat(stat, out category, out container)) + if (!TryGetStatParents(stat, out category, out container)) return false; - newContainer = new Dictionary(container); + newContainer = new SortedDictionary(container); newContainer.Remove(stat.ShortName); - newCategory = new Dictionary>(category); + newCategory = new SortedDictionary>(category); newCategory.Remove(stat.Container); newCategory[stat.Container] = newContainer; @@ -220,15 +418,70 @@ namespace OpenSim.Framework.Monitoring } } - public static bool TryGetStats(string category, out Dictionary> stats) + public static bool TryGetStat(string category, string container, string statShortName, out Stat stat) { - return RegisteredStats.TryGetValue(category, out stats); + stat = null; + SortedDictionary> categoryStats; + + lock (RegisteredStats) + { + if (!TryGetStatsForCategory(category, out categoryStats)) + return false; + + SortedDictionary containerStats; + + if (!categoryStats.TryGetValue(container, out containerStats)) + return false; + + return containerStats.TryGetValue(statShortName, out stat); + } + } + + public static bool TryGetStatsForCategory( + string category, out SortedDictionary> stats) + { + lock (RegisteredStats) + return RegisteredStats.TryGetValue(category, out stats); + } + + /// + /// Get the same stat for each container in a given category. + /// + /// + /// The stats if there were any to fetch. Otherwise null. + /// + /// + /// + public static List GetStatsFromEachContainer(string category, string statShortName) + { + SortedDictionary> categoryStats; + + lock (RegisteredStats) + { + if (!RegisteredStats.TryGetValue(category, out categoryStats)) + return null; + + List stats = null; + + foreach (SortedDictionary containerStats in categoryStats.Values) + { + if (containerStats.ContainsKey(statShortName)) + { + if (stats == null) + stats = new List(); + + stats.Add(containerStats[statShortName]); + } + } + + return stats; + } } - public static bool TryGetStat( + public static bool TryGetStatParents( Stat stat, - out Dictionary> category, - out Dictionary container) + out SortedDictionary> category, + out SortedDictionary container) { category = null; container = null; @@ -252,9 +505,9 @@ namespace OpenSim.Framework.Monitoring { lock (RegisteredStats) { - foreach (Dictionary> category in RegisteredStats.Values) + foreach (SortedDictionary> category in RegisteredStats.Values) { - foreach (Dictionary container in category.Values) + foreach (SortedDictionary container in category.Values) { foreach (Stat stat in container.Values) { diff --git a/OpenSim/Framework/Monitoring/UserStatsCollector.cs b/OpenSim/Framework/Monitoring/UserStatsCollector.cs index e89c8e6..81e0fa4 100644 --- a/OpenSim/Framework/Monitoring/UserStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/UserStatsCollector.cs @@ -27,6 +27,8 @@ using System.Timers; +using OpenMetaverse.StructuredData; + namespace OpenSim.Framework.Monitoring { /// @@ -88,5 +90,21 @@ namespace OpenSim.Framework.Monitoring Logouts total : {3}", SuccessfulLogins, SuccessfulLoginsToday, SuccessfulLoginsYesterday, Logouts); } + + public override string XReport(string uptime, string version) + { + return OSDParser.SerializeJsonString(OReport(uptime, version)); + } + + public override OSDMap OReport(string uptime, string version) + { + OSDMap ret = new OSDMap(); + ret.Add("SuccessfulLogins", OSD.FromInteger(SuccessfulLogins)); + ret.Add("SuccessfulLoginsToday", OSD.FromInteger(SuccessfulLoginsToday)); + ret.Add("SuccessfulLoginsYesterday", OSD.FromInteger(SuccessfulLoginsYesterday)); + ret.Add("Logouts", OSD.FromInteger(Logouts)); + + return ret; + } } } diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index 3f992b1..a644fa5 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -38,6 +38,8 @@ namespace OpenSim.Framework.Monitoring /// public static class Watchdog { + private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); + /// Timer interval in milliseconds for the watchdog timer public const double WATCHDOG_INTERVAL_MS = 2500.0d; @@ -82,12 +84,32 @@ namespace OpenSim.Framework.Monitoring /// public Func AlarmMethod { get; set; } - public ThreadWatchdogInfo(Thread thread, int timeout) + /// + /// Stat structure associated with this thread. + /// + public Stat Stat { get; set; } + + public ThreadWatchdogInfo(Thread thread, int timeout, string name) { Thread = thread; Timeout = timeout; FirstTick = Environment.TickCount & Int32.MaxValue; LastTick = FirstTick; + + Stat + = new Stat( + name, + string.Format("Last update of thread {0}", name), + "", + "ms", + "server", + "thread", + StatType.Pull, + MeasuresOfInterest.None, + stat => stat.Value = Environment.TickCount & Int32.MaxValue - LastTick, + StatVerbosity.Debug); + + StatsManager.RegisterStat(Stat); } public ThreadWatchdogInfo(ThreadWatchdogInfo previousTwi) @@ -100,6 +122,11 @@ namespace OpenSim.Framework.Monitoring AlarmIfTimeout = previousTwi.AlarmIfTimeout; AlarmMethod = previousTwi.AlarmMethod; } + + public void Cleanup() + { + StatsManager.DeregisterStat(Stat); + } } /// @@ -116,7 +143,7 @@ namespace OpenSim.Framework.Monitoring get { return m_enabled; } set { -// m_log.DebugFormat("[MEMORY WATCHDOG]: Setting MemoryWatchdog.Enabled to {0}", value); + // m_log.DebugFormat("[MEMORY WATCHDOG]: Setting MemoryWatchdog.Enabled to {0}", value); if (value == m_enabled) return; @@ -132,9 +159,8 @@ namespace OpenSim.Framework.Monitoring m_watchdogTimer.Enabled = m_enabled; } } - private static bool m_enabled; - private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); + private static bool m_enabled; private static Dictionary m_threads; private static System.Timers.Timer m_watchdogTimer; @@ -155,57 +181,19 @@ namespace OpenSim.Framework.Monitoring } /// - /// Start a new thread that is tracked by the watchdog timer. - /// - /// The method that will be executed in a new thread - /// A name to give to the new thread - /// Priority to run the thread at - /// True to run this thread as a background thread, otherwise false - /// Trigger an alarm function is we have timed out - /// The newly created Thread object - public static Thread StartThread( - ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout) - { - return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS); - } - - /// - /// Start a new thread that is tracked by the watchdog timer + /// Add a thread to the watchdog tracker. /// - /// The method that will be executed in a new thread - /// A name to give to the new thread - /// Priority to run the thread at - /// True to run this thread as a background - /// thread, otherwise false - /// Trigger an alarm function is we have timed out - /// - /// Alarm method to call if alarmIfTimeout is true and there is a timeout. - /// Normally, this will just return some useful debugging information. - /// - /// Number of milliseconds to wait until we issue a warning about timeout. - /// The newly created Thread object - public static Thread StartThread( - ThreadStart start, string name, ThreadPriority priority, bool isBackground, - bool alarmIfTimeout, Func alarmMethod, int timeout) + /// Information about the thread. + /// Name of the thread. + /// If true then creation of thread is logged. + public static void AddThread(ThreadWatchdogInfo info, string name, bool log = true) { - Thread thread = new Thread(start); - thread.Name = name; - thread.Priority = priority; - thread.IsBackground = isBackground; - - ThreadWatchdogInfo twi - = new ThreadWatchdogInfo(thread, timeout) - { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod }; - - m_log.DebugFormat( - "[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId); + if (log) + m_log.DebugFormat( + "[WATCHDOG]: Started tracking thread {0}, ID {1}", name, info.Thread.ManagedThreadId); lock (m_threads) - m_threads.Add(twi.Thread.ManagedThreadId, twi); - - thread.Start(); - - return thread; + m_threads.Add(info.Thread.ManagedThreadId, info); } /// @@ -219,25 +207,28 @@ namespace OpenSim.Framework.Monitoring /// /// Stops watchdog tracking on the current thread /// + /// If true then normal events in thread removal are not logged. /// /// True if the thread was removed from the list of tracked /// threads, otherwise false /// - public static bool RemoveThread() + public static bool RemoveThread(bool log = true) { - return RemoveThread(Thread.CurrentThread.ManagedThreadId); + return RemoveThread(Thread.CurrentThread.ManagedThreadId, log); } - private static bool RemoveThread(int threadID) + private static bool RemoveThread(int threadID, bool log = true) { lock (m_threads) { ThreadWatchdogInfo twi; if (m_threads.TryGetValue(threadID, out twi)) { - m_log.DebugFormat( - "[WATCHDOG]: Removing thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId); + if (log) + m_log.DebugFormat( + "[WATCHDOG]: Removing thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId); + twi.Cleanup(); m_threads.Remove(threadID); return true; @@ -293,7 +284,7 @@ namespace OpenSim.Framework.Monitoring } catch { } } - + /// /// Get currently watched threads for diagnostic purposes /// @@ -380,6 +371,7 @@ namespace OpenSim.Framework.Monitoring if (MemoryWatchdog.Enabled) MemoryWatchdog.Update(); + ChecksManager.CheckChecks(); StatsManager.RecordStats(); m_watchdogTimer.Start(); diff --git a/OpenSim/Framework/Monitoring/WorkManager.cs b/OpenSim/Framework/Monitoring/WorkManager.cs new file mode 100644 index 0000000..d1a74ce --- /dev/null +++ b/OpenSim/Framework/Monitoring/WorkManager.cs @@ -0,0 +1,290 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Reflection; +using System.Threading; +using log4net; + +namespace OpenSim.Framework.Monitoring +{ + /// + /// Manages various work items in the simulator. + /// + /// + /// Currently, here work can be started + /// * As a long-running and monitored thread. + /// * In a thread that will never timeout but where the job is expected to eventually complete. + /// * In a threadpool thread that will timeout if it takes a very long time to complete (> 10 mins). + /// * As a job which will be run in a single-threaded job engine. Such jobs must not incorporate delays (sleeps, + /// network waits, etc.). + /// + /// This is an evolving approach to better manage the work that OpenSimulator is asked to do from a very diverse + /// range of sources (client actions, incoming network, outgoing network calls, etc.). + /// + /// Util.FireAndForget is still available to insert jobs in the threadpool, though this is equivalent to + /// WorkManager.RunInThreadPool(). + /// + public static class WorkManager + { + private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); + + public static JobEngine JobEngine { get; private set; } + + static WorkManager() + { + JobEngine = new JobEngine("Non-blocking non-critical job engine", "JOB ENGINE"); + + StatsManager.RegisterStat( + new Stat( + "JobsWaiting", + "Number of jobs waiting for processing.", + "", + "", + "server", + "jobengine", + StatType.Pull, + MeasuresOfInterest.None, + stat => stat.Value = JobEngine.JobsWaiting, + StatVerbosity.Debug)); + + MainConsole.Instance.Commands.AddCommand( + "Debug", + false, + "debug jobengine", + "debug jobengine ", + "Start, stop, get status or set logging level of the job engine.", + "If stopped then all outstanding jobs are processed immediately.", + HandleControlCommand); + } + + /// + /// Start a new long-lived thread. + /// + /// The method that will be executed in a new thread + /// A name to give to the new thread + /// Priority to run the thread at + /// True to run this thread as a background thread, otherwise false + /// Trigger an alarm function is we have timed out + /// If true then creation of thread is logged. + /// The newly created Thread object + public static Thread StartThread( + ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, bool log = true) + { + return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, Watchdog.DEFAULT_WATCHDOG_TIMEOUT_MS, log); + } + + /// + /// Start a new thread that is tracked by the watchdog + /// + /// The method that will be executed in a new thread + /// A name to give to the new thread + /// Priority to run the thread at + /// True to run this thread as a background + /// thread, otherwise false + /// Trigger an alarm function is we have timed out + /// + /// Alarm method to call if alarmIfTimeout is true and there is a timeout. + /// Normally, this will just return some useful debugging information. + /// + /// Number of milliseconds to wait until we issue a warning about timeout. + /// If true then creation of thread is logged. + /// The newly created Thread object + public static Thread StartThread( + ThreadStart start, string name, ThreadPriority priority, bool isBackground, + bool alarmIfTimeout, Func alarmMethod, int timeout, bool log = true) + { + Thread thread = new Thread(start); + thread.Priority = priority; + thread.IsBackground = isBackground; + + Watchdog.ThreadWatchdogInfo twi + = new Watchdog.ThreadWatchdogInfo(thread, timeout, name) + { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod }; + + Watchdog.AddThread(twi, name, log:log); + + thread.Start(); + thread.Name = name; + + return thread; + } + + /// + /// Run the callback in a new thread immediately. If the thread exits with an exception log it but do + /// not propogate it. + /// + /// Code for the thread to execute. + /// Object to pass to the thread. + /// Name of the thread + public static void RunInThread(WaitCallback callback, object obj, string name, bool log = false) + { + if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest) + { + Culture.SetCurrentCulture(); + callback(obj); + return; + } + + ThreadStart ts = new ThreadStart(delegate() + { + try + { + Culture.SetCurrentCulture(); + callback(obj); + Watchdog.RemoveThread(log:false); + } + catch (Exception e) + { + m_log.Error(string.Format("[WATCHDOG]: Exception in thread {0}.", name), e); + } + }); + + StartThread(ts, name, ThreadPriority.Normal, true, false, log:log); + } + + /// + /// Run the callback via a threadpool thread. + /// + /// + /// Such jobs may run after some delay but must always complete. + /// + /// + /// + /// The name of the job. This is used in monitoring and debugging. + public static void RunInThreadPool(System.Threading.WaitCallback callback, object obj, string name) + { + Util.FireAndForget(callback, obj, name); + } + + /// + /// Run a job. + /// + /// + /// This differs from direct scheduling (e.g. Util.FireAndForget) in that a job can be run in the job + /// engine if it is running, where all jobs are currently performed in sequence on a single thread. This is + /// to prevent observed overload and server freeze problems when there are hundreds of connections which all attempt to + /// perform work at once (e.g. in conference situations). With lower numbers of connections, the small + /// delay in performing jobs in sequence rather than concurrently has not been notiecable in testing, though a future more + /// sophisticated implementation could perform jobs concurrently when the server is under low load. + /// + /// However, be advised that some callers of this function rely on all jobs being performed in sequence if any + /// jobs are performed in sequence (i.e. if jobengine is active or not). Therefore, expanding the jobengine + /// beyond a single thread will require considerable thought. + /// + /// Also, any jobs submitted must be guaranteed to complete within a reasonable timeframe (e.g. they cannot + /// incorporate a network delay with a long timeout). At the moment, work that could suffer such issues + /// should still be run directly with RunInThread(), Util.FireAndForget(), etc. This is another area where + /// the job engine could be improved and so CPU utilization improved by better management of concurrency within + /// OpenSimulator. + /// + /// General classification for the job (e.g. "RezAttachments"). + /// Callback for job. + /// Object to pass to callback when run + /// Specific name of job (e.g. "RezAttachments for Joe Bloggs" + /// If set to true then the job may be run in ths calling thread. + /// If the true then the job must never timeout. + /// If set to true then extra logging is performed. + public static void RunJob( + string jobType, WaitCallback callback, object obj, string name, + bool canRunInThisThread = false, bool mustNotTimeout = false, + bool log = false) + { + if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest) + { + Culture.SetCurrentCulture(); + callback(obj); + return; + } + + if (JobEngine.IsRunning) + JobEngine.QueueJob(name, () => callback(obj)); + else if (canRunInThisThread) + callback(obj); + else if (mustNotTimeout) + RunInThread(callback, obj, name, log); + else + Util.FireAndForget(callback, obj, name); + } + + private static void HandleControlCommand(string module, string[] args) + { + // if (SceneManager.Instance.CurrentScene != null && SceneManager.Instance.CurrentScene != m_udpServer.Scene) + // return; + + if (args.Length < 3) + { + MainConsole.Instance.Output("Usage: debug jobengine "); + return; + } + + string subCommand = args[2]; + + if (subCommand == "stop") + { + JobEngine.Stop(); + MainConsole.Instance.OutputFormat("Stopped job engine."); + } + else if (subCommand == "start") + { + JobEngine.Start(); + MainConsole.Instance.OutputFormat("Started job engine."); + } + else if (subCommand == "status") + { + MainConsole.Instance.OutputFormat("Job engine running: {0}", JobEngine.IsRunning); + + JobEngine.Job job = JobEngine.CurrentJob; + MainConsole.Instance.OutputFormat("Current job {0}", job != null ? job.Name : "none"); + + MainConsole.Instance.OutputFormat( + "Jobs waiting: {0}", JobEngine.IsRunning ? JobEngine.JobsWaiting.ToString() : "n/a"); + MainConsole.Instance.OutputFormat("Log Level: {0}", JobEngine.LogLevel); + } + else if (subCommand == "log") + { + if (args.Length < 4) + { + MainConsole.Instance.Output("Usage: debug jobengine log "); + return; + } + + // int logLevel; + int logLevel = int.Parse(args[3]); + // if (ConsoleUtil.TryParseConsoleInt(MainConsole.Instance, args[4], out logLevel)) + // { + JobEngine.LogLevel = logLevel; + MainConsole.Instance.OutputFormat("Set debug log level to {0}", JobEngine.LogLevel); + // } + } + else + { + MainConsole.Instance.OutputFormat("Unrecognized job engine subcommand {0}", subCommand); + } + } + } +} \ No newline at end of file -- cgit v1.1