From b1b46872500476cf97b5de8c16012b8545fed0c7 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 10 Jan 2013 00:57:49 +0000 Subject: Add "show script timers" command to show script timers. For debug purposes. Also, "show sensors" changes to "show script sensors". --- OpenSim/Framework/Monitoring/MemoryWatchdog.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/MemoryWatchdog.cs b/OpenSim/Framework/Monitoring/MemoryWatchdog.cs index c6010cd..bc5ed97 100644 --- a/OpenSim/Framework/Monitoring/MemoryWatchdog.cs +++ b/OpenSim/Framework/Monitoring/MemoryWatchdog.cs @@ -72,7 +72,7 @@ namespace OpenSim.Framework.Monitoring /// public static double LastMemoryChurn { - get { if (m_samples.Count > 0) return m_samples.Last(); else return 0; } + get { if (m_samples.Count > 0) return m_samples.First(); else return 0; } } /// -- cgit v1.1 From 84407e322f6469a3001b390f7c516f4eabaad1e6 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 10 Jan 2013 01:30:00 +0000 Subject: revert accidental change to MemoryWatchdog stat calculation in previous b1b4687 --- OpenSim/Framework/Monitoring/MemoryWatchdog.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/MemoryWatchdog.cs b/OpenSim/Framework/Monitoring/MemoryWatchdog.cs index bc5ed97..c6010cd 100644 --- a/OpenSim/Framework/Monitoring/MemoryWatchdog.cs +++ b/OpenSim/Framework/Monitoring/MemoryWatchdog.cs @@ -72,7 +72,7 @@ namespace OpenSim.Framework.Monitoring /// public static double LastMemoryChurn { - get { if (m_samples.Count > 0) return m_samples.First(); else return 0; } + get { if (m_samples.Count > 0) return m_samples.Last(); else return 0; } } /// -- cgit v1.1 From 1f1da230976451d30d920c237d53c699ba96b9d9 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 5 Feb 2013 00:23:17 +0000 Subject: Bump version and assembly version numbers from 0.7.5 to 0.7.6 This is mostly Bluewall's work but I am also bumping the general version number OpenSimulator 0.7.5 remains in the release candidate stage. I'm doing this because master is significantly adding things that will not be in 0.7.5 This update should not cause issues with existing external binary DLLs because our DLLs do not have strong names and so the exact version match requirement is not in force. --- OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs index 1f2bb40..bb83db1 100644 --- a/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs +++ b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs @@ -29,5 +29,5 @@ using System.Runtime.InteropServices; // Build Number // Revision // -[assembly: AssemblyVersion("0.7.5.*")] +[assembly: AssemblyVersion("0.7.6.*")] [assembly: AssemblyFileVersion("1.0.0.0")] -- cgit v1.1 From afeb5d4917506ced2a1e4098aeb4bc94ae64fc06 Mon Sep 17 00:00:00 2001 From: Dan Lake Date: Thu, 14 Feb 2013 20:05:42 -0800 Subject: Use SortedDictionary in StatsManager instead of regular Dictionary so stats will interate and print in a defined order --- .../Framework/Monitoring/SimExtraStatsCollector.cs | 4 +-- OpenSim/Framework/Monitoring/StatsManager.cs | 40 +++++++++++----------- 2 files changed, 22 insertions(+), 22 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs index aa86202..3765efb 100644 --- a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs @@ -359,11 +359,11 @@ Asset service request failures: {3}" + Environment.NewLine, inPacketsPerSecond, outPacketsPerSecond, pendingDownloads, pendingUploads, unackedBytes, totalFrameTime, netFrameTime, physicsFrameTime, otherFrameTime, agentFrameTime, imageFrameTime)); - Dictionary> sceneStats; + SortedDictionary> sceneStats; if (StatsManager.TryGetStats("scene", out sceneStats)) { - foreach (KeyValuePair> kvp in sceneStats) + foreach (KeyValuePair> kvp in sceneStats) { foreach (Stat stat in kvp.Value.Values) { diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index 0762b01..910907e 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -51,8 +51,8 @@ namespace OpenSim.Framework.Monitoring /// /// Do not add or remove directly from this dictionary. /// - public static Dictionary>> RegisteredStats - = new Dictionary>>(); + public static SortedDictionary>> RegisteredStats + = new SortedDictionary>>(); private static AssetStatsCollector assetStats; private static UserStatsCollector userStats; @@ -101,7 +101,7 @@ namespace OpenSim.Framework.Monitoring } else { - Dictionary> category; + SortedDictionary> category; if (!RegisteredStats.TryGetValue(categoryName, out category)) { con.OutputFormat("No such category as {0}", categoryName); @@ -120,7 +120,7 @@ namespace OpenSim.Framework.Monitoring } private static void OutputCategoryStatsToConsole( - ICommandConsole con, Dictionary> category) + ICommandConsole con, SortedDictionary> category) { foreach (var container in category.Values) { @@ -160,8 +160,8 @@ namespace OpenSim.Framework.Monitoring /// public static bool RegisterStat(Stat stat) { - Dictionary> category = null, newCategory; - Dictionary container = null, newContainer; + SortedDictionary> category = null, newCategory; + SortedDictionary container = null, newContainer; lock (RegisteredStats) { @@ -175,14 +175,14 @@ namespace OpenSim.Framework.Monitoring // This means that we don't need to lock or copy them on iteration, which will be a much more // common operation after startup. if (container != null) - newContainer = new Dictionary(container); + newContainer = new SortedDictionary(container); else - newContainer = new Dictionary(); + newContainer = new SortedDictionary(); if (category != null) - newCategory = new Dictionary>(category); + newCategory = new SortedDictionary>(category); else - newCategory = new Dictionary>(); + newCategory = new SortedDictionary>(); newContainer[stat.ShortName] = stat; newCategory[stat.Container] = newContainer; @@ -196,21 +196,21 @@ namespace OpenSim.Framework.Monitoring /// Deregister a statistic /// > /// - /// public static bool DeregisterStat(Stat stat) { - Dictionary> category = null, newCategory; - Dictionary container = null, newContainer; + SortedDictionary> category = null, newCategory; + SortedDictionary container = null, newContainer; lock (RegisteredStats) { if (!TryGetStat(stat, out category, out container)) return false; - newContainer = new Dictionary(container); + newContainer = new SortedDictionary(container); newContainer.Remove(stat.ShortName); - newCategory = new Dictionary>(category); + newCategory = new SortedDictionary>(category); newCategory.Remove(stat.Container); newCategory[stat.Container] = newContainer; @@ -220,15 +220,15 @@ namespace OpenSim.Framework.Monitoring } } - public static bool TryGetStats(string category, out Dictionary> stats) + public static bool TryGetStats(string category, out SortedDictionary> stats) { return RegisteredStats.TryGetValue(category, out stats); } public static bool TryGetStat( Stat stat, - out Dictionary> category, - out Dictionary container) + out SortedDictionary> category, + out SortedDictionary container) { category = null; container = null; @@ -252,9 +252,9 @@ namespace OpenSim.Framework.Monitoring { lock (RegisteredStats) { - foreach (Dictionary> category in RegisteredStats.Values) + foreach (SortedDictionary> category in RegisteredStats.Values) { - foreach (Dictionary container in category.Values) + foreach (SortedDictionary container in category.Values) { foreach (Stat stat in container.Values) { -- cgit v1.1 From 4779f7d7d5ce0e284d9ed15104389f8479b11545 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Tue, 19 Feb 2013 17:14:55 -0800 Subject: Deleted all AssemblyFileVersion directives --- OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs index bb83db1..36678bb 100644 --- a/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs +++ b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs @@ -30,4 +30,4 @@ using System.Runtime.InteropServices; // Revision // [assembly: AssemblyVersion("0.7.6.*")] -[assembly: AssemblyFileVersion("1.0.0.0")] + -- cgit v1.1 From 681653ca130eaf15c62aae6fd1a7c5276036a0e9 Mon Sep 17 00:00:00 2001 From: Robert Adams Date: Wed, 20 Feb 2013 14:11:02 -0800 Subject: Add a method to IStatsCollector for returning stats as an OSDMap. Extend implementors of IStatsCollector to return an OSDMap of stats. Update UserStatsCollector and AssetStatsCollector to return both string and OSDMap data (as well as console format). --- .../Framework/Monitoring/AssetStatsCollector.cs | 26 ++++++++++++++++++++++ OpenSim/Framework/Monitoring/BaseStatsCollector.cs | 7 ++++++ .../Monitoring/Interfaces/IStatsCollector.cs | 9 ++++++++ .../Framework/Monitoring/SimExtraStatsCollector.cs | 21 +++++++++++++---- OpenSim/Framework/Monitoring/UserStatsCollector.cs | 18 +++++++++++++++ 5 files changed, 77 insertions(+), 4 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/AssetStatsCollector.cs b/OpenSim/Framework/Monitoring/AssetStatsCollector.cs index 2a4d45b..6a0f676 100644 --- a/OpenSim/Framework/Monitoring/AssetStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/AssetStatsCollector.cs @@ -28,6 +28,8 @@ using System; using System.Timers; +using OpenMetaverse.StructuredData; + namespace OpenSim.Framework.Monitoring { /// @@ -100,5 +102,29 @@ Asset requests yesterday : {3} ({4} per hour) of which {5} were not found", AssetRequestsToday, assetRequestsTodayPerHour, AssetRequestsNotFoundToday, AssetRequestsYesterday, assetRequestsYesterdayPerHour, AssetRequestsNotFoundYesterday); } + + public override string XReport(string uptime, string version) + { + return OSDParser.SerializeJsonString(OReport(uptime, version)); + } + + public override OSDMap OReport(string uptime, string version) + { + double elapsedHours = (DateTime.Now - startTime).TotalHours; + if (elapsedHours <= 0) { elapsedHours = 1; } // prevent divide by zero + + long assetRequestsTodayPerHour = (long)Math.Round(AssetRequestsToday / elapsedHours); + long assetRequestsYesterdayPerHour = (long)Math.Round(AssetRequestsYesterday / 24.0); + + OSDMap ret = new OSDMap(); + ret.Add("AssetRequestsToday", OSD.FromLong(AssetRequestsToday)); + ret.Add("AssetRequestsTodayPerHour", OSD.FromLong(assetRequestsTodayPerHour)); + ret.Add("AssetRequestsNotFoundToday", OSD.FromLong(AssetRequestsNotFoundToday)); + ret.Add("AssetRequestsYesterday", OSD.FromLong(AssetRequestsYesterday)); + ret.Add("AssetRequestsYesterdayPerHour", OSD.FromLong(assetRequestsYesterdayPerHour)); + ret.Add("AssetRequestsNotFoundYesterday", OSD.FromLong(assetRequestsNotFoundYesterday)); + + return ret; + } } } diff --git a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs index 2903b6e..be1d02b 100644 --- a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs @@ -67,5 +67,12 @@ namespace OpenSim.Framework.Monitoring { return (string) Math.Round(GC.GetTotalMemory(false) / 1024.0 / 1024.0).ToString() ; } + + public virtual OSDMap OReport(string uptime, string version) + { + OSDMap ret = new OSDMap(); + ret.Add("TotalMemory", new OSDReal(Math.Round(GC.GetTotalMemory(false) / 1024.0 / 1024.0))); + return ret; + } } } diff --git a/OpenSim/Framework/Monitoring/Interfaces/IStatsCollector.cs b/OpenSim/Framework/Monitoring/Interfaces/IStatsCollector.cs index 99f75e3..40df562 100644 --- a/OpenSim/Framework/Monitoring/Interfaces/IStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/Interfaces/IStatsCollector.cs @@ -25,6 +25,8 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +using OpenMetaverse.StructuredData; + namespace OpenSim.Framework.Monitoring { /// @@ -45,5 +47,12 @@ namespace OpenSim.Framework.Monitoring /// A /// string XReport(string uptime, string version); + + /// + /// Report back collected statistical information as an OSDMap of key/values + /// + /// + /// + OSDMap OReport(string uptime, string version); } } diff --git a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs index 3765efb..109a58f 100644 --- a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs @@ -405,6 +405,15 @@ Asset service request failures: {3}" + Environment.NewLine, /// public override string XReport(string uptime, string version) { + return OSDParser.SerializeJsonString(OReport(uptime, version)); + } + + /// + /// Report back collected statistical information as an OSDMap + /// + /// + public override OSDMap OReport(string uptime, string version) + { OSDMap args = new OSDMap(30); // args["AssetsInCache"] = OSD.FromString (String.Format ("{0:0.##}", AssetsInCache)); // args["TimeAfterCacheMiss"] = OSD.FromString (String.Format ("{0:0.##}", @@ -442,13 +451,11 @@ Asset service request failures: {3}" + Environment.NewLine, args["Uptime"] = OSD.FromString (uptime); args["Version"] = OSD.FromString (version); - string strBuffer = ""; - strBuffer = OSDParser.SerializeJsonString(args); - - return strBuffer; + return args; } } + /// /// Pull packet queue stats from packet queues and report /// @@ -474,5 +481,11 @@ Asset service request failures: {3}" + Environment.NewLine, { return ""; } + + public OSDMap OReport(string uptime, string version) + { + OSDMap ret = new OSDMap(); + return ret; + } } } diff --git a/OpenSim/Framework/Monitoring/UserStatsCollector.cs b/OpenSim/Framework/Monitoring/UserStatsCollector.cs index e89c8e6..81e0fa4 100644 --- a/OpenSim/Framework/Monitoring/UserStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/UserStatsCollector.cs @@ -27,6 +27,8 @@ using System.Timers; +using OpenMetaverse.StructuredData; + namespace OpenSim.Framework.Monitoring { /// @@ -88,5 +90,21 @@ namespace OpenSim.Framework.Monitoring Logouts total : {3}", SuccessfulLogins, SuccessfulLoginsToday, SuccessfulLoginsYesterday, Logouts); } + + public override string XReport(string uptime, string version) + { + return OSDParser.SerializeJsonString(OReport(uptime, version)); + } + + public override OSDMap OReport(string uptime, string version) + { + OSDMap ret = new OSDMap(); + ret.Add("SuccessfulLogins", OSD.FromInteger(SuccessfulLogins)); + ret.Add("SuccessfulLoginsToday", OSD.FromInteger(SuccessfulLoginsToday)); + ret.Add("SuccessfulLoginsYesterday", OSD.FromInteger(SuccessfulLoginsYesterday)); + ret.Add("Logouts", OSD.FromInteger(Logouts)); + + return ret; + } } } -- cgit v1.1 From 303e21babe48e9daf39dad6b3d721fd5b9bbfb8f Mon Sep 17 00:00:00 2001 From: Robert Adams Date: Thu, 21 Feb 2013 13:36:31 -0800 Subject: Add CounterStat which is a wrapper for a counter stat but, because the 'count' event is internal, historical data can be built it. Also includes EventHistogram class for building time based, bucketed history of event occurances. Make Stat implement IDisposable for subclasses that might need it. --- OpenSim/Framework/Monitoring/Stats/CounterStat.cs | 211 ++++++++++++++++++++++ OpenSim/Framework/Monitoring/Stats/Stat.cs | 8 +- 2 files changed, 218 insertions(+), 1 deletion(-) create mode 100755 OpenSim/Framework/Monitoring/Stats/CounterStat.cs (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Stats/CounterStat.cs b/OpenSim/Framework/Monitoring/Stats/CounterStat.cs new file mode 100755 index 0000000..d81f182 --- /dev/null +++ b/OpenSim/Framework/Monitoring/Stats/CounterStat.cs @@ -0,0 +1,211 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using OpenMetaverse.StructuredData; + +namespace OpenSim.Framework.Monitoring +{ +// Create a time histogram of events. The histogram is built in a wrap-around +// array of equally distributed buckets. +// For instance, a minute long histogram of second sized buckets would be: +// new EventHistogram(60, 1000) +public class EventHistogram +{ + private int m_timeBase; + private int m_numBuckets; + private int m_bucketMilliseconds; + private int m_lastBucket; + private int m_totalHistogramMilliseconds; + private long[] m_histogram; + private object histoLock = new object(); + + public EventHistogram(int numberOfBuckets, int millisecondsPerBucket) + { + m_numBuckets = numberOfBuckets; + m_bucketMilliseconds = millisecondsPerBucket; + m_totalHistogramMilliseconds = m_numBuckets * m_bucketMilliseconds; + + m_histogram = new long[m_numBuckets]; + Zero(); + m_lastBucket = 0; + m_timeBase = Util.EnvironmentTickCount(); + } + + public void Event() + { + this.Event(1); + } + + // Record an event at time 'now' in the histogram. + public void Event(int cnt) + { + lock (histoLock) + { + // The time as displaced from the base of the histogram + int bucketTime = Util.EnvironmentTickCountSubtract(m_timeBase); + + // If more than the total time of the histogram, we just start over + if (bucketTime > m_totalHistogramMilliseconds) + { + Zero(); + m_lastBucket = 0; + m_timeBase = Util.EnvironmentTickCount(); + } + else + { + // To which bucket should we add this event? + int bucket = bucketTime / m_bucketMilliseconds; + + // Advance m_lastBucket to the new bucket. Zero any buckets skipped over. + while (bucket != m_lastBucket) + { + // Zero from just after the last bucket to the new bucket or the end + for (int jj = m_lastBucket + 1; jj <= Math.Min(bucket, m_numBuckets - 1); jj++) + { + m_histogram[jj] = 0; + } + m_lastBucket = bucket; + // If the new bucket is off the end, wrap around to the beginning + if (bucket > m_numBuckets) + { + bucket -= m_numBuckets; + m_lastBucket = 0; + m_histogram[m_lastBucket] = 0; + m_timeBase += m_totalHistogramMilliseconds; + } + } + } + m_histogram[m_lastBucket] += cnt; + } + } + + // Get a copy of the current histogram + public long[] GetHistogram() + { + long[] ret = new long[m_numBuckets]; + lock (histoLock) + { + int indx = m_lastBucket + 1; + for (int ii = 0; ii < m_numBuckets; ii++, indx++) + { + if (indx >= m_numBuckets) + indx = 0; + ret[ii] = m_histogram[indx]; + } + } + return ret; + } + + // Get a copy of the current histogram + public OSDArray GetHistogramAsOSDArray() + { + OSDArray ret = new OSDArray(m_numBuckets); + lock (histoLock) + { + int indx = m_lastBucket + 1; + for (int ii = 0; ii < m_numBuckets; ii++, indx++) + { + if (indx >= m_numBuckets) + indx = 0; + ret[ii] = OSD.FromLong(m_histogram[indx]); + } + } + return ret; + } + + // Zero out the histogram + public void Zero() + { + lock (histoLock) + { + for (int ii = 0; ii < m_numBuckets; ii++) + m_histogram[ii] = 0; + } + } +} + +// A statistic that wraps a counter. +// Built this way mostly so histograms and history can be created. +public class CounterStat : Stat +{ + private SortedDictionary m_histograms; + private object counterLock = new object(); + + public CounterStat( + string shortName, + string name, + string description, + string unitName, + string category, + string container, + StatVerbosity verbosity) + : base(shortName, name, description, unitName, category, container, StatType.Push, null, verbosity) + { + m_histograms = new SortedDictionary(); + } + + // Histograms are presumably added at intialization time and the list does not change thereafter. + // Thus no locking of the histogram list. + public void AddHistogram(string histoName, EventHistogram histo) + { + m_histograms.Add(histoName, histo); + } + + public delegate void ProcessHistogram(string name, EventHistogram histo); + public void ForEachHistogram(ProcessHistogram process) + { + foreach (KeyValuePair kvp in m_histograms) + { + process(kvp.Key, kvp.Value); + } + } + + public void Event() + { + this.Event(1); + } + + // Count the underlying counter. + public void Event(int cnt) + { + lock (counterLock) + { + base.Value += cnt; + + foreach (EventHistogram histo in m_histograms.Values) + { + histo.Event(cnt); + } + } + } +} +} diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs index f91251b..fccc460 100644 --- a/OpenSim/Framework/Monitoring/Stats/Stat.cs +++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs @@ -34,7 +34,7 @@ namespace OpenSim.Framework.Monitoring /// /// Holds individual statistic details /// - public class Stat + public class Stat : IDisposable { /// /// Category of this stat (e.g. cache, scene, etc). @@ -181,6 +181,12 @@ namespace OpenSim.Framework.Monitoring Verbosity = verbosity; } + // IDisposable.Dispose() + public virtual void Dispose() + { + return; + } + /// /// Record a value in the sample set. /// -- cgit v1.1 From 76ee47c7972969d7dbadff2c66d050ad84b10078 Mon Sep 17 00:00:00 2001 From: Robert Adams Date: Thu, 21 Feb 2013 15:48:19 -0800 Subject: Add a virtual ToOSDMap() function to Monitoring.Stat for future overloading by aggregation statistics and browser based stat viewers. --- OpenSim/Framework/Monitoring/Stats/Stat.cs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs index fccc460..c8d9174 100644 --- a/OpenSim/Framework/Monitoring/Stats/Stat.cs +++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs @@ -29,6 +29,8 @@ using System; using System.Collections.Generic; using System.Text; +using OpenMetaverse.StructuredData; + namespace OpenSim.Framework.Monitoring { /// @@ -216,6 +218,20 @@ namespace OpenSim.Framework.Monitoring return sb.ToString(); } + public virtual OSDMap ToOSDMap() + { + OSDMap ret = new OSDMap(); + ret.Add("Category", OSD.FromString(Category)); + ret.Add("Container", OSD.FromString(Container)); + ret.Add("ShortName", OSD.FromString(ShortName)); + ret.Add("Name", OSD.FromString(Name)); + ret.Add("Description", OSD.FromString(Description)); + ret.Add("UnitName", OSD.FromString(UnitName)); + ret.Add("Value", OSD.FromReal(Value)); + + return ret; + } + protected void AppendMeasuresOfInterest(StringBuilder sb) { if ((MeasuresOfInterest & MeasuresOfInterest.AverageChangeOverTime) -- cgit v1.1 From ee8d726ec5cbcbaca8aebbcbfd25cfba963c43f8 Mon Sep 17 00:00:00 2001 From: Robert Adams Date: Sat, 23 Feb 2013 17:04:19 -0800 Subject: Modify StatsManager so console command "show stats category container" only outputs the statistics in the specified container in the category. --- OpenSim/Framework/Monitoring/StatsManager.cs | 29 +++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index 910907e..24db6d4 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -85,6 +85,7 @@ namespace OpenSim.Framework.Monitoring if (cmd.Length > 2) { var categoryName = cmd[2]; + var containerName = cmd.Length > 3 ? cmd[3] : String.Empty; if (categoryName == AllSubCommand) { @@ -108,7 +109,20 @@ namespace OpenSim.Framework.Monitoring } else { - OutputCategoryStatsToConsole(con, category); + if (String.IsNullOrEmpty(containerName)) + OutputCategoryStatsToConsole(con, category); + else + { + SortedDictionary container; + if (category.TryGetValue(containerName, out container)) + { + OutputContainerStatsToConsole(con, container); + } + else + { + con.OutputFormat("No such container {0} in category {1}", containerName, categoryName); + } + } } } } @@ -124,10 +138,15 @@ namespace OpenSim.Framework.Monitoring { foreach (var container in category.Values) { - foreach (Stat stat in container.Values) - { - con.Output(stat.ToConsoleString()); - } + OutputContainerStatsToConsole(con, container); + } + } + + private static void OutputContainerStatsToConsole( ICommandConsole con, SortedDictionary container) + { + foreach (Stat stat in container.Values) + { + con.Output(stat.ToConsoleString()); } } -- cgit v1.1 From 9f213892ea0441e0e4fb9a967850a3f0a07912b3 Mon Sep 17 00:00:00 2001 From: Robert Adams Date: Sat, 23 Feb 2013 17:50:24 -0800 Subject: Add EventHistogram.GetHistogramAsOSDMap that returns that parameters about the histogram as well as the values. --- OpenSim/Framework/Monitoring/Stats/CounterStat.cs | 439 +++++++++++----------- 1 file changed, 228 insertions(+), 211 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Stats/CounterStat.cs b/OpenSim/Framework/Monitoring/Stats/CounterStat.cs index d81f182..caea30d 100755 --- a/OpenSim/Framework/Monitoring/Stats/CounterStat.cs +++ b/OpenSim/Framework/Monitoring/Stats/CounterStat.cs @@ -1,211 +1,228 @@ -/* - * Copyright (c) Contributors, http://opensimulator.org/ - * See CONTRIBUTORS.TXT for a full list of copyright holders. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the OpenSimulator Project nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -using OpenMetaverse.StructuredData; - -namespace OpenSim.Framework.Monitoring -{ -// Create a time histogram of events. The histogram is built in a wrap-around -// array of equally distributed buckets. -// For instance, a minute long histogram of second sized buckets would be: -// new EventHistogram(60, 1000) -public class EventHistogram -{ - private int m_timeBase; - private int m_numBuckets; - private int m_bucketMilliseconds; - private int m_lastBucket; - private int m_totalHistogramMilliseconds; - private long[] m_histogram; - private object histoLock = new object(); - - public EventHistogram(int numberOfBuckets, int millisecondsPerBucket) - { - m_numBuckets = numberOfBuckets; - m_bucketMilliseconds = millisecondsPerBucket; - m_totalHistogramMilliseconds = m_numBuckets * m_bucketMilliseconds; - - m_histogram = new long[m_numBuckets]; - Zero(); - m_lastBucket = 0; - m_timeBase = Util.EnvironmentTickCount(); - } - - public void Event() - { - this.Event(1); - } - - // Record an event at time 'now' in the histogram. - public void Event(int cnt) - { - lock (histoLock) - { - // The time as displaced from the base of the histogram - int bucketTime = Util.EnvironmentTickCountSubtract(m_timeBase); - - // If more than the total time of the histogram, we just start over - if (bucketTime > m_totalHistogramMilliseconds) - { - Zero(); - m_lastBucket = 0; - m_timeBase = Util.EnvironmentTickCount(); - } - else - { - // To which bucket should we add this event? - int bucket = bucketTime / m_bucketMilliseconds; - - // Advance m_lastBucket to the new bucket. Zero any buckets skipped over. - while (bucket != m_lastBucket) - { - // Zero from just after the last bucket to the new bucket or the end - for (int jj = m_lastBucket + 1; jj <= Math.Min(bucket, m_numBuckets - 1); jj++) - { - m_histogram[jj] = 0; - } - m_lastBucket = bucket; - // If the new bucket is off the end, wrap around to the beginning - if (bucket > m_numBuckets) - { - bucket -= m_numBuckets; - m_lastBucket = 0; - m_histogram[m_lastBucket] = 0; - m_timeBase += m_totalHistogramMilliseconds; - } - } - } - m_histogram[m_lastBucket] += cnt; - } - } - - // Get a copy of the current histogram - public long[] GetHistogram() - { - long[] ret = new long[m_numBuckets]; - lock (histoLock) - { - int indx = m_lastBucket + 1; - for (int ii = 0; ii < m_numBuckets; ii++, indx++) - { - if (indx >= m_numBuckets) - indx = 0; - ret[ii] = m_histogram[indx]; - } - } - return ret; - } - - // Get a copy of the current histogram - public OSDArray GetHistogramAsOSDArray() - { - OSDArray ret = new OSDArray(m_numBuckets); - lock (histoLock) - { - int indx = m_lastBucket + 1; - for (int ii = 0; ii < m_numBuckets; ii++, indx++) - { - if (indx >= m_numBuckets) - indx = 0; - ret[ii] = OSD.FromLong(m_histogram[indx]); - } - } - return ret; - } - - // Zero out the histogram - public void Zero() - { - lock (histoLock) - { - for (int ii = 0; ii < m_numBuckets; ii++) - m_histogram[ii] = 0; - } - } -} - -// A statistic that wraps a counter. -// Built this way mostly so histograms and history can be created. -public class CounterStat : Stat -{ - private SortedDictionary m_histograms; - private object counterLock = new object(); - - public CounterStat( - string shortName, - string name, - string description, - string unitName, - string category, - string container, - StatVerbosity verbosity) - : base(shortName, name, description, unitName, category, container, StatType.Push, null, verbosity) - { - m_histograms = new SortedDictionary(); - } - - // Histograms are presumably added at intialization time and the list does not change thereafter. - // Thus no locking of the histogram list. - public void AddHistogram(string histoName, EventHistogram histo) - { - m_histograms.Add(histoName, histo); - } - - public delegate void ProcessHistogram(string name, EventHistogram histo); - public void ForEachHistogram(ProcessHistogram process) - { - foreach (KeyValuePair kvp in m_histograms) - { - process(kvp.Key, kvp.Value); - } - } - - public void Event() - { - this.Event(1); - } - - // Count the underlying counter. - public void Event(int cnt) - { - lock (counterLock) - { - base.Value += cnt; - - foreach (EventHistogram histo in m_histograms.Values) - { - histo.Event(cnt); - } - } - } -} -} +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using OpenMetaverse.StructuredData; + +namespace OpenSim.Framework.Monitoring +{ +// Create a time histogram of events. The histogram is built in a wrap-around +// array of equally distributed buckets. +// For instance, a minute long histogram of second sized buckets would be: +// new EventHistogram(60, 1000) +public class EventHistogram +{ + private int m_timeBase; + private int m_numBuckets; + private int m_bucketMilliseconds; + private int m_lastBucket; + private int m_totalHistogramMilliseconds; + private long[] m_histogram; + private object histoLock = new object(); + + public EventHistogram(int numberOfBuckets, int millisecondsPerBucket) + { + m_numBuckets = numberOfBuckets; + m_bucketMilliseconds = millisecondsPerBucket; + m_totalHistogramMilliseconds = m_numBuckets * m_bucketMilliseconds; + + m_histogram = new long[m_numBuckets]; + Zero(); + m_lastBucket = 0; + m_timeBase = Util.EnvironmentTickCount(); + } + + public void Event() + { + this.Event(1); + } + + // Record an event at time 'now' in the histogram. + public void Event(int cnt) + { + lock (histoLock) + { + // The time as displaced from the base of the histogram + int bucketTime = Util.EnvironmentTickCountSubtract(m_timeBase); + + // If more than the total time of the histogram, we just start over + if (bucketTime > m_totalHistogramMilliseconds) + { + Zero(); + m_lastBucket = 0; + m_timeBase = Util.EnvironmentTickCount(); + } + else + { + // To which bucket should we add this event? + int bucket = bucketTime / m_bucketMilliseconds; + + // Advance m_lastBucket to the new bucket. Zero any buckets skipped over. + while (bucket != m_lastBucket) + { + // Zero from just after the last bucket to the new bucket or the end + for (int jj = m_lastBucket + 1; jj <= Math.Min(bucket, m_numBuckets - 1); jj++) + { + m_histogram[jj] = 0; + } + m_lastBucket = bucket; + // If the new bucket is off the end, wrap around to the beginning + if (bucket > m_numBuckets) + { + bucket -= m_numBuckets; + m_lastBucket = 0; + m_histogram[m_lastBucket] = 0; + m_timeBase += m_totalHistogramMilliseconds; + } + } + } + m_histogram[m_lastBucket] += cnt; + } + } + + // Get a copy of the current histogram + public long[] GetHistogram() + { + long[] ret = new long[m_numBuckets]; + lock (histoLock) + { + int indx = m_lastBucket + 1; + for (int ii = 0; ii < m_numBuckets; ii++, indx++) + { + if (indx >= m_numBuckets) + indx = 0; + ret[ii] = m_histogram[indx]; + } + } + return ret; + } + + public OSDMap GetHistogramAsOSDMap() + { + OSDMap ret = new OSDMap(); + + ret.Add("Buckets", OSD.FromInteger(m_numBuckets)); + ret.Add("BucketMilliseconds", OSD.FromInteger(m_bucketMilliseconds)); + ret.Add("TotalMilliseconds", OSD.FromInteger(m_totalHistogramMilliseconds)); + + // Compute a number for the first bucket in the histogram. + // This will allow readers to know how this histogram relates to any previously read histogram. + int baseBucketNum = (m_timeBase / m_bucketMilliseconds) + m_lastBucket + 1; + ret.Add("BaseNumber", OSD.FromInteger(baseBucketNum)); + + ret.Add("Values", GetHistogramAsOSDArray()); + + return ret; + } + // Get a copy of the current histogram + public OSDArray GetHistogramAsOSDArray() + { + OSDArray ret = new OSDArray(m_numBuckets); + lock (histoLock) + { + int indx = m_lastBucket + 1; + for (int ii = 0; ii < m_numBuckets; ii++, indx++) + { + if (indx >= m_numBuckets) + indx = 0; + ret[ii] = OSD.FromLong(m_histogram[indx]); + } + } + return ret; + } + + // Zero out the histogram + public void Zero() + { + lock (histoLock) + { + for (int ii = 0; ii < m_numBuckets; ii++) + m_histogram[ii] = 0; + } + } +} + +// A statistic that wraps a counter. +// Built this way mostly so histograms and history can be created. +public class CounterStat : Stat +{ + private SortedDictionary m_histograms; + private object counterLock = new object(); + + public CounterStat( + string shortName, + string name, + string description, + string unitName, + string category, + string container, + StatVerbosity verbosity) + : base(shortName, name, description, unitName, category, container, StatType.Push, null, verbosity) + { + m_histograms = new SortedDictionary(); + } + + // Histograms are presumably added at intialization time and the list does not change thereafter. + // Thus no locking of the histogram list. + public void AddHistogram(string histoName, EventHistogram histo) + { + m_histograms.Add(histoName, histo); + } + + public delegate void ProcessHistogram(string name, EventHistogram histo); + public void ForEachHistogram(ProcessHistogram process) + { + foreach (KeyValuePair kvp in m_histograms) + { + process(kvp.Key, kvp.Value); + } + } + + public void Event() + { + this.Event(1); + } + + // Count the underlying counter. + public void Event(int cnt) + { + lock (counterLock) + { + base.Value += cnt; + + foreach (EventHistogram histo in m_histograms.Values) + { + histo.Event(cnt); + } + } + } +} +} -- cgit v1.1 From 2aae046b95cc028f0eb495857d5659971e06f604 Mon Sep 17 00:00:00 2001 From: Robert Adams Date: Sun, 24 Feb 2013 07:38:24 -0800 Subject: Make StatsManager default output a little more readable --- OpenSim/Framework/Monitoring/Stats/Stat.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs index c8d9174..2e7665f 100644 --- a/OpenSim/Framework/Monitoring/Stats/Stat.cs +++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs @@ -211,7 +211,7 @@ namespace OpenSim.Framework.Monitoring public virtual string ToConsoleString() { StringBuilder sb = new StringBuilder(); - sb.AppendFormat("{0}.{1}.{2} : {3}{4}", Category, Container, ShortName, Value, UnitName); + sb.AppendFormat("{0}.{1}.{2} : {3} {4}", Category, Container, ShortName, Value, UnitName); AppendMeasuresOfInterest(sb); -- cgit v1.1 From c2e4f8aed5dee4a35679d646326537a15153f19d Mon Sep 17 00:00:00 2001 From: Robert Adams Date: Tue, 19 Mar 2013 12:37:44 -0700 Subject: For the moment, disable the output of the 'scene' statistics in SimExtraStatsCollector and thus for the command 'show stats' because it is ugly and most of the information is already output in the formatted printout that appears before. --- OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs index 109a58f..6a68322 100644 --- a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs @@ -359,8 +359,9 @@ Asset service request failures: {3}" + Environment.NewLine, inPacketsPerSecond, outPacketsPerSecond, pendingDownloads, pendingUploads, unackedBytes, totalFrameTime, netFrameTime, physicsFrameTime, otherFrameTime, agentFrameTime, imageFrameTime)); + /* 20130319 RA: For the moment, disable the dump of 'scene' catagory as they are mostly output by + * the two formatted printouts above. SortedDictionary> sceneStats; - if (StatsManager.TryGetStats("scene", out sceneStats)) { foreach (KeyValuePair> kvp in sceneStats) @@ -374,6 +375,7 @@ Asset service request failures: {3}" + Environment.NewLine, } } } + */ /* sb.Append(Environment.NewLine); -- cgit v1.1 From 0d2fd0d914581f755661455b8db2b9e399154632 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Mon, 17 Jun 2013 22:39:00 +0100 Subject: Make general server stats available on the robust console as well as the simulator console This means the "show stats" command is now active on the robust console. --- .../Framework/Monitoring/ServerStatsCollector.cs | 309 +++++++++++++++++++++ OpenSim/Framework/Monitoring/StatsManager.cs | 72 ++--- 2 files changed, 349 insertions(+), 32 deletions(-) create mode 100644 OpenSim/Framework/Monitoring/ServerStatsCollector.cs (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs new file mode 100644 index 0000000..80d0a89 --- /dev/null +++ b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs @@ -0,0 +1,309 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Net.NetworkInformation; +using System.Text; +using System.Threading; +using log4net; +using Nini.Config; +using OpenMetaverse.StructuredData; +using OpenSim.Framework; + +namespace OpenSim.Framework.Monitoring +{ + public class ServerStatsCollector + { + private readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); + private readonly string LogHeader = "[SERVER STATS]"; + + public bool Enabled = false; + private static Dictionary RegisteredStats = new Dictionary(); + + public readonly string CategoryServer = "server"; + + public readonly string ContainerProcessor = "processor"; + public readonly string ContainerMemory = "memory"; + public readonly string ContainerNetwork = "network"; + public readonly string ContainerProcess = "process"; + + public string NetworkInterfaceTypes = "Ethernet"; + + readonly int performanceCounterSampleInterval = 500; +// int lastperformanceCounterSampleTime = 0; + + private class PerfCounterControl + { + public PerformanceCounter perfCounter; + public int lastFetch; + public string name; + public PerfCounterControl(PerformanceCounter pPc) + : this(pPc, String.Empty) + { + } + public PerfCounterControl(PerformanceCounter pPc, string pName) + { + perfCounter = pPc; + lastFetch = 0; + name = pName; + } + } + + PerfCounterControl processorPercentPerfCounter = null; + + // IRegionModuleBase.Initialize + public void Initialise(IConfigSource source) + { + IConfig cfg = source.Configs["Monitoring"]; + + if (cfg != null) + Enabled = cfg.GetBoolean("ServerStatsEnabled", true); + + if (Enabled) + { + NetworkInterfaceTypes = cfg.GetString("NetworkInterfaceTypes", "Ethernet"); + } + } + + public void Start() + { + if (RegisteredStats.Count == 0) + RegisterServerStats(); + } + + public void Close() + { + if (RegisteredStats.Count > 0) + { + foreach (Stat stat in RegisteredStats.Values) + { + StatsManager.DeregisterStat(stat); + stat.Dispose(); + } + RegisteredStats.Clear(); + } + } + + private void MakeStat(string pName, string pDesc, string pUnit, string pContainer, Action act) + { + string desc = pDesc; + if (desc == null) + desc = pName; + Stat stat = new Stat(pName, pName, desc, pUnit, CategoryServer, pContainer, StatType.Pull, act, StatVerbosity.Info); + StatsManager.RegisterStat(stat); + RegisteredStats.Add(pName, stat); + } + + public void RegisterServerStats() + { +// lastperformanceCounterSampleTime = Util.EnvironmentTickCount(); + PerformanceCounter tempPC; + Stat tempStat; + string tempName; + + try + { + tempName = "CPUPercent"; + tempPC = new PerformanceCounter("Processor", "% Processor Time", "_Total"); + processorPercentPerfCounter = new PerfCounterControl(tempPC); + // A long time bug in mono is that CPU percent is reported as CPU percent idle. Windows reports CPU percent busy. + tempStat = new Stat(tempName, tempName, "", "percent", CategoryServer, ContainerProcessor, + StatType.Pull, (s) => { GetNextValue(s, processorPercentPerfCounter, Util.IsWindows() ? 1 : -1); }, + StatVerbosity.Info); + StatsManager.RegisterStat(tempStat); + RegisteredStats.Add(tempName, tempStat); + + MakeStat("TotalProcessorTime", null, "sec", ContainerProcessor, + (s) => { s.Value = Process.GetCurrentProcess().TotalProcessorTime.TotalSeconds; }); + + MakeStat("UserProcessorTime", null, "sec", ContainerProcessor, + (s) => { s.Value = Process.GetCurrentProcess().UserProcessorTime.TotalSeconds; }); + + MakeStat("PrivilegedProcessorTime", null, "sec", ContainerProcessor, + (s) => { s.Value = Process.GetCurrentProcess().PrivilegedProcessorTime.TotalSeconds; }); + + MakeStat("Threads", null, "threads", ContainerProcessor, + (s) => { s.Value = Process.GetCurrentProcess().Threads.Count; }); + } + catch (Exception e) + { + m_log.ErrorFormat("{0} Exception creating 'Process': {1}", LogHeader, e); + } + + try + { + List okInterfaceTypes = new List(NetworkInterfaceTypes.Split(',')); + + IEnumerable nics = NetworkInterface.GetAllNetworkInterfaces(); + foreach (NetworkInterface nic in nics) + { + if (nic.OperationalStatus != OperationalStatus.Up) + continue; + + string nicInterfaceType = nic.NetworkInterfaceType.ToString(); + if (!okInterfaceTypes.Contains(nicInterfaceType)) + { + m_log.DebugFormat("{0} Not including stats for network interface '{1}' of type '{2}'.", + LogHeader, nic.Name, nicInterfaceType); + m_log.DebugFormat("{0} To include, add to comma separated list in [Monitoring]NetworkInterfaceTypes={1}", + LogHeader, NetworkInterfaceTypes); + continue; + } + + if (nic.Supports(NetworkInterfaceComponent.IPv4)) + { + IPv4InterfaceStatistics nicStats = nic.GetIPv4Statistics(); + if (nicStats != null) + { + MakeStat("BytesRcvd/" + nic.Name, nic.Name, "KB", ContainerNetwork, + (s) => { LookupNic(s, (ns) => { return ns.BytesReceived; }, 1024.0); }); + MakeStat("BytesSent/" + nic.Name, nic.Name, "KB", ContainerNetwork, + (s) => { LookupNic(s, (ns) => { return ns.BytesSent; }, 1024.0); }); + MakeStat("TotalBytes/" + nic.Name, nic.Name, "KB", ContainerNetwork, + (s) => { LookupNic(s, (ns) => { return ns.BytesSent + ns.BytesReceived; }, 1024.0); }); + } + } + // TODO: add IPv6 (it may actually happen someday) + } + } + catch (Exception e) + { + m_log.ErrorFormat("{0} Exception creating 'Network Interface': {1}", LogHeader, e); + } + + MakeStat("ProcessMemory", null, "MB", ContainerMemory, + (s) => { s.Value = Process.GetCurrentProcess().WorkingSet64 / 1024d / 1024d; }); + MakeStat("ObjectMemory", null, "MB", ContainerMemory, + (s) => { s.Value = GC.GetTotalMemory(false) / 1024d / 1024d; }); + MakeStat("LastMemoryChurn", null, "MB/sec", ContainerMemory, + (s) => { s.Value = Math.Round(MemoryWatchdog.LastMemoryChurn * 1000d / 1024d / 1024d, 3); }); + MakeStat("AverageMemoryChurn", null, "MB/sec", ContainerMemory, + (s) => { s.Value = Math.Round(MemoryWatchdog.AverageMemoryChurn * 1000d / 1024d / 1024d, 3); }); + } + + // Notes on performance counters: + // "How To Read Performance Counters": http://blogs.msdn.com/b/bclteam/archive/2006/06/02/618156.aspx + // "How to get the CPU Usage in C#": http://stackoverflow.com/questions/278071/how-to-get-the-cpu-usage-in-c + // "Mono Performance Counters": http://www.mono-project.com/Mono_Performance_Counters + private delegate double PerfCounterNextValue(); + private void GetNextValue(Stat stat, PerfCounterControl perfControl) + { + GetNextValue(stat, perfControl, 1.0); + } + private void GetNextValue(Stat stat, PerfCounterControl perfControl, double factor) + { + if (Util.EnvironmentTickCountSubtract(perfControl.lastFetch) > performanceCounterSampleInterval) + { + if (perfControl != null && perfControl.perfCounter != null) + { + try + { + // Kludge for factor to run double duty. If -1, subtract the value from one + if (factor == -1) + stat.Value = 1 - perfControl.perfCounter.NextValue(); + else + stat.Value = perfControl.perfCounter.NextValue() / factor; + } + catch (Exception e) + { + m_log.ErrorFormat("{0} Exception on NextValue fetching {1}: {2}", LogHeader, stat.Name, e); + } + perfControl.lastFetch = Util.EnvironmentTickCount(); + } + } + } + + // Lookup the nic that goes with this stat and set the value by using a fetch action. + // Not sure about closure with delegates inside delegates. + private delegate double GetIPv4StatValue(IPv4InterfaceStatistics interfaceStat); + private void LookupNic(Stat stat, GetIPv4StatValue getter, double factor) + { + // Get the one nic that has the name of this stat + IEnumerable nics = NetworkInterface.GetAllNetworkInterfaces().Where( + (network) => network.Name == stat.Description); + try + { + foreach (NetworkInterface nic in nics) + { + IPv4InterfaceStatistics intrStats = nic.GetIPv4Statistics(); + if (intrStats != null) + { + double newVal = Math.Round(getter(intrStats) / factor, 3); + stat.Value = newVal; + } + break; + } + } + catch + { + // There are times interfaces go away so we just won't update the stat for this + m_log.ErrorFormat("{0} Exception fetching stat on interface '{1}'", LogHeader, stat.Description); + } + } + } + + public class ServerStatsAggregator : Stat + { + public ServerStatsAggregator( + string shortName, + string name, + string description, + string unitName, + string category, + string container + ) + : base( + shortName, + name, + description, + unitName, + category, + container, + StatType.Push, + MeasuresOfInterest.None, + null, + StatVerbosity.Info) + { + } + public override string ToConsoleString() + { + StringBuilder sb = new StringBuilder(); + + return sb.ToString(); + } + + public override OSDMap ToOSDMap() + { + OSDMap ret = new OSDMap(); + + return ret; + } + } +} diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index 24db6d4..3aee984 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -54,13 +54,13 @@ namespace OpenSim.Framework.Monitoring public static SortedDictionary>> RegisteredStats = new SortedDictionary>>(); - private static AssetStatsCollector assetStats; - private static UserStatsCollector userStats; - private static SimExtraStatsCollector simExtraStats = new SimExtraStatsCollector(); +// private static AssetStatsCollector assetStats; +// private static UserStatsCollector userStats; +// private static SimExtraStatsCollector simExtraStats = new SimExtraStatsCollector(); - public static AssetStatsCollector AssetStats { get { return assetStats; } } - public static UserStatsCollector UserStats { get { return userStats; } } - public static SimExtraStatsCollector SimExtraStats { get { return simExtraStats; } } +// public static AssetStatsCollector AssetStats { get { return assetStats; } } +// public static UserStatsCollector UserStats { get { return userStats; } } + public static SimExtraStatsCollector SimExtraStats { get; set; } public static void RegisterConsoleCommands(ICommandConsole console) { @@ -89,10 +89,7 @@ namespace OpenSim.Framework.Monitoring if (categoryName == AllSubCommand) { - foreach (var category in RegisteredStats.Values) - { - OutputCategoryStatsToConsole(con, category); - } + OutputAllStatsToConsole(con); } else if (categoryName == ListSubCommand) { @@ -129,7 +126,18 @@ namespace OpenSim.Framework.Monitoring else { // Legacy - con.Output(SimExtraStats.Report()); + if (SimExtraStats != null) + con.Output(SimExtraStats.Report()); + else + OutputAllStatsToConsole(con); + } + } + + private static void OutputAllStatsToConsole(ICommandConsole con) + { + foreach (var category in RegisteredStats.Values) + { + OutputCategoryStatsToConsole(con, category); } } @@ -150,27 +158,27 @@ namespace OpenSim.Framework.Monitoring } } - /// - /// Start collecting statistics related to assets. - /// Should only be called once. - /// - public static AssetStatsCollector StartCollectingAssetStats() - { - assetStats = new AssetStatsCollector(); - - return assetStats; - } - - /// - /// Start collecting statistics related to users. - /// Should only be called once. - /// - public static UserStatsCollector StartCollectingUserStats() - { - userStats = new UserStatsCollector(); - - return userStats; - } +// /// +// /// Start collecting statistics related to assets. +// /// Should only be called once. +// /// +// public static AssetStatsCollector StartCollectingAssetStats() +// { +// assetStats = new AssetStatsCollector(); +// +// return assetStats; +// } +// +// /// +// /// Start collecting statistics related to users. +// /// Should only be called once. +// /// +// public static UserStatsCollector StartCollectingUserStats() +// { +// userStats = new UserStatsCollector(); +// +// return userStats; +// } /// /// Registers a statistic. -- cgit v1.1 From 2c9bb0f9738534732e147c090fac43f3373af1bb Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Mon, 17 Jun 2013 22:55:25 +0100 Subject: Add server stats for available builtin threadpool and iocp workers --- OpenSim/Framework/Monitoring/ServerStatsCollector.cs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs index 80d0a89..fe74ef7 100644 --- a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs @@ -157,6 +157,22 @@ namespace OpenSim.Framework.Monitoring m_log.ErrorFormat("{0} Exception creating 'Process': {1}", LogHeader, e); } + MakeStat("BuiltinThreadpoolWorkerThreadsAvailable", null, "threads", ContainerProcessor, + s => + { + int workerThreads, iocpThreads; + ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads); + s.Value = workerThreads; + }); + + MakeStat("BuiltinThreadpoolIOCPThreadsAvailable", null, "threads", ContainerProcessor, + s => + { + int workerThreads, iocpThreads; + ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads); + s.Value = iocpThreads; + }); + try { List okInterfaceTypes = new List(NetworkInterfaceTypes.Split(',')); -- cgit v1.1 From 865d46ae1e0477f957ad674ed2a1a11ef85d885a Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Mon, 17 Jun 2013 22:56:39 +0100 Subject: Drop server level stats to debug instead of info. This was the original intention with these stats, as I didn't believe they would be useful to ordinary users if everything is working as it should. Please amend if this is an issue. Just for now, levels actually have no impact on what is displayed via the "show stats" command. --- OpenSim/Framework/Monitoring/ServerStatsCollector.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs index fe74ef7..c601c17 100644 --- a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs @@ -116,7 +116,7 @@ namespace OpenSim.Framework.Monitoring string desc = pDesc; if (desc == null) desc = pName; - Stat stat = new Stat(pName, pName, desc, pUnit, CategoryServer, pContainer, StatType.Pull, act, StatVerbosity.Info); + Stat stat = new Stat(pName, pName, desc, pUnit, CategoryServer, pContainer, StatType.Pull, act, StatVerbosity.Debug); StatsManager.RegisterStat(stat); RegisteredStats.Add(pName, stat); } -- cgit v1.1 From a1e32b843745581f7296359427a59e988814106b Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 18 Jun 2013 00:10:21 +0100 Subject: If SmartThreadPool is active, display statistical information about it in "show stats server" Also puts these and previous builtin threadpool stats in the "threadpool" stat container rather than "processor" --- OpenSim/Framework/Monitoring/ServerStatsCollector.cs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs index c601c17..0ab4b93 100644 --- a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs @@ -49,6 +49,7 @@ namespace OpenSim.Framework.Monitoring public readonly string CategoryServer = "server"; + public readonly string ContainerThreadpool = "threadpool"; public readonly string ContainerProcessor = "processor"; public readonly string ContainerMemory = "memory"; public readonly string ContainerNetwork = "network"; @@ -157,7 +158,7 @@ namespace OpenSim.Framework.Monitoring m_log.ErrorFormat("{0} Exception creating 'Process': {1}", LogHeader, e); } - MakeStat("BuiltinThreadpoolWorkerThreadsAvailable", null, "threads", ContainerProcessor, + MakeStat("BuiltinThreadpoolWorkerThreadsAvailable", null, "threads", ContainerThreadpool, s => { int workerThreads, iocpThreads; @@ -165,7 +166,7 @@ namespace OpenSim.Framework.Monitoring s.Value = workerThreads; }); - MakeStat("BuiltinThreadpoolIOCPThreadsAvailable", null, "threads", ContainerProcessor, + MakeStat("BuiltinThreadpoolIOCPThreadsAvailable", null, "threads", ContainerThreadpool, s => { int workerThreads, iocpThreads; @@ -173,6 +174,16 @@ namespace OpenSim.Framework.Monitoring s.Value = iocpThreads; }); + if (Util.FireAndForgetMethod != null && Util.GetSmartThreadPoolInfo() != null) + { + MakeStat("STPMaxThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().MaxThreads); + MakeStat("STPMinThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().MinThreads); + MakeStat("STPConcurrency", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().MaxConcurrentWorkItems); + MakeStat("STPActiveThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().ActiveThreads); + MakeStat("STPInUseThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().InUseThreads); + MakeStat("STPWorkItemsWaiting", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().WaitingCallbacks); + } + try { List okInterfaceTypes = new List(NetworkInterfaceTypes.Split(',')); -- cgit v1.1 From 768e8e363bfb0aedc6edc2e6c1dbf35e1e66b531 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 18 Jun 2013 22:49:49 +0100 Subject: Fix issue where stat samples were accidentally static, so that any additional stat with sampling would produce wrong results --- OpenSim/Framework/Monitoring/Stats/Stat.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs index 2e7665f..69ac0a5 100644 --- a/OpenSim/Framework/Monitoring/Stats/Stat.cs +++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs @@ -95,7 +95,7 @@ namespace OpenSim.Framework.Monitoring /// /// Will be null if no measures of interest require samples. /// - private static Queue m_samples; + private Queue m_samples; /// /// Maximum number of statistical samples. -- cgit v1.1 From 9501a583cbc8fd819d2b13db4f9ad76958520ce7 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 18 Jun 2013 23:07:18 +0100 Subject: Make number of inbound http requests handled available as a httpserver..IncomingHTTPRequestsProcessed stat --- OpenSim/Framework/Monitoring/Stats/Stat.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs index 69ac0a5..ca71911 100644 --- a/OpenSim/Framework/Monitoring/Stats/Stat.cs +++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs @@ -27,8 +27,9 @@ using System; using System.Collections.Generic; +using System.Reflection; using System.Text; - +using log4net; using OpenMetaverse.StructuredData; namespace OpenSim.Framework.Monitoring @@ -38,6 +39,8 @@ namespace OpenSim.Framework.Monitoring /// public class Stat : IDisposable { +// private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + /// /// Category of this stat (e.g. cache, scene, etc). /// @@ -204,6 +207,8 @@ namespace OpenSim.Framework.Monitoring if (m_samples.Count >= m_maxSamples) m_samples.Dequeue(); +// m_log.DebugFormat("[STAT]: Recording value {0} for {1}", newValue, Name); + m_samples.Enqueue(newValue); } } -- cgit v1.1 From dda44e31e3f649875a7fc7438c4a6880c56dbb31 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 18 Jun 2013 23:10:50 +0100 Subject: minor: tidy up spacing if display a unit for additional stat information --- OpenSim/Framework/Monitoring/Stats/Stat.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs index ca71911..eb5599f 100644 --- a/OpenSim/Framework/Monitoring/Stats/Stat.cs +++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs @@ -258,7 +258,7 @@ namespace OpenSim.Framework.Monitoring int divisor = m_samples.Count <= 1 ? 1 : m_samples.Count - 1; - sb.AppendFormat(", {0:0.##}{1}/s", totalChange / divisor / (Watchdog.WATCHDOG_INTERVAL_MS / 1000), UnitName); + sb.AppendFormat(", {0:0.##} {1}/s", totalChange / divisor / (Watchdog.WATCHDOG_INTERVAL_MS / 1000), UnitName); } } } -- cgit v1.1 From 84af1cab9ba991b2356f1025cab7f58433e2ec19 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Wed, 19 Jun 2013 20:48:12 +0100 Subject: Display existing statistic of how many http requests a server is making as server.network.HTTPRequestsMade in "show stats all" --- .../Framework/Monitoring/ServerStatsCollector.cs | 21 +++++++++++++++++---- OpenSim/Framework/Monitoring/Stats/Stat.cs | 5 +++++ 2 files changed, 22 insertions(+), 4 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs index 0ab4b93..07ca14b 100644 --- a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs @@ -114,10 +114,15 @@ namespace OpenSim.Framework.Monitoring private void MakeStat(string pName, string pDesc, string pUnit, string pContainer, Action act) { + MakeStat(pName, pDesc, pUnit, pContainer, act, MeasuresOfInterest.None); + } + + private void MakeStat(string pName, string pDesc, string pUnit, string pContainer, Action act, MeasuresOfInterest moi) + { string desc = pDesc; if (desc == null) desc = pName; - Stat stat = new Stat(pName, pName, desc, pUnit, CategoryServer, pContainer, StatType.Pull, act, StatVerbosity.Debug); + Stat stat = new Stat(pName, pName, desc, pUnit, CategoryServer, pContainer, StatType.Pull, moi, act, StatVerbosity.Debug); StatsManager.RegisterStat(stat); RegisteredStats.Add(pName, stat); } @@ -141,7 +146,7 @@ namespace OpenSim.Framework.Monitoring StatsManager.RegisterStat(tempStat); RegisteredStats.Add(tempName, tempStat); - MakeStat("TotalProcessorTime", null, "sec", ContainerProcessor, + MakeStat("TotalProcessorTime", null, "sec", ContainerProcessor, (s) => { s.Value = Process.GetCurrentProcess().TotalProcessorTime.TotalSeconds; }); MakeStat("UserProcessorTime", null, "sec", ContainerProcessor, @@ -158,7 +163,7 @@ namespace OpenSim.Framework.Monitoring m_log.ErrorFormat("{0} Exception creating 'Process': {1}", LogHeader, e); } - MakeStat("BuiltinThreadpoolWorkerThreadsAvailable", null, "threads", ContainerThreadpool, + MakeStat("BuiltinThreadpoolWorkerThreadsAvailable", null, "threads", ContainerThreadpool, s => { int workerThreads, iocpThreads; @@ -166,7 +171,7 @@ namespace OpenSim.Framework.Monitoring s.Value = workerThreads; }); - MakeStat("BuiltinThreadpoolIOCPThreadsAvailable", null, "threads", ContainerThreadpool, + MakeStat("BuiltinThreadpoolIOCPThreadsAvailable", null, "threads", ContainerThreadpool, s => { int workerThreads, iocpThreads; @@ -184,6 +189,14 @@ namespace OpenSim.Framework.Monitoring MakeStat("STPWorkItemsWaiting", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().WaitingCallbacks); } + MakeStat( + "HTTPRequestsMade", + "Number of outbound HTTP requests made", + "requests", + ContainerNetwork, + s => s.Value = WebUtil.RequestNumber, + MeasuresOfInterest.AverageChangeOverTime); + try { List okInterfaceTypes = new List(NetworkInterfaceTypes.Split(',')); diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs index eb5599f..85d1a78 100644 --- a/OpenSim/Framework/Monitoring/Stats/Stat.cs +++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs @@ -27,6 +27,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Reflection; using System.Text; using log4net; @@ -247,6 +248,10 @@ namespace OpenSim.Framework.Monitoring lock (m_samples) { +// m_log.DebugFormat( +// "[STAT]: Samples for {0} are {1}", +// Name, string.Join(",", m_samples.Select(s => s.ToString()).ToArray())); + foreach (double s in m_samples) { if (lastSample != null) -- cgit v1.1 From 086fd70a5fdfd9b3a0e56201596f6d6bb20f391e Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 20 Jun 2013 00:00:39 +0100 Subject: Make it possible to specify display of stats in a particular 'container' by separating category and container with a period. e.g. "show stats server.network" I failed to realize this had already been implemented without the period in the show stats command (as the command help had not been updated). However, I would prefer the . approach as it will allow specifying multiple stats, easier wildcarding, etc. This commit also prevents any stat from having a period in its short name. --- OpenSim/Framework/Monitoring/Stats/Stat.cs | 8 ++++++++ OpenSim/Framework/Monitoring/StatsManager.cs | 12 +++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs index 85d1a78..c57ee0c 100644 --- a/OpenSim/Framework/Monitoring/Stats/Stat.cs +++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs @@ -42,6 +42,8 @@ namespace OpenSim.Framework.Monitoring { // private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + public static readonly char[] DisallowedShortNameCharacters = { '.' }; + /// /// Category of this stat (e.g. cache, scene, etc). /// @@ -166,6 +168,12 @@ namespace OpenSim.Framework.Monitoring throw new Exception( string.Format("Stat cannot be in category '{0}' since this is reserved for a subcommand", category)); + foreach (char c in DisallowedShortNameCharacters) + { + if (shortName.IndexOf(c) != -1) + throw new Exception(string.Format("Stat name {0} cannot contain character {1}", shortName, c)); + } + ShortName = shortName; Name = name; Description = description; diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index 3aee984..af9f5ba 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -68,12 +68,13 @@ namespace OpenSim.Framework.Monitoring "General", false, "show stats", - "show stats [list|all|]", + "show stats [list|all|[.]", "Show statistical information for this server", "If no final argument is specified then legacy statistics information is currently shown.\n" + "If list is specified then statistic categories are shown.\n" + "If all is specified then all registered statistics are shown.\n" + "If a category name is specified then only statistics from that category are shown.\n" + + "If a category container is also specified then only statistics from that category in that container are shown.\n" + "THIS STATS FACILITY IS EXPERIMENTAL AND DOES NOT YET CONTAIN ALL STATS", HandleShowStatsCommand); } @@ -84,8 +85,11 @@ namespace OpenSim.Framework.Monitoring if (cmd.Length > 2) { - var categoryName = cmd[2]; - var containerName = cmd.Length > 3 ? cmd[3] : String.Empty; + string name = cmd[2]; + string[] components = name.Split('.'); + + string categoryName = components[0]; + string containerName = components.Length > 1 ? components[1] : null; if (categoryName == AllSubCommand) { @@ -107,7 +111,9 @@ namespace OpenSim.Framework.Monitoring else { if (String.IsNullOrEmpty(containerName)) + { OutputCategoryStatsToConsole(con, category); + } else { SortedDictionary container; -- cgit v1.1 From d97333255d45da686bb71d85ef492c5631c4dc31 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 20 Jun 2013 00:20:04 +0100 Subject: Fix minor bug where the check whether to display SmartThreadPool stats was accidentally != null rather than == FireAndForgetMethod.SmartThreadPool Due to another check this had no practical effect --- OpenSim/Framework/Monitoring/ServerStatsCollector.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs index 07ca14b..5d1c270 100644 --- a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs @@ -179,7 +179,7 @@ namespace OpenSim.Framework.Monitoring s.Value = iocpThreads; }); - if (Util.FireAndForgetMethod != null && Util.GetSmartThreadPoolInfo() != null) + if (Util.FireAndForgetMethod == FireAndForgetMethod.SmartThreadPool && Util.GetSmartThreadPoolInfo() != null) { MakeStat("STPMaxThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().MaxThreads); MakeStat("STPMinThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().MinThreads); -- cgit v1.1 From 5b1a9f84fdf4f9938180b403dd002b44e25c6efb Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 20 Jun 2013 00:32:12 +0100 Subject: minor: Change "memory churn" terminology in statistics to "heap allocation rate" since this is more generally meaningful --- OpenSim/Framework/Monitoring/BaseStatsCollector.cs | 12 ++++++------ OpenSim/Framework/Monitoring/MemoryWatchdog.cs | 8 ++++---- OpenSim/Framework/Monitoring/ServerStatsCollector.cs | 8 ++++---- 3 files changed, 14 insertions(+), 14 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs index be1d02b..20495f6 100644 --- a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs @@ -1,4 +1,4 @@ -/* +/* * Copyright (c) Contributors, http://opensimulator.org/ * See CONTRIBUTORS.TXT for a full list of copyright holders. * @@ -45,16 +45,16 @@ namespace OpenSim.Framework.Monitoring sb.Append(Environment.NewLine); sb.AppendFormat( - "Allocated to OpenSim objects: {0} MB\n", + "Heap allocated to OpenSim : {0} MB\n", Math.Round(GC.GetTotalMemory(false) / 1024.0 / 1024.0)); sb.AppendFormat( - "OpenSim last object memory churn : {0} MB/s\n", - Math.Round((MemoryWatchdog.LastMemoryChurn * 1000) / 1024.0 / 1024, 3)); + "Last heap allocation rate : {0} MB/s\n", + Math.Round((MemoryWatchdog.LastHeapAllocationRate * 1000) / 1024.0 / 1024, 3)); sb.AppendFormat( - "OpenSim average object memory churn : {0} MB/s\n", - Math.Round((MemoryWatchdog.AverageMemoryChurn * 1000) / 1024.0 / 1024, 3)); + "Average heap allocation rate: {0} MB/s\n", + Math.Round((MemoryWatchdog.AverageHeapAllocationRate * 1000) / 1024.0 / 1024, 3)); sb.AppendFormat( "Process memory : {0} MB\n", diff --git a/OpenSim/Framework/Monitoring/MemoryWatchdog.cs b/OpenSim/Framework/Monitoring/MemoryWatchdog.cs index c6010cd..c474622 100644 --- a/OpenSim/Framework/Monitoring/MemoryWatchdog.cs +++ b/OpenSim/Framework/Monitoring/MemoryWatchdog.cs @@ -60,17 +60,17 @@ namespace OpenSim.Framework.Monitoring private static bool m_enabled; /// - /// Last memory churn in bytes per millisecond. + /// Average heap allocation rate in bytes per millisecond. /// - public static double AverageMemoryChurn + public static double AverageHeapAllocationRate { get { if (m_samples.Count > 0) return m_samples.Average(); else return 0; } } /// - /// Average memory churn in bytes per millisecond. + /// Last heap allocation in bytes /// - public static double LastMemoryChurn + public static double LastHeapAllocationRate { get { if (m_samples.Count > 0) return m_samples.Last(); else return 0; } } diff --git a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs index 5d1c270..b22da1d 100644 --- a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs @@ -242,10 +242,10 @@ namespace OpenSim.Framework.Monitoring (s) => { s.Value = Process.GetCurrentProcess().WorkingSet64 / 1024d / 1024d; }); MakeStat("ObjectMemory", null, "MB", ContainerMemory, (s) => { s.Value = GC.GetTotalMemory(false) / 1024d / 1024d; }); - MakeStat("LastMemoryChurn", null, "MB/sec", ContainerMemory, - (s) => { s.Value = Math.Round(MemoryWatchdog.LastMemoryChurn * 1000d / 1024d / 1024d, 3); }); - MakeStat("AverageMemoryChurn", null, "MB/sec", ContainerMemory, - (s) => { s.Value = Math.Round(MemoryWatchdog.AverageMemoryChurn * 1000d / 1024d / 1024d, 3); }); + MakeStat("LastHeapAllocationRate", null, "MB/sec", ContainerMemory, + (s) => { s.Value = Math.Round(MemoryWatchdog.LastHeapAllocationRate * 1000d / 1024d / 1024d, 3); }); + MakeStat("AverageHeapAllocationRate", null, "MB/sec", ContainerMemory, + (s) => { s.Value = Math.Round(MemoryWatchdog.AverageHeapAllocationRate * 1000d / 1024d / 1024d, 3); }); } // Notes on performance counters: -- cgit v1.1 From 05790ba1cf7dd07f9f11ce248262041eb300ea49 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 20 Jun 2013 00:45:56 +0100 Subject: Allow more than one stat category to be specified in "show stats" e.g. "show stats httpserver.9000 server.network" --- OpenSim/Framework/Monitoring/StatsManager.cs | 64 +++++++++++++++------------- 1 file changed, 34 insertions(+), 30 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index af9f5ba..12d3a75 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -27,6 +27,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Text; namespace OpenSim.Framework.Monitoring @@ -68,13 +69,14 @@ namespace OpenSim.Framework.Monitoring "General", false, "show stats", - "show stats [list|all|[.]", + "show stats [list|all|([.])+", "Show statistical information for this server", "If no final argument is specified then legacy statistics information is currently shown.\n" - + "If list is specified then statistic categories are shown.\n" - + "If all is specified then all registered statistics are shown.\n" - + "If a category name is specified then only statistics from that category are shown.\n" - + "If a category container is also specified then only statistics from that category in that container are shown.\n" + + "'list' argument will show statistic categories.\n" + + "'all' will show all statistics.\n" + + "A name will show statistics from that category.\n" + + "A . name will show statistics from that category in that container.\n" + + "More than one name can be given separated by spaces.\n" + "THIS STATS FACILITY IS EXPERIMENTAL AND DOES NOT YET CONTAIN ALL STATS", HandleShowStatsCommand); } @@ -85,45 +87,47 @@ namespace OpenSim.Framework.Monitoring if (cmd.Length > 2) { - string name = cmd[2]; - string[] components = name.Split('.'); + foreach (string name in cmd.Skip(2)) + { + string[] components = name.Split('.'); - string categoryName = components[0]; - string containerName = components.Length > 1 ? components[1] : null; + string categoryName = components[0]; + string containerName = components.Length > 1 ? components[1] : null; - if (categoryName == AllSubCommand) - { - OutputAllStatsToConsole(con); - } - else if (categoryName == ListSubCommand) - { - con.Output("Statistic categories available are:"); - foreach (string category in RegisteredStats.Keys) - con.OutputFormat(" {0}", category); - } - else - { - SortedDictionary> category; - if (!RegisteredStats.TryGetValue(categoryName, out category)) + if (categoryName == AllSubCommand) + { + OutputAllStatsToConsole(con); + } + else if (categoryName == ListSubCommand) { - con.OutputFormat("No such category as {0}", categoryName); + con.Output("Statistic categories available are:"); + foreach (string category in RegisteredStats.Keys) + con.OutputFormat(" {0}", category); } else { - if (String.IsNullOrEmpty(containerName)) + SortedDictionary> category; + if (!RegisteredStats.TryGetValue(categoryName, out category)) { - OutputCategoryStatsToConsole(con, category); + con.OutputFormat("No such category as {0}", categoryName); } else { - SortedDictionary container; - if (category.TryGetValue(containerName, out container)) + if (String.IsNullOrEmpty(containerName)) { - OutputContainerStatsToConsole(con, container); + OutputCategoryStatsToConsole(con, category); } else { - con.OutputFormat("No such container {0} in category {1}", containerName, categoryName); + SortedDictionary container; + if (category.TryGetValue(containerName, out container)) + { + OutputContainerStatsToConsole(con, container); + } + else + { + con.OutputFormat("No such container {0} in category {1}", containerName, categoryName); + } } } } -- cgit v1.1 From 085a87060a2ccb4910305b7c5f5374d1cc779d1a Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 20 Jun 2013 00:52:39 +0100 Subject: Change "ObjectMemory" stat to "HeapMemory" to align with other stat names. Also round this and ProcessMemory to three decimal places in common with other memory stats. I believe leaving out such minor info makes stats easier to read --- OpenSim/Framework/Monitoring/ServerStatsCollector.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs index b22da1d..ac0f0bc 100644 --- a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs @@ -239,9 +239,9 @@ namespace OpenSim.Framework.Monitoring } MakeStat("ProcessMemory", null, "MB", ContainerMemory, - (s) => { s.Value = Process.GetCurrentProcess().WorkingSet64 / 1024d / 1024d; }); - MakeStat("ObjectMemory", null, "MB", ContainerMemory, - (s) => { s.Value = GC.GetTotalMemory(false) / 1024d / 1024d; }); + (s) => { s.Value = Math.Round(Process.GetCurrentProcess().WorkingSet64 / 1024d / 1024d, 3); }); + MakeStat("HeapMemory", null, "MB", ContainerMemory, + (s) => { s.Value = Math.Round(GC.GetTotalMemory(false) / 1024d / 1024d, 3); }); MakeStat("LastHeapAllocationRate", null, "MB/sec", ContainerMemory, (s) => { s.Value = Math.Round(MemoryWatchdog.LastHeapAllocationRate * 1000d / 1024d / 1024d, 3); }); MakeStat("AverageHeapAllocationRate", null, "MB/sec", ContainerMemory, -- cgit v1.1 From fa02f28dbfef9b9dc3621f5bbd6b026c827459a5 Mon Sep 17 00:00:00 2001 From: Robert Adams Date: Fri, 12 Jul 2013 14:04:14 -0700 Subject: Add ToOSDMap() overrides to the Stat subclass CounterStat. Add a GetStatsAsOSDMap method to StatsManager which allows the filtered fetching of stats for eventual returning over the internets. --- OpenSim/Framework/Monitoring/Stats/CounterStat.cs | 21 ++++++++ OpenSim/Framework/Monitoring/Stats/Stat.cs | 1 + OpenSim/Framework/Monitoring/StatsManager.cs | 66 +++++++++++++++++++++++ 3 files changed, 88 insertions(+) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Stats/CounterStat.cs b/OpenSim/Framework/Monitoring/Stats/CounterStat.cs index caea30d..04442c3 100755 --- a/OpenSim/Framework/Monitoring/Stats/CounterStat.cs +++ b/OpenSim/Framework/Monitoring/Stats/CounterStat.cs @@ -224,5 +224,26 @@ public class CounterStat : Stat } } } + + // CounterStat is a basic stat plus histograms + public override OSDMap ToOSDMap() + { + // Get the foundational instance + OSDMap map = base.ToOSDMap(); + + map["StatType"] = "CounterStat"; + + // If there are any histograms, add a new field that is an array of histograms as OSDMaps + if (m_histograms.Count > 0) + { + OSDArray histos = new OSDArray(); + foreach (EventHistogram histo in m_histograms.Values) + { + histos.Add(histo.GetHistogramAsOSDMap()); + } + map.Add("Histograms", histos); + } + return map; + } } } diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs index c57ee0c..9629b6e 100644 --- a/OpenSim/Framework/Monitoring/Stats/Stat.cs +++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs @@ -242,6 +242,7 @@ namespace OpenSim.Framework.Monitoring ret.Add("Description", OSD.FromString(Description)); ret.Add("UnitName", OSD.FromString(UnitName)); ret.Add("Value", OSD.FromReal(Value)); + ret.Add("StatType", "Stat"); // used by overloading classes to denote type of stat return ret; } diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index 12d3a75..a5b54c9 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -30,6 +30,8 @@ using System.Collections.Generic; using System.Linq; using System.Text; +using OpenMetaverse.StructuredData; + namespace OpenSim.Framework.Monitoring { /// @@ -168,6 +170,70 @@ namespace OpenSim.Framework.Monitoring } } + // Creates an OSDMap of the format: + // { categoryName: { + // containerName: { + // statName: { + // "Name": name, + // "ShortName": shortName, + // ... + // }, + // statName: { + // "Name": name, + // "ShortName": shortName, + // ... + // }, + // ... + // }, + // containerName: { + // ... + // }, + // ... + // }, + // categoryName: { + // ... + // }, + // ... + // } + // The passed in parameters will filter the categories, containers and stats returned. If any of the + // parameters are either EmptyOrNull or the AllSubCommand value, all of that type will be returned. + // Case matters. + public static OSDMap GetStatsAsOSDMap(string pCategoryName, string pContainerName, string pStatName) + { + OSDMap map = new OSDMap(); + + foreach (string catName in RegisteredStats.Keys) + { + // Do this category if null spec, "all" subcommand or category name matches passed parameter. + // Skip category if none of the above. + if (!(String.IsNullOrEmpty(pCategoryName) || pCategoryName == AllSubCommand || pCategoryName == catName)) + continue; + + OSDMap contMap = new OSDMap(); + foreach (string contName in RegisteredStats[catName].Keys) + { + if (!(string.IsNullOrEmpty(pContainerName) || pContainerName == AllSubCommand || pContainerName == contName)) + continue; + + OSDMap statMap = new OSDMap(); + + SortedDictionary theStats = RegisteredStats[catName][contName]; + foreach (string statName in theStats.Keys) + { + if (!(String.IsNullOrEmpty(pStatName) || pStatName == AllSubCommand || pStatName == statName)) + continue; + + statMap.Add(statName, theStats[statName].ToOSDMap()); + } + + contMap.Add(contName, statMap); + } + map.Add(catName, contMap); + } + + return map; + } + // /// // /// Start collecting statistics related to assets. // /// Should only be called once. -- cgit v1.1 From 90528c23d991cd1fc77823be49e4135cf412b92e Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 23 Jul 2013 01:13:13 +0100 Subject: For stats which can show average change over time, show the last sample as well as the average. This is somewhat cryptic at the moment, need to improve documentation. --- OpenSim/Framework/Monitoring/Stats/Stat.cs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs index 9629b6e..cc2c947 100644 --- a/OpenSim/Framework/Monitoring/Stats/Stat.cs +++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs @@ -253,6 +253,8 @@ namespace OpenSim.Framework.Monitoring == MeasuresOfInterest.AverageChangeOverTime) { double totalChange = 0; + double lastChangeOverTime = 0; + double? penultimateSample = null; double? lastSample = null; lock (m_samples) @@ -266,13 +268,21 @@ namespace OpenSim.Framework.Monitoring if (lastSample != null) totalChange += s - (double)lastSample; + penultimateSample = lastSample; lastSample = s; } } + if (lastSample != null && penultimateSample != null) + lastChangeOverTime = (double)lastSample - (double)penultimateSample; + int divisor = m_samples.Count <= 1 ? 1 : m_samples.Count - 1; - sb.AppendFormat(", {0:0.##} {1}/s", totalChange / divisor / (Watchdog.WATCHDOG_INTERVAL_MS / 1000), UnitName); + double averageChangeOverTime = totalChange / divisor / (Watchdog.WATCHDOG_INTERVAL_MS / 1000); + + sb.AppendFormat( + ", {0:0.##} {1}/s, {2:0.##} {3}/s", + lastChangeOverTime, UnitName, averageChangeOverTime, UnitName); } } } -- cgit v1.1 From 76e46d0158461a29e6ef358ba8cb7b7323cc0c05 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 23 Jul 2013 17:23:16 +0100 Subject: Improve spacing between data and units on console stats display --- OpenSim/Framework/Monitoring/Stats/Stat.cs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs index cc2c947..f2329ce 100644 --- a/OpenSim/Framework/Monitoring/Stats/Stat.cs +++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs @@ -225,7 +225,13 @@ namespace OpenSim.Framework.Monitoring public virtual string ToConsoleString() { StringBuilder sb = new StringBuilder(); - sb.AppendFormat("{0}.{1}.{2} : {3} {4}", Category, Container, ShortName, Value, UnitName); + sb.AppendFormat( + "{0}.{1}.{2} : {3}{4}", + Category, + Container, + ShortName, + Value, + UnitName == null || UnitName == "" ? "" : string.Format(" {0}", UnitName)); AppendMeasuresOfInterest(sb); @@ -281,8 +287,11 @@ namespace OpenSim.Framework.Monitoring double averageChangeOverTime = totalChange / divisor / (Watchdog.WATCHDOG_INTERVAL_MS / 1000); sb.AppendFormat( - ", {0:0.##} {1}/s, {2:0.##} {3}/s", - lastChangeOverTime, UnitName, averageChangeOverTime, UnitName); + ", {0:0.##}{1}/s, {2:0.##}{3}/s", + lastChangeOverTime, + UnitName == null || UnitName == "" ? "" : string.Format(" {0}", UnitName), + averageChangeOverTime, + UnitName == null || UnitName == "" ? "" : string.Format(" {0}", UnitName)); } } } -- cgit v1.1 From 9a4a513b5e4e2496b32113dcffbeeae776eabb89 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 23 Jul 2013 23:31:35 +0100 Subject: Correct issue where the last instance of a sampled stat was shown 3x larger than it should have been (though internal use was correct) --- OpenSim/Framework/Monitoring/Stats/Stat.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs index f2329ce..ffd5132 100644 --- a/OpenSim/Framework/Monitoring/Stats/Stat.cs +++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs @@ -280,7 +280,8 @@ namespace OpenSim.Framework.Monitoring } if (lastSample != null && penultimateSample != null) - lastChangeOverTime = (double)lastSample - (double)penultimateSample; + lastChangeOverTime + = ((double)lastSample - (double)penultimateSample) / (Watchdog.WATCHDOG_INTERVAL_MS / 1000); int divisor = m_samples.Count <= 1 ? 1 : m_samples.Count - 1; -- cgit v1.1 From 8efe4bfc2ed7086e9fdf4812297e6525f955f6ac Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Mon, 29 Jul 2013 23:18:29 +0100 Subject: Make "abnormal thread terminations" into "ClientLogoutsDueToNoReceives" and add this to the StatsManager This reflects the actual use of this stat - it hasn't recorded general exceptions for some time. Make the sim extra stats collector draw the data from the stats manager rather than maintaing this data itself. --- .../Framework/Monitoring/SimExtraStatsCollector.cs | 25 +++------ OpenSim/Framework/Monitoring/StatsManager.cs | 65 ++++++++++++++++++++-- 2 files changed, 68 insertions(+), 22 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs index 6a68322..f6f458d 100644 --- a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs @@ -27,6 +27,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Text; using OpenMetaverse; using OpenMetaverse.StructuredData; @@ -39,8 +40,6 @@ namespace OpenSim.Framework.Monitoring /// public class SimExtraStatsCollector : BaseStatsCollector { - private long abnormalClientThreadTerminations; - // private long assetsInCache; // private long texturesInCache; // private long assetCacheMemoryUsage; @@ -73,11 +72,6 @@ namespace OpenSim.Framework.Monitoring private volatile float activeScripts; private volatile float scriptLinesPerSecond; - /// - /// Number of times that a client thread terminated because of an exception - /// - public long AbnormalClientThreadTerminations { get { return abnormalClientThreadTerminations; } } - // /// // /// These statistics are being collected by push rather than pull. Pull would be simpler, but I had the // /// notion of providing some flow statistics (which pull wouldn't give us). Though admittedly these @@ -166,11 +160,6 @@ namespace OpenSim.Framework.Monitoring private IDictionary packetQueueStatsCollectors = new Dictionary(); - public void AddAbnormalClientThreadTermination() - { - abnormalClientThreadTerminations++; - } - // public void AddAsset(AssetBase asset) // { // assetsInCache++; @@ -324,10 +313,12 @@ Asset service request failures: {3}" + Environment.NewLine, sb.Append(Environment.NewLine); sb.Append("CONNECTION STATISTICS"); sb.Append(Environment.NewLine); - sb.Append( - string.Format( - "Abnormal client thread terminations: {0}" + Environment.NewLine, - abnormalClientThreadTerminations)); + + List stats = StatsManager.GetStatsFromEachContainer("clientstack", "ClientLogoutsDueToNoReceives"); + + sb.AppendFormat( + "Client logouts due to no data receive timeout: {0}\n\n", + stats != null ? stats.Sum(s => s.Value).ToString() : "unknown"); // sb.Append(Environment.NewLine); // sb.Append("INVENTORY STATISTICS"); @@ -338,7 +329,7 @@ Asset service request failures: {3}" + Environment.NewLine, // InventoryServiceRetrievalFailures)); sb.Append(Environment.NewLine); - sb.Append("FRAME STATISTICS"); + sb.Append("SAMPLE FRAME STATISTICS"); sb.Append(Environment.NewLine); sb.Append("Dilatn SimFPS PhyFPS AgntUp RootAg ChldAg Prims AtvPrm AtvScr ScrLPS"); sb.Append(Environment.NewLine); diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index a5b54c9..e6a2304 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -271,7 +271,7 @@ namespace OpenSim.Framework.Monitoring // Stat name is not unique across category/container/shortname key. // XXX: For now just return false. This is to avoid problems in regression tests where all tests // in a class are run in the same instance of the VM. - if (TryGetStat(stat, out category, out container)) + if (TryGetStatParents(stat, out category, out container)) return false; // We take a copy-on-write approach here of replacing dictionaries when keys are added or removed. @@ -307,7 +307,7 @@ namespace OpenSim.Framework.Monitoring lock (RegisteredStats) { - if (!TryGetStat(stat, out category, out container)) + if (!TryGetStatParents(stat, out category, out container)) return false; newContainer = new SortedDictionary(container); @@ -323,12 +323,67 @@ namespace OpenSim.Framework.Monitoring } } - public static bool TryGetStats(string category, out SortedDictionary> stats) + public static bool TryGetStat(string category, string container, string statShortName, out Stat stat) { - return RegisteredStats.TryGetValue(category, out stats); + stat = null; + SortedDictionary> categoryStats; + + lock (RegisteredStats) + { + if (!TryGetStatsForCategory(category, out categoryStats)) + return false; + + SortedDictionary containerStats; + + if (!categoryStats.TryGetValue(container, out containerStats)) + return false; + + return containerStats.TryGetValue(statShortName, out stat); + } + } + + public static bool TryGetStatsForCategory( + string category, out SortedDictionary> stats) + { + lock (RegisteredStats) + return RegisteredStats.TryGetValue(category, out stats); + } + + /// + /// Get the same stat for each container in a given category. + /// + /// + /// The stats if there were any to fetch. Otherwise null. + /// + /// + /// + public static List GetStatsFromEachContainer(string category, string statShortName) + { + SortedDictionary> categoryStats; + + lock (RegisteredStats) + { + if (!RegisteredStats.TryGetValue(category, out categoryStats)) + return null; + + List stats = null; + + foreach (SortedDictionary containerStats in categoryStats.Values) + { + if (containerStats.ContainsKey(statShortName)) + { + if (stats == null) + stats = new List(); + + stats.Add(containerStats[statShortName]); + } + } + + return stats; + } } - public static bool TryGetStat( + public static bool TryGetStatParents( Stat stat, out SortedDictionary> category, out SortedDictionary container) -- cgit v1.1 From 76bd3de2fd243d0c910404af8a9998de746b04c4 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Mon, 5 Aug 2013 19:22:47 +0100 Subject: Add checks monitoring framework to provide alerts if certain conditions do not hold. Not yet in use. --- OpenSim/Framework/Monitoring/Checks/Check.cs | 118 ++++++++++++ OpenSim/Framework/Monitoring/ChecksManager.cs | 262 ++++++++++++++++++++++++++ OpenSim/Framework/Monitoring/StatsManager.cs | 6 +- OpenSim/Framework/Monitoring/Watchdog.cs | 1 + 4 files changed, 384 insertions(+), 3 deletions(-) create mode 100644 OpenSim/Framework/Monitoring/Checks/Check.cs create mode 100644 OpenSim/Framework/Monitoring/ChecksManager.cs (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Checks/Check.cs b/OpenSim/Framework/Monitoring/Checks/Check.cs new file mode 100644 index 0000000..594386a --- /dev/null +++ b/OpenSim/Framework/Monitoring/Checks/Check.cs @@ -0,0 +1,118 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Text; + +namespace OpenSim.Framework.Monitoring +{ + public class Check + { +// private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + public static readonly char[] DisallowedShortNameCharacters = { '.' }; + + /// + /// Category of this stat (e.g. cache, scene, etc). + /// + public string Category { get; private set; } + + /// + /// Containing name for this stat. + /// FIXME: In the case of a scene, this is currently the scene name (though this leaves + /// us with a to-be-resolved problem of non-unique region names). + /// + /// + /// The container. + /// + public string Container { get; private set; } + + /// + /// Action used to check whether alert should go off. + /// + /// + /// Should return true if check passes. False otherwise. + /// + public Func CheckFunc { get; private set; } + + /// + /// Message from the last failure, if any. If there is no message or no failure then will be null. + /// + /// + /// Should be set by the CheckFunc when applicable. + /// + public string LastFailureMessage { get; set; } + + public StatVerbosity Verbosity { get; private set; } + public string ShortName { get; private set; } + public string Name { get; private set; } + public string Description { get; private set; } + + public Check( + string shortName, + string name, + string description, + string category, + string container, + Func checkFunc, + StatVerbosity verbosity) + { + if (ChecksManager.SubCommands.Contains(category)) + throw new Exception( + string.Format("Alert cannot be in category '{0}' since this is reserved for a subcommand", category)); + + foreach (char c in DisallowedShortNameCharacters) + { + if (shortName.IndexOf(c) != -1) + throw new Exception(string.Format("Alert name {0} cannot contain character {1}", shortName, c)); + } + + ShortName = shortName; + Name = name; + Description = description; + Category = category; + Container = container; + CheckFunc = checkFunc; + Verbosity = verbosity; + } + + public bool CheckIt() + { + return CheckFunc(this); + } + + public virtual string ToConsoleString() + { + return string.Format( + "{0}.{1}.{2} - {3}", + Category, + Container, + ShortName, + Description); + } + } +} \ No newline at end of file diff --git a/OpenSim/Framework/Monitoring/ChecksManager.cs b/OpenSim/Framework/Monitoring/ChecksManager.cs new file mode 100644 index 0000000..e4a7f8c --- /dev/null +++ b/OpenSim/Framework/Monitoring/ChecksManager.cs @@ -0,0 +1,262 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; +using log4net; + +namespace OpenSim.Framework.Monitoring +{ + /// + /// Static class used to register/deregister checks on runtime conditions. + /// + public static class ChecksManager + { + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + // Subcommand used to list other stats. + public const string ListSubCommand = "list"; + + // All subcommands + public static HashSet SubCommands = new HashSet { ListSubCommand }; + + /// + /// Checks categorized by category/container/shortname + /// + /// + /// Do not add or remove directly from this dictionary. + /// + public static SortedDictionary>> RegisteredChecks + = new SortedDictionary>>(); + + public static void RegisterConsoleCommands(ICommandConsole console) + { + console.Commands.AddCommand( + "General", + false, + "show checks", + "show checks", + "Show checks configured for this server", + "If no argument is specified then info on all checks will be shown.\n" + + "'list' argument will show check categories.\n" + + "THIS FACILITY IS EXPERIMENTAL", + HandleShowchecksCommand); + } + + public static void HandleShowchecksCommand(string module, string[] cmd) + { + ICommandConsole con = MainConsole.Instance; + + if (cmd.Length > 2) + { + foreach (string name in cmd.Skip(2)) + { + string[] components = name.Split('.'); + + string categoryName = components[0]; +// string containerName = components.Length > 1 ? components[1] : null; + + if (categoryName == ListSubCommand) + { + con.Output("check categories available are:"); + + foreach (string category in RegisteredChecks.Keys) + con.OutputFormat(" {0}", category); + } +// else +// { +// SortedDictionary> category; +// if (!Registeredchecks.TryGetValue(categoryName, out category)) +// { +// con.OutputFormat("No such category as {0}", categoryName); +// } +// else +// { +// if (String.IsNullOrEmpty(containerName)) +// { +// OutputConfiguredToConsole(con, category); +// } +// else +// { +// SortedDictionary container; +// if (category.TryGetValue(containerName, out container)) +// { +// OutputContainerChecksToConsole(con, container); +// } +// else +// { +// con.OutputFormat("No such container {0} in category {1}", containerName, categoryName); +// } +// } +// } +// } + } + } + else + { + OutputAllChecksToConsole(con); + } + } + + /// + /// Registers a statistic. + /// + /// + /// + public static bool RegisterCheck(Check check) + { + SortedDictionary> category = null, newCategory; + SortedDictionary container = null, newContainer; + + lock (RegisteredChecks) + { + // Check name is not unique across category/container/shortname key. + // XXX: For now just return false. This is to avoid problems in regression tests where all tests + // in a class are run in the same instance of the VM. + if (TryGetCheckParents(check, out category, out container)) + return false; + + // We take a copy-on-write approach here of replacing dictionaries when keys are added or removed. + // This means that we don't need to lock or copy them on iteration, which will be a much more + // common operation after startup. + if (container != null) + newContainer = new SortedDictionary(container); + else + newContainer = new SortedDictionary(); + + if (category != null) + newCategory = new SortedDictionary>(category); + else + newCategory = new SortedDictionary>(); + + newContainer[check.ShortName] = check; + newCategory[check.Container] = newContainer; + RegisteredChecks[check.Category] = newCategory; + } + + return true; + } + + /// + /// Deregister an check + /// > + /// + /// + public static bool DeregisterCheck(Check check) + { + SortedDictionary> category = null, newCategory; + SortedDictionary container = null, newContainer; + + lock (RegisteredChecks) + { + if (!TryGetCheckParents(check, out category, out container)) + return false; + + newContainer = new SortedDictionary(container); + newContainer.Remove(check.ShortName); + + newCategory = new SortedDictionary>(category); + newCategory.Remove(check.Container); + + newCategory[check.Container] = newContainer; + RegisteredChecks[check.Category] = newCategory; + + return true; + } + } + + public static bool TryGetCheckParents( + Check check, + out SortedDictionary> category, + out SortedDictionary container) + { + category = null; + container = null; + + lock (RegisteredChecks) + { + if (RegisteredChecks.TryGetValue(check.Category, out category)) + { + if (category.TryGetValue(check.Container, out container)) + { + if (container.ContainsKey(check.ShortName)) + return true; + } + } + } + + return false; + } + + public static void CheckChecks() + { + lock (RegisteredChecks) + { + foreach (SortedDictionary> category in RegisteredChecks.Values) + { + foreach (SortedDictionary container in category.Values) + { + foreach (Check check in container.Values) + { + if (!check.CheckIt()) + m_log.WarnFormat( + "[CHECKS MANAGER]: Check {0}.{1}.{2} failed with message {3}", check.Category, check.Container, check.ShortName, check.LastFailureMessage); + } + } + } + } + } + + private static void OutputAllChecksToConsole(ICommandConsole con) + { + foreach (var category in RegisteredChecks.Values) + { + OutputCategoryChecksToConsole(con, category); + } + } + + private static void OutputCategoryChecksToConsole( + ICommandConsole con, SortedDictionary> category) + { + foreach (var container in category.Values) + { + OutputContainerChecksToConsole(con, container); + } + } + + private static void OutputContainerChecksToConsole(ICommandConsole con, SortedDictionary container) + { + foreach (Check check in container.Values) + { + con.Output(check.ToConsoleString()); + } + } + } +} \ No newline at end of file diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index e6a2304..87197f4 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -35,9 +35,9 @@ using OpenMetaverse.StructuredData; namespace OpenSim.Framework.Monitoring { /// - /// Singleton used to provide access to statistics reporters + /// Static class used to register/deregister/fetch statistics /// - public class StatsManager + public static class StatsManager { // Subcommand used to list other stats. public const string AllSubCommand = "all"; @@ -257,7 +257,7 @@ namespace OpenSim.Framework.Monitoring // } /// - /// Registers a statistic. + /// Register a statistic. /// /// /// diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index 3f992b1..45762a6 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -380,6 +380,7 @@ namespace OpenSim.Framework.Monitoring if (MemoryWatchdog.Enabled) MemoryWatchdog.Update(); + ChecksManager.CheckChecks(); StatsManager.RecordStats(); m_watchdogTimer.Start(); -- cgit v1.1 From 4c2f6de8e4957df3c7186437089ba0925edb1a08 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 6 Aug 2013 18:29:33 +0100 Subject: Add the experimental ability to dump stats (result of command "show stats all") to file OpenSimStats.log every 5 seconds. This can currently only be activated with the console command "debug stats record start". Off by default. Records to file OpenSimStats.log for simulator and RobustStats.log for ROBUST --- OpenSim/Framework/Monitoring/StatsManager.cs | 52 +++++++++++++++++++++------- 1 file changed, 40 insertions(+), 12 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index 87197f4..c8e838c 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -81,6 +81,8 @@ namespace OpenSim.Framework.Monitoring + "More than one name can be given separated by spaces.\n" + "THIS STATS FACILITY IS EXPERIMENTAL AND DOES NOT YET CONTAIN ALL STATS", HandleShowStatsCommand); + + StatsLogger.RegisterConsoleCommands(console); } public static void HandleShowStatsCommand(string module, string[] cmd) @@ -145,29 +147,55 @@ namespace OpenSim.Framework.Monitoring } } - private static void OutputAllStatsToConsole(ICommandConsole con) + public static List GetAllStatsReports() { + List reports = new List(); + foreach (var category in RegisteredStats.Values) - { - OutputCategoryStatsToConsole(con, category); - } + reports.AddRange(GetCategoryStatsReports(category)); + + return reports; + } + + private static void OutputAllStatsToConsole(ICommandConsole con) + { + foreach (string report in GetAllStatsReports()) + con.Output(report); + } + + private static List GetCategoryStatsReports( + SortedDictionary> category) + { + List reports = new List(); + + foreach (var container in category.Values) + reports.AddRange(GetContainerStatsReports(container)); + + return reports; } private static void OutputCategoryStatsToConsole( ICommandConsole con, SortedDictionary> category) { - foreach (var container in category.Values) - { - OutputContainerStatsToConsole(con, container); - } + foreach (string report in GetCategoryStatsReports(category)) + con.Output(report); } - private static void OutputContainerStatsToConsole( ICommandConsole con, SortedDictionary container) + private static List GetContainerStatsReports(SortedDictionary container) { + List reports = new List(); + foreach (Stat stat in container.Values) - { - con.Output(stat.ToConsoleString()); - } + reports.Add(stat.ToConsoleString()); + + return reports; + } + + private static void OutputContainerStatsToConsole( + ICommandConsole con, SortedDictionary container) + { + foreach (string report in GetContainerStatsReports(container)) + con.Output(report); } // Creates an OSDMap of the format: -- cgit v1.1 From d6d5d4ebd033d47886c6e31ae87c0be527f5545e Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 6 Aug 2013 18:32:16 +0100 Subject: Add file missing from last commit 4c2f6de --- OpenSim/Framework/Monitoring/StatsLogger.cs | 108 ++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 OpenSim/Framework/Monitoring/StatsLogger.cs (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsLogger.cs b/OpenSim/Framework/Monitoring/StatsLogger.cs new file mode 100644 index 0000000..fa2e1b6 --- /dev/null +++ b/OpenSim/Framework/Monitoring/StatsLogger.cs @@ -0,0 +1,108 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Reflection; +using System.Timers; +using log4net; + +namespace OpenSim.Framework.Monitoring +{ + /// + /// Provides a means to continuously log stats for debugging purposes. + /// + public static class StatsLogger + { + private static readonly ILog m_statsLog = LogManager.GetLogger("special.StatsLogger"); + + private static Timer m_loggingTimer; + private static int m_statsLogIntervalMs = 5000; + + public static void RegisterConsoleCommands(ICommandConsole console) + { + console.Commands.AddCommand( + "Debug", + false, + "debug stats record", + "debug stats record start|stop", + "Control whether stats are being regularly recorded to a separate file.", + "For debug purposes. Experimental.", + HandleStatsRecordCommand); + } + + public static void HandleStatsRecordCommand(string module, string[] cmd) + { + ICommandConsole con = MainConsole.Instance; + + if (cmd.Length != 4) + { + con.Output("Usage: debug stats record start|stop"); + return; + } + + if (cmd[3] == "start") + { + Start(); + con.OutputFormat("Now recording all stats very {0}ms to file", m_statsLogIntervalMs); + } + else if (cmd[3] == "stop") + { + Stop(); + con.Output("Stopped recording stats to file."); + } + } + + public static void Start() + { + if (m_loggingTimer != null) + Stop(); + + m_loggingTimer = new Timer(m_statsLogIntervalMs); + m_loggingTimer.AutoReset = false; + m_loggingTimer.Elapsed += Log; + m_loggingTimer.Start(); + } + + public static void Stop() + { + if (m_loggingTimer != null) + { + m_loggingTimer.Stop(); + } + } + + private static void Log(object sender, ElapsedEventArgs e) + { + m_statsLog.InfoFormat("*** STATS REPORT AT {0} ***", DateTime.Now); + + foreach (string report in StatsManager.GetAllStatsReports()) + m_statsLog.Info(report); + + m_loggingTimer.Start(); + } + } +} \ No newline at end of file -- cgit v1.1 From 50c163ae6ca734610694f4edcc109ff0bdc65ba1 Mon Sep 17 00:00:00 2001 From: Robert Adams Date: Tue, 6 Aug 2013 08:21:16 -0700 Subject: Add a JSON web fetch of the statististics managed by StatsManager. Disabled by default. Enable by setting [Startup]ManagedStatsRemoteFetchURI="Something" and thereafter "http://ServerHTTPPort/Something/" will return all the managed stats (equivilent to "show stats all" console command). Accepts queries "cat=", "cont=" and "stat=" to specify statistic category, container and statistic names. The special name "all" is the default and returns all values in that group. --- OpenSim/Framework/Monitoring/StatsManager.cs | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index c8e838c..23c6f18 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -26,10 +26,12 @@ */ using System; +using System.Collections; using System.Collections.Generic; using System.Linq; using System.Text; +using OpenSim.Framework; using OpenMetaverse.StructuredData; namespace OpenSim.Framework.Monitoring @@ -262,6 +264,35 @@ namespace OpenSim.Framework.Monitoring return map; } + public static Hashtable HandleStatsRequest(Hashtable request) + { + Hashtable responsedata = new Hashtable(); + string regpath = request["uri"].ToString(); + int response_code = 200; + string contenttype = "text/json"; + + string pCategoryName = StatsManager.AllSubCommand; + string pContainerName = StatsManager.AllSubCommand; + string pStatName = StatsManager.AllSubCommand; + + if (request.ContainsKey("cat")) pCategoryName = request["cat"].ToString(); + if (request.ContainsKey("cont")) pContainerName = request["cat"].ToString(); + if (request.ContainsKey("stat")) pStatName = request["cat"].ToString(); + + string strOut = StatsManager.GetStatsAsOSDMap(pCategoryName, pContainerName, pStatName).ToString(); + + // m_log.DebugFormat("{0} StatFetch: uri={1}, cat={2}, cont={3}, stat={4}, resp={5}", + // LogHeader, regpath, pCategoryName, pContainerName, pStatName, strOut); + + responsedata["int_response_code"] = response_code; + responsedata["content_type"] = contenttype; + responsedata["keepalive"] = false; + responsedata["str_response_string"] = strOut; + responsedata["access_control_allow_origin"] = "*"; + + return responsedata; + } + // /// // /// Start collecting statistics related to assets. // /// Should only be called once. -- cgit v1.1 From d9bd6e6b5be3100141a3b1202f859c65a302d4ee Mon Sep 17 00:00:00 2001 From: Robert Adams Date: Thu, 8 Aug 2013 09:41:11 -0700 Subject: Add parameter and explanation of ManagedStats return to OpenSimDefaults.ini. Add 'callback' query parameter to managed stats return to return function form of JSON data. --- OpenSim/Framework/Monitoring/StatsManager.cs | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index 23c6f18..7cf1fa7 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -281,6 +281,12 @@ namespace OpenSim.Framework.Monitoring string strOut = StatsManager.GetStatsAsOSDMap(pCategoryName, pContainerName, pStatName).ToString(); + // If requestor wants it as a callback function, build response as a function rather than just the JSON string. + if (request.ContainsKey("callback")) + { + strOut = request["callback"].ToString() + "(" + strOut + ");"; + } + // m_log.DebugFormat("{0} StatFetch: uri={1}, cat={2}, cont={3}, stat={4}, resp={5}", // LogHeader, regpath, pCategoryName, pContainerName, pStatName, strOut); -- cgit v1.1 From 216f5afe54576c4852974b8479ac95654dc9e08e Mon Sep 17 00:00:00 2001 From: Robert Adams Date: Sat, 10 Aug 2013 09:09:52 -0700 Subject: Stats treaking. Update ToOSDMap for Stat and PercentageStat to return all the various numbers that have been added to the console output. Break out EventHistogram from CounterStat. --- OpenSim/Framework/Monitoring/Stats/CounterStat.cs | 150 ++---------------- .../Framework/Monitoring/Stats/EventHistogram.cs | 173 +++++++++++++++++++++ .../Framework/Monitoring/Stats/PercentageStat.cs | 16 ++ OpenSim/Framework/Monitoring/Stats/Stat.cs | 46 ++++-- 4 files changed, 235 insertions(+), 150 deletions(-) create mode 100755 OpenSim/Framework/Monitoring/Stats/EventHistogram.cs (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Stats/CounterStat.cs b/OpenSim/Framework/Monitoring/Stats/CounterStat.cs index 04442c3..318cf1c 100755 --- a/OpenSim/Framework/Monitoring/Stats/CounterStat.cs +++ b/OpenSim/Framework/Monitoring/Stats/CounterStat.cs @@ -34,142 +34,6 @@ using OpenMetaverse.StructuredData; namespace OpenSim.Framework.Monitoring { -// Create a time histogram of events. The histogram is built in a wrap-around -// array of equally distributed buckets. -// For instance, a minute long histogram of second sized buckets would be: -// new EventHistogram(60, 1000) -public class EventHistogram -{ - private int m_timeBase; - private int m_numBuckets; - private int m_bucketMilliseconds; - private int m_lastBucket; - private int m_totalHistogramMilliseconds; - private long[] m_histogram; - private object histoLock = new object(); - - public EventHistogram(int numberOfBuckets, int millisecondsPerBucket) - { - m_numBuckets = numberOfBuckets; - m_bucketMilliseconds = millisecondsPerBucket; - m_totalHistogramMilliseconds = m_numBuckets * m_bucketMilliseconds; - - m_histogram = new long[m_numBuckets]; - Zero(); - m_lastBucket = 0; - m_timeBase = Util.EnvironmentTickCount(); - } - - public void Event() - { - this.Event(1); - } - - // Record an event at time 'now' in the histogram. - public void Event(int cnt) - { - lock (histoLock) - { - // The time as displaced from the base of the histogram - int bucketTime = Util.EnvironmentTickCountSubtract(m_timeBase); - - // If more than the total time of the histogram, we just start over - if (bucketTime > m_totalHistogramMilliseconds) - { - Zero(); - m_lastBucket = 0; - m_timeBase = Util.EnvironmentTickCount(); - } - else - { - // To which bucket should we add this event? - int bucket = bucketTime / m_bucketMilliseconds; - - // Advance m_lastBucket to the new bucket. Zero any buckets skipped over. - while (bucket != m_lastBucket) - { - // Zero from just after the last bucket to the new bucket or the end - for (int jj = m_lastBucket + 1; jj <= Math.Min(bucket, m_numBuckets - 1); jj++) - { - m_histogram[jj] = 0; - } - m_lastBucket = bucket; - // If the new bucket is off the end, wrap around to the beginning - if (bucket > m_numBuckets) - { - bucket -= m_numBuckets; - m_lastBucket = 0; - m_histogram[m_lastBucket] = 0; - m_timeBase += m_totalHistogramMilliseconds; - } - } - } - m_histogram[m_lastBucket] += cnt; - } - } - - // Get a copy of the current histogram - public long[] GetHistogram() - { - long[] ret = new long[m_numBuckets]; - lock (histoLock) - { - int indx = m_lastBucket + 1; - for (int ii = 0; ii < m_numBuckets; ii++, indx++) - { - if (indx >= m_numBuckets) - indx = 0; - ret[ii] = m_histogram[indx]; - } - } - return ret; - } - - public OSDMap GetHistogramAsOSDMap() - { - OSDMap ret = new OSDMap(); - - ret.Add("Buckets", OSD.FromInteger(m_numBuckets)); - ret.Add("BucketMilliseconds", OSD.FromInteger(m_bucketMilliseconds)); - ret.Add("TotalMilliseconds", OSD.FromInteger(m_totalHistogramMilliseconds)); - - // Compute a number for the first bucket in the histogram. - // This will allow readers to know how this histogram relates to any previously read histogram. - int baseBucketNum = (m_timeBase / m_bucketMilliseconds) + m_lastBucket + 1; - ret.Add("BaseNumber", OSD.FromInteger(baseBucketNum)); - - ret.Add("Values", GetHistogramAsOSDArray()); - - return ret; - } - // Get a copy of the current histogram - public OSDArray GetHistogramAsOSDArray() - { - OSDArray ret = new OSDArray(m_numBuckets); - lock (histoLock) - { - int indx = m_lastBucket + 1; - for (int ii = 0; ii < m_numBuckets; ii++, indx++) - { - if (indx >= m_numBuckets) - indx = 0; - ret[ii] = OSD.FromLong(m_histogram[indx]); - } - } - return ret; - } - - // Zero out the histogram - public void Zero() - { - lock (histoLock) - { - for (int ii = 0; ii < m_numBuckets; ii++) - m_histogram[ii] = 0; - } - } -} - // A statistic that wraps a counter. // Built this way mostly so histograms and history can be created. public class CounterStat : Stat @@ -236,12 +100,18 @@ public class CounterStat : Stat // If there are any histograms, add a new field that is an array of histograms as OSDMaps if (m_histograms.Count > 0) { - OSDArray histos = new OSDArray(); - foreach (EventHistogram histo in m_histograms.Values) + lock (counterLock) { - histos.Add(histo.GetHistogramAsOSDMap()); + if (m_histograms.Count > 0) + { + OSDArray histos = new OSDArray(); + foreach (EventHistogram histo in m_histograms.Values) + { + histos.Add(histo.GetHistogramAsOSDMap()); + } + map.Add("Histograms", histos); + } } - map.Add("Histograms", histos); } return map; } diff --git a/OpenSim/Framework/Monitoring/Stats/EventHistogram.cs b/OpenSim/Framework/Monitoring/Stats/EventHistogram.cs new file mode 100755 index 0000000..f51f322 --- /dev/null +++ b/OpenSim/Framework/Monitoring/Stats/EventHistogram.cs @@ -0,0 +1,173 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using OpenMetaverse.StructuredData; + +namespace OpenSim.Framework.Monitoring +{ +// Create a time histogram of events. The histogram is built in a wrap-around +// array of equally distributed buckets. +// For instance, a minute long histogram of second sized buckets would be: +// new EventHistogram(60, 1000) +public class EventHistogram +{ + private int m_timeBase; + private int m_numBuckets; + private int m_bucketMilliseconds; + private int m_lastBucket; + private int m_totalHistogramMilliseconds; + private long[] m_histogram; + private object histoLock = new object(); + + public EventHistogram(int numberOfBuckets, int millisecondsPerBucket) + { + m_numBuckets = numberOfBuckets; + m_bucketMilliseconds = millisecondsPerBucket; + m_totalHistogramMilliseconds = m_numBuckets * m_bucketMilliseconds; + + m_histogram = new long[m_numBuckets]; + Zero(); + m_lastBucket = 0; + m_timeBase = Util.EnvironmentTickCount(); + } + + public void Event() + { + this.Event(1); + } + + // Record an event at time 'now' in the histogram. + public void Event(int cnt) + { + lock (histoLock) + { + // The time as displaced from the base of the histogram + int bucketTime = Util.EnvironmentTickCountSubtract(m_timeBase); + + // If more than the total time of the histogram, we just start over + if (bucketTime > m_totalHistogramMilliseconds) + { + Zero(); + m_lastBucket = 0; + m_timeBase = Util.EnvironmentTickCount(); + } + else + { + // To which bucket should we add this event? + int bucket = bucketTime / m_bucketMilliseconds; + + // Advance m_lastBucket to the new bucket. Zero any buckets skipped over. + while (bucket != m_lastBucket) + { + // Zero from just after the last bucket to the new bucket or the end + for (int jj = m_lastBucket + 1; jj <= Math.Min(bucket, m_numBuckets - 1); jj++) + { + m_histogram[jj] = 0; + } + m_lastBucket = bucket; + // If the new bucket is off the end, wrap around to the beginning + if (bucket > m_numBuckets) + { + bucket -= m_numBuckets; + m_lastBucket = 0; + m_histogram[m_lastBucket] = 0; + m_timeBase += m_totalHistogramMilliseconds; + } + } + } + m_histogram[m_lastBucket] += cnt; + } + } + + // Get a copy of the current histogram + public long[] GetHistogram() + { + long[] ret = new long[m_numBuckets]; + lock (histoLock) + { + int indx = m_lastBucket + 1; + for (int ii = 0; ii < m_numBuckets; ii++, indx++) + { + if (indx >= m_numBuckets) + indx = 0; + ret[ii] = m_histogram[indx]; + } + } + return ret; + } + + public OSDMap GetHistogramAsOSDMap() + { + OSDMap ret = new OSDMap(); + + ret.Add("Buckets", OSD.FromInteger(m_numBuckets)); + ret.Add("BucketMilliseconds", OSD.FromInteger(m_bucketMilliseconds)); + ret.Add("TotalMilliseconds", OSD.FromInteger(m_totalHistogramMilliseconds)); + + // Compute a number for the first bucket in the histogram. + // This will allow readers to know how this histogram relates to any previously read histogram. + int baseBucketNum = (m_timeBase / m_bucketMilliseconds) + m_lastBucket + 1; + ret.Add("BaseNumber", OSD.FromInteger(baseBucketNum)); + + ret.Add("Values", GetHistogramAsOSDArray()); + + return ret; + } + // Get a copy of the current histogram + public OSDArray GetHistogramAsOSDArray() + { + OSDArray ret = new OSDArray(m_numBuckets); + lock (histoLock) + { + int indx = m_lastBucket + 1; + for (int ii = 0; ii < m_numBuckets; ii++, indx++) + { + if (indx >= m_numBuckets) + indx = 0; + ret[ii] = OSD.FromLong(m_histogram[indx]); + } + } + return ret; + } + + // Zero out the histogram + public void Zero() + { + lock (histoLock) + { + for (int ii = 0; ii < m_numBuckets; ii++) + m_histogram[ii] = 0; + } + } +} + +} diff --git a/OpenSim/Framework/Monitoring/Stats/PercentageStat.cs b/OpenSim/Framework/Monitoring/Stats/PercentageStat.cs index 60bed55..55ddf06 100644 --- a/OpenSim/Framework/Monitoring/Stats/PercentageStat.cs +++ b/OpenSim/Framework/Monitoring/Stats/PercentageStat.cs @@ -29,6 +29,8 @@ using System; using System.Collections.Generic; using System.Text; +using OpenMetaverse.StructuredData; + namespace OpenSim.Framework.Monitoring { public class PercentageStat : Stat @@ -84,5 +86,19 @@ namespace OpenSim.Framework.Monitoring return sb.ToString(); } + + // PercentageStat is a basic stat plus percent calc + public override OSDMap ToOSDMap() + { + // Get the foundational instance + OSDMap map = base.ToOSDMap(); + + map["StatType"] = "PercentageStat"; + + map.Add("Antecedent", OSD.FromLong(Antecedent)); + map.Add("Consequent", OSD.FromLong(Consequent)); + + return map; + } } } \ No newline at end of file diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs index ffd5132..2b34493 100644 --- a/OpenSim/Framework/Monitoring/Stats/Stat.cs +++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs @@ -241,6 +241,8 @@ namespace OpenSim.Framework.Monitoring public virtual OSDMap ToOSDMap() { OSDMap ret = new OSDMap(); + ret.Add("StatType", "Stat"); // used by overloading classes to denote type of stat + ret.Add("Category", OSD.FromString(Category)); ret.Add("Container", OSD.FromString(Container)); ret.Add("ShortName", OSD.FromString(ShortName)); @@ -248,26 +250,36 @@ namespace OpenSim.Framework.Monitoring ret.Add("Description", OSD.FromString(Description)); ret.Add("UnitName", OSD.FromString(UnitName)); ret.Add("Value", OSD.FromReal(Value)); - ret.Add("StatType", "Stat"); // used by overloading classes to denote type of stat + + double lastChangeOverTime, averageChangeOverTime; + if (ComputeMeasuresOfInterest(out lastChangeOverTime, out averageChangeOverTime)) + { + ret.Add("LastChangeOverTime", OSD.FromReal(lastChangeOverTime)); + ret.Add("AverageChangeOverTime", OSD.FromReal(averageChangeOverTime)); + } return ret; } - protected void AppendMeasuresOfInterest(StringBuilder sb) + // Compute the averages over time and return same. + // Return 'true' if averages were actually computed. 'false' if no average info. + public bool ComputeMeasuresOfInterest(out double lastChangeOverTime, out double averageChangeOverTime) { - if ((MeasuresOfInterest & MeasuresOfInterest.AverageChangeOverTime) - == MeasuresOfInterest.AverageChangeOverTime) + bool ret = false; + lastChangeOverTime = 0; + averageChangeOverTime = 0; + + if ((MeasuresOfInterest & MeasuresOfInterest.AverageChangeOverTime) == MeasuresOfInterest.AverageChangeOverTime) { double totalChange = 0; - double lastChangeOverTime = 0; double? penultimateSample = null; double? lastSample = null; lock (m_samples) { -// m_log.DebugFormat( -// "[STAT]: Samples for {0} are {1}", -// Name, string.Join(",", m_samples.Select(s => s.ToString()).ToArray())); + // m_log.DebugFormat( + // "[STAT]: Samples for {0} are {1}", + // Name, string.Join(",", m_samples.Select(s => s.ToString()).ToArray())); foreach (double s in m_samples) { @@ -280,13 +292,27 @@ namespace OpenSim.Framework.Monitoring } if (lastSample != null && penultimateSample != null) - lastChangeOverTime + { + lastChangeOverTime = ((double)lastSample - (double)penultimateSample) / (Watchdog.WATCHDOG_INTERVAL_MS / 1000); + } int divisor = m_samples.Count <= 1 ? 1 : m_samples.Count - 1; - double averageChangeOverTime = totalChange / divisor / (Watchdog.WATCHDOG_INTERVAL_MS / 1000); + averageChangeOverTime = totalChange / divisor / (Watchdog.WATCHDOG_INTERVAL_MS / 1000); + ret = true; + } + + return ret; + } + protected void AppendMeasuresOfInterest(StringBuilder sb) + { + double lastChangeOverTime = 0; + double averageChangeOverTime = 0; + + if (ComputeMeasuresOfInterest(out lastChangeOverTime, out averageChangeOverTime)) + { sb.AppendFormat( ", {0:0.##}{1}/s, {2:0.##}{3}/s", lastChangeOverTime, -- cgit v1.1 From 217c8deae5fab5aa025932b027dfc4a4b629cc58 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Sat, 17 Aug 2013 00:51:21 +0100 Subject: minor: remove mono compiler warning in StatsManager --- OpenSim/Framework/Monitoring/StatsManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index 7cf1fa7..05ee4c5 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -267,7 +267,7 @@ namespace OpenSim.Framework.Monitoring public static Hashtable HandleStatsRequest(Hashtable request) { Hashtable responsedata = new Hashtable(); - string regpath = request["uri"].ToString(); +// string regpath = request["uri"].ToString(); int response_code = 200; string contenttype = "text/json"; -- cgit v1.1 From a9f9b0da9d6c42aaa0c28c78c27f078f1a80c874 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 23 Aug 2013 00:13:31 +0100 Subject: minor: Correct typo on "debug stats record start" message --- OpenSim/Framework/Monitoring/StatsLogger.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsLogger.cs b/OpenSim/Framework/Monitoring/StatsLogger.cs index fa2e1b6..1e4fa11 100644 --- a/OpenSim/Framework/Monitoring/StatsLogger.cs +++ b/OpenSim/Framework/Monitoring/StatsLogger.cs @@ -67,7 +67,7 @@ namespace OpenSim.Framework.Monitoring if (cmd[3] == "start") { Start(); - con.OutputFormat("Now recording all stats very {0}ms to file", m_statsLogIntervalMs); + con.OutputFormat("Now recording all stats to file every {0}ms", m_statsLogIntervalMs); } else if (cmd[3] == "stop") { -- cgit v1.1 From 42bdf446585007029faf4cd21abd289487f0f797 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 4 Oct 2013 23:33:47 +0100 Subject: Bump OPenSimulator version and assembly versions up to 0.8.0 Dev --- OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs index 36678bb..07971bf 100644 --- a/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs +++ b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs @@ -29,5 +29,5 @@ using System.Runtime.InteropServices; // Build Number // Revision // -[assembly: AssemblyVersion("0.7.6.*")] +[assembly: AssemblyVersion("0.8.0.*")] -- cgit v1.1 From 7cab41f4223b7febd3fdd42fa7cfefef25e4a9c9 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 15 Nov 2013 21:45:08 +0000 Subject: refactor: replace verbose checks with String.IsNullOrEmpty where applicable. Thanks to Kira for this patch from http://opensimulator.org/mantis/view.php?id=6845 --- OpenSim/Framework/Monitoring/Stats/Stat.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs index 2b34493..e095801 100644 --- a/OpenSim/Framework/Monitoring/Stats/Stat.cs +++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs @@ -231,7 +231,7 @@ namespace OpenSim.Framework.Monitoring Container, ShortName, Value, - UnitName == null || UnitName == "" ? "" : string.Format(" {0}", UnitName)); + string.IsNullOrEmpty(UnitName) ? "" : string.Format(" {0}", UnitName)); AppendMeasuresOfInterest(sb); @@ -316,9 +316,9 @@ namespace OpenSim.Framework.Monitoring sb.AppendFormat( ", {0:0.##}{1}/s, {2:0.##}{3}/s", lastChangeOverTime, - UnitName == null || UnitName == "" ? "" : string.Format(" {0}", UnitName), + string.IsNullOrEmpty(UnitName) ? "" : string.Format(" {0}", UnitName), averageChangeOverTime, - UnitName == null || UnitName == "" ? "" : string.Format(" {0}", UnitName)); + string.IsNullOrEmpty(UnitName) ? "" : string.Format(" {0}", UnitName)); } } } -- cgit v1.1 From 8ce3fa646b4da171d96784935ba5a39ea65dd70a Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 18 Mar 2014 20:21:47 +0000 Subject: Allow "show stats" console command to take a full stat name and display only that stat. For example, scene.test.RootAgents will show only the RootAgents stat in the scene named "test" --- OpenSim/Framework/Monitoring/StatsManager.cs | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index 05ee4c5..8fcbdca 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -99,6 +99,7 @@ namespace OpenSim.Framework.Monitoring string categoryName = components[0]; string containerName = components.Length > 1 ? components[1] : null; + string statName = components.Length > 2 ? components[2] : null; if (categoryName == AllSubCommand) { @@ -128,7 +129,23 @@ namespace OpenSim.Framework.Monitoring SortedDictionary container; if (category.TryGetValue(containerName, out container)) { - OutputContainerStatsToConsole(con, container); + if (String.IsNullOrEmpty(statName)) + { + OutputContainerStatsToConsole(con, container); + } + else + { + Stat stat; + if (container.TryGetValue(statName, out stat)) + { + OutputStatToConsole(con, stat); + } + else + { + con.OutputFormat( + "No such stat {0} in {1}.{2}", statName, categoryName, containerName); + } + } } else { @@ -200,6 +217,11 @@ namespace OpenSim.Framework.Monitoring con.Output(report); } + private static void OutputStatToConsole(ICommandConsole con, Stat stat) + { + con.Output(stat.ToConsoleString()); + } + // Creates an OSDMap of the format: // { categoryName: { // containerName: { -- cgit v1.1 From 1d0a9e521c57c95987008fd48b8a44e3a966fe54 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Wed, 19 Mar 2014 00:28:57 +0000 Subject: Allow a snapshot of stats to be dumped to a file with a "stats save" command --- OpenSim/Framework/Monitoring/StatsLogger.cs | 49 +++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsLogger.cs b/OpenSim/Framework/Monitoring/StatsLogger.cs index 1e4fa11..5822794 100644 --- a/OpenSim/Framework/Monitoring/StatsLogger.cs +++ b/OpenSim/Framework/Monitoring/StatsLogger.cs @@ -26,7 +26,10 @@ */ using System; +using System.Collections.Generic; +using System.IO; using System.Reflection; +using System.Text; using System.Timers; using log4net; @@ -52,6 +55,15 @@ namespace OpenSim.Framework.Monitoring "Control whether stats are being regularly recorded to a separate file.", "For debug purposes. Experimental.", HandleStatsRecordCommand); + + console.Commands.AddCommand( + "General", + false, + "stats save", + "stats save ", + "Save stats snapshot to a file. If the file already exists, then the report is appended.", + "For debug purposes. Experimental.", + HandleStatsSaveCommand); } public static void HandleStatsRecordCommand(string module, string[] cmd) @@ -76,6 +88,27 @@ namespace OpenSim.Framework.Monitoring } } + public static void HandleStatsSaveCommand(string module, string[] cmd) + { + ICommandConsole con = MainConsole.Instance; + + if (cmd.Length != 3) + { + con.Output("Usage: stats save "); + return; + } + + string path = cmd[2]; + + using (StreamWriter sw = new StreamWriter(path, true)) + { + foreach (string line in GetReport()) + sw.WriteLine(line); + } + + MainConsole.Instance.OutputFormat("Stats saved to file {0}", path); + } + public static void Start() { if (m_loggingTimer != null) @@ -97,12 +130,22 @@ namespace OpenSim.Framework.Monitoring private static void Log(object sender, ElapsedEventArgs e) { - m_statsLog.InfoFormat("*** STATS REPORT AT {0} ***", DateTime.Now); + foreach (string line in GetReport()) + m_statsLog.Info(line); + + m_loggingTimer.Start(); + } + + private static List GetReport() + { + List lines = new List(); + + lines.Add(string.Format("*** STATS REPORT AT {0} ***", DateTime.Now)); foreach (string report in StatsManager.GetAllStatsReports()) - m_statsLog.Info(report); + lines.Add(report); - m_loggingTimer.Start(); + return lines; } } } \ No newline at end of file -- cgit v1.1 From e6080a38c5d91f84b123fe510f4a7fcb155de8bf Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Wed, 19 Mar 2014 00:29:36 +0000 Subject: Rename "debug stats record" command to "stats record" for consistency. Move from debug to general command category --- OpenSim/Framework/Monitoring/StatsLogger.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsLogger.cs b/OpenSim/Framework/Monitoring/StatsLogger.cs index 5822794..f23107c 100644 --- a/OpenSim/Framework/Monitoring/StatsLogger.cs +++ b/OpenSim/Framework/Monitoring/StatsLogger.cs @@ -48,10 +48,10 @@ namespace OpenSim.Framework.Monitoring public static void RegisterConsoleCommands(ICommandConsole console) { console.Commands.AddCommand( - "Debug", + "General", false, - "debug stats record", - "debug stats record start|stop", + "stats record", + "stats record start|stop", "Control whether stats are being regularly recorded to a separate file.", "For debug purposes. Experimental.", HandleStatsRecordCommand); -- cgit v1.1 From 98587665160ae82fcc26c7cd03bded8e0e81751c Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Wed, 19 Mar 2014 00:35:49 +0000 Subject: minor: Make "stats show" an alias for "show stats" command. --- OpenSim/Framework/Monitoring/StatsManager.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index 8fcbdca..0bac247 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -72,8 +72,8 @@ namespace OpenSim.Framework.Monitoring console.Commands.AddCommand( "General", false, - "show stats", - "show stats [list|all|([.])+", + "stats show", + "stats show [list|all|([.])+", "Show statistical information for this server", "If no final argument is specified then legacy statistics information is currently shown.\n" + "'list' argument will show statistic categories.\n" @@ -84,6 +84,14 @@ namespace OpenSim.Framework.Monitoring + "THIS STATS FACILITY IS EXPERIMENTAL AND DOES NOT YET CONTAIN ALL STATS", HandleShowStatsCommand); + console.Commands.AddCommand( + "General", + false, + "show stats", + "show stats [list|all|([.])+", + "Alias for 'stats show' command", + HandleShowStatsCommand); + StatsLogger.RegisterConsoleCommands(console); } -- cgit v1.1 From 47e7febebcbd12d534b0ba248cb4bfc7da4c9d43 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 21 Mar 2014 02:06:10 +0000 Subject: Add monitored thread last update times as stats in "show stats all" These have the format server.thread. --- OpenSim/Framework/Monitoring/Watchdog.cs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index 45762a6..427ed7b 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -82,12 +82,32 @@ namespace OpenSim.Framework.Monitoring /// public Func AlarmMethod { get; set; } + /// + /// Stat structure associated with this thread. + /// + public Stat Stat { get; set; } + public ThreadWatchdogInfo(Thread thread, int timeout) { Thread = thread; Timeout = timeout; FirstTick = Environment.TickCount & Int32.MaxValue; LastTick = FirstTick; + + Stat + = new Stat( + thread.Name, + string.Format("Last update of thread {0}", thread.Name), + "", + "ms", + "server", + "thread", + StatType.Pull, + MeasuresOfInterest.None, + stat => stat.Value = Environment.TickCount & Int32.MaxValue - LastTick, + StatVerbosity.Debug); + + StatsManager.RegisterStat(Stat); } public ThreadWatchdogInfo(ThreadWatchdogInfo previousTwi) @@ -100,6 +120,11 @@ namespace OpenSim.Framework.Monitoring AlarmIfTimeout = previousTwi.AlarmIfTimeout; AlarmMethod = previousTwi.AlarmMethod; } + + public void Cleanup() + { + StatsManager.DeregisterStat(Stat); + } } /// @@ -238,6 +263,7 @@ namespace OpenSim.Framework.Monitoring m_log.DebugFormat( "[WATCHDOG]: Removing thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId); + twi.Cleanup(); m_threads.Remove(threadID); return true; -- cgit v1.1 From 5450b1b0247bb3907f60f2b3f9b0582903de4f83 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 17 Jun 2014 18:37:15 +0100 Subject: Change assembly versions to 0.8.1 --- OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs index 07971bf..ee82753 100644 --- a/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs +++ b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs @@ -29,5 +29,5 @@ using System.Runtime.InteropServices; // Build Number // Revision // -[assembly: AssemblyVersion("0.8.0.*")] +[assembly: AssemblyVersion("0.8.1.*")] -- cgit v1.1 From 8ae5ab809fa4efe692537500258b407c3ed152a6 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 8 Jul 2014 18:49:57 +0100 Subject: Fix issue with running "stats record start|stop" console command In commit e6080a38 (Wed Mar 19 00:29:36 2014) I renamed this from "debug stats record start|stop" Unfortunately, I didn't do this fully so before this commit "stats record start|stop" will report a usage failure with the old debug text. Unfortunately this is in the 0.8 release. The workaround is to repeat the last command twice (e.g. "stats record start start") --- OpenSim/Framework/Monitoring/StatsLogger.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsLogger.cs b/OpenSim/Framework/Monitoring/StatsLogger.cs index f23107c..15a37aa 100644 --- a/OpenSim/Framework/Monitoring/StatsLogger.cs +++ b/OpenSim/Framework/Monitoring/StatsLogger.cs @@ -70,18 +70,18 @@ namespace OpenSim.Framework.Monitoring { ICommandConsole con = MainConsole.Instance; - if (cmd.Length != 4) + if (cmd.Length != 3) { - con.Output("Usage: debug stats record start|stop"); + con.Output("Usage: stats record start|stop"); return; } - if (cmd[3] == "start") + if (cmd[2] == "start") { Start(); con.OutputFormat("Now recording all stats to file every {0}ms", m_statsLogIntervalMs); } - else if (cmd[3] == "stop") + else if (cmd[2] == "stop") { Stop(); con.Output("Stopped recording stats to file."); -- cgit v1.1 From 5d534127663899cd5592c865b1d00855fce25854 Mon Sep 17 00:00:00 2001 From: Oren Hurvitz Date: Sun, 29 Jun 2014 16:40:11 +0300 Subject: Write UDP statistics to the log, not just the console (e.g., "show queues") --- OpenSim/Framework/Monitoring/StatsManager.cs | 53 ++++++++++++---------------- 1 file changed, 22 insertions(+), 31 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index 0bac247..5c8d934 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -30,6 +30,8 @@ using System.Collections; using System.Collections.Generic; using System.Linq; using System.Text; +using System.Reflection; +using log4net; using OpenSim.Framework; using OpenMetaverse.StructuredData; @@ -41,6 +43,8 @@ namespace OpenSim.Framework.Monitoring /// public static class StatsManager { + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + // Subcommand used to list other stats. public const string AllSubCommand = "all"; @@ -98,6 +102,7 @@ namespace OpenSim.Framework.Monitoring public static void HandleShowStatsCommand(string module, string[] cmd) { ICommandConsole con = MainConsole.Instance; + StringBuilder report = new StringBuilder(); if (cmd.Length > 2) { @@ -111,7 +116,8 @@ namespace OpenSim.Framework.Monitoring if (categoryName == AllSubCommand) { - OutputAllStatsToConsole(con); + foreach (string report2 in GetAllStatsReports()) + report.AppendLine(report2); } else if (categoryName == ListSubCommand) { @@ -130,7 +136,8 @@ namespace OpenSim.Framework.Monitoring { if (String.IsNullOrEmpty(containerName)) { - OutputCategoryStatsToConsole(con, category); + foreach (string report2 in GetCategoryStatsReports(category)) + report.AppendLine(report2); } else { @@ -139,14 +146,15 @@ namespace OpenSim.Framework.Monitoring { if (String.IsNullOrEmpty(statName)) { - OutputContainerStatsToConsole(con, container); + foreach (string report2 in GetContainerStatsReports(container)) + report.AppendLine(report2); } else { Stat stat; if (container.TryGetValue(statName, out stat)) { - OutputStatToConsole(con, stat); + report.AppendLine(stat.ToConsoleString()); } else { @@ -168,10 +176,18 @@ namespace OpenSim.Framework.Monitoring { // Legacy if (SimExtraStats != null) - con.Output(SimExtraStats.Report()); + { + report.Append(SimExtraStats.Report()); + } else - OutputAllStatsToConsole(con); + { + foreach (string report2 in GetAllStatsReports()) + report.AppendLine(report2); + } } + + if (report.Length > 0) + m_log.Debug(string.Join(" ", cmd) + "\n" + report.ToString()); } public static List GetAllStatsReports() @@ -184,12 +200,6 @@ namespace OpenSim.Framework.Monitoring return reports; } - private static void OutputAllStatsToConsole(ICommandConsole con) - { - foreach (string report in GetAllStatsReports()) - con.Output(report); - } - private static List GetCategoryStatsReports( SortedDictionary> category) { @@ -201,13 +211,6 @@ namespace OpenSim.Framework.Monitoring return reports; } - private static void OutputCategoryStatsToConsole( - ICommandConsole con, SortedDictionary> category) - { - foreach (string report in GetCategoryStatsReports(category)) - con.Output(report); - } - private static List GetContainerStatsReports(SortedDictionary container) { List reports = new List(); @@ -218,18 +221,6 @@ namespace OpenSim.Framework.Monitoring return reports; } - private static void OutputContainerStatsToConsole( - ICommandConsole con, SortedDictionary container) - { - foreach (string report in GetContainerStatsReports(container)) - con.Output(report); - } - - private static void OutputStatToConsole(ICommandConsole con, Stat stat) - { - con.Output(stat.ToConsoleString()); - } - // Creates an OSDMap of the format: // { categoryName: { // containerName: { -- cgit v1.1 From 200dcee1b7fa799936aaf43a18a34bfbf8b94449 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Mon, 21 Jul 2014 22:51:11 +0100 Subject: Fix CPU processor use reporting on Mono. Despite the comments in the code, it appears that the issue where the .NET performance counter was wrongly idle time time on Mono was fixed in 2009. https://bugzilla.novell.com/show_bug.cgi?id=468625 Which means that the workaround is no longer necessary and produces bad results instead. --- OpenSim/Framework/Monitoring/ServerStatsCollector.cs | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs index ac0f0bc..b4e0b16 100644 --- a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs @@ -141,7 +141,7 @@ namespace OpenSim.Framework.Monitoring processorPercentPerfCounter = new PerfCounterControl(tempPC); // A long time bug in mono is that CPU percent is reported as CPU percent idle. Windows reports CPU percent busy. tempStat = new Stat(tempName, tempName, "", "percent", CategoryServer, ContainerProcessor, - StatType.Pull, (s) => { GetNextValue(s, processorPercentPerfCounter, Util.IsWindows() ? 1 : -1); }, + StatType.Pull, (s) => { GetNextValue(s, processorPercentPerfCounter); }, StatVerbosity.Info); StatsManager.RegisterStat(tempStat); RegisteredStats.Add(tempName, tempStat); @@ -253,28 +253,22 @@ namespace OpenSim.Framework.Monitoring // "How to get the CPU Usage in C#": http://stackoverflow.com/questions/278071/how-to-get-the-cpu-usage-in-c // "Mono Performance Counters": http://www.mono-project.com/Mono_Performance_Counters private delegate double PerfCounterNextValue(); + private void GetNextValue(Stat stat, PerfCounterControl perfControl) { - GetNextValue(stat, perfControl, 1.0); - } - private void GetNextValue(Stat stat, PerfCounterControl perfControl, double factor) - { if (Util.EnvironmentTickCountSubtract(perfControl.lastFetch) > performanceCounterSampleInterval) { if (perfControl != null && perfControl.perfCounter != null) { try { - // Kludge for factor to run double duty. If -1, subtract the value from one - if (factor == -1) - stat.Value = 1 - perfControl.perfCounter.NextValue(); - else - stat.Value = perfControl.perfCounter.NextValue() / factor; + stat.Value = perfControl.perfCounter.NextValue(); } catch (Exception e) { m_log.ErrorFormat("{0} Exception on NextValue fetching {1}: {2}", LogHeader, stat.Name, e); } + perfControl.lastFetch = Util.EnvironmentTickCount(); } } -- cgit v1.1 From 3355bedaeb26b07b6b91c5de194991acaf2cdf36 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Mon, 21 Jul 2014 23:08:15 +0100 Subject: minor: Limit processor related stats to 3 decimal places instead of all the places. Easier to read and analyze, and probably still too much detail (1 dp would probably be fine) --- OpenSim/Framework/Monitoring/ServerStatsCollector.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs index b4e0b16..ff0fcd5 100644 --- a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs @@ -147,13 +147,13 @@ namespace OpenSim.Framework.Monitoring RegisteredStats.Add(tempName, tempStat); MakeStat("TotalProcessorTime", null, "sec", ContainerProcessor, - (s) => { s.Value = Process.GetCurrentProcess().TotalProcessorTime.TotalSeconds; }); + (s) => { s.Value = Math.Round(Process.GetCurrentProcess().TotalProcessorTime.TotalSeconds, 3); }); MakeStat("UserProcessorTime", null, "sec", ContainerProcessor, - (s) => { s.Value = Process.GetCurrentProcess().UserProcessorTime.TotalSeconds; }); + (s) => { s.Value = Math.Round(Process.GetCurrentProcess().UserProcessorTime.TotalSeconds, 3); }); MakeStat("PrivilegedProcessorTime", null, "sec", ContainerProcessor, - (s) => { s.Value = Process.GetCurrentProcess().PrivilegedProcessorTime.TotalSeconds; }); + (s) => { s.Value = Math.Round(Process.GetCurrentProcess().PrivilegedProcessorTime.TotalSeconds, 3); }); MakeStat("Threads", null, "threads", ContainerProcessor, (s) => { s.Value = Process.GetCurrentProcess().Threads.Count; }); @@ -262,7 +262,7 @@ namespace OpenSim.Framework.Monitoring { try { - stat.Value = perfControl.perfCounter.NextValue(); + stat.Value = Math.Round(perfControl.perfCounter.NextValue(), 3); } catch (Exception e) { -- cgit v1.1 From cc61681484185f3c450fc0ab7efeb093c672d194 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 25 Jul 2014 01:56:41 +0100 Subject: Revert "Write UDP statistics to the log, not just the console (e.g., "show queues")" Fixes http://opensimulator.org/mantis/view.php?id=7280 It can't be done this way because the stats data needs to show up on the console at all log levels, not just debug. But this means setting it to log at fatal, which is not appropriate for this stuff in the log. I understand the desire but this has to be done some other way, perhaps by (yet another) config parameter. Also, this was already being done with the ClientStatsReport but that also should be done in another way, I think. This reverts commit 5d534127663899cd5592c865b1d00855fce25854. --- OpenSim/Framework/Monitoring/StatsManager.cs | 53 ++++++++++++++++------------ 1 file changed, 31 insertions(+), 22 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index 5c8d934..0bac247 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -30,8 +30,6 @@ using System.Collections; using System.Collections.Generic; using System.Linq; using System.Text; -using System.Reflection; -using log4net; using OpenSim.Framework; using OpenMetaverse.StructuredData; @@ -43,8 +41,6 @@ namespace OpenSim.Framework.Monitoring /// public static class StatsManager { - private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - // Subcommand used to list other stats. public const string AllSubCommand = "all"; @@ -102,7 +98,6 @@ namespace OpenSim.Framework.Monitoring public static void HandleShowStatsCommand(string module, string[] cmd) { ICommandConsole con = MainConsole.Instance; - StringBuilder report = new StringBuilder(); if (cmd.Length > 2) { @@ -116,8 +111,7 @@ namespace OpenSim.Framework.Monitoring if (categoryName == AllSubCommand) { - foreach (string report2 in GetAllStatsReports()) - report.AppendLine(report2); + OutputAllStatsToConsole(con); } else if (categoryName == ListSubCommand) { @@ -136,8 +130,7 @@ namespace OpenSim.Framework.Monitoring { if (String.IsNullOrEmpty(containerName)) { - foreach (string report2 in GetCategoryStatsReports(category)) - report.AppendLine(report2); + OutputCategoryStatsToConsole(con, category); } else { @@ -146,15 +139,14 @@ namespace OpenSim.Framework.Monitoring { if (String.IsNullOrEmpty(statName)) { - foreach (string report2 in GetContainerStatsReports(container)) - report.AppendLine(report2); + OutputContainerStatsToConsole(con, container); } else { Stat stat; if (container.TryGetValue(statName, out stat)) { - report.AppendLine(stat.ToConsoleString()); + OutputStatToConsole(con, stat); } else { @@ -176,18 +168,10 @@ namespace OpenSim.Framework.Monitoring { // Legacy if (SimExtraStats != null) - { - report.Append(SimExtraStats.Report()); - } + con.Output(SimExtraStats.Report()); else - { - foreach (string report2 in GetAllStatsReports()) - report.AppendLine(report2); - } + OutputAllStatsToConsole(con); } - - if (report.Length > 0) - m_log.Debug(string.Join(" ", cmd) + "\n" + report.ToString()); } public static List GetAllStatsReports() @@ -200,6 +184,12 @@ namespace OpenSim.Framework.Monitoring return reports; } + private static void OutputAllStatsToConsole(ICommandConsole con) + { + foreach (string report in GetAllStatsReports()) + con.Output(report); + } + private static List GetCategoryStatsReports( SortedDictionary> category) { @@ -211,6 +201,13 @@ namespace OpenSim.Framework.Monitoring return reports; } + private static void OutputCategoryStatsToConsole( + ICommandConsole con, SortedDictionary> category) + { + foreach (string report in GetCategoryStatsReports(category)) + con.Output(report); + } + private static List GetContainerStatsReports(SortedDictionary container) { List reports = new List(); @@ -221,6 +218,18 @@ namespace OpenSim.Framework.Monitoring return reports; } + private static void OutputContainerStatsToConsole( + ICommandConsole con, SortedDictionary container) + { + foreach (string report in GetContainerStatsReports(container)) + con.Output(report); + } + + private static void OutputStatToConsole(ICommandConsole con, Stat stat) + { + con.Output(stat.ToConsoleString()); + } + // Creates an OSDMap of the format: // { categoryName: { // containerName: { -- cgit v1.1 From f1f935ed9543141cc5861a7204b6df85a7358ab4 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Wed, 13 Aug 2014 19:53:42 +0100 Subject: Add 'server' stats information to pCampbot, as used elsewhere in OpenSimulator This adds the "show stats", "stats record", etc. commands and information on available Threadpool threads, etc. It also adds the Watchdog which logs warnings if time between executions is unexpectedly large. --- OpenSim/Framework/Monitoring/ServerStatsCollector.cs | 3 +++ 1 file changed, 3 insertions(+) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs index ff0fcd5..77315bb 100644 --- a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs @@ -82,6 +82,9 @@ namespace OpenSim.Framework.Monitoring // IRegionModuleBase.Initialize public void Initialise(IConfigSource source) { + if (source == null) + return; + IConfig cfg = source.Configs["Monitoring"]; if (cfg != null) -- cgit v1.1 From 41f2f3132bdcbfb8020c7fd6e5f3b7e48c75b1cf Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 5 Sep 2014 23:20:59 +0100 Subject: For monitoring purposes, start non-timeout tasks (which do not currently use a threadpool) via Watchdog.RunInThread() rather than Util.RunThreadNoTimeout() The functionality is the same but this allow us to monitor such tasks via "show threads" and abort them for test purposes, etc. Also extends thread names to provide more info (e.g. SendInitialDataToClient says what client the task is for). --- OpenSim/Framework/Monitoring/Watchdog.cs | 60 ++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 11 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index 427ed7b..e9e7bd2 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -187,15 +187,16 @@ namespace OpenSim.Framework.Monitoring /// Priority to run the thread at /// True to run this thread as a background thread, otherwise false /// Trigger an alarm function is we have timed out + /// If true then creation of thread is logged. /// The newly created Thread object public static Thread StartThread( - ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout) + ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, bool log = true) { - return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS); + return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS, log); } /// - /// Start a new thread that is tracked by the watchdog timer + /// Start a new thread that is tracked by the watchdog /// /// The method that will be executed in a new thread /// A name to give to the new thread @@ -208,10 +209,11 @@ namespace OpenSim.Framework.Monitoring /// Normally, this will just return some useful debugging information. /// /// Number of milliseconds to wait until we issue a warning about timeout. + /// If true then creation of thread is logged. /// The newly created Thread object public static Thread StartThread( ThreadStart start, string name, ThreadPriority priority, bool isBackground, - bool alarmIfTimeout, Func alarmMethod, int timeout) + bool alarmIfTimeout, Func alarmMethod, int timeout, bool log = true) { Thread thread = new Thread(start); thread.Name = name; @@ -222,8 +224,9 @@ namespace OpenSim.Framework.Monitoring = new ThreadWatchdogInfo(thread, timeout) { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod }; - m_log.DebugFormat( - "[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId); + if (log) + m_log.DebugFormat( + "[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId); lock (m_threads) m_threads.Add(twi.Thread.ManagedThreadId, twi); @@ -234,6 +237,39 @@ namespace OpenSim.Framework.Monitoring } /// + /// Run the callback in a new thread immediately. If the thread exits with an exception log it but do + /// not propogate it. + /// + /// Code for the thread to execute. + /// Name of the thread + /// Object to pass to the thread. + public static void RunInThread(WaitCallback callback, string name, object obj, bool log = false) + { + if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest) + { + Culture.SetCurrentCulture(); + callback(obj); + return; + } + + ThreadStart ts = new ThreadStart(delegate() + { + try + { + Culture.SetCurrentCulture(); + callback(obj); + Watchdog.RemoveThread(log:false); + } + catch (Exception e) + { + m_log.Error(string.Format("[WATCHDOG]: Exception in thread {0}.", name), e); + } + }); + + StartThread(ts, name, ThreadPriority.Normal, true, false, log:log); + } + + /// /// Marks the current thread as alive /// public static void UpdateThread() @@ -244,24 +280,26 @@ namespace OpenSim.Framework.Monitoring /// /// Stops watchdog tracking on the current thread /// + /// If true then normal events in thread removal are not logged. /// /// True if the thread was removed from the list of tracked /// threads, otherwise false /// - public static bool RemoveThread() + public static bool RemoveThread(bool log = true) { - return RemoveThread(Thread.CurrentThread.ManagedThreadId); + return RemoveThread(Thread.CurrentThread.ManagedThreadId, log); } - private static bool RemoveThread(int threadID) + private static bool RemoveThread(int threadID, bool log = true) { lock (m_threads) { ThreadWatchdogInfo twi; if (m_threads.TryGetValue(threadID, out twi)) { - m_log.DebugFormat( - "[WATCHDOG]: Removing thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId); + if (log) + m_log.DebugFormat( + "[WATCHDOG]: Removing thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId); twi.Cleanup(); m_threads.Remove(threadID); -- cgit v1.1 From e271607befc952c8b9a000a7405f871fa10e92a0 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 9 Sep 2014 18:52:07 +0100 Subject: For stat names containing periods, replace with '#' rather than throw exception In relation to http://opensimulator.org/mantis/view.php?id=7317 --- OpenSim/Framework/Monitoring/Stats/Stat.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs index e095801..df55314 100644 --- a/OpenSim/Framework/Monitoring/Stats/Stat.cs +++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs @@ -171,7 +171,8 @@ namespace OpenSim.Framework.Monitoring foreach (char c in DisallowedShortNameCharacters) { if (shortName.IndexOf(c) != -1) - throw new Exception(string.Format("Stat name {0} cannot contain character {1}", shortName, c)); + shortName = shortName.Replace('c', '#'); +// throw new Exception(string.Format("Stat name {0} cannot contain character {1}", shortName, c)); } ShortName = shortName; -- cgit v1.1 From b4ce71df1e02c86c6b8470dd7429b21af559d644 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 9 Sep 2014 18:55:38 +0100 Subject: Make proper fix for last commit wrt Mantis 7317 by replacing disallowed c char and not literal 'c' --- OpenSim/Framework/Monitoring/Stats/Stat.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs index df55314..a7cb2a6 100644 --- a/OpenSim/Framework/Monitoring/Stats/Stat.cs +++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs @@ -171,7 +171,7 @@ namespace OpenSim.Framework.Monitoring foreach (char c in DisallowedShortNameCharacters) { if (shortName.IndexOf(c) != -1) - shortName = shortName.Replace('c', '#'); + shortName = shortName.Replace(c, '#'); // throw new Exception(string.Format("Stat name {0} cannot contain character {1}", shortName, c)); } -- cgit v1.1 From af286d5fcb688e8b64202b6deca4f249e9a2b6b8 Mon Sep 17 00:00:00 2001 From: BlueWall Date: Wed, 17 Sep 2014 17:56:10 -0400 Subject: Small changes to threading to send thread names to unmanaged threads. Needs Mono 3.6+ to see thread names in utilities like top -H . Some formatting of the thread name to fin in the 16 byte limit on Linux. Please test on Windows to see if the work has any adverse effects. --- OpenSim/Framework/Monitoring/Watchdog.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index e9e7bd2..e9f22f1 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -87,7 +87,7 @@ namespace OpenSim.Framework.Monitoring /// public Stat Stat { get; set; } - public ThreadWatchdogInfo(Thread thread, int timeout) + public ThreadWatchdogInfo(Thread thread, int timeout, string name) { Thread = thread; Timeout = timeout; @@ -96,8 +96,8 @@ namespace OpenSim.Framework.Monitoring Stat = new Stat( - thread.Name, - string.Format("Last update of thread {0}", thread.Name), + name, + string.Format("Last update of thread {0}", name), "", "ms", "server", @@ -216,12 +216,11 @@ namespace OpenSim.Framework.Monitoring bool alarmIfTimeout, Func alarmMethod, int timeout, bool log = true) { Thread thread = new Thread(start); - thread.Name = name; thread.Priority = priority; thread.IsBackground = isBackground; ThreadWatchdogInfo twi - = new ThreadWatchdogInfo(thread, timeout) + = new ThreadWatchdogInfo(thread, timeout, name) { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod }; if (log) @@ -230,8 +229,10 @@ namespace OpenSim.Framework.Monitoring lock (m_threads) m_threads.Add(twi.Thread.ManagedThreadId, twi); - + thread.Start(); + thread.Name = name; + return thread; } -- cgit v1.1 From 376fab140227e92dbd841436509a97b87c9e7792 Mon Sep 17 00:00:00 2001 From: BlueWall Date: Wed, 17 Sep 2014 18:11:41 -0400 Subject: Revert "Small changes to threading to send thread names to unmanaged threads. Needs Mono 3.6+ to see thread names in utilities like top -H . Some formatting of the thread name to fin in the 16 byte limit on Linux. Please test on Windows to see if the work has any adverse effects." This reverts commit af286d5fcb688e8b64202b6deca4f249e9a2b6b8. Issue with Jenkins builds --- OpenSim/Framework/Monitoring/Watchdog.cs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index e9f22f1..e9e7bd2 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -87,7 +87,7 @@ namespace OpenSim.Framework.Monitoring /// public Stat Stat { get; set; } - public ThreadWatchdogInfo(Thread thread, int timeout, string name) + public ThreadWatchdogInfo(Thread thread, int timeout) { Thread = thread; Timeout = timeout; @@ -96,8 +96,8 @@ namespace OpenSim.Framework.Monitoring Stat = new Stat( - name, - string.Format("Last update of thread {0}", name), + thread.Name, + string.Format("Last update of thread {0}", thread.Name), "", "ms", "server", @@ -216,11 +216,12 @@ namespace OpenSim.Framework.Monitoring bool alarmIfTimeout, Func alarmMethod, int timeout, bool log = true) { Thread thread = new Thread(start); + thread.Name = name; thread.Priority = priority; thread.IsBackground = isBackground; ThreadWatchdogInfo twi - = new ThreadWatchdogInfo(thread, timeout, name) + = new ThreadWatchdogInfo(thread, timeout) { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod }; if (log) @@ -229,10 +230,8 @@ namespace OpenSim.Framework.Monitoring lock (m_threads) m_threads.Add(twi.Thread.ManagedThreadId, twi); - - thread.Start(); - thread.Name = name; + thread.Start(); return thread; } -- cgit v1.1 From 96a24491222e29ba8d615d9ffaf63e72742a9cc7 Mon Sep 17 00:00:00 2001 From: BlueWall Date: Wed, 17 Sep 2014 20:03:54 -0400 Subject: Undo "Revert "Small changes to threading to send thread names to unmanaged threads. Needs Mono 3.6+ to see thread names in utilities like top -H . Some formatting of the thread name to fin in the 16 byte limit on Linux. Please test on Windows to see if the work has any adverse effects."" Fix for break in next commit This reverts commit 376fab140227e92dbd841436509a97b87c9e7792. --- OpenSim/Framework/Monitoring/Watchdog.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index e9e7bd2..e9f22f1 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -87,7 +87,7 @@ namespace OpenSim.Framework.Monitoring /// public Stat Stat { get; set; } - public ThreadWatchdogInfo(Thread thread, int timeout) + public ThreadWatchdogInfo(Thread thread, int timeout, string name) { Thread = thread; Timeout = timeout; @@ -96,8 +96,8 @@ namespace OpenSim.Framework.Monitoring Stat = new Stat( - thread.Name, - string.Format("Last update of thread {0}", thread.Name), + name, + string.Format("Last update of thread {0}", name), "", "ms", "server", @@ -216,12 +216,11 @@ namespace OpenSim.Framework.Monitoring bool alarmIfTimeout, Func alarmMethod, int timeout, bool log = true) { Thread thread = new Thread(start); - thread.Name = name; thread.Priority = priority; thread.IsBackground = isBackground; ThreadWatchdogInfo twi - = new ThreadWatchdogInfo(thread, timeout) + = new ThreadWatchdogInfo(thread, timeout, name) { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod }; if (log) @@ -230,8 +229,10 @@ namespace OpenSim.Framework.Monitoring lock (m_threads) m_threads.Add(twi.Thread.ManagedThreadId, twi); - + thread.Start(); + thread.Name = name; + return thread; } -- cgit v1.1 From 23a0946e3658a4671b330e086c487cc9d1340e2e Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 26 Sep 2014 01:10:49 +0100 Subject: Fix long-lived thread name logging. --- OpenSim/Framework/Monitoring/Watchdog.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index e9f22f1..0fcb195 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -225,7 +225,7 @@ namespace OpenSim.Framework.Monitoring if (log) m_log.DebugFormat( - "[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId); + "[WATCHDOG]: Started tracking thread {0}, ID {1}", name, twi.Thread.ManagedThreadId); lock (m_threads) m_threads.Add(twi.Thread.ManagedThreadId, twi); -- cgit v1.1 From 859b1c5ba41198c57e1e84cf6bbf3e3634cc3d58 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 9 Sep 2014 18:14:56 +0100 Subject: Add experimental job engine to see if queueing some existing async work during root agent entry to a region improves perf rather than always attempting to execute everything concurrently Job engine is controlled via "debug jobengine start|stop|status". Can only currently be enabled and disabled dynamically at runtime. Currently only applies to code sending initial region data (objects, other avatar data) to a client. --- OpenSim/Framework/Monitoring/JobEngine.cs | 292 ++++++++++++++++++++++++++++++ OpenSim/Framework/Monitoring/Watchdog.cs | 18 ++ 2 files changed, 310 insertions(+) create mode 100644 OpenSim/Framework/Monitoring/JobEngine.cs (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs new file mode 100644 index 0000000..b401a5c --- /dev/null +++ b/OpenSim/Framework/Monitoring/JobEngine.cs @@ -0,0 +1,292 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Concurrent; +using System.Reflection; +using System.Threading; +using log4net; +using OpenSim.Framework; + +namespace OpenSim.Framework.Monitoring +{ + public class Job + { + public string Name; + public WaitCallback Callback; + public object O; + + public Job(string name, WaitCallback callback, object o) + { + Name = name; + Callback = callback; + O = o; + } + } + + public class JobEngine + { + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + public bool IsRunning { get; private set; } + + /// + /// The timeout in milliseconds to wait for at least one event to be written when the recorder is stopping. + /// + public int RequestProcessTimeoutOnStop { get; set; } + + /// + /// Controls whether we need to warn in the log about exceeding the max queue size. + /// + /// + /// This is flipped to false once queue max has been exceeded and back to true when it falls below max, in + /// order to avoid spamming the log with lots of warnings. + /// + private bool m_warnOverMaxQueue = true; + + private BlockingCollection m_requestQueue; + + private CancellationTokenSource m_cancelSource = new CancellationTokenSource(); + + private Stat m_requestsWaitingStat; + + private Job m_currentJob; + + /// + /// Used to signal that we are ready to complete stop. + /// + private ManualResetEvent m_finishedProcessingAfterStop = new ManualResetEvent(false); + + public JobEngine() + { + RequestProcessTimeoutOnStop = 5000; + + MainConsole.Instance.Commands.AddCommand( + "Debug", + false, + "debug jobengine", + "debug jobengine ", + "Start, stop or get status of the job engine.", + "If stopped then all jobs are processed immediately.", + HandleControlCommand); + } + + public void Start() + { + lock (this) + { + if (IsRunning) + return; + + IsRunning = true; + + m_finishedProcessingAfterStop.Reset(); + + m_requestQueue = new BlockingCollection(new ConcurrentQueue(), 5000); + + m_requestsWaitingStat = + new Stat( + "JobsWaiting", + "Number of jobs waiting for processing.", + "", + "", + "server", + "jobengine", + StatType.Pull, + MeasuresOfInterest.None, + stat => stat.Value = m_requestQueue.Count, + StatVerbosity.Debug); + + StatsManager.RegisterStat(m_requestsWaitingStat); + + Watchdog.StartThread( + ProcessRequests, + "JobEngineThread", + ThreadPriority.Normal, + false, + true, + null, + int.MaxValue); + } + } + + public void Stop() + { + lock (this) + { + try + { + if (!IsRunning) + return; + + IsRunning = false; + + int requestsLeft = m_requestQueue.Count; + + if (requestsLeft <= 0) + { + m_cancelSource.Cancel(); + } + else + { + m_log.InfoFormat("[JOB ENGINE]: Waiting to write {0} events after stop.", requestsLeft); + + while (requestsLeft > 0) + { + if (!m_finishedProcessingAfterStop.WaitOne(RequestProcessTimeoutOnStop)) + { + // After timeout no events have been written + if (requestsLeft == m_requestQueue.Count) + { + m_log.WarnFormat( + "[JOB ENGINE]: No requests processed after {0} ms wait. Discarding remaining {1} requests", + RequestProcessTimeoutOnStop, requestsLeft); + + break; + } + } + + requestsLeft = m_requestQueue.Count; + } + } + } + finally + { + m_cancelSource.Dispose(); + StatsManager.DeregisterStat(m_requestsWaitingStat); + m_requestsWaitingStat = null; + m_requestQueue = null; + } + } + } + + public bool QueueRequest(string name, WaitCallback req, object o) + { + m_log.DebugFormat("[JOB ENGINE]: Queued job {0}", name); + + if (m_requestQueue.Count < m_requestQueue.BoundedCapacity) + { + // m_log.DebugFormat( + // "[OUTGOING QUEUE REFILL ENGINE]: Adding request for categories {0} for {1} in {2}", + // categories, client.AgentID, m_udpServer.Scene.Name); + + m_requestQueue.Add(new Job(name, req, o)); + + if (!m_warnOverMaxQueue) + m_warnOverMaxQueue = true; + + return true; + } + else + { + if (m_warnOverMaxQueue) + { +// m_log.WarnFormat( +// "[JOB ENGINE]: Request queue at maximum capacity, not recording request from {0} in {1}", +// client.AgentID, m_udpServer.Scene.Name); + + m_log.WarnFormat("[JOB ENGINE]: Request queue at maximum capacity, not recording job"); + + m_warnOverMaxQueue = false; + } + + return false; + } + } + + private void ProcessRequests() + { + try + { + while (IsRunning || m_requestQueue.Count > 0) + { + m_currentJob = m_requestQueue.Take(m_cancelSource.Token); + + // QueueEmpty callback = req.Client.OnQueueEmpty; + // + // if (callback != null) + // { + // try + // { + // callback(req.Categories); + // } + // catch (Exception e) + // { + // m_log.Error("[OUTGOING QUEUE REFILL ENGINE]: ProcessRequests(" + req.Categories + ") threw an exception: " + e.Message, e); + // } + // } + + m_log.DebugFormat("[JOB ENGINE]: Processing job {0}", m_currentJob.Name); + m_currentJob.Callback.Invoke(m_currentJob.O); + m_log.DebugFormat("[JOB ENGINE]: Processed job {0}", m_currentJob.Name); + m_currentJob = null; + } + } + catch (OperationCanceledException) + { + } + + m_finishedProcessingAfterStop.Set(); + } + + private void HandleControlCommand(string module, string[] args) + { +// if (SceneManager.Instance.CurrentScene != null && SceneManager.Instance.CurrentScene != m_udpServer.Scene) +// return; + + if (args.Length != 3) + { + MainConsole.Instance.Output("Usage: debug jobengine "); + return; + } + + string subCommand = args[2]; + + if (subCommand == "stop") + { + Stop(); + MainConsole.Instance.OutputFormat("Stopped job engine."); + } + else if (subCommand == "start") + { + Start(); + MainConsole.Instance.OutputFormat("Started job engine."); + } + else if (subCommand == "status") + { + MainConsole.Instance.OutputFormat("Job engine running: {0}", IsRunning); + MainConsole.Instance.OutputFormat("Current job {0}", m_currentJob != null ? m_currentJob.Name : "none"); + MainConsole.Instance.OutputFormat( + "Jobs waiting: {0}", IsRunning ? m_requestQueue.Count.ToString() : "n/a"); + } + else + { + MainConsole.Instance.OutputFormat("Unrecognized job engine subcommand {0}", subCommand); + } + } + } +} \ No newline at end of file diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index 0fcb195..b00aa5c 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -133,6 +133,8 @@ namespace OpenSim.Framework.Monitoring /// /summary> public static event Action OnWatchdogTimeout; + private static JobEngine m_jobEngine; + /// /// Is this watchdog active? /// @@ -173,6 +175,7 @@ namespace OpenSim.Framework.Monitoring static Watchdog() { + m_jobEngine = new JobEngine(); m_threads = new Dictionary(); m_watchdogTimer = new System.Timers.Timer(WATCHDOG_INTERVAL_MS); m_watchdogTimer.AutoReset = false; @@ -450,5 +453,20 @@ namespace OpenSim.Framework.Monitoring m_watchdogTimer.Start(); } + + public static void RunWhenPossible(string jobType, WaitCallback callback, string name, object obj, bool log = false) + { + if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest) + { + Culture.SetCurrentCulture(); + callback(obj); + return; + } + + if (m_jobEngine.IsRunning) + m_jobEngine.QueueRequest(name, callback, obj); + else + RunInThread(callback, name, obj, log); + } } } \ No newline at end of file -- cgit v1.1 From 9fb3065d336d1907b5946c30010c9066e99e7597 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 9 Sep 2014 18:26:41 +0100 Subject: Temporarily add root agent rez attachments work to job engine if it is running rather than as a fire and forget. Experiment to see if serializing attachment rez and send initial data jobs improves other parts of sim performance. --- OpenSim/Framework/Monitoring/Watchdog.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index b00aa5c..ebbf095 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -133,7 +133,7 @@ namespace OpenSim.Framework.Monitoring /// /summary> public static event Action OnWatchdogTimeout; - private static JobEngine m_jobEngine; + public static JobEngine JobEngine { get; private set; } /// /// Is this watchdog active? @@ -175,7 +175,7 @@ namespace OpenSim.Framework.Monitoring static Watchdog() { - m_jobEngine = new JobEngine(); + JobEngine = new JobEngine(); m_threads = new Dictionary(); m_watchdogTimer = new System.Timers.Timer(WATCHDOG_INTERVAL_MS); m_watchdogTimer.AutoReset = false; @@ -463,8 +463,8 @@ namespace OpenSim.Framework.Monitoring return; } - if (m_jobEngine.IsRunning) - m_jobEngine.QueueRequest(name, callback, obj); + if (JobEngine.IsRunning) + JobEngine.QueueRequest(name, callback, obj); else RunInThread(callback, name, obj, log); } -- cgit v1.1 From d3ed5de77d60ed87aff5b9caddd6dbd3c04c3ba9 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 9 Sep 2014 18:42:02 +0100 Subject: Add loglevel to jobengine that can be controlled via "debug jobengine loglevel ". Defaults to 0 Level 1 currently does verbose logging about every queued and processed job. --- OpenSim/Framework/Monitoring/JobEngine.cs | 32 +++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs index b401a5c..ea2203f 100644 --- a/OpenSim/Framework/Monitoring/JobEngine.cs +++ b/OpenSim/Framework/Monitoring/JobEngine.cs @@ -52,6 +52,8 @@ namespace OpenSim.Framework.Monitoring { private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + public int LogLevel { get; set; } + public bool IsRunning { get; private set; } /// @@ -186,7 +188,8 @@ namespace OpenSim.Framework.Monitoring public bool QueueRequest(string name, WaitCallback req, object o) { - m_log.DebugFormat("[JOB ENGINE]: Queued job {0}", name); + if (LogLevel >= 1) + m_log.DebugFormat("[JOB ENGINE]: Queued job {0}", name); if (m_requestQueue.Count < m_requestQueue.BoundedCapacity) { @@ -240,9 +243,14 @@ namespace OpenSim.Framework.Monitoring // } // } - m_log.DebugFormat("[JOB ENGINE]: Processing job {0}", m_currentJob.Name); + if (LogLevel >= 1) + m_log.DebugFormat("[JOB ENGINE]: Processing job {0}", m_currentJob.Name); + m_currentJob.Callback.Invoke(m_currentJob.O); - m_log.DebugFormat("[JOB ENGINE]: Processed job {0}", m_currentJob.Name); + + if (LogLevel >= 1) + m_log.DebugFormat("[JOB ENGINE]: Processed job {0}", m_currentJob.Name); + m_currentJob = null; } } @@ -258,9 +266,9 @@ namespace OpenSim.Framework.Monitoring // if (SceneManager.Instance.CurrentScene != null && SceneManager.Instance.CurrentScene != m_udpServer.Scene) // return; - if (args.Length != 3) + if (args.Length < 3) { - MainConsole.Instance.Output("Usage: debug jobengine "); + MainConsole.Instance.Output("Usage: debug jobengine "); return; } @@ -282,6 +290,18 @@ namespace OpenSim.Framework.Monitoring MainConsole.Instance.OutputFormat("Current job {0}", m_currentJob != null ? m_currentJob.Name : "none"); MainConsole.Instance.OutputFormat( "Jobs waiting: {0}", IsRunning ? m_requestQueue.Count.ToString() : "n/a"); + MainConsole.Instance.OutputFormat("Log Level: {0}", LogLevel); + } + + else if (subCommand == "loglevel") + { +// int logLevel; + int logLevel = int.Parse(args[3]); +// if (ConsoleUtil.TryParseConsoleInt(MainConsole.Instance, args[4], out logLevel)) +// { + LogLevel = logLevel; + MainConsole.Instance.OutputFormat("Set log level to {0}", LogLevel); +// } } else { @@ -289,4 +309,4 @@ namespace OpenSim.Framework.Monitoring } } } -} \ No newline at end of file +} -- cgit v1.1 From 6c043dd6ffe673ec3c9b1deb5519d12b19b32f72 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 14 Oct 2014 20:42:41 +0100 Subject: If an exception makes it to the top of a JobEngine request, catch and log instead of letting it terminate the simulator... --- OpenSim/Framework/Monitoring/JobEngine.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs index ea2203f..7dcce6a 100644 --- a/OpenSim/Framework/Monitoring/JobEngine.cs +++ b/OpenSim/Framework/Monitoring/JobEngine.cs @@ -246,7 +246,16 @@ namespace OpenSim.Framework.Monitoring if (LogLevel >= 1) m_log.DebugFormat("[JOB ENGINE]: Processing job {0}", m_currentJob.Name); - m_currentJob.Callback.Invoke(m_currentJob.O); + try + { + m_currentJob.Callback.Invoke(m_currentJob.O); + } + catch (Exception e) + { + m_log.Error( + string.Format( + "[JOB ENGINE]: Job {0} failed, continuing. Exception ", m_currentJob.Name), e); + } if (LogLevel >= 1) m_log.DebugFormat("[JOB ENGINE]: Processed job {0}", m_currentJob.Name); -- cgit v1.1 From 28d1dbfee4256a97744ed1cebe21971adf5fdaef Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 21 Nov 2014 01:08:58 +0000 Subject: Move conditionals which control whether a task is placed in the JobEngine inside Watchdog.RunJob() (renamed from RunWhenPossible) and generalize them. --- OpenSim/Framework/Monitoring/Watchdog.cs | 39 ++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index ebbf095..0feec7c 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -454,7 +454,38 @@ namespace OpenSim.Framework.Monitoring m_watchdogTimer.Start(); } - public static void RunWhenPossible(string jobType, WaitCallback callback, string name, object obj, bool log = false) + /// + /// Run a job. + /// + /// + /// This differs from direct scheduling (e.g. Util.FireAndForget) in that a job can be run in the job + /// engine if it is running, where all jobs are currently performed in sequence on a single thread. This is + /// to prevent observed overload and server freeze problems when there are hundreds of connections which all attempt to + /// perform work at once (e.g. in conference situations). With lower numbers of connections, the small + /// delay in performing jobs in sequence rather than concurrently has not been notiecable in testing, though a future more + /// sophisticated implementation could perform jobs concurrently when the server is under low load. + /// + /// However, be advised that some callers of this function rely on all jobs being performed in sequence if any + /// jobs are performed in sequence (i.e. if jobengine is active or not). Therefore, expanding the jobengine + /// beyond a single thread will require considerable thought. + /// + /// Also, any jobs submitted must be guaranteed to complete within a reasonable timeframe (e.g. they cannot + /// incorporate a network delay with a long timeout). At the moment, work that could suffer such issues + /// should still be run directly with RunInThread(), Util.FireAndForget(), etc. This is another area where + /// the job engine could be improved and so CPU utilization improved by better management of concurrency within + /// OpenSimulator. + /// + /// General classification for the job (e.g. "RezAttachments"). + /// Callback for job. + /// Specific name of job (e.g. "RezAttachments for Joe Bloggs" + /// Object to pass to callback when run + /// If set to true then the job may be run in ths calling thread. + /// If the true then the job must never timeout. + /// If set to true then extra logging is performed. + public static void RunJob( + string jobType, WaitCallback callback, string name, object obj, + bool canRunInThisThread = false, bool mustNotTimeout = false, + bool log = false) { if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest) { @@ -465,8 +496,12 @@ namespace OpenSim.Framework.Monitoring if (JobEngine.IsRunning) JobEngine.QueueRequest(name, callback, obj); - else + else if (canRunInThisThread) + callback(obj); + else if (mustNotTimeout) RunInThread(callback, name, obj, log); + else + Util.FireAndForget(callback, obj, name); } } } \ No newline at end of file -- cgit v1.1 From 4b45a1b6adda6483ce25dfd8da8b4b94557779dc Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 21 Nov 2014 21:25:55 +0000 Subject: Change jobengine logging command to "debug jobengine log " rather than loglevel, in common with similar commands. --- OpenSim/Framework/Monitoring/JobEngine.cs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs index 7dcce6a..4a46328 100644 --- a/OpenSim/Framework/Monitoring/JobEngine.cs +++ b/OpenSim/Framework/Monitoring/JobEngine.cs @@ -91,9 +91,9 @@ namespace OpenSim.Framework.Monitoring "Debug", false, "debug jobengine", - "debug jobengine ", - "Start, stop or get status of the job engine.", - "If stopped then all jobs are processed immediately.", + "debug jobengine ", + "Start, stop, get status or set logging level of the job engine.", + "If stopped then all outstanding jobs are processed immediately.", HandleControlCommand); } @@ -277,7 +277,7 @@ namespace OpenSim.Framework.Monitoring if (args.Length < 3) { - MainConsole.Instance.Output("Usage: debug jobengine "); + MainConsole.Instance.Output("Usage: debug jobengine "); return; } @@ -301,15 +301,14 @@ namespace OpenSim.Framework.Monitoring "Jobs waiting: {0}", IsRunning ? m_requestQueue.Count.ToString() : "n/a"); MainConsole.Instance.OutputFormat("Log Level: {0}", LogLevel); } - - else if (subCommand == "loglevel") + else if (subCommand == "log") { // int logLevel; int logLevel = int.Parse(args[3]); // if (ConsoleUtil.TryParseConsoleInt(MainConsole.Instance, args[4], out logLevel)) // { LogLevel = logLevel; - MainConsole.Instance.OutputFormat("Set log level to {0}", LogLevel); + MainConsole.Instance.OutputFormat("Set debug log level to {0}", LogLevel); // } } else -- cgit v1.1 From 86367d7219b3bd52f63045b2b17bcbde328844ed Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 25 Nov 2014 23:56:32 +0000 Subject: refactor: Move methods to start a monitored thread, start work in its own thread and run work in the jobengine from Watchdog to a WorkManager class. This is to achieve a clean separation of concerns - the watchdog is an inappropriate place for work management. Also adds a WorkManager.RunInThreadPool() class which feeds through to Util.FireAndForget. Also switches around the name and obj arguments to the new RunInThread() and RunJob() methods so that the callback obj comes after the callback as seen in the SDK and elsewhere --- OpenSim/Framework/Monitoring/JobEngine.cs | 2 +- OpenSim/Framework/Monitoring/Watchdog.cs | 149 ++----------------- OpenSim/Framework/Monitoring/WorkManager.cs | 212 ++++++++++++++++++++++++++++ 3 files changed, 224 insertions(+), 139 deletions(-) create mode 100644 OpenSim/Framework/Monitoring/WorkManager.cs (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs index 4a46328..5925867 100644 --- a/OpenSim/Framework/Monitoring/JobEngine.cs +++ b/OpenSim/Framework/Monitoring/JobEngine.cs @@ -125,7 +125,7 @@ namespace OpenSim.Framework.Monitoring StatsManager.RegisterStat(m_requestsWaitingStat); - Watchdog.StartThread( + WorkManager.StartThread( ProcessRequests, "JobEngineThread", ThreadPriority.Normal, diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index 0feec7c..a644fa5 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -38,6 +38,8 @@ namespace OpenSim.Framework.Monitoring /// public static class Watchdog { + private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); + /// Timer interval in milliseconds for the watchdog timer public const double WATCHDOG_INTERVAL_MS = 2500.0d; @@ -133,8 +135,6 @@ namespace OpenSim.Framework.Monitoring /// /summary> public static event Action OnWatchdogTimeout; - public static JobEngine JobEngine { get; private set; } - /// /// Is this watchdog active? /// @@ -143,7 +143,7 @@ namespace OpenSim.Framework.Monitoring get { return m_enabled; } set { -// m_log.DebugFormat("[MEMORY WATCHDOG]: Setting MemoryWatchdog.Enabled to {0}", value); + // m_log.DebugFormat("[MEMORY WATCHDOG]: Setting MemoryWatchdog.Enabled to {0}", value); if (value == m_enabled) return; @@ -159,9 +159,8 @@ namespace OpenSim.Framework.Monitoring m_watchdogTimer.Enabled = m_enabled; } } - private static bool m_enabled; - private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); + private static bool m_enabled; private static Dictionary m_threads; private static System.Timers.Timer m_watchdogTimer; @@ -175,7 +174,6 @@ namespace OpenSim.Framework.Monitoring static Watchdog() { - JobEngine = new JobEngine(); m_threads = new Dictionary(); m_watchdogTimer = new System.Timers.Timer(WATCHDOG_INTERVAL_MS); m_watchdogTimer.AutoReset = false; @@ -183,94 +181,19 @@ namespace OpenSim.Framework.Monitoring } /// - /// Start a new thread that is tracked by the watchdog timer. - /// - /// The method that will be executed in a new thread - /// A name to give to the new thread - /// Priority to run the thread at - /// True to run this thread as a background thread, otherwise false - /// Trigger an alarm function is we have timed out - /// If true then creation of thread is logged. - /// The newly created Thread object - public static Thread StartThread( - ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, bool log = true) - { - return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS, log); - } - - /// - /// Start a new thread that is tracked by the watchdog + /// Add a thread to the watchdog tracker. /// - /// The method that will be executed in a new thread - /// A name to give to the new thread - /// Priority to run the thread at - /// True to run this thread as a background - /// thread, otherwise false - /// Trigger an alarm function is we have timed out - /// - /// Alarm method to call if alarmIfTimeout is true and there is a timeout. - /// Normally, this will just return some useful debugging information. - /// - /// Number of milliseconds to wait until we issue a warning about timeout. + /// Information about the thread. + /// Name of the thread. /// If true then creation of thread is logged. - /// The newly created Thread object - public static Thread StartThread( - ThreadStart start, string name, ThreadPriority priority, bool isBackground, - bool alarmIfTimeout, Func alarmMethod, int timeout, bool log = true) + public static void AddThread(ThreadWatchdogInfo info, string name, bool log = true) { - Thread thread = new Thread(start); - thread.Priority = priority; - thread.IsBackground = isBackground; - - ThreadWatchdogInfo twi - = new ThreadWatchdogInfo(thread, timeout, name) - { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod }; - if (log) m_log.DebugFormat( - "[WATCHDOG]: Started tracking thread {0}, ID {1}", name, twi.Thread.ManagedThreadId); + "[WATCHDOG]: Started tracking thread {0}, ID {1}", name, info.Thread.ManagedThreadId); lock (m_threads) - m_threads.Add(twi.Thread.ManagedThreadId, twi); - - thread.Start(); - thread.Name = name; - - - return thread; - } - - /// - /// Run the callback in a new thread immediately. If the thread exits with an exception log it but do - /// not propogate it. - /// - /// Code for the thread to execute. - /// Name of the thread - /// Object to pass to the thread. - public static void RunInThread(WaitCallback callback, string name, object obj, bool log = false) - { - if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest) - { - Culture.SetCurrentCulture(); - callback(obj); - return; - } - - ThreadStart ts = new ThreadStart(delegate() - { - try - { - Culture.SetCurrentCulture(); - callback(obj); - Watchdog.RemoveThread(log:false); - } - catch (Exception e) - { - m_log.Error(string.Format("[WATCHDOG]: Exception in thread {0}.", name), e); - } - }); - - StartThread(ts, name, ThreadPriority.Normal, true, false, log:log); + m_threads.Add(info.Thread.ManagedThreadId, info); } /// @@ -361,7 +284,7 @@ namespace OpenSim.Framework.Monitoring } catch { } } - + /// /// Get currently watched threads for diagnostic purposes /// @@ -453,55 +376,5 @@ namespace OpenSim.Framework.Monitoring m_watchdogTimer.Start(); } - - /// - /// Run a job. - /// - /// - /// This differs from direct scheduling (e.g. Util.FireAndForget) in that a job can be run in the job - /// engine if it is running, where all jobs are currently performed in sequence on a single thread. This is - /// to prevent observed overload and server freeze problems when there are hundreds of connections which all attempt to - /// perform work at once (e.g. in conference situations). With lower numbers of connections, the small - /// delay in performing jobs in sequence rather than concurrently has not been notiecable in testing, though a future more - /// sophisticated implementation could perform jobs concurrently when the server is under low load. - /// - /// However, be advised that some callers of this function rely on all jobs being performed in sequence if any - /// jobs are performed in sequence (i.e. if jobengine is active or not). Therefore, expanding the jobengine - /// beyond a single thread will require considerable thought. - /// - /// Also, any jobs submitted must be guaranteed to complete within a reasonable timeframe (e.g. they cannot - /// incorporate a network delay with a long timeout). At the moment, work that could suffer such issues - /// should still be run directly with RunInThread(), Util.FireAndForget(), etc. This is another area where - /// the job engine could be improved and so CPU utilization improved by better management of concurrency within - /// OpenSimulator. - /// - /// General classification for the job (e.g. "RezAttachments"). - /// Callback for job. - /// Specific name of job (e.g. "RezAttachments for Joe Bloggs" - /// Object to pass to callback when run - /// If set to true then the job may be run in ths calling thread. - /// If the true then the job must never timeout. - /// If set to true then extra logging is performed. - public static void RunJob( - string jobType, WaitCallback callback, string name, object obj, - bool canRunInThisThread = false, bool mustNotTimeout = false, - bool log = false) - { - if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest) - { - Culture.SetCurrentCulture(); - callback(obj); - return; - } - - if (JobEngine.IsRunning) - JobEngine.QueueRequest(name, callback, obj); - else if (canRunInThisThread) - callback(obj); - else if (mustNotTimeout) - RunInThread(callback, name, obj, log); - else - Util.FireAndForget(callback, obj, name); - } } } \ No newline at end of file diff --git a/OpenSim/Framework/Monitoring/WorkManager.cs b/OpenSim/Framework/Monitoring/WorkManager.cs new file mode 100644 index 0000000..9d0eefc --- /dev/null +++ b/OpenSim/Framework/Monitoring/WorkManager.cs @@ -0,0 +1,212 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Reflection; +using System.Threading; +using log4net; + +namespace OpenSim.Framework.Monitoring +{ + /// + /// Manages various work items in the simulator. + /// + /// + /// Currently, here work can be started + /// * As a long-running and monitored thread. + /// * In a thread that will never timeout but where the job is expected to eventually complete. + /// * In a threadpool thread that will timeout if it takes a very long time to complete (> 10 mins). + /// * As a job which will be run in a single-threaded job engine. Such jobs must not incorporate delays (sleeps, + /// network waits, etc.). + /// + /// This is an evolving approach to better manage the work that OpenSimulator is asked to do from a very diverse + /// range of sources (client actions, incoming network, outgoing network calls, etc.). + /// + /// Util.FireAndForget is still available to insert jobs in the threadpool, though this is equivalent to + /// WorkManager.RunInThreadPool(). + /// + public static class WorkManager + { + private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); + + public static JobEngine JobEngine { get; private set; } + + static WorkManager() + { + JobEngine = new JobEngine(); + } + + /// + /// Start a new long-lived thread. + /// + /// The method that will be executed in a new thread + /// A name to give to the new thread + /// Priority to run the thread at + /// True to run this thread as a background thread, otherwise false + /// Trigger an alarm function is we have timed out + /// If true then creation of thread is logged. + /// The newly created Thread object + public static Thread StartThread( + ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, bool log = true) + { + return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, Watchdog.DEFAULT_WATCHDOG_TIMEOUT_MS, log); + } + + /// + /// Start a new thread that is tracked by the watchdog + /// + /// The method that will be executed in a new thread + /// A name to give to the new thread + /// Priority to run the thread at + /// True to run this thread as a background + /// thread, otherwise false + /// Trigger an alarm function is we have timed out + /// + /// Alarm method to call if alarmIfTimeout is true and there is a timeout. + /// Normally, this will just return some useful debugging information. + /// + /// Number of milliseconds to wait until we issue a warning about timeout. + /// If true then creation of thread is logged. + /// The newly created Thread object + public static Thread StartThread( + ThreadStart start, string name, ThreadPriority priority, bool isBackground, + bool alarmIfTimeout, Func alarmMethod, int timeout, bool log = true) + { + Thread thread = new Thread(start); + thread.Priority = priority; + thread.IsBackground = isBackground; + + Watchdog.ThreadWatchdogInfo twi + = new Watchdog.ThreadWatchdogInfo(thread, timeout, name) + { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod }; + + Watchdog.AddThread(twi, name, log:log); + + thread.Start(); + thread.Name = name; + + return thread; + } + + /// + /// Run the callback in a new thread immediately. If the thread exits with an exception log it but do + /// not propogate it. + /// + /// Code for the thread to execute. + /// Object to pass to the thread. + /// Name of the thread + public static void RunInThread(WaitCallback callback, object obj, string name, bool log = false) + { + if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest) + { + Culture.SetCurrentCulture(); + callback(obj); + return; + } + + ThreadStart ts = new ThreadStart(delegate() + { + try + { + Culture.SetCurrentCulture(); + callback(obj); + Watchdog.RemoveThread(log:false); + } + catch (Exception e) + { + m_log.Error(string.Format("[WATCHDOG]: Exception in thread {0}.", name), e); + } + }); + + StartThread(ts, name, ThreadPriority.Normal, true, false, log:log); + } + + /// + /// Run the callback via a threadpool thread. + /// + /// + /// Such jobs may run after some delay but must always complete. + /// + /// + /// + /// The name of the job. This is used in monitoring and debugging. + public static void RunInThreadPool(System.Threading.WaitCallback callback, object obj, string name) + { + Util.FireAndForget(callback, obj, name); + } + + /// + /// Run a job. + /// + /// + /// This differs from direct scheduling (e.g. Util.FireAndForget) in that a job can be run in the job + /// engine if it is running, where all jobs are currently performed in sequence on a single thread. This is + /// to prevent observed overload and server freeze problems when there are hundreds of connections which all attempt to + /// perform work at once (e.g. in conference situations). With lower numbers of connections, the small + /// delay in performing jobs in sequence rather than concurrently has not been notiecable in testing, though a future more + /// sophisticated implementation could perform jobs concurrently when the server is under low load. + /// + /// However, be advised that some callers of this function rely on all jobs being performed in sequence if any + /// jobs are performed in sequence (i.e. if jobengine is active or not). Therefore, expanding the jobengine + /// beyond a single thread will require considerable thought. + /// + /// Also, any jobs submitted must be guaranteed to complete within a reasonable timeframe (e.g. they cannot + /// incorporate a network delay with a long timeout). At the moment, work that could suffer such issues + /// should still be run directly with RunInThread(), Util.FireAndForget(), etc. This is another area where + /// the job engine could be improved and so CPU utilization improved by better management of concurrency within + /// OpenSimulator. + /// + /// General classification for the job (e.g. "RezAttachments"). + /// Callback for job. + /// Object to pass to callback when run + /// Specific name of job (e.g. "RezAttachments for Joe Bloggs" + /// If set to true then the job may be run in ths calling thread. + /// If the true then the job must never timeout. + /// If set to true then extra logging is performed. + public static void RunJob( + string jobType, WaitCallback callback, object obj, string name, + bool canRunInThisThread = false, bool mustNotTimeout = false, + bool log = false) + { + if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest) + { + Culture.SetCurrentCulture(); + callback(obj); + return; + } + + if (JobEngine.IsRunning) + JobEngine.QueueRequest(name, callback, obj); + else if (canRunInThisThread) + callback(obj); + else if (mustNotTimeout) + RunInThread(callback, obj, name, log); + else + Util.FireAndForget(callback, obj, name); + } + } +} \ No newline at end of file -- cgit v1.1 From 7c2aeb9e8e72fe0a1168558b7810ff496f5a96ec Mon Sep 17 00:00:00 2001 From: Robert Adams Date: Thu, 1 Jan 2015 09:39:07 -0800 Subject: Fix cut-and-paste error that made StatsManager web fetch queries fail for container specification. --- OpenSim/Framework/Monitoring/StatsManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index 0bac247..3136ee8 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -307,7 +307,7 @@ namespace OpenSim.Framework.Monitoring if (request.ContainsKey("cat")) pCategoryName = request["cat"].ToString(); if (request.ContainsKey("cont")) pContainerName = request["cat"].ToString(); - if (request.ContainsKey("stat")) pStatName = request["cat"].ToString(); + if (request.ContainsKey("stat")) pStatName = request["stat"].ToString(); string strOut = StatsManager.GetStatsAsOSDMap(pCategoryName, pContainerName, pStatName).ToString(); -- cgit v1.1 From 8e1e8a0920a9e94305619e9afb8e053b4daefb89 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Mon, 12 Jan 2015 20:56:37 +0000 Subject: Make the performance controlling job processing threads introduced in conference code use a generic JobEngine class rather than 4 slightly different copy/pasted versions. --- OpenSim/Framework/Monitoring/JobEngine.cs | 320 ---------------------------- OpenSim/Framework/Monitoring/WorkManager.cs | 76 ++++++- 2 files changed, 74 insertions(+), 322 deletions(-) delete mode 100644 OpenSim/Framework/Monitoring/JobEngine.cs (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs deleted file mode 100644 index 5925867..0000000 --- a/OpenSim/Framework/Monitoring/JobEngine.cs +++ /dev/null @@ -1,320 +0,0 @@ -/* - * Copyright (c) Contributors, http://opensimulator.org/ - * See CONTRIBUTORS.TXT for a full list of copyright holders. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the OpenSimulator Project nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -using System; -using System.Collections.Concurrent; -using System.Reflection; -using System.Threading; -using log4net; -using OpenSim.Framework; - -namespace OpenSim.Framework.Monitoring -{ - public class Job - { - public string Name; - public WaitCallback Callback; - public object O; - - public Job(string name, WaitCallback callback, object o) - { - Name = name; - Callback = callback; - O = o; - } - } - - public class JobEngine - { - private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - - public int LogLevel { get; set; } - - public bool IsRunning { get; private set; } - - /// - /// The timeout in milliseconds to wait for at least one event to be written when the recorder is stopping. - /// - public int RequestProcessTimeoutOnStop { get; set; } - - /// - /// Controls whether we need to warn in the log about exceeding the max queue size. - /// - /// - /// This is flipped to false once queue max has been exceeded and back to true when it falls below max, in - /// order to avoid spamming the log with lots of warnings. - /// - private bool m_warnOverMaxQueue = true; - - private BlockingCollection m_requestQueue; - - private CancellationTokenSource m_cancelSource = new CancellationTokenSource(); - - private Stat m_requestsWaitingStat; - - private Job m_currentJob; - - /// - /// Used to signal that we are ready to complete stop. - /// - private ManualResetEvent m_finishedProcessingAfterStop = new ManualResetEvent(false); - - public JobEngine() - { - RequestProcessTimeoutOnStop = 5000; - - MainConsole.Instance.Commands.AddCommand( - "Debug", - false, - "debug jobengine", - "debug jobengine ", - "Start, stop, get status or set logging level of the job engine.", - "If stopped then all outstanding jobs are processed immediately.", - HandleControlCommand); - } - - public void Start() - { - lock (this) - { - if (IsRunning) - return; - - IsRunning = true; - - m_finishedProcessingAfterStop.Reset(); - - m_requestQueue = new BlockingCollection(new ConcurrentQueue(), 5000); - - m_requestsWaitingStat = - new Stat( - "JobsWaiting", - "Number of jobs waiting for processing.", - "", - "", - "server", - "jobengine", - StatType.Pull, - MeasuresOfInterest.None, - stat => stat.Value = m_requestQueue.Count, - StatVerbosity.Debug); - - StatsManager.RegisterStat(m_requestsWaitingStat); - - WorkManager.StartThread( - ProcessRequests, - "JobEngineThread", - ThreadPriority.Normal, - false, - true, - null, - int.MaxValue); - } - } - - public void Stop() - { - lock (this) - { - try - { - if (!IsRunning) - return; - - IsRunning = false; - - int requestsLeft = m_requestQueue.Count; - - if (requestsLeft <= 0) - { - m_cancelSource.Cancel(); - } - else - { - m_log.InfoFormat("[JOB ENGINE]: Waiting to write {0} events after stop.", requestsLeft); - - while (requestsLeft > 0) - { - if (!m_finishedProcessingAfterStop.WaitOne(RequestProcessTimeoutOnStop)) - { - // After timeout no events have been written - if (requestsLeft == m_requestQueue.Count) - { - m_log.WarnFormat( - "[JOB ENGINE]: No requests processed after {0} ms wait. Discarding remaining {1} requests", - RequestProcessTimeoutOnStop, requestsLeft); - - break; - } - } - - requestsLeft = m_requestQueue.Count; - } - } - } - finally - { - m_cancelSource.Dispose(); - StatsManager.DeregisterStat(m_requestsWaitingStat); - m_requestsWaitingStat = null; - m_requestQueue = null; - } - } - } - - public bool QueueRequest(string name, WaitCallback req, object o) - { - if (LogLevel >= 1) - m_log.DebugFormat("[JOB ENGINE]: Queued job {0}", name); - - if (m_requestQueue.Count < m_requestQueue.BoundedCapacity) - { - // m_log.DebugFormat( - // "[OUTGOING QUEUE REFILL ENGINE]: Adding request for categories {0} for {1} in {2}", - // categories, client.AgentID, m_udpServer.Scene.Name); - - m_requestQueue.Add(new Job(name, req, o)); - - if (!m_warnOverMaxQueue) - m_warnOverMaxQueue = true; - - return true; - } - else - { - if (m_warnOverMaxQueue) - { -// m_log.WarnFormat( -// "[JOB ENGINE]: Request queue at maximum capacity, not recording request from {0} in {1}", -// client.AgentID, m_udpServer.Scene.Name); - - m_log.WarnFormat("[JOB ENGINE]: Request queue at maximum capacity, not recording job"); - - m_warnOverMaxQueue = false; - } - - return false; - } - } - - private void ProcessRequests() - { - try - { - while (IsRunning || m_requestQueue.Count > 0) - { - m_currentJob = m_requestQueue.Take(m_cancelSource.Token); - - // QueueEmpty callback = req.Client.OnQueueEmpty; - // - // if (callback != null) - // { - // try - // { - // callback(req.Categories); - // } - // catch (Exception e) - // { - // m_log.Error("[OUTGOING QUEUE REFILL ENGINE]: ProcessRequests(" + req.Categories + ") threw an exception: " + e.Message, e); - // } - // } - - if (LogLevel >= 1) - m_log.DebugFormat("[JOB ENGINE]: Processing job {0}", m_currentJob.Name); - - try - { - m_currentJob.Callback.Invoke(m_currentJob.O); - } - catch (Exception e) - { - m_log.Error( - string.Format( - "[JOB ENGINE]: Job {0} failed, continuing. Exception ", m_currentJob.Name), e); - } - - if (LogLevel >= 1) - m_log.DebugFormat("[JOB ENGINE]: Processed job {0}", m_currentJob.Name); - - m_currentJob = null; - } - } - catch (OperationCanceledException) - { - } - - m_finishedProcessingAfterStop.Set(); - } - - private void HandleControlCommand(string module, string[] args) - { -// if (SceneManager.Instance.CurrentScene != null && SceneManager.Instance.CurrentScene != m_udpServer.Scene) -// return; - - if (args.Length < 3) - { - MainConsole.Instance.Output("Usage: debug jobengine "); - return; - } - - string subCommand = args[2]; - - if (subCommand == "stop") - { - Stop(); - MainConsole.Instance.OutputFormat("Stopped job engine."); - } - else if (subCommand == "start") - { - Start(); - MainConsole.Instance.OutputFormat("Started job engine."); - } - else if (subCommand == "status") - { - MainConsole.Instance.OutputFormat("Job engine running: {0}", IsRunning); - MainConsole.Instance.OutputFormat("Current job {0}", m_currentJob != null ? m_currentJob.Name : "none"); - MainConsole.Instance.OutputFormat( - "Jobs waiting: {0}", IsRunning ? m_requestQueue.Count.ToString() : "n/a"); - MainConsole.Instance.OutputFormat("Log Level: {0}", LogLevel); - } - else if (subCommand == "log") - { -// int logLevel; - int logLevel = int.Parse(args[3]); -// if (ConsoleUtil.TryParseConsoleInt(MainConsole.Instance, args[4], out logLevel)) -// { - LogLevel = logLevel; - MainConsole.Instance.OutputFormat("Set debug log level to {0}", LogLevel); -// } - } - else - { - MainConsole.Instance.OutputFormat("Unrecognized job engine subcommand {0}", subCommand); - } - } - } -} diff --git a/OpenSim/Framework/Monitoring/WorkManager.cs b/OpenSim/Framework/Monitoring/WorkManager.cs index 9d0eefc..134661b 100644 --- a/OpenSim/Framework/Monitoring/WorkManager.cs +++ b/OpenSim/Framework/Monitoring/WorkManager.cs @@ -57,7 +57,29 @@ namespace OpenSim.Framework.Monitoring static WorkManager() { - JobEngine = new JobEngine(); + JobEngine = new JobEngine("Non-blocking non-critical job engine", "JOB ENGINE"); + + StatsManager.RegisterStat( + new Stat( + "JobsWaiting", + "Number of jobs waiting for processing.", + "", + "", + "server", + "jobengine", + StatType.Pull, + MeasuresOfInterest.None, + stat => stat.Value = JobEngine.JobsWaiting, + StatVerbosity.Debug)); + + MainConsole.Instance.Commands.AddCommand( + "Debug", + false, + "debug jobengine", + "debug jobengine ", + "Start, stop, get status or set logging level of the job engine.", + "If stopped then all outstanding jobs are processed immediately.", + HandleControlCommand); } /// @@ -200,7 +222,7 @@ namespace OpenSim.Framework.Monitoring } if (JobEngine.IsRunning) - JobEngine.QueueRequest(name, callback, obj); + JobEngine.QueueJob(name, () => callback(obj)); else if (canRunInThisThread) callback(obj); else if (mustNotTimeout) @@ -208,5 +230,55 @@ namespace OpenSim.Framework.Monitoring else Util.FireAndForget(callback, obj, name); } + + private static void HandleControlCommand(string module, string[] args) + { + // if (SceneManager.Instance.CurrentScene != null && SceneManager.Instance.CurrentScene != m_udpServer.Scene) + // return; + + if (args.Length < 3) + { + MainConsole.Instance.Output("Usage: debug jobengine "); + return; + } + + string subCommand = args[2]; + + if (subCommand == "stop") + { + JobEngine.Stop(); + MainConsole.Instance.OutputFormat("Stopped job engine."); + } + else if (subCommand == "start") + { + JobEngine.Start(); + MainConsole.Instance.OutputFormat("Started job engine."); + } + else if (subCommand == "status") + { + MainConsole.Instance.OutputFormat("Job engine running: {0}", JobEngine.IsRunning); + + JobEngine.Job job = JobEngine.CurrentJob; + MainConsole.Instance.OutputFormat("Current job {0}", job != null ? job.Name : "none"); + + MainConsole.Instance.OutputFormat( + "Jobs waiting: {0}", JobEngine.IsRunning ? JobEngine.JobsWaiting.ToString() : "n/a"); + MainConsole.Instance.OutputFormat("Log Level: {0}", JobEngine.LogLevel); + } + else if (subCommand == "log") + { + // int logLevel; + int logLevel = int.Parse(args[3]); + // if (ConsoleUtil.TryParseConsoleInt(MainConsole.Instance, args[4], out logLevel)) + // { + JobEngine.LogLevel = logLevel; + MainConsole.Instance.OutputFormat("Set debug log level to {0}", JobEngine.LogLevel); + // } + } + else + { + MainConsole.Instance.OutputFormat("Unrecognized job engine subcommand {0}", subCommand); + } + } } } \ No newline at end of file -- cgit v1.1 From 0c31eb0a5d0a124e0cd7c3db53464e62e87b8fb3 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Mon, 12 Jan 2015 20:59:58 +0000 Subject: Add the missing deleted and re-added JobEngine class from the previous commit 8e1e8a0 --- OpenSim/Framework/Monitoring/JobEngine.cs | 329 ++++++++++++++++++++++++++++++ 1 file changed, 329 insertions(+) create mode 100644 OpenSim/Framework/Monitoring/JobEngine.cs (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs new file mode 100644 index 0000000..44f5d9a --- /dev/null +++ b/OpenSim/Framework/Monitoring/JobEngine.cs @@ -0,0 +1,329 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Concurrent; +using System.Reflection; +using System.Threading; +using log4net; +using OpenSim.Framework; + +namespace OpenSim.Framework.Monitoring +{ + public class JobEngine + { + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + public int LogLevel { get; set; } + + public string Name { get; private set; } + + public string LoggingName { get; private set; } + + /// + /// Is this engine running? + /// + public bool IsRunning { get; private set; } + + /// + /// The current job that the engine is running. + /// + /// + /// Will be null if no job is currently running. + /// + public Job CurrentJob { get; private set; } + + /// + /// Number of jobs waiting to be processed. + /// + public int JobsWaiting { get { return m_jobQueue.Count; } } + + /// + /// The timeout in milliseconds to wait for at least one event to be written when the recorder is stopping. + /// + public int RequestProcessTimeoutOnStop { get; set; } + + /// + /// Controls whether we need to warn in the log about exceeding the max queue size. + /// + /// + /// This is flipped to false once queue max has been exceeded and back to true when it falls below max, in + /// order to avoid spamming the log with lots of warnings. + /// + private bool m_warnOverMaxQueue = true; + + private BlockingCollection m_jobQueue; + + private CancellationTokenSource m_cancelSource = new CancellationTokenSource(); + + /// + /// Used to signal that we are ready to complete stop. + /// + private ManualResetEvent m_finishedProcessingAfterStop = new ManualResetEvent(false); + + public JobEngine(string name, string loggingName) + { + Name = name; + LoggingName = loggingName; + + RequestProcessTimeoutOnStop = 5000; + } + + public void Start() + { + lock (this) + { + if (IsRunning) + return; + + IsRunning = true; + + m_finishedProcessingAfterStop.Reset(); + + m_jobQueue = new BlockingCollection(new ConcurrentQueue(), 5000); + + WorkManager.StartThread( + ProcessRequests, + Name, + ThreadPriority.Normal, + false, + true, + null, + int.MaxValue); + } + } + + public void Stop() + { + lock (this) + { + try + { + if (!IsRunning) + return; + + IsRunning = false; + + int requestsLeft = m_jobQueue.Count; + + if (requestsLeft <= 0) + { + m_cancelSource.Cancel(); + } + else + { + m_log.InfoFormat("[{0}]: Waiting to write {1} events after stop.", LoggingName, requestsLeft); + + while (requestsLeft > 0) + { + if (!m_finishedProcessingAfterStop.WaitOne(RequestProcessTimeoutOnStop)) + { + // After timeout no events have been written + if (requestsLeft == m_jobQueue.Count) + { + m_log.WarnFormat( + "[{0}]: No requests processed after {1} ms wait. Discarding remaining {2} requests", + LoggingName, RequestProcessTimeoutOnStop, requestsLeft); + + break; + } + } + + requestsLeft = m_jobQueue.Count; + } + } + } + finally + { + m_cancelSource.Dispose(); + m_jobQueue = null; + } + } + } + + /// + /// Make a job. + /// + /// + /// We provide this method to replace the constructor so that we can later pool job objects if necessary to + /// reduce memory churn. Normally one would directly call QueueJob() with parameters anyway. + /// + /// + /// Name. + /// Action. + /// Common identifier. + public static Job MakeJob(string name, Action action, string commonId = null) + { + return Job.MakeJob(name, action, commonId); + } + + /// + /// Remove the next job queued for processing. + /// + /// + /// Returns null if there is no next job. + /// Will not remove a job currently being performed. + /// + public Job RemoveNextJob() + { + Job nextJob; + m_jobQueue.TryTake(out nextJob); + + return nextJob; + } + + /// + /// Queue the job for processing. + /// + /// true, if job was queued, false otherwise. + /// Name of job. This appears on the console and in logging. + /// Action to perform. + /// + /// Common identifier for a set of jobs. This is allows a set of jobs to be removed + /// if required (e.g. all jobs for a given agent. Optional. + /// + public bool QueueJob(string name, Action action, string commonId = null) + { + return QueueJob(MakeJob(name, action, commonId)); + } + + /// + /// Queue the job for processing. + /// + /// true, if job was queued, false otherwise. + /// The job + /// + public bool QueueJob(Job job) + { + if (m_jobQueue.Count < m_jobQueue.BoundedCapacity) + { + m_jobQueue.Add(job); + + if (!m_warnOverMaxQueue) + m_warnOverMaxQueue = true; + + return true; + } + else + { + if (m_warnOverMaxQueue) + { + m_log.WarnFormat( + "[{0}]: Job queue at maximum capacity, not recording job from {1} in {2}", + LoggingName, job.Name, Name); + + m_warnOverMaxQueue = false; + } + + return false; + } + } + + private void ProcessRequests() + { + try + { + while (IsRunning || m_jobQueue.Count > 0) + { + CurrentJob = m_jobQueue.Take(m_cancelSource.Token); + + if (LogLevel >= 1) + m_log.DebugFormat("[{0}]: Processing job {1}", LoggingName, CurrentJob.Name); + + try + { + CurrentJob.Action(); + } + catch (Exception e) + { + m_log.Error( + string.Format( + "[{0}]: Job {1} failed, continuing. Exception ", LoggingName, CurrentJob.Name), e); + } + + if (LogLevel >= 1) + m_log.DebugFormat("[{0}]: Processed job {1}", LoggingName, CurrentJob.Name); + + CurrentJob = null; + } + } + catch (OperationCanceledException) + { + } + + m_finishedProcessingAfterStop.Set(); + } + + public class Job + { + /// + /// Name of the job. + /// + /// + /// This appears on console and debug output. + /// + public string Name { get; private set; } + + /// + /// Common ID for this job. + /// + /// + /// This allows all jobs with a certain common ID (e.g. a client UUID) to be removed en-masse if required. + /// Can be null if this is not required. + /// + public string CommonId { get; private set; } + + /// + /// Action to perform when this job is processed. + /// + public Action Action { get; private set; } + + private Job(string name, string commonId, Action action) + { + Name = name; + CommonId = commonId; + Action = action; + } + + /// + /// Make a job. It needs to be separately queued. + /// + /// + /// We provide this method to replace the constructor so that we can pool job objects if necessary to + /// to reduce memory churn. Normally one would directly call JobEngine.QueueJob() with parameters anyway. + /// + /// + /// Name. + /// Action. + /// Common identifier. + public static Job MakeJob(string name, Action action, string commonId = null) + { + return new Job(name, commonId, action); + } + } + } +} \ No newline at end of file -- cgit v1.1 From 68343938215a438cd446a7db819a4f7d93ccacd4 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Wed, 4 Feb 2015 00:10:13 +0000 Subject: Fix bug in JobEngine where an attempt to restart after stop would trigger an exception because the cancellation source was not recreated. --- OpenSim/Framework/Monitoring/JobEngine.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs index 44f5d9a..a32e4aa 100644 --- a/OpenSim/Framework/Monitoring/JobEngine.cs +++ b/OpenSim/Framework/Monitoring/JobEngine.cs @@ -78,7 +78,7 @@ namespace OpenSim.Framework.Monitoring private BlockingCollection m_jobQueue; - private CancellationTokenSource m_cancelSource = new CancellationTokenSource(); + private CancellationTokenSource m_cancelSource; /// /// Used to signal that we are ready to complete stop. @@ -105,6 +105,7 @@ namespace OpenSim.Framework.Monitoring m_finishedProcessingAfterStop.Reset(); m_jobQueue = new BlockingCollection(new ConcurrentQueue(), 5000); + m_cancelSource = new CancellationTokenSource(); WorkManager.StartThread( ProcessRequests, -- cgit v1.1 From fed566b8d3bb480ec89615e011934c10023a4dad Mon Sep 17 00:00:00 2001 From: Melanie Thielker Date: Wed, 18 Feb 2015 01:14:08 +0100 Subject: Abbreviate the stats by removing unneeded and redundant elements. Human readability is overrated. Also add a (hardcoded) password. --- .../Framework/Monitoring/ServerStatsCollector.cs | 43 ++++++++++++++++++++++ OpenSim/Framework/Monitoring/Stats/Stat.cs | 13 ++++++- OpenSim/Framework/Monitoring/StatsManager.cs | 15 +++++++- 3 files changed, 68 insertions(+), 3 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs index ac0f0bc..e6c73d3 100644 --- a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs @@ -246,6 +246,49 @@ namespace OpenSim.Framework.Monitoring (s) => { s.Value = Math.Round(MemoryWatchdog.LastHeapAllocationRate * 1000d / 1024d / 1024d, 3); }); MakeStat("AverageHeapAllocationRate", null, "MB/sec", ContainerMemory, (s) => { s.Value = Math.Round(MemoryWatchdog.AverageHeapAllocationRate * 1000d / 1024d / 1024d, 3); }); + + MakeStat("ProcessResident", null, "MB", ContainerProcess, + (s) => + { + Process myprocess = Process.GetCurrentProcess(); + myprocess.Refresh(); + s.Value = Math.Round(Process.GetCurrentProcess().WorkingSet64 / 1024.0 / 1024.0); + }); + MakeStat("ProcessPaged", null, "MB", ContainerProcess, + (s) => + { + Process myprocess = Process.GetCurrentProcess(); + myprocess.Refresh(); + s.Value = Math.Round(Process.GetCurrentProcess().PagedMemorySize64 / 1024.0 / 1024.0); + }); + MakeStat("ProcessVirtual", null, "MB", ContainerProcess, + (s) => + { + Process myprocess = Process.GetCurrentProcess(); + myprocess.Refresh(); + s.Value = Math.Round(Process.GetCurrentProcess().VirtualMemorySize64 / 1024.0 / 1024.0); + }); + MakeStat("PeakProcessResident", null, "MB", ContainerProcess, + (s) => + { + Process myprocess = Process.GetCurrentProcess(); + myprocess.Refresh(); + s.Value = Math.Round(Process.GetCurrentProcess().PeakWorkingSet64 / 1024.0 / 1024.0); + }); + MakeStat("PeakProcessPaged", null, "MB", ContainerProcess, + (s) => + { + Process myprocess = Process.GetCurrentProcess(); + myprocess.Refresh(); + s.Value = Math.Round(Process.GetCurrentProcess().PeakPagedMemorySize64 / 1024.0 / 1024.0); + }); + MakeStat("PeakProcessVirtual", null, "MB", ContainerProcess, + (s) => + { + Process myprocess = Process.GetCurrentProcess(); + myprocess.Refresh(); + s.Value = Math.Round(Process.GetCurrentProcess().PeakVirtualMemorySize64 / 1024.0 / 1024.0); + }); } // Notes on performance counters: diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs index e095801..bd757d0 100644 --- a/OpenSim/Framework/Monitoring/Stats/Stat.cs +++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs @@ -238,6 +238,17 @@ namespace OpenSim.Framework.Monitoring return sb.ToString(); } + public virtual OSDMap ToBriefOSDMap() + { + OSDMap ret = new OSDMap(); + + ret.Add("Value", OSD.FromReal(Value)); + + double lastChangeOverTime, averageChangeOverTime; + + return ret; + } + public virtual OSDMap ToOSDMap() { OSDMap ret = new OSDMap(); @@ -322,4 +333,4 @@ namespace OpenSim.Framework.Monitoring } } } -} \ No newline at end of file +} diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index 05ee4c5..249cef6 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -253,7 +253,7 @@ namespace OpenSim.Framework.Monitoring if (!(String.IsNullOrEmpty(pStatName) || pStatName == AllSubCommand || pStatName == statName)) continue; - statMap.Add(statName, theStats[statName].ToOSDMap()); + statMap.Add(statName, theStats[statName].ToBriefOSDMap()); } contMap.Add(contName, statMap); @@ -275,6 +275,17 @@ namespace OpenSim.Framework.Monitoring string pContainerName = StatsManager.AllSubCommand; string pStatName = StatsManager.AllSubCommand; + if (!request.ContainsKey("pass") || request["pass"].ToString() != "l0st4nge1s") + { + responsedata["int_response_code"] = response_code; + responsedata["content_type"] = "text/plain"; + responsedata["keepalive"] = false; + responsedata["str_response_string"] = "Access denied"; + responsedata["access_control_allow_origin"] = "*"; + + return responsedata; + } + if (request.ContainsKey("cat")) pCategoryName = request["cat"].ToString(); if (request.ContainsKey("cont")) pContainerName = request["cat"].ToString(); if (request.ContainsKey("stat")) pStatName = request["cat"].ToString(); @@ -524,4 +535,4 @@ namespace OpenSim.Framework.Monitoring Debug, Info } -} \ No newline at end of file +} -- cgit v1.1 From 686b22da6e460c7869586e88332c981d7fbaf627 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Wed, 25 Feb 2015 20:01:34 +0000 Subject: On shutdown (job engine stop), don't allow the ObjectDisposedException on BlockingCollection.Take() to propogate if the running thread checked IsRunning before the stop thread set it and disposed of the canellation source. Looks to address http://opensimulator.org/mantis/view.php?id=7453 --- OpenSim/Framework/Monitoring/JobEngine.cs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs index a32e4aa..6db9a67 100644 --- a/OpenSim/Framework/Monitoring/JobEngine.cs +++ b/OpenSim/Framework/Monitoring/JobEngine.cs @@ -161,7 +161,6 @@ namespace OpenSim.Framework.Monitoring finally { m_cancelSource.Dispose(); - m_jobQueue = null; } } } @@ -250,7 +249,19 @@ namespace OpenSim.Framework.Monitoring { while (IsRunning || m_jobQueue.Count > 0) { - CurrentJob = m_jobQueue.Take(m_cancelSource.Token); + try + { + CurrentJob = m_jobQueue.Take(m_cancelSource.Token); + } + catch (ObjectDisposedException e) + { + // If we see this whilst not running then it may be due to a race where this thread checks + // IsRunning after the stopping thread sets it to false and disposes of the cancellation source. + if (IsRunning) + throw e; + else + break; + } if (LogLevel >= 1) m_log.DebugFormat("[{0}]: Processing job {1}", LoggingName, CurrentJob.Name); -- cgit v1.1 From 1959eb8372b6c8e35e1afa435504e9ef41dec958 Mon Sep 17 00:00:00 2001 From: Glenn Martin Date: Mon, 20 Apr 2015 14:55:00 -0400 Subject: Moved over metrics from previous OpenSim 0.8.0.3 repository (this new repository is now the master branch from OpenSim). --- .../Framework/Monitoring/SimExtraStatsCollector.cs | 47 ++++++++++++++++++++++ 1 file changed, 47 insertions(+) mode change 100644 => 100755 OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs old mode 100644 new mode 100755 index f6f458d..98fa65a --- a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs @@ -27,6 +27,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Text; using OpenMetaverse; @@ -71,6 +72,10 @@ namespace OpenSim.Framework.Monitoring private volatile float pendingUploads; private volatile float activeScripts; private volatile float scriptLinesPerSecond; + private volatile float m_usersLoggingIn; + private volatile float m_totalGeoPrims; + private volatile float m_totalMeshes; + private volatile float m_inUseThreads; // /// // /// These statistics are being collected by push rather than pull. Pull would be simpler, but I had the @@ -249,6 +254,10 @@ namespace OpenSim.Framework.Monitoring { // FIXME: SimStats shouldn't allow an arbitrary stat packing order (which is inherited from the original // SimStatsPacket that was being used). + + // For an unknown reason the original designers decided not to + // include the spare MS statistic inside of this class, this is + // located inside the StatsBlock at location 21 thus it is skipped timeDilation = stats.StatsBlock[0].StatValue; simFps = stats.StatsBlock[1].StatValue; physicsFps = stats.StatsBlock[2].StatValue; @@ -270,6 +279,10 @@ namespace OpenSim.Framework.Monitoring pendingUploads = stats.StatsBlock[18].StatValue; activeScripts = stats.StatsBlock[19].StatValue; scriptLinesPerSecond = stats.StatsBlock[20].StatValue; + m_usersLoggingIn = stats.StatsBlock[22].StatValue; + m_totalGeoPrims = stats.StatsBlock[23].StatValue; + m_totalMeshes = stats.StatsBlock[24].StatValue; + m_inUseThreads = stats.StatsBlock[25].StatValue; } /// @@ -407,6 +420,23 @@ Asset service request failures: {3}" + Environment.NewLine, /// public override OSDMap OReport(string uptime, string version) { + // Get the amount of physical memory, allocated with the instance of this program, in kilobytes; + // the working set is the set of memory pages currently visible to this program in physical RAM + // memory and includes both shared (e.g. system libraries) and private data + double memUsage = Process.GetCurrentProcess().WorkingSet64 / 1024.0; + + // Get the number of threads from the system that are currently + // running + int numberThreadsRunning = 0; + foreach (ProcessThread currentThread in + Process.GetCurrentProcess().Threads) + { + if (currentThread.ThreadState == ThreadState.Running) + { + numberThreadsRunning++; + } + } + OSDMap args = new OSDMap(30); // args["AssetsInCache"] = OSD.FromString (String.Format ("{0:0.##}", AssetsInCache)); // args["TimeAfterCacheMiss"] = OSD.FromString (String.Format ("{0:0.##}", @@ -443,6 +473,23 @@ Asset service request failures: {3}" + Environment.NewLine, args["Memory"] = OSD.FromString (base.XReport (uptime, version)); args["Uptime"] = OSD.FromString (uptime); args["Version"] = OSD.FromString (version); + + args["Logging in Users"] = OSD.FromString(String.Format("{0:0.##}", + m_usersLoggingIn)); + args["GeoPrims"] = OSD.FromString(String.Format("{0:0.##}", + m_totalGeoPrims)); + args["Mesh Objects"] = OSD.FromString(String.Format("{0:0.##}", + m_totalMeshes)); + args["Polygon Count"] = OSD.FromString(String.Format("{0:0.##}", 0)); + args["Texture Count"] = OSD.FromString(String.Format("{0:0.##}", 0)); + args["XEngine Thread Count"] = OSD.FromString(String.Format("{0:0.##}", + m_inUseThreads)); + args["Util Thread Count"] = OSD.FromString(String.Format("{0:0.##}", + Util.GetSmartThreadPoolInfo().InUseThreads)); + args["System Thread Count"] = OSD.FromString(String.Format( + "{0:0.##}", numberThreadsRunning)); + args["ProcMem"] = OSD.FromString(String.Format("{0:#,###,###.##}", + memUsage)); return args; } -- cgit v1.1 From c3138f9f38420ae370078df3b0990a953f43b087 Mon Sep 17 00:00:00 2001 From: Robert Louden Date: Tue, 21 Apr 2015 16:15:48 -0400 Subject: Phase 2 additons with Frame Dilation metric. --- .../Framework/Monitoring/SimExtraStatsCollector.cs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs index 98fa65a..432efdb 100755 --- a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs @@ -72,6 +72,7 @@ namespace OpenSim.Framework.Monitoring private volatile float pendingUploads; private volatile float activeScripts; private volatile float scriptLinesPerSecond; + private volatile float m_frameDilation; private volatile float m_usersLoggingIn; private volatile float m_totalGeoPrims; private volatile float m_totalMeshes; @@ -257,7 +258,7 @@ namespace OpenSim.Framework.Monitoring // For an unknown reason the original designers decided not to // include the spare MS statistic inside of this class, this is - // located inside the StatsBlock at location 21 thus it is skipped + // located inside the StatsBlock at location 21, thus it is skipped timeDilation = stats.StatsBlock[0].StatValue; simFps = stats.StatsBlock[1].StatValue; physicsFps = stats.StatsBlock[2].StatValue; @@ -279,10 +280,11 @@ namespace OpenSim.Framework.Monitoring pendingUploads = stats.StatsBlock[18].StatValue; activeScripts = stats.StatsBlock[19].StatValue; scriptLinesPerSecond = stats.StatsBlock[20].StatValue; - m_usersLoggingIn = stats.StatsBlock[22].StatValue; - m_totalGeoPrims = stats.StatsBlock[23].StatValue; - m_totalMeshes = stats.StatsBlock[24].StatValue; - m_inUseThreads = stats.StatsBlock[25].StatValue; + m_frameDilation = stats.StatsBlock[22].StatValue; + m_usersLoggingIn = stats.StatsBlock[23].StatValue; + m_totalGeoPrims = stats.StatsBlock[24].StatValue; + m_totalMeshes = stats.StatsBlock[25].StatValue; + m_inUseThreads = stats.StatsBlock[26].StatValue; } /// @@ -474,14 +476,13 @@ Asset service request failures: {3}" + Environment.NewLine, args["Uptime"] = OSD.FromString (uptime); args["Version"] = OSD.FromString (version); + args["FrameDilatn"] = OSD.FromString(String.Format("{0:0.##}", m_frameDilation)); args["Logging in Users"] = OSD.FromString(String.Format("{0:0.##}", - m_usersLoggingIn)); + m_usersLoggingIn)); args["GeoPrims"] = OSD.FromString(String.Format("{0:0.##}", m_totalGeoPrims)); args["Mesh Objects"] = OSD.FromString(String.Format("{0:0.##}", m_totalMeshes)); - args["Polygon Count"] = OSD.FromString(String.Format("{0:0.##}", 0)); - args["Texture Count"] = OSD.FromString(String.Format("{0:0.##}", 0)); args["XEngine Thread Count"] = OSD.FromString(String.Format("{0:0.##}", m_inUseThreads)); args["Util Thread Count"] = OSD.FromString(String.Format("{0:0.##}", @@ -489,7 +490,7 @@ Asset service request failures: {3}" + Environment.NewLine, args["System Thread Count"] = OSD.FromString(String.Format( "{0:0.##}", numberThreadsRunning)); args["ProcMem"] = OSD.FromString(String.Format("{0:#,###,###.##}", - memUsage)); + memUsage)); return args; } -- cgit v1.1 From da32512ea449c2de2d4a6069f899fbd4a8bb03fa Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Wed, 29 Apr 2015 18:47:17 -0700 Subject: Updated all occurrences of AssemblyVersion("0.8.1.*") to AssemblyVersion("0.8.2.*") --- OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs index ee82753..b08e4f7 100644 --- a/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs +++ b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs @@ -29,5 +29,5 @@ using System.Runtime.InteropServices; // Build Number // Revision // -[assembly: AssemblyVersion("0.8.1.*")] +[assembly: AssemblyVersion("0.8.2.*")] -- cgit v1.1 From c4a3c93097c9a1c1b93f5b86dfc0f17062e42c30 Mon Sep 17 00:00:00 2001 From: Steven Zielinski Date: Fri, 1 May 2015 17:07:19 -0400 Subject: Fixed a bug that would cause the sim extra stats reporter to fail in reporting stats. The bug was caused by the current process threads which can return null references. Test Plan: Tested on windows using opensim standalone and the json stats. Reviewers: rlouden, kboswell, clattin, martin, ssirigam Differential Revision: http://cr.irl.ucf.edu/D277 --- OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs index 432efdb..83d9e85 100755 --- a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs @@ -433,7 +433,11 @@ Asset service request failures: {3}" + Environment.NewLine, foreach (ProcessThread currentThread in Process.GetCurrentProcess().Threads) { - if (currentThread.ThreadState == ThreadState.Running) + // A known issue with the current process .threads method is + // that it can return null threads, thus don't count those as + // running threads and prevent the program function from failing + if (currentThread != null && + currentThread.ThreadState == ThreadState.Running) { numberThreadsRunning++; } -- cgit v1.1 From cb517839e9e258d154e17fe85e97f1462b06536a Mon Sep 17 00:00:00 2001 From: Steven Zielinski Date: Mon, 4 May 2015 10:05:36 -0400 Subject: Fixed vocabulary in a comment to match the code. --- OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs index 83d9e85..3791fff 100755 --- a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs @@ -433,7 +433,7 @@ Asset service request failures: {3}" + Environment.NewLine, foreach (ProcessThread currentThread in Process.GetCurrentProcess().Threads) { - // A known issue with the current process .threads method is + // A known issue with the current process .threads property is // that it can return null threads, thus don't count those as // running threads and prevent the program function from failing if (currentThread != null && -- cgit v1.1 From 96a86e7d5a4a4f0d712522190a1ff19fc44aac80 Mon Sep 17 00:00:00 2001 From: Steven Zielinski Date: Mon, 4 May 2015 10:40:36 -0400 Subject: Fixed a bug that would cause the sim extra stats reporter to fail in reporting stats. The bug was caused by the current process threads which can return null references. Test Plan: Tested on windows using opensim standalone and the json stats. Reviewers: rlouden, ssirigam, clattin, martin, kboswell Reviewed By: rlouden, ssirigam, clattin, martin, kboswell Differential Revision: http://cr.irl.ucf.edu/D277 --- OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs index 3791fff..e4df7ee 100755 --- a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs @@ -433,7 +433,7 @@ Asset service request failures: {3}" + Environment.NewLine, foreach (ProcessThread currentThread in Process.GetCurrentProcess().Threads) { - // A known issue with the current process .threads property is + // A known issue with the current process .Threads property is // that it can return null threads, thus don't count those as // running threads and prevent the program function from failing if (currentThread != null && -- cgit v1.1 From c8135834db90dc438543a3906fffe1024f4c9a03 Mon Sep 17 00:00:00 2001 From: Oren Hurvitz Date: Fri, 17 Jul 2015 16:46:19 +0300 Subject: Prevent a crash if "debug jobengine log" is entered without a log level --- OpenSim/Framework/Monitoring/WorkManager.cs | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/WorkManager.cs b/OpenSim/Framework/Monitoring/WorkManager.cs index 134661b..d1a74ce 100644 --- a/OpenSim/Framework/Monitoring/WorkManager.cs +++ b/OpenSim/Framework/Monitoring/WorkManager.cs @@ -267,6 +267,12 @@ namespace OpenSim.Framework.Monitoring } else if (subCommand == "log") { + if (args.Length < 4) + { + MainConsole.Instance.Output("Usage: debug jobengine log "); + return; + } + // int logLevel; int logLevel = int.Parse(args[3]); // if (ConsoleUtil.TryParseConsoleInt(MainConsole.Instance, args[4], out logLevel)) -- cgit v1.1 From cdaed113872f0615a0d2864cc54064a4432054c6 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Thu, 3 Sep 2015 18:39:08 +0100 Subject: at last we can login and see objects ( friends is dead and disable in scenepresence) --- OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs index e4df7ee..b0fdc81 100755 --- a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs @@ -281,10 +281,10 @@ namespace OpenSim.Framework.Monitoring activeScripts = stats.StatsBlock[19].StatValue; scriptLinesPerSecond = stats.StatsBlock[20].StatValue; m_frameDilation = stats.StatsBlock[22].StatValue; - m_usersLoggingIn = stats.StatsBlock[23].StatValue; - m_totalGeoPrims = stats.StatsBlock[24].StatValue; - m_totalMeshes = stats.StatsBlock[25].StatValue; - m_inUseThreads = stats.StatsBlock[26].StatValue; +// m_usersLoggingIn = stats.StatsBlock[23].StatValue; +// m_totalGeoPrims = stats.StatsBlock[24].StatValue; +// m_totalMeshes = stats.StatsBlock[25].StatValue; +// m_inUseThreads = stats.StatsBlock[26].StatValue; } /// -- cgit v1.1 From 4b1ecc90fcb1e577d1f02f40fac1e53c66c63496 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Sat, 5 Sep 2015 21:23:22 +0100 Subject: fix modifing m_threads on a foreach scanning it --- OpenSim/Framework/Monitoring/Watchdog.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index 0cab427..4485a9c 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -335,7 +335,9 @@ namespace OpenSim.Framework.Monitoring lock (m_threads) { - foreach (ThreadWatchdogInfo threadInfo in m_threads.Values) + // get a copy since we may change m_threads + List threadsInfo = m_threads.Values.ToList(); + foreach (ThreadWatchdogInfo threadInfo in threadsInfo) { if (threadInfo.Thread.ThreadState == ThreadState.Stopped) { -- cgit v1.1 From 0ddcc55166961d6e10b0250fc521c0663fedd250 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Sun, 6 Sep 2015 00:32:20 +0100 Subject: let extra stats travel side by side with viewers ones... try to fix damm array indexes all over the place --- .../Framework/Monitoring/SimExtraStatsCollector.cs | 28 +++++++++++++++------- 1 file changed, 19 insertions(+), 9 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs index b0fdc81..08c2409 100755 --- a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs @@ -34,6 +34,7 @@ using OpenMetaverse; using OpenMetaverse.StructuredData; using OpenSim.Framework.Monitoring.Interfaces; + namespace OpenSim.Framework.Monitoring { /// @@ -71,6 +72,11 @@ namespace OpenSim.Framework.Monitoring private volatile float pendingDownloads; private volatile float pendingUploads; private volatile float activeScripts; + private volatile float spareTime; + private volatile float sleepTime; + private volatile float physicsStep; + + private volatile float scriptLinesPerSecond; private volatile float m_frameDilation; private volatile float m_usersLoggingIn; @@ -253,7 +259,7 @@ namespace OpenSim.Framework.Monitoring /// public void ReceiveClassicSimStatsPacket(SimStats stats) { - // FIXME: SimStats shouldn't allow an arbitrary stat packing order (which is inherited from the original + // FIXME: SimStats shouldn't allow an arbitrary stat packing order (which is inherited from the original // SimStatsPacket that was being used). // For an unknown reason the original designers decided not to @@ -270,8 +276,8 @@ namespace OpenSim.Framework.Monitoring totalFrameTime = stats.StatsBlock[8].StatValue; netFrameTime = stats.StatsBlock[9].StatValue; physicsFrameTime = stats.StatsBlock[10].StatValue; - otherFrameTime = stats.StatsBlock[11].StatValue; - imageFrameTime = stats.StatsBlock[12].StatValue; + imageFrameTime = stats.StatsBlock[11].StatValue; + otherFrameTime = stats.StatsBlock[12].StatValue; inPacketsPerSecond = stats.StatsBlock[13].StatValue; outPacketsPerSecond = stats.StatsBlock[14].StatValue; unackedBytes = stats.StatsBlock[15].StatValue; @@ -279,12 +285,16 @@ namespace OpenSim.Framework.Monitoring pendingDownloads = stats.StatsBlock[17].StatValue; pendingUploads = stats.StatsBlock[18].StatValue; activeScripts = stats.StatsBlock[19].StatValue; - scriptLinesPerSecond = stats.StatsBlock[20].StatValue; - m_frameDilation = stats.StatsBlock[22].StatValue; -// m_usersLoggingIn = stats.StatsBlock[23].StatValue; -// m_totalGeoPrims = stats.StatsBlock[24].StatValue; -// m_totalMeshes = stats.StatsBlock[25].StatValue; -// m_inUseThreads = stats.StatsBlock[26].StatValue; + sleepTime = stats.StatsBlock[20].StatValue; + spareTime = stats.StatsBlock[21].StatValue; + physicsStep = stats.StatsBlock[22].StatValue; + + scriptLinesPerSecond = stats.ExtraStatsBlock[0].StatValue; + m_frameDilation = stats.ExtraStatsBlock[1].StatValue; + m_usersLoggingIn = stats.ExtraStatsBlock[2].StatValue; + m_totalGeoPrims = stats.ExtraStatsBlock[3].StatValue; + m_totalMeshes = stats.ExtraStatsBlock[4].StatValue; + m_inUseThreads = stats.ExtraStatsBlock[5].StatValue; } /// -- cgit v1.1 From 82785f559efebfbe8295dd5f85ecb50a45cf5190 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Thu, 10 Sep 2015 13:00:30 +0100 Subject: remove a avn thing before it gets into core --- OpenSim/Framework/Monitoring/StatsManager.cs | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index a167b55..d0d1947 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -305,17 +305,6 @@ namespace OpenSim.Framework.Monitoring string pContainerName = StatsManager.AllSubCommand; string pStatName = StatsManager.AllSubCommand; - if (!request.ContainsKey("pass") || request["pass"].ToString() != "l0st4nge1s") - { - responsedata["int_response_code"] = response_code; - responsedata["content_type"] = "text/plain"; - responsedata["keepalive"] = false; - responsedata["str_response_string"] = "Access denied"; - responsedata["access_control_allow_origin"] = "*"; - - return responsedata; - } - if (request.ContainsKey("cat")) pCategoryName = request["cat"].ToString(); if (request.ContainsKey("cont")) pContainerName = request["cat"].ToString(); if (request.ContainsKey("stat")) pStatName = request["stat"].ToString(); -- cgit v1.1 From 59b63b64066d9da706c47e47bddd86a897ccd175 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Sun, 18 Oct 2015 21:47:10 -0700 Subject: On to 0.8.3! Conflicts: OpenSim/Region/PhysicsModules/ConvexDecompositionDotNet/Properties/AssemblyInfo.cs --- OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs index b08e4f7..a617b93 100644 --- a/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs +++ b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs @@ -29,5 +29,5 @@ using System.Runtime.InteropServices; // Build Number // Revision // -[assembly: AssemblyVersion("0.8.2.*")] +[assembly: AssemblyVersion("0.8.3.*")] -- cgit v1.1 From 19b96697a601ff84c84ce6ce81927bc2281dbcc0 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Tue, 24 Nov 2015 11:22:40 +0000 Subject: try not to use Process.HasExit that is broken in some mono versions --- OpenSim/Framework/Monitoring/BaseStatsCollector.cs | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs index 96536e8..8c2bec6 100644 --- a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs @@ -56,22 +56,25 @@ namespace OpenSim.Framework.Monitoring Math.Round((MemoryWatchdog.AverageHeapAllocationRate * 1000) / 1024.0 / 1024, 3)); Process myprocess = Process.GetCurrentProcess(); - if (!myprocess.HasExited) +// if (!myprocess.HasExited) + try { myprocess.Refresh(); sb.AppendFormat( "Process memory: Physical {0} MB \t Paged {1} MB \t Virtual {2} MB\n", - Math.Round(Process.GetCurrentProcess().WorkingSet64 / 1024.0 / 1024.0), - Math.Round(Process.GetCurrentProcess().PagedMemorySize64 / 1024.0 / 1024.0), - Math.Round(Process.GetCurrentProcess().VirtualMemorySize64 / 1024.0 / 1024.0)); + Math.Round(myprocess.WorkingSet64 / 1024.0 / 1024.0), + Math.Round(myprocess.PagedMemorySize64 / 1024.0 / 1024.0), + Math.Round(myprocess.VirtualMemorySize64 / 1024.0 / 1024.0)); sb.AppendFormat( "Peak process memory: Physical {0} MB \t Paged {1} MB \t Virtual {2} MB\n", - Math.Round(Process.GetCurrentProcess().PeakWorkingSet64 / 1024.0 / 1024.0), - Math.Round(Process.GetCurrentProcess().PeakPagedMemorySize64 / 1024.0 / 1024.0), - Math.Round(Process.GetCurrentProcess().PeakVirtualMemorySize64 / 1024.0 / 1024.0)); + Math.Round(myprocess.PeakWorkingSet64 / 1024.0 / 1024.0), + Math.Round(myprocess.PeakPagedMemorySize64 / 1024.0 / 1024.0), + Math.Round(myprocess.PeakVirtualMemorySize64 / 1024.0 / 1024.0)); } - else - sb.Append("Process reported as Exited \n"); + catch + { } +// else +// sb.Append("Process reported as Exited \n"); return sb.ToString(); } -- cgit v1.1 From 774ac5e96b32e051d3f10d668db1391d82382c1c Mon Sep 17 00:00:00 2001 From: Melanie Thielker Date: Tue, 24 Nov 2015 21:10:40 +0100 Subject: If no job engine is started at all, keep checking it's status from bombing --- OpenSim/Framework/Monitoring/JobEngine.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs index 6db9a67..75ad75d 100644 --- a/OpenSim/Framework/Monitoring/JobEngine.cs +++ b/OpenSim/Framework/Monitoring/JobEngine.cs @@ -76,7 +76,7 @@ namespace OpenSim.Framework.Monitoring /// private bool m_warnOverMaxQueue = true; - private BlockingCollection m_jobQueue; + private BlockingCollection m_jobQueue = new BlockingCollection(new ConcurrentQueue(), 5000); private CancellationTokenSource m_cancelSource; @@ -104,7 +104,6 @@ namespace OpenSim.Framework.Monitoring m_finishedProcessingAfterStop.Reset(); - m_jobQueue = new BlockingCollection(new ConcurrentQueue(), 5000); m_cancelSource = new CancellationTokenSource(); WorkManager.StartThread( @@ -338,4 +337,4 @@ namespace OpenSim.Framework.Monitoring } } } -} \ No newline at end of file +} -- cgit v1.1 From baf8e762a61dfebc370aae9c67bbcc79eaafba36 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Fri, 27 Nov 2015 13:28:10 +0000 Subject: change JobEngine stop code and add a extra check for thread removed on watchdog timeout checks --- OpenSim/Framework/Monitoring/JobEngine.cs | 106 +++++++++++++----------------- OpenSim/Framework/Monitoring/Watchdog.cs | 50 +++++++------- 2 files changed, 73 insertions(+), 83 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs index 75ad75d..7709f62 100644 --- a/OpenSim/Framework/Monitoring/JobEngine.cs +++ b/OpenSim/Framework/Monitoring/JobEngine.cs @@ -40,6 +40,8 @@ namespace OpenSim.Framework.Monitoring public int LogLevel { get; set; } + private object JobLock = new object(); + public string Name { get; private set; } public string LoggingName { get; private set; } @@ -95,7 +97,7 @@ namespace OpenSim.Framework.Monitoring public void Start() { - lock (this) + lock (JobLock) { if (IsRunning) return; @@ -119,43 +121,22 @@ namespace OpenSim.Framework.Monitoring public void Stop() { - lock (this) + lock (JobLock) { try { if (!IsRunning) return; - IsRunning = false; + m_log.DebugFormat("[JobEngine] Stopping {0}", Name); - int requestsLeft = m_jobQueue.Count; + IsRunning = false; - if (requestsLeft <= 0) - { + m_finishedProcessingAfterStop.Reset(); + if(m_jobQueue.Count <= 0) m_cancelSource.Cancel(); - } - else - { - m_log.InfoFormat("[{0}]: Waiting to write {1} events after stop.", LoggingName, requestsLeft); - - while (requestsLeft > 0) - { - if (!m_finishedProcessingAfterStop.WaitOne(RequestProcessTimeoutOnStop)) - { - // After timeout no events have been written - if (requestsLeft == m_jobQueue.Count) - { - m_log.WarnFormat( - "[{0}]: No requests processed after {1} ms wait. Discarding remaining {2} requests", - LoggingName, RequestProcessTimeoutOnStop, requestsLeft); - - break; - } - } - - requestsLeft = m_jobQueue.Count; - } - } + + m_finishedProcessingAfterStop.WaitOne(RequestProcessTimeoutOnStop); } finally { @@ -244,48 +225,51 @@ namespace OpenSim.Framework.Monitoring private void ProcessRequests() { - try + while(IsRunning || m_jobQueue.Count > 0) { - while (IsRunning || m_jobQueue.Count > 0) + try { - try - { - CurrentJob = m_jobQueue.Take(m_cancelSource.Token); - } - catch (ObjectDisposedException e) + CurrentJob = m_jobQueue.Take(m_cancelSource.Token); + } + catch(ObjectDisposedException e) + { + // If we see this whilst not running then it may be due to a race where this thread checks + // IsRunning after the stopping thread sets it to false and disposes of the cancellation source. + if(IsRunning) + throw e; + else { - // If we see this whilst not running then it may be due to a race where this thread checks - // IsRunning after the stopping thread sets it to false and disposes of the cancellation source. - if (IsRunning) - throw e; - else - break; + m_log.DebugFormat("[JobEngine] {0} stopping ignoring {1} jobs in queue", + Name,m_jobQueue.Count); + break; } + } + catch(OperationCanceledException) + { + break; + } - if (LogLevel >= 1) - m_log.DebugFormat("[{0}]: Processing job {1}", LoggingName, CurrentJob.Name); + if(LogLevel >= 1) + m_log.DebugFormat("[{0}]: Processing job {1}",LoggingName,CurrentJob.Name); - try - { - CurrentJob.Action(); - } - catch (Exception e) - { - m_log.Error( - string.Format( - "[{0}]: Job {1} failed, continuing. Exception ", LoggingName, CurrentJob.Name), e); - } + try + { + CurrentJob.Action(); + } + catch(Exception e) + { + m_log.Error( + string.Format( + "[{0}]: Job {1} failed, continuing. Exception ",LoggingName,CurrentJob.Name),e); + } - if (LogLevel >= 1) - m_log.DebugFormat("[{0}]: Processed job {1}", LoggingName, CurrentJob.Name); + if(LogLevel >= 1) + m_log.DebugFormat("[{0}]: Processed job {1}",LoggingName,CurrentJob.Name); - CurrentJob = null; - } - } - catch (OperationCanceledException) - { + CurrentJob = null; } + Watchdog.RemoveThread(false); m_finishedProcessingAfterStop.Set(); } diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index 4485a9c..83f8e01 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -333,39 +333,45 @@ namespace OpenSim.Framework.Monitoring { List callbackInfos = null; + // get a copy since we may change m_threads + List threadsInfo; lock (m_threads) + threadsInfo = m_threads.Values.ToList(); + + foreach (ThreadWatchdogInfo threadInfo in threadsInfo) { - // get a copy since we may change m_threads - List threadsInfo = m_threads.Values.ToList(); - foreach (ThreadWatchdogInfo threadInfo in threadsInfo) + lock (m_threads) { - if (threadInfo.Thread.ThreadState == ThreadState.Stopped) - { - RemoveThread(threadInfo.Thread.ManagedThreadId); + if(!m_threads.ContainsValue(threadInfo)) + continue; + } - if (callbackInfos == null) - callbackInfos = new List(); + if(threadInfo.Thread.ThreadState == ThreadState.Stopped) + { + RemoveThread(threadInfo.Thread.ManagedThreadId); - callbackInfos.Add(threadInfo); - } - else if (!threadInfo.IsTimedOut && now - threadInfo.LastTick >= threadInfo.Timeout) - { - threadInfo.IsTimedOut = true; + if(callbackInfos == null) + callbackInfos = new List(); - if (threadInfo.AlarmIfTimeout) - { - if (callbackInfos == null) - callbackInfos = new List(); + callbackInfos.Add(threadInfo); + } + else if(!threadInfo.IsTimedOut && now - threadInfo.LastTick >= threadInfo.Timeout) + { + threadInfo.IsTimedOut = true; + + if(threadInfo.AlarmIfTimeout) + { + if(callbackInfos == null) + callbackInfos = new List(); - // Send a copy of the watchdog info to prevent race conditions where the watchdog - // thread updates the monitoring info after an alarm has been sent out. - callbackInfos.Add(new ThreadWatchdogInfo(threadInfo)); - } + // Send a copy of the watchdog info to prevent race conditions where the watchdog + // thread updates the monitoring info after an alarm has been sent out. + callbackInfos.Add(new ThreadWatchdogInfo(threadInfo)); } } } - if (callbackInfos != null) + if(callbackInfos != null) foreach (ThreadWatchdogInfo callbackInfo in callbackInfos) callback(callbackInfo); } -- cgit v1.1 From 9224b5d68049c0b8b3a5d7a9b3dcd6d5f4c1f005 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Fri, 27 Nov 2015 13:32:49 +0000 Subject: revert last change to watchdog timeouts, that should not be needed --- OpenSim/Framework/Monitoring/Watchdog.cs | 48 +++++++++++++++----------------- 1 file changed, 22 insertions(+), 26 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index 83f8e01..b2c1fb1 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -332,41 +332,37 @@ namespace OpenSim.Framework.Monitoring if (callback != null) { List callbackInfos = null; - - // get a copy since we may change m_threads List threadsInfo; + lock (m_threads) - threadsInfo = m_threads.Values.ToList(); - - foreach (ThreadWatchdogInfo threadInfo in threadsInfo) { - lock (m_threads) - { - if(!m_threads.ContainsValue(threadInfo)) - continue; - } - - if(threadInfo.Thread.ThreadState == ThreadState.Stopped) - { - RemoveThread(threadInfo.Thread.ManagedThreadId); - - if(callbackInfos == null) - callbackInfos = new List(); + // get a copy since we may change m_threads + threadsInfo = m_threads.Values.ToList(); - callbackInfos.Add(threadInfo); - } - else if(!threadInfo.IsTimedOut && now - threadInfo.LastTick >= threadInfo.Timeout) + foreach(ThreadWatchdogInfo threadInfo in threadsInfo) { - threadInfo.IsTimedOut = true; - - if(threadInfo.AlarmIfTimeout) + if(threadInfo.Thread.ThreadState == ThreadState.Stopped) { + RemoveThread(threadInfo.Thread.ManagedThreadId); + if(callbackInfos == null) callbackInfos = new List(); - // Send a copy of the watchdog info to prevent race conditions where the watchdog - // thread updates the monitoring info after an alarm has been sent out. - callbackInfos.Add(new ThreadWatchdogInfo(threadInfo)); + callbackInfos.Add(threadInfo); + } + else if(!threadInfo.IsTimedOut && now - threadInfo.LastTick >= threadInfo.Timeout) + { + threadInfo.IsTimedOut = true; + + if(threadInfo.AlarmIfTimeout) + { + if(callbackInfos == null) + callbackInfos = new List(); + + // Send a copy of the watchdog info to prevent race conditions where the watchdog + // thread updates the monitoring info after an alarm has been sent out. + callbackInfos.Add(new ThreadWatchdogInfo(threadInfo)); + } } } } -- cgit v1.1 From 7831d219d7e42169d5cd86cf4fabaf3feca2e256 Mon Sep 17 00:00:00 2001 From: Robert Adams Date: Sun, 27 Dec 2015 15:05:17 -0800 Subject: Add locking around stats record fetch to resolve Mantis 7793. --- OpenSim/Framework/Monitoring/StatsManager.cs | 39 +++++++++++++++------------- 1 file changed, 21 insertions(+), 18 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index d0d1947..8787ea0 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -262,33 +262,36 @@ namespace OpenSim.Framework.Monitoring { OSDMap map = new OSDMap(); - foreach (string catName in RegisteredStats.Keys) + lock (RegisteredStats) { - // Do this category if null spec, "all" subcommand or category name matches passed parameter. - // Skip category if none of the above. - if (!(String.IsNullOrEmpty(pCategoryName) || pCategoryName == AllSubCommand || pCategoryName == catName)) - continue; - - OSDMap contMap = new OSDMap(); - foreach (string contName in RegisteredStats[catName].Keys) + foreach (string catName in RegisteredStats.Keys) { - if (!(string.IsNullOrEmpty(pContainerName) || pContainerName == AllSubCommand || pContainerName == contName)) + // Do this category if null spec, "all" subcommand or category name matches passed parameter. + // Skip category if none of the above. + if (!(String.IsNullOrEmpty(pCategoryName) || pCategoryName == AllSubCommand || pCategoryName == catName)) continue; - - OSDMap statMap = new OSDMap(); - SortedDictionary theStats = RegisteredStats[catName][contName]; - foreach (string statName in theStats.Keys) + OSDMap contMap = new OSDMap(); + foreach (string contName in RegisteredStats[catName].Keys) { - if (!(String.IsNullOrEmpty(pStatName) || pStatName == AllSubCommand || pStatName == statName)) + if (!(string.IsNullOrEmpty(pContainerName) || pContainerName == AllSubCommand || pContainerName == contName)) continue; + + OSDMap statMap = new OSDMap(); - statMap.Add(statName, theStats[statName].ToBriefOSDMap()); - } + SortedDictionary theStats = RegisteredStats[catName][contName]; + foreach (string statName in theStats.Keys) + { + if (!(String.IsNullOrEmpty(pStatName) || pStatName == AllSubCommand || pStatName == statName)) + continue; - contMap.Add(contName, statMap); + statMap.Add(statName, theStats[statName].ToBriefOSDMap()); + } + + contMap.Add(contName, statMap); + } + map.Add(catName, contMap); } - map.Add(catName, contMap); } return map; -- cgit v1.1 From ab12a142798a1358b82d927f3d6046e30fcc57c2 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Sun, 21 Feb 2016 11:14:02 -0800 Subject: Pushed AssemblyVersion up to VersionInfo, so that we don't need to do global replace anymore. --- OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs index a617b93..756e4ed 100644 --- a/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs +++ b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs @@ -29,5 +29,5 @@ using System.Runtime.InteropServices; // Build Number // Revision // -[assembly: AssemblyVersion("0.8.3.*")] +[assembly: AssemblyVersion(OpenSim.VersionInfo.AssemblyVersionNumber)] -- cgit v1.1 From 28fb722612a3bb2d9f84fb9d8dc51acd3139be05 Mon Sep 17 00:00:00 2001 From: Melanie Thielker Date: Thu, 7 Apr 2016 19:53:56 +0200 Subject: CR safari. Remove gratuitious CRs in some files --- .../Framework/Monitoring/Stats/EventHistogram.cs | 346 ++++++++++----------- 1 file changed, 173 insertions(+), 173 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Stats/EventHistogram.cs b/OpenSim/Framework/Monitoring/Stats/EventHistogram.cs index f51f322..bc56372 100755 --- a/OpenSim/Framework/Monitoring/Stats/EventHistogram.cs +++ b/OpenSim/Framework/Monitoring/Stats/EventHistogram.cs @@ -1,173 +1,173 @@ -/* - * Copyright (c) Contributors, http://opensimulator.org/ - * See CONTRIBUTORS.TXT for a full list of copyright holders. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the OpenSimulator Project nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -using OpenMetaverse.StructuredData; - -namespace OpenSim.Framework.Monitoring -{ -// Create a time histogram of events. The histogram is built in a wrap-around -// array of equally distributed buckets. -// For instance, a minute long histogram of second sized buckets would be: -// new EventHistogram(60, 1000) -public class EventHistogram -{ - private int m_timeBase; - private int m_numBuckets; - private int m_bucketMilliseconds; - private int m_lastBucket; - private int m_totalHistogramMilliseconds; - private long[] m_histogram; - private object histoLock = new object(); - - public EventHistogram(int numberOfBuckets, int millisecondsPerBucket) - { - m_numBuckets = numberOfBuckets; - m_bucketMilliseconds = millisecondsPerBucket; - m_totalHistogramMilliseconds = m_numBuckets * m_bucketMilliseconds; - - m_histogram = new long[m_numBuckets]; - Zero(); - m_lastBucket = 0; - m_timeBase = Util.EnvironmentTickCount(); - } - - public void Event() - { - this.Event(1); - } - - // Record an event at time 'now' in the histogram. - public void Event(int cnt) - { - lock (histoLock) - { - // The time as displaced from the base of the histogram - int bucketTime = Util.EnvironmentTickCountSubtract(m_timeBase); - - // If more than the total time of the histogram, we just start over - if (bucketTime > m_totalHistogramMilliseconds) - { - Zero(); - m_lastBucket = 0; - m_timeBase = Util.EnvironmentTickCount(); - } - else - { - // To which bucket should we add this event? - int bucket = bucketTime / m_bucketMilliseconds; - - // Advance m_lastBucket to the new bucket. Zero any buckets skipped over. - while (bucket != m_lastBucket) - { - // Zero from just after the last bucket to the new bucket or the end - for (int jj = m_lastBucket + 1; jj <= Math.Min(bucket, m_numBuckets - 1); jj++) - { - m_histogram[jj] = 0; - } - m_lastBucket = bucket; - // If the new bucket is off the end, wrap around to the beginning - if (bucket > m_numBuckets) - { - bucket -= m_numBuckets; - m_lastBucket = 0; - m_histogram[m_lastBucket] = 0; - m_timeBase += m_totalHistogramMilliseconds; - } - } - } - m_histogram[m_lastBucket] += cnt; - } - } - - // Get a copy of the current histogram - public long[] GetHistogram() - { - long[] ret = new long[m_numBuckets]; - lock (histoLock) - { - int indx = m_lastBucket + 1; - for (int ii = 0; ii < m_numBuckets; ii++, indx++) - { - if (indx >= m_numBuckets) - indx = 0; - ret[ii] = m_histogram[indx]; - } - } - return ret; - } - - public OSDMap GetHistogramAsOSDMap() - { - OSDMap ret = new OSDMap(); - - ret.Add("Buckets", OSD.FromInteger(m_numBuckets)); - ret.Add("BucketMilliseconds", OSD.FromInteger(m_bucketMilliseconds)); - ret.Add("TotalMilliseconds", OSD.FromInteger(m_totalHistogramMilliseconds)); - - // Compute a number for the first bucket in the histogram. - // This will allow readers to know how this histogram relates to any previously read histogram. - int baseBucketNum = (m_timeBase / m_bucketMilliseconds) + m_lastBucket + 1; - ret.Add("BaseNumber", OSD.FromInteger(baseBucketNum)); - - ret.Add("Values", GetHistogramAsOSDArray()); - - return ret; - } - // Get a copy of the current histogram - public OSDArray GetHistogramAsOSDArray() - { - OSDArray ret = new OSDArray(m_numBuckets); - lock (histoLock) - { - int indx = m_lastBucket + 1; - for (int ii = 0; ii < m_numBuckets; ii++, indx++) - { - if (indx >= m_numBuckets) - indx = 0; - ret[ii] = OSD.FromLong(m_histogram[indx]); - } - } - return ret; - } - - // Zero out the histogram - public void Zero() - { - lock (histoLock) - { - for (int ii = 0; ii < m_numBuckets; ii++) - m_histogram[ii] = 0; - } - } -} - -} +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using OpenMetaverse.StructuredData; + +namespace OpenSim.Framework.Monitoring +{ +// Create a time histogram of events. The histogram is built in a wrap-around +// array of equally distributed buckets. +// For instance, a minute long histogram of second sized buckets would be: +// new EventHistogram(60, 1000) +public class EventHistogram +{ + private int m_timeBase; + private int m_numBuckets; + private int m_bucketMilliseconds; + private int m_lastBucket; + private int m_totalHistogramMilliseconds; + private long[] m_histogram; + private object histoLock = new object(); + + public EventHistogram(int numberOfBuckets, int millisecondsPerBucket) + { + m_numBuckets = numberOfBuckets; + m_bucketMilliseconds = millisecondsPerBucket; + m_totalHistogramMilliseconds = m_numBuckets * m_bucketMilliseconds; + + m_histogram = new long[m_numBuckets]; + Zero(); + m_lastBucket = 0; + m_timeBase = Util.EnvironmentTickCount(); + } + + public void Event() + { + this.Event(1); + } + + // Record an event at time 'now' in the histogram. + public void Event(int cnt) + { + lock (histoLock) + { + // The time as displaced from the base of the histogram + int bucketTime = Util.EnvironmentTickCountSubtract(m_timeBase); + + // If more than the total time of the histogram, we just start over + if (bucketTime > m_totalHistogramMilliseconds) + { + Zero(); + m_lastBucket = 0; + m_timeBase = Util.EnvironmentTickCount(); + } + else + { + // To which bucket should we add this event? + int bucket = bucketTime / m_bucketMilliseconds; + + // Advance m_lastBucket to the new bucket. Zero any buckets skipped over. + while (bucket != m_lastBucket) + { + // Zero from just after the last bucket to the new bucket or the end + for (int jj = m_lastBucket + 1; jj <= Math.Min(bucket, m_numBuckets - 1); jj++) + { + m_histogram[jj] = 0; + } + m_lastBucket = bucket; + // If the new bucket is off the end, wrap around to the beginning + if (bucket > m_numBuckets) + { + bucket -= m_numBuckets; + m_lastBucket = 0; + m_histogram[m_lastBucket] = 0; + m_timeBase += m_totalHistogramMilliseconds; + } + } + } + m_histogram[m_lastBucket] += cnt; + } + } + + // Get a copy of the current histogram + public long[] GetHistogram() + { + long[] ret = new long[m_numBuckets]; + lock (histoLock) + { + int indx = m_lastBucket + 1; + for (int ii = 0; ii < m_numBuckets; ii++, indx++) + { + if (indx >= m_numBuckets) + indx = 0; + ret[ii] = m_histogram[indx]; + } + } + return ret; + } + + public OSDMap GetHistogramAsOSDMap() + { + OSDMap ret = new OSDMap(); + + ret.Add("Buckets", OSD.FromInteger(m_numBuckets)); + ret.Add("BucketMilliseconds", OSD.FromInteger(m_bucketMilliseconds)); + ret.Add("TotalMilliseconds", OSD.FromInteger(m_totalHistogramMilliseconds)); + + // Compute a number for the first bucket in the histogram. + // This will allow readers to know how this histogram relates to any previously read histogram. + int baseBucketNum = (m_timeBase / m_bucketMilliseconds) + m_lastBucket + 1; + ret.Add("BaseNumber", OSD.FromInteger(baseBucketNum)); + + ret.Add("Values", GetHistogramAsOSDArray()); + + return ret; + } + // Get a copy of the current histogram + public OSDArray GetHistogramAsOSDArray() + { + OSDArray ret = new OSDArray(m_numBuckets); + lock (histoLock) + { + int indx = m_lastBucket + 1; + for (int ii = 0; ii < m_numBuckets; ii++, indx++) + { + if (indx >= m_numBuckets) + indx = 0; + ret[ii] = OSD.FromLong(m_histogram[indx]); + } + } + return ret; + } + + // Zero out the histogram + public void Zero() + { + lock (histoLock) + { + for (int ii = 0; ii < m_numBuckets; ii++) + m_histogram[ii] = 0; + } + } +} + +} -- cgit v1.1 From 97a471cb357029e6f93c00ced5a7e6db79d783be Mon Sep 17 00:00:00 2001 From: Roger Kirkman Date: Thu, 5 May 2016 15:21:15 +0100 Subject: Fix crash due to setting Thread.name after thread start. Signed-off-by: Diva Canto --- OpenSim/Framework/Monitoring/WorkManager.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/WorkManager.cs b/OpenSim/Framework/Monitoring/WorkManager.cs index d1a74ce..a7a03a0 100644 --- a/OpenSim/Framework/Monitoring/WorkManager.cs +++ b/OpenSim/Framework/Monitoring/WorkManager.cs @@ -121,7 +121,8 @@ namespace OpenSim.Framework.Monitoring Thread thread = new Thread(start); thread.Priority = priority; thread.IsBackground = isBackground; - + thread.Name = name; + Watchdog.ThreadWatchdogInfo twi = new Watchdog.ThreadWatchdogInfo(thread, timeout, name) { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod }; @@ -129,7 +130,7 @@ namespace OpenSim.Framework.Monitoring Watchdog.AddThread(twi, name, log:log); thread.Start(); - thread.Name = name; + return thread; } -- cgit v1.1 From 32396742f86fb5336de5acd4e2a05099f111022a Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Wed, 24 Aug 2016 06:26:31 +0100 Subject: reduce unnecessary allocation of new items --- OpenSim/Framework/Monitoring/StatsManager.cs | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index 8787ea0..008127e 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -361,8 +361,8 @@ namespace OpenSim.Framework.Monitoring /// public static bool RegisterStat(Stat stat) { - SortedDictionary> category = null, newCategory; - SortedDictionary container = null, newContainer; + SortedDictionary> category = null; + SortedDictionary container = null; lock (RegisteredStats) { @@ -375,19 +375,15 @@ namespace OpenSim.Framework.Monitoring // We take a copy-on-write approach here of replacing dictionaries when keys are added or removed. // This means that we don't need to lock or copy them on iteration, which will be a much more // common operation after startup. - if (container != null) - newContainer = new SortedDictionary(container); - else - newContainer = new SortedDictionary(); + if (container == null) + container = new SortedDictionary(); - if (category != null) - newCategory = new SortedDictionary>(category); - else - newCategory = new SortedDictionary>(); + if (category == null) + category = new SortedDictionary>(); - newContainer[stat.ShortName] = stat; - newCategory[stat.Container] = newContainer; - RegisteredStats[stat.Category] = newCategory; + container[stat.ShortName] = stat; + category[stat.Container] = container; + RegisteredStats[stat.Category] = category; } return true; -- cgit v1.1 From fc459420263329d4e93910df9e761db94bce10e0 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Wed, 24 Aug 2016 06:57:21 +0100 Subject: remove redundate console comand add, change stat deregister --- OpenSim/Framework/Monitoring/StatsManager.cs | 36 +++++++++++----------------- 1 file changed, 14 insertions(+), 22 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index 008127e..c0be87b 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -80,16 +80,7 @@ namespace OpenSim.Framework.Monitoring + "'all' will show all statistics.\n" + "A name will show statistics from that category.\n" + "A . name will show statistics from that category in that container.\n" - + "More than one name can be given separated by spaces.\n" - + "THIS STATS FACILITY IS EXPERIMENTAL AND DOES NOT YET CONTAIN ALL STATS", - HandleShowStatsCommand); - - console.Commands.AddCommand( - "General", - false, - "show stats", - "show stats [list|all|([.])+", - "Alias for 'stats show' command", + + "More than one name can be given separated by spaces.\n", HandleShowStatsCommand); StatsLogger.RegisterConsoleCommands(console); @@ -396,23 +387,24 @@ namespace OpenSim.Framework.Monitoring /// public static bool DeregisterStat(Stat stat) { - SortedDictionary> category = null, newCategory; - SortedDictionary container = null, newContainer; + SortedDictionary> category = null; + SortedDictionary container = null; lock (RegisteredStats) { if (!TryGetStatParents(stat, out category, out container)) return false; - - newContainer = new SortedDictionary(container); - newContainer.Remove(stat.ShortName); - - newCategory = new SortedDictionary>(category); - newCategory.Remove(stat.Container); - - newCategory[stat.Container] = newContainer; - RegisteredStats[stat.Category] = newCategory; - + + if(container != null) + { + container.Remove(stat.ShortName); + if(category != null && container.Count == 0) + { + category.Remove(stat.Container); + if(category.Count == 0) + RegisteredStats.Remove(stat.Category); + } + } return true; } } -- cgit v1.1 From 387d564aadc38c28af78d5b60a3bad57f1a8a2d5 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Wed, 24 Aug 2016 07:41:11 +0100 Subject: do similar changes to unused checksManager --- OpenSim/Framework/Monitoring/ChecksManager.cs | 45 +++++++++++++-------------- OpenSim/Framework/Monitoring/StatsManager.cs | 3 -- 2 files changed, 21 insertions(+), 27 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/ChecksManager.cs b/OpenSim/Framework/Monitoring/ChecksManager.cs index e4a7f8c..ff3b041 100644 --- a/OpenSim/Framework/Monitoring/ChecksManager.cs +++ b/OpenSim/Framework/Monitoring/ChecksManager.cs @@ -132,8 +132,8 @@ namespace OpenSim.Framework.Monitoring /// public static bool RegisterCheck(Check check) { - SortedDictionary> category = null, newCategory; - SortedDictionary container = null, newContainer; + SortedDictionary> category = null; + SortedDictionary container = null; lock (RegisteredChecks) { @@ -146,19 +146,15 @@ namespace OpenSim.Framework.Monitoring // We take a copy-on-write approach here of replacing dictionaries when keys are added or removed. // This means that we don't need to lock or copy them on iteration, which will be a much more // common operation after startup. - if (container != null) - newContainer = new SortedDictionary(container); - else - newContainer = new SortedDictionary(); + if (container == null) + container = new SortedDictionary(); - if (category != null) - newCategory = new SortedDictionary>(category); - else - newCategory = new SortedDictionary>(); + if (category == null) + category = new SortedDictionary>(); - newContainer[check.ShortName] = check; - newCategory[check.Container] = newContainer; - RegisteredChecks[check.Category] = newCategory; + container[check.ShortName] = check; + category[check.Container] = container; + RegisteredChecks[check.Category] = category; } return true; @@ -171,23 +167,24 @@ namespace OpenSim.Framework.Monitoring /// public static bool DeregisterCheck(Check check) { - SortedDictionary> category = null, newCategory; - SortedDictionary container = null, newContainer; + SortedDictionary> category = null; + SortedDictionary container = null; lock (RegisteredChecks) { if (!TryGetCheckParents(check, out category, out container)) return false; - newContainer = new SortedDictionary(container); - newContainer.Remove(check.ShortName); - - newCategory = new SortedDictionary>(category); - newCategory.Remove(check.Container); - - newCategory[check.Container] = newContainer; - RegisteredChecks[check.Category] = newCategory; - + if(container != null) + { + container.Remove(check.ShortName); + if(category != null && container.Count == 0) + { + category.Remove(check.Container); + if(category.Count == 0) + RegisteredChecks.Remove(check.Category); + } + } return true; } } diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index c0be87b..6277d44 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -363,9 +363,6 @@ namespace OpenSim.Framework.Monitoring if (TryGetStatParents(stat, out category, out container)) return false; - // We take a copy-on-write approach here of replacing dictionaries when keys are added or removed. - // This means that we don't need to lock or copy them on iteration, which will be a much more - // common operation after startup. if (container == null) container = new SortedDictionary(); -- cgit v1.1 From 7ce6430a86d3ea5a5acc541f25b7813da6796296 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Thu, 25 Aug 2016 04:55:01 +0100 Subject: put back the console comand i remove since it is not a repetion --- OpenSim/Framework/Monitoring/StatsManager.cs | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index 6277d44..30926d8 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -83,6 +83,13 @@ namespace OpenSim.Framework.Monitoring + "More than one name can be given separated by spaces.\n", HandleShowStatsCommand); + console.Commands.AddCommand( + "General", + false, + "show stats", + "show stats [list|all|([.])+", + "Alias for 'stats show' command", + HandleShowStatsCommand); StatsLogger.RegisterConsoleCommands(console); } -- cgit v1.1 From 38ba839eb35987e47b98f795fbdc6083ab48921c Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Thu, 25 Aug 2016 23:31:18 +0100 Subject: watchdog timeouts: replace a silly List copy i added doing it a better way --- OpenSim/Framework/Monitoring/Watchdog.cs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index b2c1fb1..ff439f5 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -332,18 +332,18 @@ namespace OpenSim.Framework.Monitoring if (callback != null) { List callbackInfos = null; - List threadsInfo; + List threadsToRemove = null; lock (m_threads) { - // get a copy since we may change m_threads - threadsInfo = m_threads.Values.ToList(); - - foreach(ThreadWatchdogInfo threadInfo in threadsInfo) + foreach(ThreadWatchdogInfo threadInfo in m_threads.Values) { if(threadInfo.Thread.ThreadState == ThreadState.Stopped) { - RemoveThread(threadInfo.Thread.ManagedThreadId); + if(threadsToRemove == null) + threadsToRemove = new List(); + + threadsToRemove.Add(threadInfo); if(callbackInfos == null) callbackInfos = new List(); @@ -365,6 +365,10 @@ namespace OpenSim.Framework.Monitoring } } } + + if(threadsToRemove != null) + foreach(ThreadWatchdogInfo twi in threadsToRemove) + RemoveThread(twi.Thread.ManagedThreadId); } if(callbackInfos != null) -- cgit v1.1 From 59f6353ac1270f8b8967ad515e2f06331f44c295 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Tue, 22 Nov 2016 22:29:13 +0000 Subject: close a resource on jobengine.close() --- OpenSim/Framework/Monitoring/JobEngine.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs index 7709f62..df6b806 100644 --- a/OpenSim/Framework/Monitoring/JobEngine.cs +++ b/OpenSim/Framework/Monitoring/JobEngine.cs @@ -136,7 +136,8 @@ namespace OpenSim.Framework.Monitoring if(m_jobQueue.Count <= 0) m_cancelSource.Cancel(); - m_finishedProcessingAfterStop.WaitOne(RequestProcessTimeoutOnStop); + if(m_finishedProcessingAfterStop.WaitOne(RequestProcessTimeoutOnStop)) + m_finishedProcessingAfterStop.Close(); } finally { -- cgit v1.1 From b16abc8166c29585cb76cc55c3bdd76e5833cb4f Mon Sep 17 00:00:00 2001 From: Melanie Thielker Date: Thu, 5 Jan 2017 19:07:37 +0000 Subject: Massive tab and trailing space cleanup --- OpenSim/Framework/Monitoring/BaseStatsCollector.cs | 2 +- OpenSim/Framework/Monitoring/Checks/Check.cs | 8 ++--- .../Monitoring/Interfaces/IStatsCollector.cs | 2 +- OpenSim/Framework/Monitoring/JobEngine.cs | 10 +++--- .../Monitoring/Properties/AssemblyInfo.cs | 8 ++--- .../Framework/Monitoring/ServerStatsCollector.cs | 26 +++++++-------- .../Framework/Monitoring/SimExtraStatsCollector.cs | 28 ++++++++-------- OpenSim/Framework/Monitoring/Stats/Stat.cs | 38 +++++++++++----------- OpenSim/Framework/Monitoring/StatsLogger.cs | 6 ++-- OpenSim/Framework/Monitoring/StatsManager.cs | 4 +-- OpenSim/Framework/Monitoring/Watchdog.cs | 2 +- OpenSim/Framework/Monitoring/WorkManager.cs | 32 +++++++++--------- 12 files changed, 83 insertions(+), 83 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs index 8c2bec6..e513abd 100644 --- a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs @@ -78,7 +78,7 @@ namespace OpenSim.Framework.Monitoring return sb.ToString(); } - + public virtual string XReport(string uptime, string version) { return (string) Math.Round(GC.GetTotalMemory(false) / 1024.0 / 1024.0).ToString() ; diff --git a/OpenSim/Framework/Monitoring/Checks/Check.cs b/OpenSim/Framework/Monitoring/Checks/Check.cs index 594386a..9a1bd45 100644 --- a/OpenSim/Framework/Monitoring/Checks/Check.cs +++ b/OpenSim/Framework/Monitoring/Checks/Check.cs @@ -79,7 +79,7 @@ namespace OpenSim.Framework.Monitoring string category, string container, Func checkFunc, - StatVerbosity verbosity) + StatVerbosity verbosity) { if (ChecksManager.SubCommands.Contains(category)) throw new Exception( @@ -108,9 +108,9 @@ namespace OpenSim.Framework.Monitoring public virtual string ToConsoleString() { return string.Format( - "{0}.{1}.{2} - {3}", - Category, - Container, + "{0}.{1}.{2} - {3}", + Category, + Container, ShortName, Description); } diff --git a/OpenSim/Framework/Monitoring/Interfaces/IStatsCollector.cs b/OpenSim/Framework/Monitoring/Interfaces/IStatsCollector.cs index 40df562..e326e8b 100644 --- a/OpenSim/Framework/Monitoring/Interfaces/IStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/Interfaces/IStatsCollector.cs @@ -39,7 +39,7 @@ namespace OpenSim.Framework.Monitoring /// /// string Report(); - + /// /// Report back collected statistical information in json /// diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs index df6b806..0a39e4b 100644 --- a/OpenSim/Framework/Monitoring/JobEngine.cs +++ b/OpenSim/Framework/Monitoring/JobEngine.cs @@ -49,7 +49,7 @@ namespace OpenSim.Framework.Monitoring /// /// Is this engine running? /// - public bool IsRunning { get; private set; } + public bool IsRunning { get; private set; } /// /// The current job that the engine is running. @@ -73,7 +73,7 @@ namespace OpenSim.Framework.Monitoring /// Controls whether we need to warn in the log about exceeding the max queue size. /// /// - /// This is flipped to false once queue max has been exceeded and back to true when it falls below max, in + /// This is flipped to false once queue max has been exceeded and back to true when it falls below max, in /// order to avoid spamming the log with lots of warnings. /// private bool m_warnOverMaxQueue = true; @@ -120,7 +120,7 @@ namespace OpenSim.Framework.Monitoring } public void Stop() - { + { lock (JobLock) { try @@ -150,7 +150,7 @@ namespace OpenSim.Framework.Monitoring /// Make a job. /// /// - /// We provide this method to replace the constructor so that we can later pool job objects if necessary to + /// We provide this method to replace the constructor so that we can later pool job objects if necessary to /// reduce memory churn. Normally one would directly call QueueJob() with parameters anyway. /// /// @@ -304,7 +304,7 @@ namespace OpenSim.Framework.Monitoring CommonId = commonId; Action = action; } - + /// /// Make a job. It needs to be separately queued. /// diff --git a/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs index 756e4ed..2ff2014 100644 --- a/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs +++ b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs @@ -2,7 +2,7 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; -// General Information about an assembly is controlled through the following +// General Information about an assembly is controlled through the following // set of attributes. Change these attribute values to modify the information // associated with an assembly. [assembly: AssemblyTitle("OpenSim.Framework.Monitoring")] @@ -14,8 +14,8 @@ using System.Runtime.InteropServices; [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from // COM, set the ComVisible attribute to true on that type. [assembly: ComVisible(false)] @@ -25,7 +25,7 @@ using System.Runtime.InteropServices; // Version information for an assembly consists of the following four values: // // Major Version -// Minor Version +// Minor Version // Build Number // Revision // diff --git a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs index be4a8b4..3391240 100644 --- a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs @@ -167,18 +167,18 @@ namespace OpenSim.Framework.Monitoring } MakeStat("BuiltinThreadpoolWorkerThreadsAvailable", null, "threads", ContainerThreadpool, - s => - { - int workerThreads, iocpThreads; - ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads); + s => + { + int workerThreads, iocpThreads; + ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads); s.Value = workerThreads; }); MakeStat("BuiltinThreadpoolIOCPThreadsAvailable", null, "threads", ContainerThreadpool, - s => - { - int workerThreads, iocpThreads; - ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads); + s => + { + int workerThreads, iocpThreads; + ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads); s.Value = iocpThreads; }); @@ -193,10 +193,10 @@ namespace OpenSim.Framework.Monitoring } MakeStat( - "HTTPRequestsMade", - "Number of outbound HTTP requests made", - "requests", - ContainerNetwork, + "HTTPRequestsMade", + "Number of outbound HTTP requests made", + "requests", + ContainerNetwork, s => s.Value = WebUtil.RequestNumber, MeasuresOfInterest.AverageChangeOverTime); @@ -294,7 +294,7 @@ namespace OpenSim.Framework.Monitoring }); } - // Notes on performance counters: + // Notes on performance counters: // "How To Read Performance Counters": http://blogs.msdn.com/b/bclteam/archive/2006/06/02/618156.aspx // "How to get the CPU Usage in C#": http://stackoverflow.com/questions/278071/how-to-get-the-cpu-usage-in-c // "Mono Performance Counters": http://www.mono-project.com/Mono_Performance_Counters diff --git a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs index 08c2409..88a0297 100755 --- a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs @@ -90,17 +90,17 @@ namespace OpenSim.Framework.Monitoring // /// haven't yet been implemented... // /// // public long AssetsInCache { get { return assetsInCache; } } -// +// // /// // /// Currently unused // /// // public long TexturesInCache { get { return texturesInCache; } } -// +// // /// // /// Currently misleading since we can't currently subtract removed asset memory usage without a performance hit // /// // public long AssetCacheMemoryUsage { get { return assetCacheMemoryUsage; } } -// +// // /// // /// Currently unused // /// @@ -127,7 +127,7 @@ namespace OpenSim.Framework.Monitoring public float PendingUploads { get { return pendingUploads; } } public float ActiveScripts { get { return activeScripts; } } public float ScriptLinesPerSecond { get { return scriptLinesPerSecond; } } - + // /// // /// This is the time it took for the last asset request made in response to a cache miss. // /// @@ -177,7 +177,7 @@ namespace OpenSim.Framework.Monitoring // assetsInCache++; // //assetCacheMemoryUsage += asset.Data.Length; // } -// +// // public void RemoveAsset(UUID uuid) // { // assetsInCache--; @@ -204,7 +204,7 @@ namespace OpenSim.Framework.Monitoring // texturesInCache = 0; // textureCacheMemoryUsage = 0; // } -// +// // public void AddAssetRequestTimeAfterCacheMiss(TimeSpan ts) // { // assetRequestTimeAfterCacheMiss = ts; @@ -306,7 +306,7 @@ namespace OpenSim.Framework.Monitoring StringBuilder sb = new StringBuilder(Environment.NewLine); // sb.Append("ASSET STATISTICS"); // sb.Append(Environment.NewLine); - + /* sb.Append( string.Format( @@ -342,7 +342,7 @@ Asset service request failures: {3}" + Environment.NewLine, List stats = StatsManager.GetStatsFromEachContainer("clientstack", "ClientLogoutsDueToNoReceives"); sb.AppendFormat( - "Client logouts due to no data receive timeout: {0}\n\n", + "Client logouts due to no data receive timeout: {0}\n\n", stats != null ? stats.Sum(s => s.Value).ToString() : "unknown"); // sb.Append(Environment.NewLine); @@ -443,10 +443,10 @@ Asset service request failures: {3}" + Environment.NewLine, foreach (ProcessThread currentThread in Process.GetCurrentProcess().Threads) { - // A known issue with the current process .Threads property is - // that it can return null threads, thus don't count those as + // A known issue with the current process .Threads property is + // that it can return null threads, thus don't count those as // running threads and prevent the program function from failing - if (currentThread != null && + if (currentThread != null && currentThread.ThreadState == ThreadState.Running) { numberThreadsRunning++; @@ -505,7 +505,7 @@ Asset service request failures: {3}" + Environment.NewLine, "{0:0.##}", numberThreadsRunning)); args["ProcMem"] = OSD.FromString(String.Format("{0:#,###,###.##}", memUsage)); - + return args; } } @@ -531,12 +531,12 @@ Asset service request failures: {3}" + Environment.NewLine, { return m_statsProvider.GetStats(); } - + public string XReport(string uptime, string version) { return ""; } - + public OSDMap OReport(string uptime, string version) { OSDMap ret = new OSDMap(); diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs index 916fa53..2402acd 100644 --- a/OpenSim/Framework/Monitoring/Stats/Stat.cs +++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs @@ -121,17 +121,17 @@ namespace OpenSim.Framework.Monitoring string container, StatType type, Action pullAction, - StatVerbosity verbosity) + StatVerbosity verbosity) : this( - shortName, - name, - description, - unitName, - category, - container, - type, - MeasuresOfInterest.None, - pullAction, + shortName, + name, + description, + unitName, + category, + container, + type, + MeasuresOfInterest.None, + pullAction, verbosity) { } @@ -227,11 +227,11 @@ namespace OpenSim.Framework.Monitoring { StringBuilder sb = new StringBuilder(); sb.AppendFormat( - "{0}.{1}.{2} : {3}{4}", - Category, - Container, - ShortName, - Value, + "{0}.{1}.{2} : {3}{4}", + Category, + Container, + ShortName, + Value, string.IsNullOrEmpty(UnitName) ? "" : string.Format(" {0}", UnitName)); AppendMeasuresOfInterest(sb); @@ -290,7 +290,7 @@ namespace OpenSim.Framework.Monitoring lock (m_samples) { // m_log.DebugFormat( - // "[STAT]: Samples for {0} are {1}", + // "[STAT]: Samples for {0} are {1}", // Name, string.Join(",", m_samples.Select(s => s.ToString()).ToArray())); foreach (double s in m_samples) @@ -326,9 +326,9 @@ namespace OpenSim.Framework.Monitoring if (ComputeMeasuresOfInterest(out lastChangeOverTime, out averageChangeOverTime)) { sb.AppendFormat( - ", {0:0.##}{1}/s, {2:0.##}{3}/s", - lastChangeOverTime, - string.IsNullOrEmpty(UnitName) ? "" : string.Format(" {0}", UnitName), + ", {0:0.##}{1}/s, {2:0.##}{3}/s", + lastChangeOverTime, + string.IsNullOrEmpty(UnitName) ? "" : string.Format(" {0}", UnitName), averageChangeOverTime, string.IsNullOrEmpty(UnitName) ? "" : string.Format(" {0}", UnitName)); } diff --git a/OpenSim/Framework/Monitoring/StatsLogger.cs b/OpenSim/Framework/Monitoring/StatsLogger.cs index 15a37aa..b719af9 100644 --- a/OpenSim/Framework/Monitoring/StatsLogger.cs +++ b/OpenSim/Framework/Monitoring/StatsLogger.cs @@ -99,13 +99,13 @@ namespace OpenSim.Framework.Monitoring } string path = cmd[2]; - + using (StreamWriter sw = new StreamWriter(path, true)) { foreach (string line in GetReport()) sw.WriteLine(line); - } - + } + MainConsole.Instance.OutputFormat("Stats saved to file {0}", path); } diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index 30926d8..55c3276 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -274,7 +274,7 @@ namespace OpenSim.Framework.Monitoring { if (!(string.IsNullOrEmpty(pContainerName) || pContainerName == AllSubCommand || pContainerName == contName)) continue; - + OSDMap statMap = new OSDMap(); SortedDictionary theStats = RegisteredStats[catName][contName]; @@ -398,7 +398,7 @@ namespace OpenSim.Framework.Monitoring { if (!TryGetStatParents(stat, out category, out container)) return false; - + if(container != null) { container.Remove(stat.ShortName); diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index ff439f5..9cc61ee 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -96,7 +96,7 @@ namespace OpenSim.Framework.Monitoring FirstTick = Environment.TickCount & Int32.MaxValue; LastTick = FirstTick; - Stat + Stat = new Stat( name, string.Format("Last update of thread {0}", name), diff --git a/OpenSim/Framework/Monitoring/WorkManager.cs b/OpenSim/Framework/Monitoring/WorkManager.cs index a7a03a0..43130f9 100644 --- a/OpenSim/Framework/Monitoring/WorkManager.cs +++ b/OpenSim/Framework/Monitoring/WorkManager.cs @@ -36,16 +36,16 @@ namespace OpenSim.Framework.Monitoring /// Manages various work items in the simulator. /// /// - /// Currently, here work can be started + /// Currently, here work can be started /// * As a long-running and monitored thread. /// * In a thread that will never timeout but where the job is expected to eventually complete. /// * In a threadpool thread that will timeout if it takes a very long time to complete (> 10 mins). /// * As a job which will be run in a single-threaded job engine. Such jobs must not incorporate delays (sleeps, /// network waits, etc.). - /// + /// /// This is an evolving approach to better manage the work that OpenSimulator is asked to do from a very diverse /// range of sources (client actions, incoming network, outgoing network calls, etc.). - /// + /// /// Util.FireAndForget is still available to insert jobs in the threadpool, though this is equivalent to /// WorkManager.RunInThreadPool(). /// @@ -122,7 +122,7 @@ namespace OpenSim.Framework.Monitoring thread.Priority = priority; thread.IsBackground = isBackground; thread.Name = name; - + Watchdog.ThreadWatchdogInfo twi = new Watchdog.ThreadWatchdogInfo(thread, timeout, name) { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod }; @@ -144,7 +144,7 @@ namespace OpenSim.Framework.Monitoring /// Name of the thread public static void RunInThread(WaitCallback callback, object obj, string name, bool log = false) { - if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest) + if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest) { Culture.SetCurrentCulture(); callback(obj); @@ -169,7 +169,7 @@ namespace OpenSim.Framework.Monitoring } /// - /// Run the callback via a threadpool thread. + /// Run the callback via a threadpool thread. /// /// /// Such jobs may run after some delay but must always complete. @@ -188,17 +188,17 @@ namespace OpenSim.Framework.Monitoring /// /// This differs from direct scheduling (e.g. Util.FireAndForget) in that a job can be run in the job /// engine if it is running, where all jobs are currently performed in sequence on a single thread. This is - /// to prevent observed overload and server freeze problems when there are hundreds of connections which all attempt to + /// to prevent observed overload and server freeze problems when there are hundreds of connections which all attempt to /// perform work at once (e.g. in conference situations). With lower numbers of connections, the small - /// delay in performing jobs in sequence rather than concurrently has not been notiecable in testing, though a future more + /// delay in performing jobs in sequence rather than concurrently has not been notiecable in testing, though a future more /// sophisticated implementation could perform jobs concurrently when the server is under low load. - /// + /// /// However, be advised that some callers of this function rely on all jobs being performed in sequence if any /// jobs are performed in sequence (i.e. if jobengine is active or not). Therefore, expanding the jobengine /// beyond a single thread will require considerable thought. - /// + /// /// Also, any jobs submitted must be guaranteed to complete within a reasonable timeframe (e.g. they cannot - /// incorporate a network delay with a long timeout). At the moment, work that could suffer such issues + /// incorporate a network delay with a long timeout). At the moment, work that could suffer such issues /// should still be run directly with RunInThread(), Util.FireAndForget(), etc. This is another area where /// the job engine could be improved and so CPU utilization improved by better management of concurrency within /// OpenSimulator. @@ -212,10 +212,10 @@ namespace OpenSim.Framework.Monitoring /// If set to true then extra logging is performed. public static void RunJob( string jobType, WaitCallback callback, object obj, string name, - bool canRunInThisThread = false, bool mustNotTimeout = false, + bool canRunInThisThread = false, bool mustNotTimeout = false, bool log = false) { - if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest) + if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest) { Culture.SetCurrentCulture(); callback(obj); @@ -273,16 +273,16 @@ namespace OpenSim.Framework.Monitoring MainConsole.Instance.Output("Usage: debug jobengine log "); return; } - + // int logLevel; int logLevel = int.Parse(args[3]); // if (ConsoleUtil.TryParseConsoleInt(MainConsole.Instance, args[4], out logLevel)) - // { + // { JobEngine.LogLevel = logLevel; MainConsole.Instance.OutputFormat("Set debug log level to {0}", JobEngine.LogLevel); // } } - else + else { MainConsole.Instance.OutputFormat("Unrecognized job engine subcommand {0}", subCommand); } -- cgit v1.1 From 5ceb315e3463eaa50abf43a4c07cb37d9486b1fd Mon Sep 17 00:00:00 2001 From: Melanie Thielker Date: Wed, 1 Feb 2017 15:30:17 +0000 Subject: Secure sim stats with an (optional) password. There is way too much in these stats to expose them to the world, at least for some of us. --- OpenSim/Framework/Monitoring/StatsManager.cs | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index 55c3276..a6b341f 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -47,6 +47,8 @@ namespace OpenSim.Framework.Monitoring // Subcommand used to list other stats. public const string ListSubCommand = "list"; + public static string StatsPassword { get; set; } + // All subcommands public static HashSet SubCommands = new HashSet { AllSubCommand, ListSubCommand }; @@ -302,6 +304,17 @@ namespace OpenSim.Framework.Monitoring int response_code = 200; string contenttype = "text/json"; + if (StatsPassword != String.Empty && (!request.ContainsKey("pass") || request["pass"].ToString() != StatsPassword)) + { + responsedata["int_response_code"] = response_code; + responsedata["content_type"] = "text/plain"; + responsedata["keepalive"] = false; + responsedata["str_response_string"] = "Access denied"; + responsedata["access_control_allow_origin"] = "*"; + + return responsedata; + } + string pCategoryName = StatsManager.AllSubCommand; string pContainerName = StatsManager.AllSubCommand; string pStatName = StatsManager.AllSubCommand; -- cgit v1.1 From f3eb73926e1602f855f2c7fceb040b07f7265f3b Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Tue, 23 May 2017 04:05:31 +0100 Subject: stop a few more threads on exit --- OpenSim/Framework/Monitoring/WorkManager.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/WorkManager.cs b/OpenSim/Framework/Monitoring/WorkManager.cs index 43130f9..a3e0390 100644 --- a/OpenSim/Framework/Monitoring/WorkManager.cs +++ b/OpenSim/Framework/Monitoring/WorkManager.cs @@ -82,6 +82,11 @@ namespace OpenSim.Framework.Monitoring HandleControlCommand); } + public static void Stop() + { + JobEngine.Stop(); + } + /// /// Start a new long-lived thread. /// @@ -131,7 +136,6 @@ namespace OpenSim.Framework.Monitoring thread.Start(); - return thread; } -- cgit v1.1 From 772c5b7db030eb95da8852d711323244cfa91ab4 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Thu, 25 May 2017 06:05:02 +0100 Subject: move assetsconnector threads to whatchdog; Abort all alive threads known to watchdog before exit --- OpenSim/Framework/Monitoring/Watchdog.cs | 24 ++++++++++++++++++++++++ OpenSim/Framework/Monitoring/WorkManager.cs | 1 + 2 files changed, 25 insertions(+) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index 9cc61ee..8a4894e 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -180,6 +180,30 @@ namespace OpenSim.Framework.Monitoring m_watchdogTimer.Elapsed += WatchdogTimerElapsed; } + public static void Stop() + { + if(m_threads == null) + return; + + lock(m_threads) + { + m_enabled = false; + if(m_watchdogTimer != null) + { + m_watchdogTimer.Dispose(); + m_watchdogTimer = null; + } + + foreach(ThreadWatchdogInfo twi in m_threads.Values) + { + Thread t = twi.Thread; + if(t.IsAlive) + t.Abort(); + } + m_threads.Clear(); + } + } + /// /// Add a thread to the watchdog tracker. /// diff --git a/OpenSim/Framework/Monitoring/WorkManager.cs b/OpenSim/Framework/Monitoring/WorkManager.cs index a3e0390..9d52f71 100644 --- a/OpenSim/Framework/Monitoring/WorkManager.cs +++ b/OpenSim/Framework/Monitoring/WorkManager.cs @@ -85,6 +85,7 @@ namespace OpenSim.Framework.Monitoring public static void Stop() { JobEngine.Stop(); + Watchdog.Stop(); } /// -- cgit v1.1 From d90b68c2a637b822b826a0cf3c52991aa9ee2c97 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Fri, 26 May 2017 00:05:35 +0100 Subject: change wrong watchdog stopped thread check code, and don't log it --- OpenSim/Framework/Monitoring/Watchdog.cs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index 8a4894e..5fb725c 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -193,7 +193,7 @@ namespace OpenSim.Framework.Monitoring m_watchdogTimer.Dispose(); m_watchdogTimer = null; } - + foreach(ThreadWatchdogInfo twi in m_threads.Values) { Thread t = twi.Thread; @@ -341,6 +341,8 @@ namespace OpenSim.Framework.Monitoring /// private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e) { + if(!m_enabled) + return; int now = Environment.TickCount & Int32.MaxValue; int msElapsed = now - LastWatchdogThreadTick; @@ -358,21 +360,26 @@ namespace OpenSim.Framework.Monitoring List callbackInfos = null; List threadsToRemove = null; + const ThreadState thgone = ThreadState.Stopped | ThreadState.Aborted | ThreadState.AbortRequested; + lock (m_threads) { foreach(ThreadWatchdogInfo threadInfo in m_threads.Values) { - if(threadInfo.Thread.ThreadState == ThreadState.Stopped) + if(!m_enabled) + return; + if(!threadInfo.Thread.IsAlive || (threadInfo.Thread.ThreadState & thgone) != 0) { if(threadsToRemove == null) threadsToRemove = new List(); threadsToRemove.Add(threadInfo); - +/* if(callbackInfos == null) callbackInfos = new List(); callbackInfos.Add(threadInfo); +*/ } else if(!threadInfo.IsTimedOut && now - threadInfo.LastTick >= threadInfo.Timeout) { -- cgit v1.1 From 8d8236cfb245eaa25f81f2840175b8fe676febbd Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Fri, 26 May 2017 00:34:04 +0100 Subject: missing file change.. actually use watchdog threads on assetsconnector and avoid null refs on JobEngine --- OpenSim/Framework/Monitoring/JobEngine.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs index 0a39e4b..a6a059d 100644 --- a/OpenSim/Framework/Monitoring/JobEngine.cs +++ b/OpenSim/Framework/Monitoring/JobEngine.cs @@ -136,12 +136,15 @@ namespace OpenSim.Framework.Monitoring if(m_jobQueue.Count <= 0) m_cancelSource.Cancel(); - if(m_finishedProcessingAfterStop.WaitOne(RequestProcessTimeoutOnStop)) + m_finishedProcessingAfterStop.WaitOne(RequestProcessTimeoutOnStop); m_finishedProcessingAfterStop.Close(); } finally { - m_cancelSource.Dispose(); + if(m_cancelSource != null) + m_cancelSource.Dispose(); + if(m_finishedProcessingAfterStop != null) + m_finishedProcessingAfterStop.Dispose(); } } } -- cgit v1.1 From e8165a7b51db74ea8d283dbfa34f21bf3d7a7552 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Fri, 9 Jun 2017 20:14:56 +0100 Subject: only silent remove threads from watch list if they stopped ( ie still consider aborted etc ) --- OpenSim/Framework/Monitoring/Watchdog.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index 5fb725c..9cac451 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -254,14 +254,12 @@ namespace OpenSim.Framework.Monitoring twi.Cleanup(); m_threads.Remove(threadID); - return true; } else { m_log.WarnFormat( "[WATCHDOG]: Requested to remove thread with ID {0} but this is not being monitored", threadID); - return false; } } @@ -360,7 +358,7 @@ namespace OpenSim.Framework.Monitoring List callbackInfos = null; List threadsToRemove = null; - const ThreadState thgone = ThreadState.Stopped | ThreadState.Aborted | ThreadState.AbortRequested; + const ThreadState thgone = ThreadState.Stopped; lock (m_threads) { @@ -368,7 +366,7 @@ namespace OpenSim.Framework.Monitoring { if(!m_enabled) return; - if(!threadInfo.Thread.IsAlive || (threadInfo.Thread.ThreadState & thgone) != 0) + if((threadInfo.Thread.ThreadState & thgone) != 0) { if(threadsToRemove == null) threadsToRemove = new List(); -- cgit v1.1 From 5ce15566acf41be0d07e67e858d22eb65177816a Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Tue, 13 Jun 2017 18:39:42 +0100 Subject: add option for Fireandforget not timeout (our access to main smartThreadPool) --- OpenSim/Framework/Monitoring/WorkManager.cs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/WorkManager.cs b/OpenSim/Framework/Monitoring/WorkManager.cs index 9d52f71..50f6731 100644 --- a/OpenSim/Framework/Monitoring/WorkManager.cs +++ b/OpenSim/Framework/Monitoring/WorkManager.cs @@ -182,9 +182,9 @@ namespace OpenSim.Framework.Monitoring /// /// /// The name of the job. This is used in monitoring and debugging. - public static void RunInThreadPool(System.Threading.WaitCallback callback, object obj, string name) + public static void RunInThreadPool(System.Threading.WaitCallback callback, object obj, string name, bool timeout = true) { - Util.FireAndForget(callback, obj, name); + Util.FireAndForget(callback, obj, name, timeout); } /// @@ -231,10 +231,8 @@ namespace OpenSim.Framework.Monitoring JobEngine.QueueJob(name, () => callback(obj)); else if (canRunInThisThread) callback(obj); - else if (mustNotTimeout) - RunInThread(callback, obj, name, log); else - Util.FireAndForget(callback, obj, name); + Util.FireAndForget(callback, obj, name, !mustNotTimeout); } private static void HandleControlCommand(string module, string[] args) -- cgit v1.1 From 482ff06e13d6694027eec8d4146f733d69908658 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Tue, 13 Jun 2017 18:50:34 +0100 Subject: make JobEngine be a workitem of mail pool (smartThread), with the option to release thread after a idle time, so is free to do other service elsewhere --- OpenSim/Framework/Monitoring/JobEngine.cs | 89 ++++++++++++++----------------- 1 file changed, 41 insertions(+), 48 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs index a6a059d..115871e 100644 --- a/OpenSim/Framework/Monitoring/JobEngine.cs +++ b/OpenSim/Framework/Monitoring/JobEngine.cs @@ -57,7 +57,8 @@ namespace OpenSim.Framework.Monitoring /// /// Will be null if no job is currently running. /// - public Job CurrentJob { get; private set; } + private Job m_currentJob; + public Job CurrentJob { get { return m_currentJob;} } /// /// Number of jobs waiting to be processed. @@ -82,16 +83,15 @@ namespace OpenSim.Framework.Monitoring private CancellationTokenSource m_cancelSource; - /// - /// Used to signal that we are ready to complete stop. - /// - private ManualResetEvent m_finishedProcessingAfterStop = new ManualResetEvent(false); + private int m_timeout = -1; + + private bool m_threadRunnig = false; - public JobEngine(string name, string loggingName) + public JobEngine(string name, string loggingName, int timeout = -1) { Name = name; LoggingName = loggingName; - + m_timeout = timeout; RequestProcessTimeoutOnStop = 5000; } @@ -104,18 +104,9 @@ namespace OpenSim.Framework.Monitoring IsRunning = true; - m_finishedProcessingAfterStop.Reset(); - m_cancelSource = new CancellationTokenSource(); - - WorkManager.StartThread( - ProcessRequests, - Name, - ThreadPriority.Normal, - false, - true, - null, - int.MaxValue); + WorkManager.RunInThreadPool(ProcessRequests, null, Name, false); + m_threadRunnig = true; } } @@ -131,20 +122,16 @@ namespace OpenSim.Framework.Monitoring m_log.DebugFormat("[JobEngine] Stopping {0}", Name); IsRunning = false; - - m_finishedProcessingAfterStop.Reset(); - if(m_jobQueue.Count <= 0) + if(m_threadRunnig) + { m_cancelSource.Cancel(); - - m_finishedProcessingAfterStop.WaitOne(RequestProcessTimeoutOnStop); - m_finishedProcessingAfterStop.Close(); + m_threadRunnig = false; + } } finally { if(m_cancelSource != null) m_cancelSource.Dispose(); - if(m_finishedProcessingAfterStop != null) - m_finishedProcessingAfterStop.Dispose(); } } } @@ -203,6 +190,18 @@ namespace OpenSim.Framework.Monitoring /// public bool QueueJob(Job job) { + lock(JobLock) + { + if(!IsRunning) + return false; + + if(!m_threadRunnig) + { + WorkManager.RunInThreadPool(ProcessRequests, null, Name, false); + m_threadRunnig = true; + } + } + if (m_jobQueue.Count < m_jobQueue.BoundedCapacity) { m_jobQueue.Add(job); @@ -222,59 +221,53 @@ namespace OpenSim.Framework.Monitoring m_warnOverMaxQueue = false; } - return false; } } - private void ProcessRequests() + private void ProcessRequests(Object o) { - while(IsRunning || m_jobQueue.Count > 0) + while(IsRunning) { try { - CurrentJob = m_jobQueue.Take(m_cancelSource.Token); - } - catch(ObjectDisposedException e) - { - // If we see this whilst not running then it may be due to a race where this thread checks - // IsRunning after the stopping thread sets it to false and disposes of the cancellation source. - if(IsRunning) - throw e; - else + if(!m_jobQueue.TryTake(out m_currentJob, m_timeout, m_cancelSource.Token)) { - m_log.DebugFormat("[JobEngine] {0} stopping ignoring {1} jobs in queue", - Name,m_jobQueue.Count); + lock(JobLock) + m_threadRunnig = false; break; } } + catch(ObjectDisposedException e) + { + m_log.DebugFormat("[JobEngine] {0} stopping ignoring {1} jobs in queue", + Name,m_jobQueue.Count); + break; + } catch(OperationCanceledException) { break; } if(LogLevel >= 1) - m_log.DebugFormat("[{0}]: Processing job {1}",LoggingName,CurrentJob.Name); + m_log.DebugFormat("[{0}]: Processing job {1}",LoggingName,m_currentJob.Name); try { - CurrentJob.Action(); + m_currentJob.Action(); } catch(Exception e) { m_log.Error( string.Format( - "[{0}]: Job {1} failed, continuing. Exception ",LoggingName,CurrentJob.Name),e); + "[{0}]: Job {1} failed, continuing. Exception ",LoggingName,m_currentJob.Name),e); } if(LogLevel >= 1) - m_log.DebugFormat("[{0}]: Processed job {1}",LoggingName,CurrentJob.Name); + m_log.DebugFormat("[{0}]: Processed job {1}",LoggingName,m_currentJob.Name); - CurrentJob = null; + m_currentJob = null; } - - Watchdog.RemoveThread(false); - m_finishedProcessingAfterStop.Set(); } public class Job -- cgit v1.1 From 5e67bd5778487c3389cf972ed4a46dc20792dabb Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Wed, 14 Jun 2017 00:51:22 +0100 Subject: main generic use JobEngine also does not need a permanent thread.. actually doesn't even seem to be in use --- OpenSim/Framework/Monitoring/WorkManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/WorkManager.cs b/OpenSim/Framework/Monitoring/WorkManager.cs index 50f6731..5d9b185 100644 --- a/OpenSim/Framework/Monitoring/WorkManager.cs +++ b/OpenSim/Framework/Monitoring/WorkManager.cs @@ -57,7 +57,7 @@ namespace OpenSim.Framework.Monitoring static WorkManager() { - JobEngine = new JobEngine("Non-blocking non-critical job engine", "JOB ENGINE"); + JobEngine = new JobEngine("Non-blocking non-critical job engine", "JOB ENGINE", 30000); StatsManager.RegisterStat( new Stat( -- cgit v1.1 From 4df19ece539d6c731facd43f3eceac3c34418f05 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Fri, 16 Jun 2017 18:16:26 +0100 Subject: framework main thread pool is always active and in use ( even id hard to catch) so show in on show stats. Disable ServerStatsCollector by default, since most don't use it, Adicionally it uses shared framework performance counters system that may be affected if a region crashs --- OpenSim/Framework/Monitoring/ServerStatsCollector.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs index 3391240..a26a6e0 100644 --- a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs @@ -88,7 +88,7 @@ namespace OpenSim.Framework.Monitoring IConfig cfg = source.Configs["Monitoring"]; if (cfg != null) - Enabled = cfg.GetBoolean("ServerStatsEnabled", true); + Enabled = cfg.GetBoolean("ServerStatsEnabled", false); if (Enabled) { @@ -98,12 +98,18 @@ namespace OpenSim.Framework.Monitoring public void Start() { + if(!Enabled) + return; + if (RegisteredStats.Count == 0) RegisterServerStats(); } public void Close() { + if(!Enabled) + return; + if (RegisteredStats.Count > 0) { foreach (Stat stat in RegisteredStats.Values) -- cgit v1.1 From c92ba1cc04c36d46b3569c06d8339cf897af65cc Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Mon, 15 Jan 2018 16:23:19 +0000 Subject: shut up some pesty warnings --- OpenSim/Framework/Monitoring/JobEngine.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs index 115871e..4a831e8 100644 --- a/OpenSim/Framework/Monitoring/JobEngine.cs +++ b/OpenSim/Framework/Monitoring/JobEngine.cs @@ -238,7 +238,7 @@ namespace OpenSim.Framework.Monitoring break; } } - catch(ObjectDisposedException e) + catch(ObjectDisposedException) { m_log.DebugFormat("[JobEngine] {0} stopping ignoring {1} jobs in queue", Name,m_jobQueue.Count); -- cgit v1.1 From e9d2d818804bf0c4894ddee3aa9619cced368dce Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Wed, 17 Jan 2018 07:04:07 +0000 Subject: pesty warnings --- OpenSim/Framework/Monitoring/Stats/Stat.cs | 4 ---- 1 file changed, 4 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs index 2402acd..4b1a229 100644 --- a/OpenSim/Framework/Monitoring/Stats/Stat.cs +++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs @@ -242,11 +242,7 @@ namespace OpenSim.Framework.Monitoring public virtual OSDMap ToBriefOSDMap() { OSDMap ret = new OSDMap(); - ret.Add("Value", OSD.FromReal(Value)); - - double lastChangeOverTime, averageChangeOverTime; - return ret; } -- cgit v1.1 From fc224b444a8339aae2802fb919c34cfd994595f1 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Wed, 24 Jan 2018 10:02:18 +0000 Subject: avoid some large unnecessary strings --- OpenSim/Framework/Monitoring/JobEngine.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs index 4a831e8..6c388b3 100644 --- a/OpenSim/Framework/Monitoring/JobEngine.cs +++ b/OpenSim/Framework/Monitoring/JobEngine.cs @@ -79,7 +79,7 @@ namespace OpenSim.Framework.Monitoring /// private bool m_warnOverMaxQueue = true; - private BlockingCollection m_jobQueue = new BlockingCollection(new ConcurrentQueue(), 5000); + private BlockingCollection m_jobQueue = new BlockingCollection(5000); private CancellationTokenSource m_cancelSource; -- cgit v1.1 From ea815df6bd7234d8c3e030db0a86272c4d4197c8 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Sat, 14 Jul 2018 14:43:20 +0100 Subject: add a disabled log --- OpenSim/Framework/Monitoring/Watchdog.cs | 3 +++ 1 file changed, 3 insertions(+) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index 9cac451..ad21d93 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -197,6 +197,9 @@ namespace OpenSim.Framework.Monitoring foreach(ThreadWatchdogInfo twi in m_threads.Values) { Thread t = twi.Thread; + // m_log.DebugFormat( + // "[WATCHDOG]: Stop: Removing thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId); + if(t.IsAlive) t.Abort(); } -- cgit v1.1 From 57bd671e00535e8406050b3a3df2ed3b5c432a47 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Sun, 2 Dec 2018 17:29:13 +0000 Subject: stats heap allocation rate can negative --- OpenSim/Framework/Monitoring/MemoryWatchdog.cs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/MemoryWatchdog.cs b/OpenSim/Framework/Monitoring/MemoryWatchdog.cs index c474622..f2e47f8 100644 --- a/OpenSim/Framework/Monitoring/MemoryWatchdog.cs +++ b/OpenSim/Framework/Monitoring/MemoryWatchdog.cs @@ -110,20 +110,17 @@ namespace OpenSim.Framework.Monitoring long memoryNow = GC.GetTotalMemory(false); long memoryDiff = memoryNow - m_lastUpdateMemory; - if (memoryDiff >= 0) - { - if (m_samples.Count >= m_maxSamples) + if (m_samples.Count >= m_maxSamples) m_samples.Dequeue(); - double elapsed = Util.EnvironmentTickCountSubtract(now, m_lastUpdateTick); + double elapsed = Util.EnvironmentTickCountSubtract(now, m_lastUpdateTick); - // This should never happen since it's not useful for updates to occur with no time elapsed, but - // protect ourselves from a divide-by-zero just in case. - if (elapsed == 0) - return; + // This should never happen since it's not useful for updates to occur with no time elapsed, but + // protect ourselves from a divide-by-zero just in case. + if (elapsed == 0) + return; - m_samples.Enqueue(memoryDiff / (double)elapsed); - } + m_samples.Enqueue(memoryDiff / (double)elapsed); UpdateLastRecord(memoryNow, now); } -- cgit v1.1 From 66f3c6c73030d4f723ec9a3c36dfd32c52f0c5a3 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Mon, 3 Dec 2018 10:39:54 +0000 Subject: avoid a data copy --- OpenSim/Framework/Monitoring/BaseStatsCollector.cs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs index e513abd..b5df94b 100644 --- a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs @@ -48,12 +48,9 @@ namespace OpenSim.Framework.Monitoring Math.Round(GC.GetTotalMemory(false) / 1024.0 / 1024.0)); sb.AppendFormat( - "Last heap allocation rate : {0} MB/s\n", - Math.Round((MemoryWatchdog.LastHeapAllocationRate * 1000) / 1024.0 / 1024, 3)); - - sb.AppendFormat( - "Average heap allocation rate: {0} MB/s\n", - Math.Round((MemoryWatchdog.AverageHeapAllocationRate * 1000) / 1024.0 / 1024, 3)); + "Heap allocation rate (last/avg): {0}/{1}MB/s\n", + Math.Round((MemoryWatchdog.LastHeapAllocationRate * 1000) / 1048576.0, 3), + Math.Round((MemoryWatchdog.AverageHeapAllocationRate * 1000) / 1048576.0, 3)); Process myprocess = Process.GetCurrentProcess(); // if (!myprocess.HasExited) -- cgit v1.1 From 4a73cc81dc4e03b2b7c46829cecfc0627c3fddb4 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Fri, 28 Dec 2018 13:52:59 +0000 Subject: now break several things at same time... sog/sop updates, threads options,... --- OpenSim/Framework/Monitoring/WorkManager.cs | 68 +++++++---------------------- 1 file changed, 16 insertions(+), 52 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/WorkManager.cs b/OpenSim/Framework/Monitoring/WorkManager.cs index 5d9b185..c6d97e1 100644 --- a/OpenSim/Framework/Monitoring/WorkManager.cs +++ b/OpenSim/Framework/Monitoring/WorkManager.cs @@ -88,6 +88,11 @@ namespace OpenSim.Framework.Monitoring Watchdog.Stop(); } + public static Thread StartThread(ThreadStart start, string name, bool alarmIfTimeout = false, bool log = true) + { + return StartThread(start, name, ThreadPriority.Normal, true, alarmIfTimeout, null, Watchdog.DEFAULT_WATCHDOG_TIMEOUT_MS, log); + } + /// /// Start a new long-lived thread. /// @@ -99,9 +104,9 @@ namespace OpenSim.Framework.Monitoring /// If true then creation of thread is logged. /// The newly created Thread object public static Thread StartThread( - ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, bool log = true) + ThreadStart start, string name, ThreadPriority priority, bool alarmIfTimeout, bool log = true) { - return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, Watchdog.DEFAULT_WATCHDOG_TIMEOUT_MS, log); + return StartThread(start, name, priority, true, alarmIfTimeout, null, Watchdog.DEFAULT_WATCHDOG_TIMEOUT_MS, log); } /// @@ -162,15 +167,22 @@ namespace OpenSim.Framework.Monitoring { Culture.SetCurrentCulture(); callback(obj); - Watchdog.RemoveThread(log:false); } catch (Exception e) { m_log.Error(string.Format("[WATCHDOG]: Exception in thread {0}.", name), e); } + finally + { + try + { + Watchdog.RemoveThread(log: false); + } + catch { } + } }); - StartThread(ts, name, ThreadPriority.Normal, true, false, log:log); + StartThread(ts, name, false, log:log); } /// @@ -187,54 +199,6 @@ namespace OpenSim.Framework.Monitoring Util.FireAndForget(callback, obj, name, timeout); } - /// - /// Run a job. - /// - /// - /// This differs from direct scheduling (e.g. Util.FireAndForget) in that a job can be run in the job - /// engine if it is running, where all jobs are currently performed in sequence on a single thread. This is - /// to prevent observed overload and server freeze problems when there are hundreds of connections which all attempt to - /// perform work at once (e.g. in conference situations). With lower numbers of connections, the small - /// delay in performing jobs in sequence rather than concurrently has not been notiecable in testing, though a future more - /// sophisticated implementation could perform jobs concurrently when the server is under low load. - /// - /// However, be advised that some callers of this function rely on all jobs being performed in sequence if any - /// jobs are performed in sequence (i.e. if jobengine is active or not). Therefore, expanding the jobengine - /// beyond a single thread will require considerable thought. - /// - /// Also, any jobs submitted must be guaranteed to complete within a reasonable timeframe (e.g. they cannot - /// incorporate a network delay with a long timeout). At the moment, work that could suffer such issues - /// should still be run directly with RunInThread(), Util.FireAndForget(), etc. This is another area where - /// the job engine could be improved and so CPU utilization improved by better management of concurrency within - /// OpenSimulator. - /// - /// General classification for the job (e.g. "RezAttachments"). - /// Callback for job. - /// Object to pass to callback when run - /// Specific name of job (e.g. "RezAttachments for Joe Bloggs" - /// If set to true then the job may be run in ths calling thread. - /// If the true then the job must never timeout. - /// If set to true then extra logging is performed. - public static void RunJob( - string jobType, WaitCallback callback, object obj, string name, - bool canRunInThisThread = false, bool mustNotTimeout = false, - bool log = false) - { - if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest) - { - Culture.SetCurrentCulture(); - callback(obj); - return; - } - - if (JobEngine.IsRunning) - JobEngine.QueueJob(name, () => callback(obj)); - else if (canRunInThisThread) - callback(obj); - else - Util.FireAndForget(callback, obj, name, !mustNotTimeout); - } - private static void HandleControlCommand(string module, string[] args) { // if (SceneManager.Instance.CurrentScene != null && SceneManager.Instance.CurrentScene != m_udpServer.Scene) -- cgit v1.1 From 0fd17c08ae642fac17b24dfad06c61cfe5739483 Mon Sep 17 00:00:00 2001 From: Melanie Date: Tue, 20 Aug 2019 23:28:59 +0100 Subject: Massive console refactor. Greatly simplify interface. --- OpenSim/Framework/Monitoring/ChecksManager.cs | 2 +- OpenSim/Framework/Monitoring/StatsLogger.cs | 4 ++-- OpenSim/Framework/Monitoring/StatsManager.cs | 10 +++++----- OpenSim/Framework/Monitoring/WorkManager.cs | 18 +++++++++--------- 4 files changed, 17 insertions(+), 17 deletions(-) mode change 100644 => 100755 OpenSim/Framework/Monitoring/ChecksManager.cs mode change 100644 => 100755 OpenSim/Framework/Monitoring/StatsLogger.cs mode change 100644 => 100755 OpenSim/Framework/Monitoring/StatsManager.cs mode change 100644 => 100755 OpenSim/Framework/Monitoring/WorkManager.cs (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/ChecksManager.cs b/OpenSim/Framework/Monitoring/ChecksManager.cs old mode 100644 new mode 100755 index ff3b041..100b748 --- a/OpenSim/Framework/Monitoring/ChecksManager.cs +++ b/OpenSim/Framework/Monitoring/ChecksManager.cs @@ -88,7 +88,7 @@ namespace OpenSim.Framework.Monitoring con.Output("check categories available are:"); foreach (string category in RegisteredChecks.Keys) - con.OutputFormat(" {0}", category); + con.Output(" {0}", null, category); } // else // { diff --git a/OpenSim/Framework/Monitoring/StatsLogger.cs b/OpenSim/Framework/Monitoring/StatsLogger.cs old mode 100644 new mode 100755 index b719af9..4369b36 --- a/OpenSim/Framework/Monitoring/StatsLogger.cs +++ b/OpenSim/Framework/Monitoring/StatsLogger.cs @@ -79,7 +79,7 @@ namespace OpenSim.Framework.Monitoring if (cmd[2] == "start") { Start(); - con.OutputFormat("Now recording all stats to file every {0}ms", m_statsLogIntervalMs); + con.Output("Now recording all stats to file every {0}ms", null, m_statsLogIntervalMs); } else if (cmd[2] == "stop") { @@ -106,7 +106,7 @@ namespace OpenSim.Framework.Monitoring sw.WriteLine(line); } - MainConsole.Instance.OutputFormat("Stats saved to file {0}", path); + MainConsole.Instance.Output("Stats saved to file {0}", null, path); } public static void Start() diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs old mode 100644 new mode 100755 index a6b341f..57b9474 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -117,14 +117,14 @@ namespace OpenSim.Framework.Monitoring { con.Output("Statistic categories available are:"); foreach (string category in RegisteredStats.Keys) - con.OutputFormat(" {0}", category); + con.Output(" {0}", null, category); } else { SortedDictionary> category; if (!RegisteredStats.TryGetValue(categoryName, out category)) { - con.OutputFormat("No such category as {0}", categoryName); + con.Output("No such category as {0}", null, categoryName); } else { @@ -150,14 +150,14 @@ namespace OpenSim.Framework.Monitoring } else { - con.OutputFormat( - "No such stat {0} in {1}.{2}", statName, categoryName, containerName); + con.Output( + "No such stat {0} in {1}.{2}", null, statName, categoryName, containerName); } } } else { - con.OutputFormat("No such container {0} in category {1}", containerName, categoryName); + con.Output("No such container {0} in category {1}", null, containerName, categoryName); } } } diff --git a/OpenSim/Framework/Monitoring/WorkManager.cs b/OpenSim/Framework/Monitoring/WorkManager.cs old mode 100644 new mode 100755 index c6d97e1..f6e0799 --- a/OpenSim/Framework/Monitoring/WorkManager.cs +++ b/OpenSim/Framework/Monitoring/WorkManager.cs @@ -215,23 +215,23 @@ namespace OpenSim.Framework.Monitoring if (subCommand == "stop") { JobEngine.Stop(); - MainConsole.Instance.OutputFormat("Stopped job engine."); + MainConsole.Instance.Output("Stopped job engine."); } else if (subCommand == "start") { JobEngine.Start(); - MainConsole.Instance.OutputFormat("Started job engine."); + MainConsole.Instance.Output("Started job engine."); } else if (subCommand == "status") { - MainConsole.Instance.OutputFormat("Job engine running: {0}", JobEngine.IsRunning); + MainConsole.Instance.Output("Job engine running: {0}", null, JobEngine.IsRunning); JobEngine.Job job = JobEngine.CurrentJob; - MainConsole.Instance.OutputFormat("Current job {0}", job != null ? job.Name : "none"); + MainConsole.Instance.Output("Current job {0}", null, job != null ? job.Name : "none"); - MainConsole.Instance.OutputFormat( - "Jobs waiting: {0}", JobEngine.IsRunning ? JobEngine.JobsWaiting.ToString() : "n/a"); - MainConsole.Instance.OutputFormat("Log Level: {0}", JobEngine.LogLevel); + MainConsole.Instance.Output( + "Jobs waiting: {0}", null, JobEngine.IsRunning ? JobEngine.JobsWaiting.ToString() : "n/a"); + MainConsole.Instance.Output("Log Level: {0}", null, JobEngine.LogLevel); } else if (subCommand == "log") { @@ -246,12 +246,12 @@ namespace OpenSim.Framework.Monitoring // if (ConsoleUtil.TryParseConsoleInt(MainConsole.Instance, args[4], out logLevel)) // { JobEngine.LogLevel = logLevel; - MainConsole.Instance.OutputFormat("Set debug log level to {0}", JobEngine.LogLevel); + MainConsole.Instance.Output("Set debug log level to {0}", null, JobEngine.LogLevel); // } } else { - MainConsole.Instance.OutputFormat("Unrecognized job engine subcommand {0}", subCommand); + MainConsole.Instance.Output("Unrecognized job engine subcommand {0}", null, subCommand); } } } -- cgit v1.1 From b98ad6c53c484aa2744c5040c9f29c475c3d4f72 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Tue, 22 Oct 2019 12:23:19 +0100 Subject: remove some useless NULL arguments --- OpenSim/Framework/Monitoring/ChecksManager.cs | 2 +- OpenSim/Framework/Monitoring/StatsLogger.cs | 4 ++-- OpenSim/Framework/Monitoring/StatsManager.cs | 2 +- OpenSim/Framework/Monitoring/WorkManager.cs | 12 ++++++------ 4 files changed, 10 insertions(+), 10 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/ChecksManager.cs b/OpenSim/Framework/Monitoring/ChecksManager.cs index 100b748..f719099 100755 --- a/OpenSim/Framework/Monitoring/ChecksManager.cs +++ b/OpenSim/Framework/Monitoring/ChecksManager.cs @@ -88,7 +88,7 @@ namespace OpenSim.Framework.Monitoring con.Output("check categories available are:"); foreach (string category in RegisteredChecks.Keys) - con.Output(" {0}", null, category); + con.Output(" {0}", category); } // else // { diff --git a/OpenSim/Framework/Monitoring/StatsLogger.cs b/OpenSim/Framework/Monitoring/StatsLogger.cs index 4369b36..e87c188 100755 --- a/OpenSim/Framework/Monitoring/StatsLogger.cs +++ b/OpenSim/Framework/Monitoring/StatsLogger.cs @@ -79,7 +79,7 @@ namespace OpenSim.Framework.Monitoring if (cmd[2] == "start") { Start(); - con.Output("Now recording all stats to file every {0}ms", null, m_statsLogIntervalMs); + con.Output("Now recording all stats to file every {0}ms", m_statsLogIntervalMs); } else if (cmd[2] == "stop") { @@ -106,7 +106,7 @@ namespace OpenSim.Framework.Monitoring sw.WriteLine(line); } - MainConsole.Instance.Output("Stats saved to file {0}", null, path); + MainConsole.Instance.Output("Stats saved to file {0}", path); } public static void Start() diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index 57b9474..9646bd7 100755 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -117,7 +117,7 @@ namespace OpenSim.Framework.Monitoring { con.Output("Statistic categories available are:"); foreach (string category in RegisteredStats.Keys) - con.Output(" {0}", null, category); + con.Output(" {0}", category); } else { diff --git a/OpenSim/Framework/Monitoring/WorkManager.cs b/OpenSim/Framework/Monitoring/WorkManager.cs index f6e0799..ce639b9 100755 --- a/OpenSim/Framework/Monitoring/WorkManager.cs +++ b/OpenSim/Framework/Monitoring/WorkManager.cs @@ -224,14 +224,14 @@ namespace OpenSim.Framework.Monitoring } else if (subCommand == "status") { - MainConsole.Instance.Output("Job engine running: {0}", null, JobEngine.IsRunning); + MainConsole.Instance.Output("Job engine running: {0}", JobEngine.IsRunning); JobEngine.Job job = JobEngine.CurrentJob; - MainConsole.Instance.Output("Current job {0}", null, job != null ? job.Name : "none"); + MainConsole.Instance.Output("Current job {0}", job != null ? job.Name : "none"); MainConsole.Instance.Output( - "Jobs waiting: {0}", null, JobEngine.IsRunning ? JobEngine.JobsWaiting.ToString() : "n/a"); - MainConsole.Instance.Output("Log Level: {0}", null, JobEngine.LogLevel); + "Jobs waiting: {0}", JobEngine.IsRunning ? JobEngine.JobsWaiting.ToString() : "n/a"); + MainConsole.Instance.Output("Log Level: {0}", JobEngine.LogLevel); } else if (subCommand == "log") { @@ -246,12 +246,12 @@ namespace OpenSim.Framework.Monitoring // if (ConsoleUtil.TryParseConsoleInt(MainConsole.Instance, args[4], out logLevel)) // { JobEngine.LogLevel = logLevel; - MainConsole.Instance.Output("Set debug log level to {0}", null, JobEngine.LogLevel); + MainConsole.Instance.Output("Set debug log level to {0}", JobEngine.LogLevel); // } } else { - MainConsole.Instance.Output("Unrecognized job engine subcommand {0}", null, subCommand); + MainConsole.Instance.Output("Unrecognized job engine subcommand {0}", subCommand); } } } -- cgit v1.1 From da0a8d6c43054b1b6b7487fc5fc78f03af349c02 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Tue, 22 Oct 2019 12:39:50 +0100 Subject: remove some more useless NULL arguments --- OpenSim/Framework/Monitoring/StatsManager.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index 9646bd7..05fb07a 100755 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -124,7 +124,7 @@ namespace OpenSim.Framework.Monitoring SortedDictionary> category; if (!RegisteredStats.TryGetValue(categoryName, out category)) { - con.Output("No such category as {0}", null, categoryName); + con.Output("No such category as {0}", categoryName); } else { @@ -151,13 +151,13 @@ namespace OpenSim.Framework.Monitoring else { con.Output( - "No such stat {0} in {1}.{2}", null, statName, categoryName, containerName); + "No such stat {0} in {1}.{2}", statName, categoryName, containerName); } } } else { - con.Output("No such container {0} in category {1}", null, containerName, categoryName); + con.Output("No such container {0} in category {1}", containerName, categoryName); } } } -- cgit v1.1 From bd12d60e80b6f051a9e4f4470c7621e5ad098380 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Wed, 20 Nov 2019 23:16:20 +0000 Subject: cosmetics --- OpenSim/Framework/Monitoring/BaseStatsCollector.cs | 26 +++++++++++----------- .../Framework/Monitoring/SimExtraStatsCollector.cs | 22 +++++++++--------- 2 files changed, 23 insertions(+), 25 deletions(-) (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs index b5df94b..7119964 100644 --- a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs @@ -52,21 +52,21 @@ namespace OpenSim.Framework.Monitoring Math.Round((MemoryWatchdog.LastHeapAllocationRate * 1000) / 1048576.0, 3), Math.Round((MemoryWatchdog.AverageHeapAllocationRate * 1000) / 1048576.0, 3)); - Process myprocess = Process.GetCurrentProcess(); -// if (!myprocess.HasExited) try { - myprocess.Refresh(); - sb.AppendFormat( - "Process memory: Physical {0} MB \t Paged {1} MB \t Virtual {2} MB\n", - Math.Round(myprocess.WorkingSet64 / 1024.0 / 1024.0), - Math.Round(myprocess.PagedMemorySize64 / 1024.0 / 1024.0), - Math.Round(myprocess.VirtualMemorySize64 / 1024.0 / 1024.0)); - sb.AppendFormat( - "Peak process memory: Physical {0} MB \t Paged {1} MB \t Virtual {2} MB\n", - Math.Round(myprocess.PeakWorkingSet64 / 1024.0 / 1024.0), - Math.Round(myprocess.PeakPagedMemorySize64 / 1024.0 / 1024.0), - Math.Round(myprocess.PeakVirtualMemorySize64 / 1024.0 / 1024.0)); + using (Process myprocess = Process.GetCurrentProcess()) + { + sb.AppendFormat( + "Process memory: Physical {0} MB \t Paged {1} MB \t Virtual {2} MB\n", + Math.Round(myprocess.WorkingSet64 / 1024.0 / 1024.0), + Math.Round(myprocess.PagedMemorySize64 / 1024.0 / 1024.0), + Math.Round(myprocess.VirtualMemorySize64 / 1024.0 / 1024.0)); + sb.AppendFormat( + "Peak process memory: Physical {0} MB \t Paged {1} MB \t Virtual {2} MB\n", + Math.Round(myprocess.PeakWorkingSet64 / 1024.0 / 1024.0), + Math.Round(myprocess.PeakPagedMemorySize64 / 1024.0 / 1024.0), + Math.Round(myprocess.PeakVirtualMemorySize64 / 1024.0 / 1024.0)); + } } catch { } diff --git a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs index 88a0297..c41c4b9 100755 --- a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs @@ -435,21 +435,19 @@ Asset service request failures: {3}" + Environment.NewLine, // Get the amount of physical memory, allocated with the instance of this program, in kilobytes; // the working set is the set of memory pages currently visible to this program in physical RAM // memory and includes both shared (e.g. system libraries) and private data - double memUsage = Process.GetCurrentProcess().WorkingSet64 / 1024.0; - - // Get the number of threads from the system that are currently - // running int numberThreadsRunning = 0; - foreach (ProcessThread currentThread in - Process.GetCurrentProcess().Threads) + double memUsage = 0; + using(Process p = Process.GetCurrentProcess()) { - // A known issue with the current process .Threads property is - // that it can return null threads, thus don't count those as - // running threads and prevent the program function from failing - if (currentThread != null && - currentThread.ThreadState == ThreadState.Running) + memUsage = p.WorkingSet64 / 1024.0; + + // Get the number of threads from the system that are currently + // running + + foreach (ProcessThread currentThread in p.Threads) { - numberThreadsRunning++; + if (currentThread != null && currentThread.ThreadState == ThreadState.Running) + numberThreadsRunning++; } } -- cgit v1.1 From 237659ba30a2d6a072aae989252a1e6be0423c3c Mon Sep 17 00:00:00 2001 From: onefang Date: Wed, 9 Sep 2020 00:52:35 +1000 Subject: Remove useless executable bit that Windows adds. --- OpenSim/Framework/Monitoring/ChecksManager.cs | 0 OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs | 0 OpenSim/Framework/Monitoring/Stats/CounterStat.cs | 0 OpenSim/Framework/Monitoring/Stats/EventHistogram.cs | 0 OpenSim/Framework/Monitoring/StatsLogger.cs | 0 OpenSim/Framework/Monitoring/StatsManager.cs | 0 OpenSim/Framework/Monitoring/WorkManager.cs | 0 7 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 OpenSim/Framework/Monitoring/ChecksManager.cs mode change 100755 => 100644 OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs mode change 100755 => 100644 OpenSim/Framework/Monitoring/Stats/CounterStat.cs mode change 100755 => 100644 OpenSim/Framework/Monitoring/Stats/EventHistogram.cs mode change 100755 => 100644 OpenSim/Framework/Monitoring/StatsLogger.cs mode change 100755 => 100644 OpenSim/Framework/Monitoring/StatsManager.cs mode change 100755 => 100644 OpenSim/Framework/Monitoring/WorkManager.cs (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/ChecksManager.cs b/OpenSim/Framework/Monitoring/ChecksManager.cs old mode 100755 new mode 100644 diff --git a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs old mode 100755 new mode 100644 diff --git a/OpenSim/Framework/Monitoring/Stats/CounterStat.cs b/OpenSim/Framework/Monitoring/Stats/CounterStat.cs old mode 100755 new mode 100644 diff --git a/OpenSim/Framework/Monitoring/Stats/EventHistogram.cs b/OpenSim/Framework/Monitoring/Stats/EventHistogram.cs old mode 100755 new mode 100644 diff --git a/OpenSim/Framework/Monitoring/StatsLogger.cs b/OpenSim/Framework/Monitoring/StatsLogger.cs old mode 100755 new mode 100644 diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs old mode 100755 new mode 100644 diff --git a/OpenSim/Framework/Monitoring/WorkManager.cs b/OpenSim/Framework/Monitoring/WorkManager.cs old mode 100755 new mode 100644 -- cgit v1.1