From 5e4d6cab00cb29cd088ab7b62ab13aff103b64cb Mon Sep 17 00:00:00 2001 From: onefang Date: Sun, 19 May 2019 21:24:15 +1000 Subject: Dump OpenSim 0.9.0.1 into it's own branch. --- OpenSim/Framework/Monitoring/BaseStatsCollector.cs | 26 +- OpenSim/Framework/Monitoring/Checks/Check.cs | 8 +- OpenSim/Framework/Monitoring/ChecksManager.cs | 45 ++- .../Monitoring/Interfaces/IStatsCollector.cs | 2 +- OpenSim/Framework/Monitoring/JobEngine.cs | 162 +++++----- .../Monitoring/Properties/AssemblyInfo.cs | 10 +- .../Framework/Monitoring/ServerStatsCollector.cs | 77 ++++- .../Framework/Monitoring/SimExtraStatsCollector.cs | 56 ++-- OpenSim/Framework/Monitoring/Stats/CounterStat.cs | 0 .../Framework/Monitoring/Stats/EventHistogram.cs | 346 ++++++++++----------- OpenSim/Framework/Monitoring/Stats/Stat.cs | 51 +-- OpenSim/Framework/Monitoring/StatsLogger.cs | 6 +- OpenSim/Framework/Monitoring/StatsManager.cs | 106 ++++--- OpenSim/Framework/Monitoring/Watchdog.cs | 61 +++- OpenSim/Framework/Monitoring/WorkManager.cs | 48 +-- 15 files changed, 558 insertions(+), 446 deletions(-) mode change 100644 => 100755 OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs mode change 100644 => 100755 OpenSim/Framework/Monitoring/Stats/CounterStat.cs mode change 100644 => 100755 OpenSim/Framework/Monitoring/Stats/EventHistogram.cs (limited to 'OpenSim/Framework/Monitoring') diff --git a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs index 20495f6..e513abd 100644 --- a/OpenSim/Framework/Monitoring/BaseStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/BaseStatsCollector.cs @@ -43,7 +43,6 @@ namespace OpenSim.Framework.Monitoring StringBuilder sb = new StringBuilder(Environment.NewLine); sb.Append("MEMORY STATISTICS"); sb.Append(Environment.NewLine); - sb.AppendFormat( "Heap allocated to OpenSim : {0} MB\n", Math.Round(GC.GetTotalMemory(false) / 1024.0 / 1024.0)); @@ -56,13 +55,30 @@ namespace OpenSim.Framework.Monitoring "Average heap allocation rate: {0} MB/s\n", Math.Round((MemoryWatchdog.AverageHeapAllocationRate * 1000) / 1024.0 / 1024, 3)); - sb.AppendFormat( - "Process memory : {0} MB\n", - Math.Round(Process.GetCurrentProcess().WorkingSet64 / 1024.0 / 1024.0)); + Process myprocess = Process.GetCurrentProcess(); +// if (!myprocess.HasExited) + try + { + myprocess.Refresh(); + sb.AppendFormat( + "Process memory: Physical {0} MB \t Paged {1} MB \t Virtual {2} MB\n", + Math.Round(myprocess.WorkingSet64 / 1024.0 / 1024.0), + Math.Round(myprocess.PagedMemorySize64 / 1024.0 / 1024.0), + Math.Round(myprocess.VirtualMemorySize64 / 1024.0 / 1024.0)); + sb.AppendFormat( + "Peak process memory: Physical {0} MB \t Paged {1} MB \t Virtual {2} MB\n", + Math.Round(myprocess.PeakWorkingSet64 / 1024.0 / 1024.0), + Math.Round(myprocess.PeakPagedMemorySize64 / 1024.0 / 1024.0), + Math.Round(myprocess.PeakVirtualMemorySize64 / 1024.0 / 1024.0)); + } + catch + { } +// else +// sb.Append("Process reported as Exited \n"); return sb.ToString(); } - + public virtual string XReport(string uptime, string version) { return (string) Math.Round(GC.GetTotalMemory(false) / 1024.0 / 1024.0).ToString() ; diff --git a/OpenSim/Framework/Monitoring/Checks/Check.cs b/OpenSim/Framework/Monitoring/Checks/Check.cs index 594386a..9a1bd45 100644 --- a/OpenSim/Framework/Monitoring/Checks/Check.cs +++ b/OpenSim/Framework/Monitoring/Checks/Check.cs @@ -79,7 +79,7 @@ namespace OpenSim.Framework.Monitoring string category, string container, Func checkFunc, - StatVerbosity verbosity) + StatVerbosity verbosity) { if (ChecksManager.SubCommands.Contains(category)) throw new Exception( @@ -108,9 +108,9 @@ namespace OpenSim.Framework.Monitoring public virtual string ToConsoleString() { return string.Format( - "{0}.{1}.{2} - {3}", - Category, - Container, + "{0}.{1}.{2} - {3}", + Category, + Container, ShortName, Description); } diff --git a/OpenSim/Framework/Monitoring/ChecksManager.cs b/OpenSim/Framework/Monitoring/ChecksManager.cs index e4a7f8c..ff3b041 100644 --- a/OpenSim/Framework/Monitoring/ChecksManager.cs +++ b/OpenSim/Framework/Monitoring/ChecksManager.cs @@ -132,8 +132,8 @@ namespace OpenSim.Framework.Monitoring /// public static bool RegisterCheck(Check check) { - SortedDictionary> category = null, newCategory; - SortedDictionary container = null, newContainer; + SortedDictionary> category = null; + SortedDictionary container = null; lock (RegisteredChecks) { @@ -146,19 +146,15 @@ namespace OpenSim.Framework.Monitoring // We take a copy-on-write approach here of replacing dictionaries when keys are added or removed. // This means that we don't need to lock or copy them on iteration, which will be a much more // common operation after startup. - if (container != null) - newContainer = new SortedDictionary(container); - else - newContainer = new SortedDictionary(); + if (container == null) + container = new SortedDictionary(); - if (category != null) - newCategory = new SortedDictionary>(category); - else - newCategory = new SortedDictionary>(); + if (category == null) + category = new SortedDictionary>(); - newContainer[check.ShortName] = check; - newCategory[check.Container] = newContainer; - RegisteredChecks[check.Category] = newCategory; + container[check.ShortName] = check; + category[check.Container] = container; + RegisteredChecks[check.Category] = category; } return true; @@ -171,23 +167,24 @@ namespace OpenSim.Framework.Monitoring /// public static bool DeregisterCheck(Check check) { - SortedDictionary> category = null, newCategory; - SortedDictionary container = null, newContainer; + SortedDictionary> category = null; + SortedDictionary container = null; lock (RegisteredChecks) { if (!TryGetCheckParents(check, out category, out container)) return false; - newContainer = new SortedDictionary(container); - newContainer.Remove(check.ShortName); - - newCategory = new SortedDictionary>(category); - newCategory.Remove(check.Container); - - newCategory[check.Container] = newContainer; - RegisteredChecks[check.Category] = newCategory; - + if(container != null) + { + container.Remove(check.ShortName); + if(category != null && container.Count == 0) + { + category.Remove(check.Container); + if(category.Count == 0) + RegisteredChecks.Remove(check.Category); + } + } return true; } } diff --git a/OpenSim/Framework/Monitoring/Interfaces/IStatsCollector.cs b/OpenSim/Framework/Monitoring/Interfaces/IStatsCollector.cs index 40df562..e326e8b 100644 --- a/OpenSim/Framework/Monitoring/Interfaces/IStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/Interfaces/IStatsCollector.cs @@ -39,7 +39,7 @@ namespace OpenSim.Framework.Monitoring /// /// string Report(); - + /// /// Report back collected statistical information in json /// diff --git a/OpenSim/Framework/Monitoring/JobEngine.cs b/OpenSim/Framework/Monitoring/JobEngine.cs index 6db9a67..115871e 100644 --- a/OpenSim/Framework/Monitoring/JobEngine.cs +++ b/OpenSim/Framework/Monitoring/JobEngine.cs @@ -40,6 +40,8 @@ namespace OpenSim.Framework.Monitoring public int LogLevel { get; set; } + private object JobLock = new object(); + public string Name { get; private set; } public string LoggingName { get; private set; } @@ -47,7 +49,7 @@ namespace OpenSim.Framework.Monitoring /// /// Is this engine running? /// - public bool IsRunning { get; private set; } + public bool IsRunning { get; private set; } /// /// The current job that the engine is running. @@ -55,7 +57,8 @@ namespace OpenSim.Framework.Monitoring /// /// Will be null if no job is currently running. /// - public Job CurrentJob { get; private set; } + private Job m_currentJob; + public Job CurrentJob { get { return m_currentJob;} } /// /// Number of jobs waiting to be processed. @@ -71,96 +74,64 @@ namespace OpenSim.Framework.Monitoring /// Controls whether we need to warn in the log about exceeding the max queue size. /// /// - /// This is flipped to false once queue max has been exceeded and back to true when it falls below max, in + /// This is flipped to false once queue max has been exceeded and back to true when it falls below max, in /// order to avoid spamming the log with lots of warnings. /// private bool m_warnOverMaxQueue = true; - private BlockingCollection m_jobQueue; + private BlockingCollection m_jobQueue = new BlockingCollection(new ConcurrentQueue(), 5000); private CancellationTokenSource m_cancelSource; - /// - /// Used to signal that we are ready to complete stop. - /// - private ManualResetEvent m_finishedProcessingAfterStop = new ManualResetEvent(false); + private int m_timeout = -1; - public JobEngine(string name, string loggingName) + private bool m_threadRunnig = false; + + public JobEngine(string name, string loggingName, int timeout = -1) { Name = name; LoggingName = loggingName; - + m_timeout = timeout; RequestProcessTimeoutOnStop = 5000; } public void Start() { - lock (this) + lock (JobLock) { if (IsRunning) return; IsRunning = true; - m_finishedProcessingAfterStop.Reset(); - - m_jobQueue = new BlockingCollection(new ConcurrentQueue(), 5000); m_cancelSource = new CancellationTokenSource(); - - WorkManager.StartThread( - ProcessRequests, - Name, - ThreadPriority.Normal, - false, - true, - null, - int.MaxValue); + WorkManager.RunInThreadPool(ProcessRequests, null, Name, false); + m_threadRunnig = true; } } public void Stop() - { - lock (this) + { + lock (JobLock) { try { if (!IsRunning) return; - IsRunning = false; + m_log.DebugFormat("[JobEngine] Stopping {0}", Name); - int requestsLeft = m_jobQueue.Count; - - if (requestsLeft <= 0) + IsRunning = false; + if(m_threadRunnig) { m_cancelSource.Cancel(); - } - else - { - m_log.InfoFormat("[{0}]: Waiting to write {1} events after stop.", LoggingName, requestsLeft); - - while (requestsLeft > 0) - { - if (!m_finishedProcessingAfterStop.WaitOne(RequestProcessTimeoutOnStop)) - { - // After timeout no events have been written - if (requestsLeft == m_jobQueue.Count) - { - m_log.WarnFormat( - "[{0}]: No requests processed after {1} ms wait. Discarding remaining {2} requests", - LoggingName, RequestProcessTimeoutOnStop, requestsLeft); - - break; - } - } - - requestsLeft = m_jobQueue.Count; - } + m_threadRunnig = false; } } finally { - m_cancelSource.Dispose(); + if(m_cancelSource != null) + m_cancelSource.Dispose(); } } } @@ -169,7 +140,7 @@ namespace OpenSim.Framework.Monitoring /// Make a job. /// /// - /// We provide this method to replace the constructor so that we can later pool job objects if necessary to + /// We provide this method to replace the constructor so that we can later pool job objects if necessary to /// reduce memory churn. Normally one would directly call QueueJob() with parameters anyway. /// /// @@ -219,6 +190,18 @@ namespace OpenSim.Framework.Monitoring /// public bool QueueJob(Job job) { + lock(JobLock) + { + if(!IsRunning) + return false; + + if(!m_threadRunnig) + { + WorkManager.RunInThreadPool(ProcessRequests, null, Name, false); + m_threadRunnig = true; + } + } + if (m_jobQueue.Count < m_jobQueue.BoundedCapacity) { m_jobQueue.Add(job); @@ -238,56 +221,53 @@ namespace OpenSim.Framework.Monitoring m_warnOverMaxQueue = false; } - return false; } } - private void ProcessRequests() + private void ProcessRequests(Object o) { - try + while(IsRunning) { - while (IsRunning || m_jobQueue.Count > 0) + try { - try + if(!m_jobQueue.TryTake(out m_currentJob, m_timeout, m_cancelSource.Token)) { - CurrentJob = m_jobQueue.Take(m_cancelSource.Token); - } - catch (ObjectDisposedException e) - { - // If we see this whilst not running then it may be due to a race where this thread checks - // IsRunning after the stopping thread sets it to false and disposes of the cancellation source. - if (IsRunning) - throw e; - else - break; + lock(JobLock) + m_threadRunnig = false; + break; } + } + catch(ObjectDisposedException e) + { + m_log.DebugFormat("[JobEngine] {0} stopping ignoring {1} jobs in queue", + Name,m_jobQueue.Count); + break; + } + catch(OperationCanceledException) + { + break; + } - if (LogLevel >= 1) - m_log.DebugFormat("[{0}]: Processing job {1}", LoggingName, CurrentJob.Name); + if(LogLevel >= 1) + m_log.DebugFormat("[{0}]: Processing job {1}",LoggingName,m_currentJob.Name); - try - { - CurrentJob.Action(); - } - catch (Exception e) - { - m_log.Error( - string.Format( - "[{0}]: Job {1} failed, continuing. Exception ", LoggingName, CurrentJob.Name), e); - } + try + { + m_currentJob.Action(); + } + catch(Exception e) + { + m_log.Error( + string.Format( + "[{0}]: Job {1} failed, continuing. Exception ",LoggingName,m_currentJob.Name),e); + } - if (LogLevel >= 1) - m_log.DebugFormat("[{0}]: Processed job {1}", LoggingName, CurrentJob.Name); + if(LogLevel >= 1) + m_log.DebugFormat("[{0}]: Processed job {1}",LoggingName,m_currentJob.Name); - CurrentJob = null; - } - } - catch (OperationCanceledException) - { + m_currentJob = null; } - - m_finishedProcessingAfterStop.Set(); } public class Job @@ -320,7 +300,7 @@ namespace OpenSim.Framework.Monitoring CommonId = commonId; Action = action; } - + /// /// Make a job. It needs to be separately queued. /// @@ -338,4 +318,4 @@ namespace OpenSim.Framework.Monitoring } } } -} \ No newline at end of file +} diff --git a/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs index a617b93..2ff2014 100644 --- a/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs +++ b/OpenSim/Framework/Monitoring/Properties/AssemblyInfo.cs @@ -2,7 +2,7 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; -// General Information about an assembly is controlled through the following +// General Information about an assembly is controlled through the following // set of attributes. Change these attribute values to modify the information // associated with an assembly. [assembly: AssemblyTitle("OpenSim.Framework.Monitoring")] @@ -14,8 +14,8 @@ using System.Runtime.InteropServices; [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from // COM, set the ComVisible attribute to true on that type. [assembly: ComVisible(false)] @@ -25,9 +25,9 @@ using System.Runtime.InteropServices; // Version information for an assembly consists of the following four values: // // Major Version -// Minor Version +// Minor Version // Build Number // Revision // -[assembly: AssemblyVersion("0.8.3.*")] +[assembly: AssemblyVersion(OpenSim.VersionInfo.AssemblyVersionNumber)] diff --git a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs index 77315bb..a26a6e0 100644 --- a/OpenSim/Framework/Monitoring/ServerStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/ServerStatsCollector.cs @@ -88,7 +88,7 @@ namespace OpenSim.Framework.Monitoring IConfig cfg = source.Configs["Monitoring"]; if (cfg != null) - Enabled = cfg.GetBoolean("ServerStatsEnabled", true); + Enabled = cfg.GetBoolean("ServerStatsEnabled", false); if (Enabled) { @@ -98,12 +98,18 @@ namespace OpenSim.Framework.Monitoring public void Start() { + if(!Enabled) + return; + if (RegisteredStats.Count == 0) RegisterServerStats(); } public void Close() { + if(!Enabled) + return; + if (RegisteredStats.Count > 0) { foreach (Stat stat in RegisteredStats.Values) @@ -167,18 +173,18 @@ namespace OpenSim.Framework.Monitoring } MakeStat("BuiltinThreadpoolWorkerThreadsAvailable", null, "threads", ContainerThreadpool, - s => - { - int workerThreads, iocpThreads; - ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads); + s => + { + int workerThreads, iocpThreads; + ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads); s.Value = workerThreads; }); MakeStat("BuiltinThreadpoolIOCPThreadsAvailable", null, "threads", ContainerThreadpool, - s => - { - int workerThreads, iocpThreads; - ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads); + s => + { + int workerThreads, iocpThreads; + ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads); s.Value = iocpThreads; }); @@ -193,10 +199,10 @@ namespace OpenSim.Framework.Monitoring } MakeStat( - "HTTPRequestsMade", - "Number of outbound HTTP requests made", - "requests", - ContainerNetwork, + "HTTPRequestsMade", + "Number of outbound HTTP requests made", + "requests", + ContainerNetwork, s => s.Value = WebUtil.RequestNumber, MeasuresOfInterest.AverageChangeOverTime); @@ -249,9 +255,52 @@ namespace OpenSim.Framework.Monitoring (s) => { s.Value = Math.Round(MemoryWatchdog.LastHeapAllocationRate * 1000d / 1024d / 1024d, 3); }); MakeStat("AverageHeapAllocationRate", null, "MB/sec", ContainerMemory, (s) => { s.Value = Math.Round(MemoryWatchdog.AverageHeapAllocationRate * 1000d / 1024d / 1024d, 3); }); + + MakeStat("ProcessResident", null, "MB", ContainerProcess, + (s) => + { + Process myprocess = Process.GetCurrentProcess(); + myprocess.Refresh(); + s.Value = Math.Round(Process.GetCurrentProcess().WorkingSet64 / 1024.0 / 1024.0); + }); + MakeStat("ProcessPaged", null, "MB", ContainerProcess, + (s) => + { + Process myprocess = Process.GetCurrentProcess(); + myprocess.Refresh(); + s.Value = Math.Round(Process.GetCurrentProcess().PagedMemorySize64 / 1024.0 / 1024.0); + }); + MakeStat("ProcessVirtual", null, "MB", ContainerProcess, + (s) => + { + Process myprocess = Process.GetCurrentProcess(); + myprocess.Refresh(); + s.Value = Math.Round(Process.GetCurrentProcess().VirtualMemorySize64 / 1024.0 / 1024.0); + }); + MakeStat("PeakProcessResident", null, "MB", ContainerProcess, + (s) => + { + Process myprocess = Process.GetCurrentProcess(); + myprocess.Refresh(); + s.Value = Math.Round(Process.GetCurrentProcess().PeakWorkingSet64 / 1024.0 / 1024.0); + }); + MakeStat("PeakProcessPaged", null, "MB", ContainerProcess, + (s) => + { + Process myprocess = Process.GetCurrentProcess(); + myprocess.Refresh(); + s.Value = Math.Round(Process.GetCurrentProcess().PeakPagedMemorySize64 / 1024.0 / 1024.0); + }); + MakeStat("PeakProcessVirtual", null, "MB", ContainerProcess, + (s) => + { + Process myprocess = Process.GetCurrentProcess(); + myprocess.Refresh(); + s.Value = Math.Round(Process.GetCurrentProcess().PeakVirtualMemorySize64 / 1024.0 / 1024.0); + }); } - // Notes on performance counters: + // Notes on performance counters: // "How To Read Performance Counters": http://blogs.msdn.com/b/bclteam/archive/2006/06/02/618156.aspx // "How to get the CPU Usage in C#": http://stackoverflow.com/questions/278071/how-to-get-the-cpu-usage-in-c // "Mono Performance Counters": http://www.mono-project.com/Mono_Performance_Counters diff --git a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs old mode 100644 new mode 100755 index e4df7ee..88a0297 --- a/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs +++ b/OpenSim/Framework/Monitoring/SimExtraStatsCollector.cs @@ -34,6 +34,7 @@ using OpenMetaverse; using OpenMetaverse.StructuredData; using OpenSim.Framework.Monitoring.Interfaces; + namespace OpenSim.Framework.Monitoring { /// @@ -71,6 +72,11 @@ namespace OpenSim.Framework.Monitoring private volatile float pendingDownloads; private volatile float pendingUploads; private volatile float activeScripts; + private volatile float spareTime; + private volatile float sleepTime; + private volatile float physicsStep; + + private volatile float scriptLinesPerSecond; private volatile float m_frameDilation; private volatile float m_usersLoggingIn; @@ -84,17 +90,17 @@ namespace OpenSim.Framework.Monitoring // /// haven't yet been implemented... // /// // public long AssetsInCache { get { return assetsInCache; } } -// +// // /// // /// Currently unused // /// // public long TexturesInCache { get { return texturesInCache; } } -// +// // /// // /// Currently misleading since we can't currently subtract removed asset memory usage without a performance hit // /// // public long AssetCacheMemoryUsage { get { return assetCacheMemoryUsage; } } -// +// // /// // /// Currently unused // /// @@ -121,7 +127,7 @@ namespace OpenSim.Framework.Monitoring public float PendingUploads { get { return pendingUploads; } } public float ActiveScripts { get { return activeScripts; } } public float ScriptLinesPerSecond { get { return scriptLinesPerSecond; } } - + // /// // /// This is the time it took for the last asset request made in response to a cache miss. // /// @@ -171,7 +177,7 @@ namespace OpenSim.Framework.Monitoring // assetsInCache++; // //assetCacheMemoryUsage += asset.Data.Length; // } -// +// // public void RemoveAsset(UUID uuid) // { // assetsInCache--; @@ -198,7 +204,7 @@ namespace OpenSim.Framework.Monitoring // texturesInCache = 0; // textureCacheMemoryUsage = 0; // } -// +// // public void AddAssetRequestTimeAfterCacheMiss(TimeSpan ts) // { // assetRequestTimeAfterCacheMiss = ts; @@ -253,7 +259,7 @@ namespace OpenSim.Framework.Monitoring /// public void ReceiveClassicSimStatsPacket(SimStats stats) { - // FIXME: SimStats shouldn't allow an arbitrary stat packing order (which is inherited from the original + // FIXME: SimStats shouldn't allow an arbitrary stat packing order (which is inherited from the original // SimStatsPacket that was being used). // For an unknown reason the original designers decided not to @@ -270,8 +276,8 @@ namespace OpenSim.Framework.Monitoring totalFrameTime = stats.StatsBlock[8].StatValue; netFrameTime = stats.StatsBlock[9].StatValue; physicsFrameTime = stats.StatsBlock[10].StatValue; - otherFrameTime = stats.StatsBlock[11].StatValue; - imageFrameTime = stats.StatsBlock[12].StatValue; + imageFrameTime = stats.StatsBlock[11].StatValue; + otherFrameTime = stats.StatsBlock[12].StatValue; inPacketsPerSecond = stats.StatsBlock[13].StatValue; outPacketsPerSecond = stats.StatsBlock[14].StatValue; unackedBytes = stats.StatsBlock[15].StatValue; @@ -279,12 +285,16 @@ namespace OpenSim.Framework.Monitoring pendingDownloads = stats.StatsBlock[17].StatValue; pendingUploads = stats.StatsBlock[18].StatValue; activeScripts = stats.StatsBlock[19].StatValue; - scriptLinesPerSecond = stats.StatsBlock[20].StatValue; - m_frameDilation = stats.StatsBlock[22].StatValue; - m_usersLoggingIn = stats.StatsBlock[23].StatValue; - m_totalGeoPrims = stats.StatsBlock[24].StatValue; - m_totalMeshes = stats.StatsBlock[25].StatValue; - m_inUseThreads = stats.StatsBlock[26].StatValue; + sleepTime = stats.StatsBlock[20].StatValue; + spareTime = stats.StatsBlock[21].StatValue; + physicsStep = stats.StatsBlock[22].StatValue; + + scriptLinesPerSecond = stats.ExtraStatsBlock[0].StatValue; + m_frameDilation = stats.ExtraStatsBlock[1].StatValue; + m_usersLoggingIn = stats.ExtraStatsBlock[2].StatValue; + m_totalGeoPrims = stats.ExtraStatsBlock[3].StatValue; + m_totalMeshes = stats.ExtraStatsBlock[4].StatValue; + m_inUseThreads = stats.ExtraStatsBlock[5].StatValue; } /// @@ -296,7 +306,7 @@ namespace OpenSim.Framework.Monitoring StringBuilder sb = new StringBuilder(Environment.NewLine); // sb.Append("ASSET STATISTICS"); // sb.Append(Environment.NewLine); - + /* sb.Append( string.Format( @@ -332,7 +342,7 @@ Asset service request failures: {3}" + Environment.NewLine, List stats = StatsManager.GetStatsFromEachContainer("clientstack", "ClientLogoutsDueToNoReceives"); sb.AppendFormat( - "Client logouts due to no data receive timeout: {0}\n\n", + "Client logouts due to no data receive timeout: {0}\n\n", stats != null ? stats.Sum(s => s.Value).ToString() : "unknown"); // sb.Append(Environment.NewLine); @@ -433,10 +443,10 @@ Asset service request failures: {3}" + Environment.NewLine, foreach (ProcessThread currentThread in Process.GetCurrentProcess().Threads) { - // A known issue with the current process .Threads property is - // that it can return null threads, thus don't count those as + // A known issue with the current process .Threads property is + // that it can return null threads, thus don't count those as // running threads and prevent the program function from failing - if (currentThread != null && + if (currentThread != null && currentThread.ThreadState == ThreadState.Running) { numberThreadsRunning++; @@ -495,7 +505,7 @@ Asset service request failures: {3}" + Environment.NewLine, "{0:0.##}", numberThreadsRunning)); args["ProcMem"] = OSD.FromString(String.Format("{0:#,###,###.##}", memUsage)); - + return args; } } @@ -521,12 +531,12 @@ Asset service request failures: {3}" + Environment.NewLine, { return m_statsProvider.GetStats(); } - + public string XReport(string uptime, string version) { return ""; } - + public OSDMap OReport(string uptime, string version) { OSDMap ret = new OSDMap(); diff --git a/OpenSim/Framework/Monitoring/Stats/CounterStat.cs b/OpenSim/Framework/Monitoring/Stats/CounterStat.cs old mode 100644 new mode 100755 diff --git a/OpenSim/Framework/Monitoring/Stats/EventHistogram.cs b/OpenSim/Framework/Monitoring/Stats/EventHistogram.cs old mode 100644 new mode 100755 index f51f322..bc56372 --- a/OpenSim/Framework/Monitoring/Stats/EventHistogram.cs +++ b/OpenSim/Framework/Monitoring/Stats/EventHistogram.cs @@ -1,173 +1,173 @@ -/* - * Copyright (c) Contributors, http://opensimulator.org/ - * See CONTRIBUTORS.TXT for a full list of copyright holders. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the OpenSimulator Project nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -using OpenMetaverse.StructuredData; - -namespace OpenSim.Framework.Monitoring -{ -// Create a time histogram of events. The histogram is built in a wrap-around -// array of equally distributed buckets. -// For instance, a minute long histogram of second sized buckets would be: -// new EventHistogram(60, 1000) -public class EventHistogram -{ - private int m_timeBase; - private int m_numBuckets; - private int m_bucketMilliseconds; - private int m_lastBucket; - private int m_totalHistogramMilliseconds; - private long[] m_histogram; - private object histoLock = new object(); - - public EventHistogram(int numberOfBuckets, int millisecondsPerBucket) - { - m_numBuckets = numberOfBuckets; - m_bucketMilliseconds = millisecondsPerBucket; - m_totalHistogramMilliseconds = m_numBuckets * m_bucketMilliseconds; - - m_histogram = new long[m_numBuckets]; - Zero(); - m_lastBucket = 0; - m_timeBase = Util.EnvironmentTickCount(); - } - - public void Event() - { - this.Event(1); - } - - // Record an event at time 'now' in the histogram. - public void Event(int cnt) - { - lock (histoLock) - { - // The time as displaced from the base of the histogram - int bucketTime = Util.EnvironmentTickCountSubtract(m_timeBase); - - // If more than the total time of the histogram, we just start over - if (bucketTime > m_totalHistogramMilliseconds) - { - Zero(); - m_lastBucket = 0; - m_timeBase = Util.EnvironmentTickCount(); - } - else - { - // To which bucket should we add this event? - int bucket = bucketTime / m_bucketMilliseconds; - - // Advance m_lastBucket to the new bucket. Zero any buckets skipped over. - while (bucket != m_lastBucket) - { - // Zero from just after the last bucket to the new bucket or the end - for (int jj = m_lastBucket + 1; jj <= Math.Min(bucket, m_numBuckets - 1); jj++) - { - m_histogram[jj] = 0; - } - m_lastBucket = bucket; - // If the new bucket is off the end, wrap around to the beginning - if (bucket > m_numBuckets) - { - bucket -= m_numBuckets; - m_lastBucket = 0; - m_histogram[m_lastBucket] = 0; - m_timeBase += m_totalHistogramMilliseconds; - } - } - } - m_histogram[m_lastBucket] += cnt; - } - } - - // Get a copy of the current histogram - public long[] GetHistogram() - { - long[] ret = new long[m_numBuckets]; - lock (histoLock) - { - int indx = m_lastBucket + 1; - for (int ii = 0; ii < m_numBuckets; ii++, indx++) - { - if (indx >= m_numBuckets) - indx = 0; - ret[ii] = m_histogram[indx]; - } - } - return ret; - } - - public OSDMap GetHistogramAsOSDMap() - { - OSDMap ret = new OSDMap(); - - ret.Add("Buckets", OSD.FromInteger(m_numBuckets)); - ret.Add("BucketMilliseconds", OSD.FromInteger(m_bucketMilliseconds)); - ret.Add("TotalMilliseconds", OSD.FromInteger(m_totalHistogramMilliseconds)); - - // Compute a number for the first bucket in the histogram. - // This will allow readers to know how this histogram relates to any previously read histogram. - int baseBucketNum = (m_timeBase / m_bucketMilliseconds) + m_lastBucket + 1; - ret.Add("BaseNumber", OSD.FromInteger(baseBucketNum)); - - ret.Add("Values", GetHistogramAsOSDArray()); - - return ret; - } - // Get a copy of the current histogram - public OSDArray GetHistogramAsOSDArray() - { - OSDArray ret = new OSDArray(m_numBuckets); - lock (histoLock) - { - int indx = m_lastBucket + 1; - for (int ii = 0; ii < m_numBuckets; ii++, indx++) - { - if (indx >= m_numBuckets) - indx = 0; - ret[ii] = OSD.FromLong(m_histogram[indx]); - } - } - return ret; - } - - // Zero out the histogram - public void Zero() - { - lock (histoLock) - { - for (int ii = 0; ii < m_numBuckets; ii++) - m_histogram[ii] = 0; - } - } -} - -} +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using OpenMetaverse.StructuredData; + +namespace OpenSim.Framework.Monitoring +{ +// Create a time histogram of events. The histogram is built in a wrap-around +// array of equally distributed buckets. +// For instance, a minute long histogram of second sized buckets would be: +// new EventHistogram(60, 1000) +public class EventHistogram +{ + private int m_timeBase; + private int m_numBuckets; + private int m_bucketMilliseconds; + private int m_lastBucket; + private int m_totalHistogramMilliseconds; + private long[] m_histogram; + private object histoLock = new object(); + + public EventHistogram(int numberOfBuckets, int millisecondsPerBucket) + { + m_numBuckets = numberOfBuckets; + m_bucketMilliseconds = millisecondsPerBucket; + m_totalHistogramMilliseconds = m_numBuckets * m_bucketMilliseconds; + + m_histogram = new long[m_numBuckets]; + Zero(); + m_lastBucket = 0; + m_timeBase = Util.EnvironmentTickCount(); + } + + public void Event() + { + this.Event(1); + } + + // Record an event at time 'now' in the histogram. + public void Event(int cnt) + { + lock (histoLock) + { + // The time as displaced from the base of the histogram + int bucketTime = Util.EnvironmentTickCountSubtract(m_timeBase); + + // If more than the total time of the histogram, we just start over + if (bucketTime > m_totalHistogramMilliseconds) + { + Zero(); + m_lastBucket = 0; + m_timeBase = Util.EnvironmentTickCount(); + } + else + { + // To which bucket should we add this event? + int bucket = bucketTime / m_bucketMilliseconds; + + // Advance m_lastBucket to the new bucket. Zero any buckets skipped over. + while (bucket != m_lastBucket) + { + // Zero from just after the last bucket to the new bucket or the end + for (int jj = m_lastBucket + 1; jj <= Math.Min(bucket, m_numBuckets - 1); jj++) + { + m_histogram[jj] = 0; + } + m_lastBucket = bucket; + // If the new bucket is off the end, wrap around to the beginning + if (bucket > m_numBuckets) + { + bucket -= m_numBuckets; + m_lastBucket = 0; + m_histogram[m_lastBucket] = 0; + m_timeBase += m_totalHistogramMilliseconds; + } + } + } + m_histogram[m_lastBucket] += cnt; + } + } + + // Get a copy of the current histogram + public long[] GetHistogram() + { + long[] ret = new long[m_numBuckets]; + lock (histoLock) + { + int indx = m_lastBucket + 1; + for (int ii = 0; ii < m_numBuckets; ii++, indx++) + { + if (indx >= m_numBuckets) + indx = 0; + ret[ii] = m_histogram[indx]; + } + } + return ret; + } + + public OSDMap GetHistogramAsOSDMap() + { + OSDMap ret = new OSDMap(); + + ret.Add("Buckets", OSD.FromInteger(m_numBuckets)); + ret.Add("BucketMilliseconds", OSD.FromInteger(m_bucketMilliseconds)); + ret.Add("TotalMilliseconds", OSD.FromInteger(m_totalHistogramMilliseconds)); + + // Compute a number for the first bucket in the histogram. + // This will allow readers to know how this histogram relates to any previously read histogram. + int baseBucketNum = (m_timeBase / m_bucketMilliseconds) + m_lastBucket + 1; + ret.Add("BaseNumber", OSD.FromInteger(baseBucketNum)); + + ret.Add("Values", GetHistogramAsOSDArray()); + + return ret; + } + // Get a copy of the current histogram + public OSDArray GetHistogramAsOSDArray() + { + OSDArray ret = new OSDArray(m_numBuckets); + lock (histoLock) + { + int indx = m_lastBucket + 1; + for (int ii = 0; ii < m_numBuckets; ii++, indx++) + { + if (indx >= m_numBuckets) + indx = 0; + ret[ii] = OSD.FromLong(m_histogram[indx]); + } + } + return ret; + } + + // Zero out the histogram + public void Zero() + { + lock (histoLock) + { + for (int ii = 0; ii < m_numBuckets; ii++) + m_histogram[ii] = 0; + } + } +} + +} diff --git a/OpenSim/Framework/Monitoring/Stats/Stat.cs b/OpenSim/Framework/Monitoring/Stats/Stat.cs index a7cb2a6..2402acd 100644 --- a/OpenSim/Framework/Monitoring/Stats/Stat.cs +++ b/OpenSim/Framework/Monitoring/Stats/Stat.cs @@ -121,17 +121,17 @@ namespace OpenSim.Framework.Monitoring string container, StatType type, Action pullAction, - StatVerbosity verbosity) + StatVerbosity verbosity) : this( - shortName, - name, - description, - unitName, - category, - container, - type, - MeasuresOfInterest.None, - pullAction, + shortName, + name, + description, + unitName, + category, + container, + type, + MeasuresOfInterest.None, + pullAction, verbosity) { } @@ -227,11 +227,11 @@ namespace OpenSim.Framework.Monitoring { StringBuilder sb = new StringBuilder(); sb.AppendFormat( - "{0}.{1}.{2} : {3}{4}", - Category, - Container, - ShortName, - Value, + "{0}.{1}.{2} : {3}{4}", + Category, + Container, + ShortName, + Value, string.IsNullOrEmpty(UnitName) ? "" : string.Format(" {0}", UnitName)); AppendMeasuresOfInterest(sb); @@ -239,6 +239,17 @@ namespace OpenSim.Framework.Monitoring return sb.ToString(); } + public virtual OSDMap ToBriefOSDMap() + { + OSDMap ret = new OSDMap(); + + ret.Add("Value", OSD.FromReal(Value)); + + double lastChangeOverTime, averageChangeOverTime; + + return ret; + } + public virtual OSDMap ToOSDMap() { OSDMap ret = new OSDMap(); @@ -279,7 +290,7 @@ namespace OpenSim.Framework.Monitoring lock (m_samples) { // m_log.DebugFormat( - // "[STAT]: Samples for {0} are {1}", + // "[STAT]: Samples for {0} are {1}", // Name, string.Join(",", m_samples.Select(s => s.ToString()).ToArray())); foreach (double s in m_samples) @@ -315,12 +326,12 @@ namespace OpenSim.Framework.Monitoring if (ComputeMeasuresOfInterest(out lastChangeOverTime, out averageChangeOverTime)) { sb.AppendFormat( - ", {0:0.##}{1}/s, {2:0.##}{3}/s", - lastChangeOverTime, - string.IsNullOrEmpty(UnitName) ? "" : string.Format(" {0}", UnitName), + ", {0:0.##}{1}/s, {2:0.##}{3}/s", + lastChangeOverTime, + string.IsNullOrEmpty(UnitName) ? "" : string.Format(" {0}", UnitName), averageChangeOverTime, string.IsNullOrEmpty(UnitName) ? "" : string.Format(" {0}", UnitName)); } } } -} \ No newline at end of file +} diff --git a/OpenSim/Framework/Monitoring/StatsLogger.cs b/OpenSim/Framework/Monitoring/StatsLogger.cs index 15a37aa..b719af9 100644 --- a/OpenSim/Framework/Monitoring/StatsLogger.cs +++ b/OpenSim/Framework/Monitoring/StatsLogger.cs @@ -99,13 +99,13 @@ namespace OpenSim.Framework.Monitoring } string path = cmd[2]; - + using (StreamWriter sw = new StreamWriter(path, true)) { foreach (string line in GetReport()) sw.WriteLine(line); - } - + } + MainConsole.Instance.OutputFormat("Stats saved to file {0}", path); } diff --git a/OpenSim/Framework/Monitoring/StatsManager.cs b/OpenSim/Framework/Monitoring/StatsManager.cs index 3136ee8..a6b341f 100644 --- a/OpenSim/Framework/Monitoring/StatsManager.cs +++ b/OpenSim/Framework/Monitoring/StatsManager.cs @@ -47,6 +47,8 @@ namespace OpenSim.Framework.Monitoring // Subcommand used to list other stats. public const string ListSubCommand = "list"; + public static string StatsPassword { get; set; } + // All subcommands public static HashSet SubCommands = new HashSet { AllSubCommand, ListSubCommand }; @@ -80,8 +82,7 @@ namespace OpenSim.Framework.Monitoring + "'all' will show all statistics.\n" + "A name will show statistics from that category.\n" + "A . name will show statistics from that category in that container.\n" - + "More than one name can be given separated by spaces.\n" - + "THIS STATS FACILITY IS EXPERIMENTAL AND DOES NOT YET CONTAIN ALL STATS", + + "More than one name can be given separated by spaces.\n", HandleShowStatsCommand); console.Commands.AddCommand( @@ -91,7 +92,6 @@ namespace OpenSim.Framework.Monitoring "show stats [list|all|([.])+", "Alias for 'stats show' command", HandleShowStatsCommand); - StatsLogger.RegisterConsoleCommands(console); } @@ -262,33 +262,36 @@ namespace OpenSim.Framework.Monitoring { OSDMap map = new OSDMap(); - foreach (string catName in RegisteredStats.Keys) + lock (RegisteredStats) { - // Do this category if null spec, "all" subcommand or category name matches passed parameter. - // Skip category if none of the above. - if (!(String.IsNullOrEmpty(pCategoryName) || pCategoryName == AllSubCommand || pCategoryName == catName)) - continue; - - OSDMap contMap = new OSDMap(); - foreach (string contName in RegisteredStats[catName].Keys) + foreach (string catName in RegisteredStats.Keys) { - if (!(string.IsNullOrEmpty(pContainerName) || pContainerName == AllSubCommand || pContainerName == contName)) + // Do this category if null spec, "all" subcommand or category name matches passed parameter. + // Skip category if none of the above. + if (!(String.IsNullOrEmpty(pCategoryName) || pCategoryName == AllSubCommand || pCategoryName == catName)) continue; - - OSDMap statMap = new OSDMap(); - SortedDictionary theStats = RegisteredStats[catName][contName]; - foreach (string statName in theStats.Keys) + OSDMap contMap = new OSDMap(); + foreach (string contName in RegisteredStats[catName].Keys) { - if (!(String.IsNullOrEmpty(pStatName) || pStatName == AllSubCommand || pStatName == statName)) + if (!(string.IsNullOrEmpty(pContainerName) || pContainerName == AllSubCommand || pContainerName == contName)) continue; - statMap.Add(statName, theStats[statName].ToOSDMap()); - } + OSDMap statMap = new OSDMap(); + + SortedDictionary theStats = RegisteredStats[catName][contName]; + foreach (string statName in theStats.Keys) + { + if (!(String.IsNullOrEmpty(pStatName) || pStatName == AllSubCommand || pStatName == statName)) + continue; + + statMap.Add(statName, theStats[statName].ToBriefOSDMap()); + } - contMap.Add(contName, statMap); + contMap.Add(contName, statMap); + } + map.Add(catName, contMap); } - map.Add(catName, contMap); } return map; @@ -301,6 +304,17 @@ namespace OpenSim.Framework.Monitoring int response_code = 200; string contenttype = "text/json"; + if (StatsPassword != String.Empty && (!request.ContainsKey("pass") || request["pass"].ToString() != StatsPassword)) + { + responsedata["int_response_code"] = response_code; + responsedata["content_type"] = "text/plain"; + responsedata["keepalive"] = false; + responsedata["str_response_string"] = "Access denied"; + responsedata["access_control_allow_origin"] = "*"; + + return responsedata; + } + string pCategoryName = StatsManager.AllSubCommand; string pContainerName = StatsManager.AllSubCommand; string pStatName = StatsManager.AllSubCommand; @@ -358,8 +372,8 @@ namespace OpenSim.Framework.Monitoring /// public static bool RegisterStat(Stat stat) { - SortedDictionary> category = null, newCategory; - SortedDictionary container = null, newContainer; + SortedDictionary> category = null; + SortedDictionary container = null; lock (RegisteredStats) { @@ -369,22 +383,15 @@ namespace OpenSim.Framework.Monitoring if (TryGetStatParents(stat, out category, out container)) return false; - // We take a copy-on-write approach here of replacing dictionaries when keys are added or removed. - // This means that we don't need to lock or copy them on iteration, which will be a much more - // common operation after startup. - if (container != null) - newContainer = new SortedDictionary(container); - else - newContainer = new SortedDictionary(); + if (container == null) + container = new SortedDictionary(); - if (category != null) - newCategory = new SortedDictionary>(category); - else - newCategory = new SortedDictionary>(); + if (category == null) + category = new SortedDictionary>(); - newContainer[stat.ShortName] = stat; - newCategory[stat.Container] = newContainer; - RegisteredStats[stat.Category] = newCategory; + container[stat.ShortName] = stat; + category[stat.Container] = container; + RegisteredStats[stat.Category] = category; } return true; @@ -397,23 +404,24 @@ namespace OpenSim.Framework.Monitoring /// public static bool DeregisterStat(Stat stat) { - SortedDictionary> category = null, newCategory; - SortedDictionary container = null, newContainer; + SortedDictionary> category = null; + SortedDictionary container = null; lock (RegisteredStats) { if (!TryGetStatParents(stat, out category, out container)) return false; - newContainer = new SortedDictionary(container); - newContainer.Remove(stat.ShortName); - - newCategory = new SortedDictionary>(category); - newCategory.Remove(stat.Container); - - newCategory[stat.Container] = newContainer; - RegisteredStats[stat.Category] = newCategory; - + if(container != null) + { + container.Remove(stat.ShortName); + if(category != null && container.Count == 0) + { + category.Remove(stat.Container); + if(category.Count == 0) + RegisteredStats.Remove(stat.Category); + } + } return true; } } @@ -554,4 +562,4 @@ namespace OpenSim.Framework.Monitoring Debug, Info } -} \ No newline at end of file +} diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index a644fa5..9cac451 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -96,7 +96,7 @@ namespace OpenSim.Framework.Monitoring FirstTick = Environment.TickCount & Int32.MaxValue; LastTick = FirstTick; - Stat + Stat = new Stat( name, string.Format("Last update of thread {0}", name), @@ -180,6 +180,30 @@ namespace OpenSim.Framework.Monitoring m_watchdogTimer.Elapsed += WatchdogTimerElapsed; } + public static void Stop() + { + if(m_threads == null) + return; + + lock(m_threads) + { + m_enabled = false; + if(m_watchdogTimer != null) + { + m_watchdogTimer.Dispose(); + m_watchdogTimer = null; + } + + foreach(ThreadWatchdogInfo twi in m_threads.Values) + { + Thread t = twi.Thread; + if(t.IsAlive) + t.Abort(); + } + m_threads.Clear(); + } + } + /// /// Add a thread to the watchdog tracker. /// @@ -230,14 +254,12 @@ namespace OpenSim.Framework.Monitoring twi.Cleanup(); m_threads.Remove(threadID); - return true; } else { m_log.WarnFormat( "[WATCHDOG]: Requested to remove thread with ID {0} but this is not being monitored", threadID); - return false; } } @@ -317,6 +339,8 @@ namespace OpenSim.Framework.Monitoring /// private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e) { + if(!m_enabled) + return; int now = Environment.TickCount & Int32.MaxValue; int msElapsed = now - LastWatchdogThreadTick; @@ -332,27 +356,36 @@ namespace OpenSim.Framework.Monitoring if (callback != null) { List callbackInfos = null; + List threadsToRemove = null; + + const ThreadState thgone = ThreadState.Stopped; lock (m_threads) { - foreach (ThreadWatchdogInfo threadInfo in m_threads.Values) + foreach(ThreadWatchdogInfo threadInfo in m_threads.Values) { - if (threadInfo.Thread.ThreadState == ThreadState.Stopped) + if(!m_enabled) + return; + if((threadInfo.Thread.ThreadState & thgone) != 0) { - RemoveThread(threadInfo.Thread.ManagedThreadId); + if(threadsToRemove == null) + threadsToRemove = new List(); - if (callbackInfos == null) + threadsToRemove.Add(threadInfo); +/* + if(callbackInfos == null) callbackInfos = new List(); callbackInfos.Add(threadInfo); +*/ } - else if (!threadInfo.IsTimedOut && now - threadInfo.LastTick >= threadInfo.Timeout) + else if(!threadInfo.IsTimedOut && now - threadInfo.LastTick >= threadInfo.Timeout) { threadInfo.IsTimedOut = true; - if (threadInfo.AlarmIfTimeout) + if(threadInfo.AlarmIfTimeout) { - if (callbackInfos == null) + if(callbackInfos == null) callbackInfos = new List(); // Send a copy of the watchdog info to prevent race conditions where the watchdog @@ -361,9 +394,13 @@ namespace OpenSim.Framework.Monitoring } } } + + if(threadsToRemove != null) + foreach(ThreadWatchdogInfo twi in threadsToRemove) + RemoveThread(twi.Thread.ManagedThreadId); } - if (callbackInfos != null) + if(callbackInfos != null) foreach (ThreadWatchdogInfo callbackInfo in callbackInfos) callback(callbackInfo); } @@ -377,4 +414,4 @@ namespace OpenSim.Framework.Monitoring m_watchdogTimer.Start(); } } -} \ No newline at end of file +} diff --git a/OpenSim/Framework/Monitoring/WorkManager.cs b/OpenSim/Framework/Monitoring/WorkManager.cs index d1a74ce..5d9b185 100644 --- a/OpenSim/Framework/Monitoring/WorkManager.cs +++ b/OpenSim/Framework/Monitoring/WorkManager.cs @@ -36,16 +36,16 @@ namespace OpenSim.Framework.Monitoring /// Manages various work items in the simulator. /// /// - /// Currently, here work can be started + /// Currently, here work can be started /// * As a long-running and monitored thread. /// * In a thread that will never timeout but where the job is expected to eventually complete. /// * In a threadpool thread that will timeout if it takes a very long time to complete (> 10 mins). /// * As a job which will be run in a single-threaded job engine. Such jobs must not incorporate delays (sleeps, /// network waits, etc.). - /// + /// /// This is an evolving approach to better manage the work that OpenSimulator is asked to do from a very diverse /// range of sources (client actions, incoming network, outgoing network calls, etc.). - /// + /// /// Util.FireAndForget is still available to insert jobs in the threadpool, though this is equivalent to /// WorkManager.RunInThreadPool(). /// @@ -57,7 +57,7 @@ namespace OpenSim.Framework.Monitoring static WorkManager() { - JobEngine = new JobEngine("Non-blocking non-critical job engine", "JOB ENGINE"); + JobEngine = new JobEngine("Non-blocking non-critical job engine", "JOB ENGINE", 30000); StatsManager.RegisterStat( new Stat( @@ -82,6 +82,12 @@ namespace OpenSim.Framework.Monitoring HandleControlCommand); } + public static void Stop() + { + JobEngine.Stop(); + Watchdog.Stop(); + } + /// /// Start a new long-lived thread. /// @@ -121,6 +127,7 @@ namespace OpenSim.Framework.Monitoring Thread thread = new Thread(start); thread.Priority = priority; thread.IsBackground = isBackground; + thread.Name = name; Watchdog.ThreadWatchdogInfo twi = new Watchdog.ThreadWatchdogInfo(thread, timeout, name) @@ -129,7 +136,6 @@ namespace OpenSim.Framework.Monitoring Watchdog.AddThread(twi, name, log:log); thread.Start(); - thread.Name = name; return thread; } @@ -143,7 +149,7 @@ namespace OpenSim.Framework.Monitoring /// Name of the thread public static void RunInThread(WaitCallback callback, object obj, string name, bool log = false) { - if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest) + if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest) { Culture.SetCurrentCulture(); callback(obj); @@ -168,7 +174,7 @@ namespace OpenSim.Framework.Monitoring } /// - /// Run the callback via a threadpool thread. + /// Run the callback via a threadpool thread. /// /// /// Such jobs may run after some delay but must always complete. @@ -176,9 +182,9 @@ namespace OpenSim.Framework.Monitoring /// /// /// The name of the job. This is used in monitoring and debugging. - public static void RunInThreadPool(System.Threading.WaitCallback callback, object obj, string name) + public static void RunInThreadPool(System.Threading.WaitCallback callback, object obj, string name, bool timeout = true) { - Util.FireAndForget(callback, obj, name); + Util.FireAndForget(callback, obj, name, timeout); } /// @@ -187,17 +193,17 @@ namespace OpenSim.Framework.Monitoring /// /// This differs from direct scheduling (e.g. Util.FireAndForget) in that a job can be run in the job /// engine if it is running, where all jobs are currently performed in sequence on a single thread. This is - /// to prevent observed overload and server freeze problems when there are hundreds of connections which all attempt to + /// to prevent observed overload and server freeze problems when there are hundreds of connections which all attempt to /// perform work at once (e.g. in conference situations). With lower numbers of connections, the small - /// delay in performing jobs in sequence rather than concurrently has not been notiecable in testing, though a future more + /// delay in performing jobs in sequence rather than concurrently has not been notiecable in testing, though a future more /// sophisticated implementation could perform jobs concurrently when the server is under low load. - /// + /// /// However, be advised that some callers of this function rely on all jobs being performed in sequence if any /// jobs are performed in sequence (i.e. if jobengine is active or not). Therefore, expanding the jobengine /// beyond a single thread will require considerable thought. - /// + /// /// Also, any jobs submitted must be guaranteed to complete within a reasonable timeframe (e.g. they cannot - /// incorporate a network delay with a long timeout). At the moment, work that could suffer such issues + /// incorporate a network delay with a long timeout). At the moment, work that could suffer such issues /// should still be run directly with RunInThread(), Util.FireAndForget(), etc. This is another area where /// the job engine could be improved and so CPU utilization improved by better management of concurrency within /// OpenSimulator. @@ -211,10 +217,10 @@ namespace OpenSim.Framework.Monitoring /// If set to true then extra logging is performed. public static void RunJob( string jobType, WaitCallback callback, object obj, string name, - bool canRunInThisThread = false, bool mustNotTimeout = false, + bool canRunInThisThread = false, bool mustNotTimeout = false, bool log = false) { - if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest) + if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest) { Culture.SetCurrentCulture(); callback(obj); @@ -225,10 +231,8 @@ namespace OpenSim.Framework.Monitoring JobEngine.QueueJob(name, () => callback(obj)); else if (canRunInThisThread) callback(obj); - else if (mustNotTimeout) - RunInThread(callback, obj, name, log); else - Util.FireAndForget(callback, obj, name); + Util.FireAndForget(callback, obj, name, !mustNotTimeout); } private static void HandleControlCommand(string module, string[] args) @@ -272,16 +276,16 @@ namespace OpenSim.Framework.Monitoring MainConsole.Instance.Output("Usage: debug jobengine log "); return; } - + // int logLevel; int logLevel = int.Parse(args[3]); // if (ConsoleUtil.TryParseConsoleInt(MainConsole.Instance, args[4], out logLevel)) - // { + // { JobEngine.LogLevel = logLevel; MainConsole.Instance.OutputFormat("Set debug log level to {0}", JobEngine.LogLevel); // } } - else + else { MainConsole.Instance.OutputFormat("Unrecognized job engine subcommand {0}", subCommand); } -- cgit v1.1