aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
authorOren Hurvitz2014-01-16 15:08:45 +0200
committerOren Hurvitz2014-03-25 08:01:50 +0100
commit84d7227dfd8fc7e673b3015e2b5495d1752e16a3 (patch)
treefdacb6bcf75a2090b19d7df43bb036bb795ae9ad
parentAdded a watchdog to abort slow threads in the main thread pool (diff)
downloadopensim-SC_OLD-84d7227dfd8fc7e673b3015e2b5495d1752e16a3.zip
opensim-SC_OLD-84d7227dfd8fc7e673b3015e2b5495d1752e16a3.tar.gz
opensim-SC_OLD-84d7227dfd8fc7e673b3015e2b5495d1752e16a3.tar.bz2
opensim-SC_OLD-84d7227dfd8fc7e673b3015e2b5495d1752e16a3.tar.xz
Changed LogThreadPool to have 4 logging levels. Added console command "debug threadpool level" to set the logging level.
Resolves http://opensimulator.org/mantis/view.php?id=6945
-rw-r--r--OpenSim/Framework/Servers/ServerBase.cs38
-rw-r--r--OpenSim/Framework/Util.cs59
2 files changed, 78 insertions, 19 deletions
diff --git a/OpenSim/Framework/Servers/ServerBase.cs b/OpenSim/Framework/Servers/ServerBase.cs
index c258ff6..fd56587 100644
--- a/OpenSim/Framework/Servers/ServerBase.cs
+++ b/OpenSim/Framework/Servers/ServerBase.cs
@@ -279,6 +279,17 @@ namespace OpenSim.Framework.Servers
279 "debug threadpool status", 279 "debug threadpool status",
280 "Show current debug threadpool parameters.", 280 "Show current debug threadpool parameters.",
281 HandleDebugThreadpoolStatus); 281 HandleDebugThreadpoolStatus);
282
283 m_console.Commands.AddCommand(
284 "Debug", false, "debug threadpool level",
285 "debug threadpool level 0.." + Util.MAX_THREADPOOL_LEVEL,
286 "Turn on logging of activity in the main thread pool.",
287 "Log levels:\n"
288 + " 0 = no logging\n"
289 + " 1 = only first line of stack trace; don't log common threads\n"
290 + " 2 = full stack trace; don't log common threads\n"
291 + " 3 = full stack trace, including common threads\n",
292 HandleDebugThreadpoolLevel);
282 293
283 m_console.Commands.AddCommand( 294 m_console.Commands.AddCommand(
284 "Debug", false, "force gc", 295 "Debug", false, "force gc",
@@ -432,6 +443,33 @@ namespace OpenSim.Framework.Servers
432 } 443 }
433 } 444 }
434 445
446 private static void HandleDebugThreadpoolLevel(string module, string[] cmdparams)
447 {
448 if (cmdparams.Length < 4)
449 {
450 MainConsole.Instance.Output("Usage: debug threadpool level 0.." + Util.MAX_THREADPOOL_LEVEL);
451 return;
452 }
453
454 string rawLevel = cmdparams[3];
455 int newLevel;
456
457 if (!int.TryParse(rawLevel, out newLevel))
458 {
459 MainConsole.Instance.OutputFormat("{0} is not a valid debug level", rawLevel);
460 return;
461 }
462
463 if (newLevel < 0 || newLevel > Util.MAX_THREADPOOL_LEVEL)
464 {
465 MainConsole.Instance.OutputFormat("{0} is outside the valid debug level range of 0.." + Util.MAX_THREADPOOL_LEVEL, newLevel);
466 return;
467 }
468
469 Util.LogThreadPool = newLevel;
470 MainConsole.Instance.OutputFormat("LogThreadPool set to {0}", newLevel);
471 }
472
435 private void HandleForceGc(string module, string[] args) 473 private void HandleForceGc(string module, string[] args)
436 { 474 {
437 Notice("Manually invoking runtime garbage collection"); 475 Notice("Manually invoking runtime garbage collection");
diff --git a/OpenSim/Framework/Util.cs b/OpenSim/Framework/Util.cs
index 2de596e..b39dc5f 100644
--- a/OpenSim/Framework/Util.cs
+++ b/OpenSim/Framework/Util.cs
@@ -110,15 +110,6 @@ namespace OpenSim.Framework
110 public int MaxConcurrentWorkItems { get; set; } 110 public int MaxConcurrentWorkItems { get; set; }
111 } 111 }
112 112
113 [Flags]
114 public enum DebugFlagsEnum : uint
115 {
116 None = 0,
117
118 // Log every invocation of a thread using the threadpool
119 LogThreadPool = 0x01
120 }
121
122 /// <summary> 113 /// <summary>
123 /// Miscellaneous utility functions 114 /// Miscellaneous utility functions
124 /// </summary> 115 /// </summary>
@@ -127,13 +118,19 @@ namespace OpenSim.Framework
127 private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); 118 private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
128 119
129 /// <summary> 120 /// <summary>
130 /// Flags that enable additional debugging. 121 /// Log-level for the thread pool:
122 /// 0 = no logging
123 /// 1 = only first line of stack trace; don't log common threads
124 /// 2 = full stack trace; don't log common threads
125 /// 3 = full stack trace, including common threads
131 /// </summary> 126 /// </summary>
132 public static DebugFlagsEnum DebugFlags { get; set; } 127 public static int LogThreadPool { get; set; }
128
129 public static readonly int MAX_THREADPOOL_LEVEL = 3;
133 130
134 static Util() 131 static Util()
135 { 132 {
136 DebugFlags = DebugFlagsEnum.None; 133 LogThreadPool = 0;
137 } 134 }
138 135
139 private static uint nextXferID = 5000; 136 private static uint nextXferID = 5000;
@@ -1921,6 +1918,7 @@ namespace OpenSim.Framework
1921 public long ThreadFuncNum { get; set; } 1918 public long ThreadFuncNum { get; set; }
1922 public string StackTrace { get; set; } 1919 public string StackTrace { get; set; }
1923 private string context; 1920 private string context;
1921 public bool LogThread { get; set; }
1924 1922
1925 public IWorkItemResult WorkItem { get; set; } 1923 public IWorkItemResult WorkItem { get; set; }
1926 public Thread Thread { get; set; } 1924 public Thread Thread { get; set; }
@@ -1932,6 +1930,7 @@ namespace OpenSim.Framework
1932 { 1930 {
1933 ThreadFuncNum = threadFuncNum; 1931 ThreadFuncNum = threadFuncNum;
1934 this.context = context; 1932 this.context = context;
1933 LogThread = true;
1935 Thread = null; 1934 Thread = null;
1936 Running = false; 1935 Running = false;
1937 Aborted = false; 1936 Aborted = false;
@@ -2027,7 +2026,7 @@ namespace OpenSim.Framework
2027 { 2026 {
2028 WaitCallback realCallback; 2027 WaitCallback realCallback;
2029 2028
2030 bool loggingEnabled = (DebugFlags & DebugFlagsEnum.LogThreadPool) != 0; 2029 bool loggingEnabled = LogThreadPool > 0;
2031 2030
2032 long threadFuncNum = Interlocked.Increment(ref nextThreadFuncNum); 2031 long threadFuncNum = Interlocked.Increment(ref nextThreadFuncNum);
2033 ThreadInfo threadInfo = new ThreadInfo(threadFuncNum, context); 2032 ThreadInfo threadInfo = new ThreadInfo(threadFuncNum, context);
@@ -2051,7 +2050,7 @@ namespace OpenSim.Framework
2051 2050
2052 try 2051 try
2053 { 2052 {
2054 if (loggingEnabled || (threadFuncOverloadMode == 1)) 2053 if ((loggingEnabled || (threadFuncOverloadMode == 1)) && threadInfo.LogThread)
2055 m_log.DebugFormat("Run threadfunc {0} (Queued {1}, Running {2})", threadFuncNum, numQueued1, numRunning1); 2054 m_log.DebugFormat("Run threadfunc {0} (Queued {1}, Running {2})", threadFuncNum, numQueued1, numRunning1);
2056 2055
2057 Culture.SetCurrentCulture(); 2056 Culture.SetCurrentCulture();
@@ -2072,7 +2071,7 @@ namespace OpenSim.Framework
2072 threadInfo.Ended(); 2071 threadInfo.Ended();
2073 ThreadInfo dummy; 2072 ThreadInfo dummy;
2074 activeThreads.TryRemove(threadFuncNum, out dummy); 2073 activeThreads.TryRemove(threadFuncNum, out dummy);
2075 if (loggingEnabled || (threadFuncOverloadMode == 1)) 2074 if ((loggingEnabled || (threadFuncOverloadMode == 1)) && threadInfo.LogThread)
2076 m_log.DebugFormat("Exit threadfunc {0} ({1})", threadFuncNum, FormatDuration(threadInfo.Elapsed())); 2075 m_log.DebugFormat("Exit threadfunc {0} ({1})", threadFuncNum, FormatDuration(threadInfo.Elapsed()));
2077 } 2076 }
2078 }; 2077 };
@@ -2097,11 +2096,16 @@ namespace OpenSim.Framework
2097 { 2096 {
2098 string full, partial; 2097 string full, partial;
2099 GetFireAndForgetStackTrace(out full, out partial); 2098 GetFireAndForgetStackTrace(out full, out partial);
2100 m_log.DebugFormat("Queue threadfunc {0} (Queued {1}, Running {2}) {3}{4}",
2101 threadFuncNum, numQueued, numRunningThreadFuncs,
2102 (context == null) ? "" : ("(" + context + ") "),
2103 loggingEnabled ? full : partial);
2104 threadInfo.StackTrace = full; 2099 threadInfo.StackTrace = full;
2100 threadInfo.LogThread = ShouldLogThread(partial);
2101
2102 if (threadInfo.LogThread)
2103 {
2104 m_log.DebugFormat("Queue threadfunc {0} (Queued {1}, Running {2}) {3}{4}",
2105 threadFuncNum, numQueued, numRunningThreadFuncs,
2106 (context == null) ? "" : ("(" + context + ") "),
2107 (LogThreadPool >= 2) ? full : partial);
2108 }
2105 } 2109 }
2106 2110
2107 switch (FireAndForgetMethod) 2111 switch (FireAndForgetMethod)
@@ -2143,6 +2147,23 @@ namespace OpenSim.Framework
2143 } 2147 }
2144 2148
2145 /// <summary> 2149 /// <summary>
2150 /// Returns whether the thread should be logged. Some very common threads aren't logged,
2151 /// to avoid filling up the log.
2152 /// </summary>
2153 /// <param name="stackTrace">A partial stack trace of where the thread was queued</param>
2154 /// <returns>Whether to log this thread</returns>
2155 private static bool ShouldLogThread(string stackTrace)
2156 {
2157 if (LogThreadPool < 3)
2158 {
2159 if (stackTrace.Contains("BeginFireQueueEmpty"))
2160 return false;
2161 }
2162
2163 return true;
2164 }
2165
2166 /// <summary>
2146 /// Returns a stack trace for a thread added using FireAndForget(). 2167 /// Returns a stack trace for a thread added using FireAndForget().
2147 /// </summary> 2168 /// </summary>
2148 /// <param name="full">Will contain the full stack trace</param> 2169 /// <param name="full">Will contain the full stack trace</param>