Add "show threadpool calls active" console debug command.

This shows named threadpool calls (excluding timer and network calls) that are currently queued or running.
Also shows total of labelled and any anonymous calls.

2 files changed, 67 insertions, 6 deletions

diff --git a/OpenSim/Framework/Servers/ServerBase.cs b/OpenSim/Framework/Servers/ServerBase.cs
index eb9fb8b..c22c119 100644
--- a/OpenSim/Framework/Servers/ServerBase.cs
+++ b/OpenSim/Framework/Servers/ServerBase.cs
@@ -293,10 +293,16 @@ namespace OpenSim.Framework.Servers
                 HandleDebugThreadpoolLevel);
 
             m_console.Commands.AddCommand(
-                "Debug", false, "show threadpool calls",
-                "show threadpool calls",
-                "Show the number of labelled threadpool calls.",
-                HandleShowThreadpoolCalls);
+                "Debug", false, "show threadpool calls active",
+                "show threadpool calls active",
+                "Show details about threadpool calls that are still active (currently waiting or in progress)",
+                HandleShowThreadpoolCallsActive);
+
+            m_console.Commands.AddCommand(
+                "Debug", false, "show threadpool calls complete",
+                "show threadpool calls complete",
+                "Show details about threadpool calls that have been completed.",
+                HandleShowThreadpoolCallsComplete);
 
             m_console.Commands.AddCommand(
                 "Debug", false, "force gc",
@@ -361,7 +367,36 @@ namespace OpenSim.Framework.Servers
             Notice("serialosdreq is now {0}", setSerializeOsdRequests);
         }
 
-        private void HandleShowThreadpoolCalls(string module, string[] args)
+        private void HandleShowThreadpoolCallsActive(string module, string[] args)
+        {
+            List<KeyValuePair<string, int>> calls = Util.GetFireAndForgetCallsInProgress().ToList();
+            calls.Sort((kvp1, kvp2) => kvp2.Value.CompareTo(kvp1.Value));
+            int namedCalls = 0;
+
+            ConsoleDisplayList cdl = new ConsoleDisplayList();
+            foreach (KeyValuePair<string, int> kvp in calls)
+            {
+                if (kvp.Value > 0)
+                {
+                    cdl.AddRow(kvp.Key, kvp.Value);
+                    namedCalls += kvp.Value;
+                }
+            }
+
+            cdl.AddRow("TOTAL NAMED", namedCalls);
+
+            long allQueuedCalls = Util.TotalQueuedFireAndForgetCalls;
+            long allRunningCalls = Util.TotalRunningFireAndForgetCalls;
+
+            cdl.AddRow("TOTAL QUEUED", allQueuedCalls);
+            cdl.AddRow("TOTAL RUNNING", allRunningCalls);
+            cdl.AddRow("TOTAL ANONYMOUS", allQueuedCalls + allRunningCalls - namedCalls);
+            cdl.AddRow("TOTAL ALL", allQueuedCalls + allRunningCalls);
+
+            MainConsole.Instance.Output(cdl.ToString());
+        }
+
+        private void HandleShowThreadpoolCallsComplete(string module, string[] args)
         {
             List<KeyValuePair<string, int>> calls = Util.GetFireAndForgetCallsMade().ToList();
             calls.Sort((kvp1, kvp2) => kvp2.Value.CompareTo(kvp1.Value));
diff --git a/OpenSim/Framework/Util.cs b/OpenSim/Framework/Util.cs
index baad0b9..97c958a 100644
--- a/OpenSim/Framework/Util.cs
+++ b/OpenSim/Framework/Util.cs
@@ -2052,6 +2052,9 @@ namespace OpenSim.Framework
         private static long numTotalThreadFuncsCalled = 0;
         private static Int32 threadFuncOverloadMode = 0;
 
+        public static long TotalQueuedFireAndForgetCalls { get { return numQueuedThreadFuncs; } }
+        public static long TotalRunningFireAndForgetCalls { get { return numRunningThreadFuncs; } }
+
         // Maps (ThreadFunc number -> Thread)
         private static ConcurrentDictionary<long, ThreadInfo> activeThreads = new ConcurrentDictionary<long, ThreadInfo>();
 
@@ -2089,6 +2092,13 @@ namespace OpenSim.Framework
 
         private static Dictionary<string, int> m_fireAndForgetCallsMade = new Dictionary<string, int>();
 
+        public static Dictionary<string, int> GetFireAndForgetCallsInProgress()
+        {
+            return new Dictionary<string, int>(m_fireAndForgetCallsInProgress);
+        }
+
+        private static Dictionary<string, int> m_fireAndForgetCallsInProgress = new Dictionary<string, int>();
+
         public static void FireAndForget(System.Threading.WaitCallback callback)
         {
             FireAndForget(callback, null, null);
@@ -2109,6 +2119,11 @@ namespace OpenSim.Framework
                     m_fireAndForgetCallsMade[context] = 1;
                 else
                     m_fireAndForgetCallsMade[context]++;
+
+                if (!m_fireAndForgetCallsInProgress.ContainsKey(context))
+                    m_fireAndForgetCallsInProgress[context] = 1;
+                else
+                    m_fireAndForgetCallsInProgress[context]++;
             }
 
             WaitCallback realCallback;
@@ -2121,7 +2136,15 @@ namespace OpenSim.Framework
             if (FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
             {
                 // If we're running regression tests, then we want any exceptions to rise up to the test code.
-                realCallback = o => { Culture.SetCurrentCulture(); callback(o); };
+                realCallback = 
+                    o => 
+                    { 
+                        Culture.SetCurrentCulture(); 
+                        callback(o); 
+                        
+                        if (context != null)
+                            m_fireAndForgetCallsInProgress[context]--;
+                    };
             }
             else
             {
@@ -2160,6 +2183,9 @@ namespace OpenSim.Framework
                         activeThreads.TryRemove(threadFuncNum, out dummy);
                         if ((loggingEnabled || (threadFuncOverloadMode == 1)) && threadInfo.LogThread)
                             m_log.DebugFormat("Exit threadfunc {0} ({1})", threadFuncNum, FormatDuration(threadInfo.Elapsed()));
+
+                        if (context != null)
+                            m_fireAndForgetCallsInProgress[context]--;
                     }
                 };
             }