Add experimental "OpenSim object memory churn" statistics to output of region console "show stats" command

This aims to capture the amount of memory that OpenSim turns over whilst operating a region.
This memory is not lost - apart from leaks it is reclaimed by the garbage collector.
However, the more memory that gets turned over the more work the GC has to do to reclaim it.

5 files changed, 72 insertions, 63 deletions

diff --git a/OpenSim/Framework/Statistics/BaseStatsCollector.cs b/OpenSim/Framework/Statistics/BaseStatsCollector.cs
index c9e57ce..28ce650 100644
--- a/OpenSim/Framework/Statistics/BaseStatsCollector.cs
+++ b/OpenSim/Framework/Statistics/BaseStatsCollector.cs
@@ -44,14 +44,18 @@ namespace OpenSim.Framework.Statistics
             StringBuilder sb = new StringBuilder(Environment.NewLine);
             sb.Append("MEMORY STATISTICS");
             sb.Append(Environment.NewLine);
-            sb.Append(
-                string.Format(
-                    "Allocated to OpenSim objects: {0} MB\n",
-                    Math.Round(GC.GetTotalMemory(false) / 1024.0 / 1024.0)));
-            sb.Append(
-                string.Format(
-                    "Process memory              : {0} MB\n",
-                    Math.Round(Process.GetCurrentProcess().WorkingSet64 / 1024.0 / 1024.0)));
+
+            sb.AppendFormat(
+                "Allocated to OpenSim objects: {0} MB\n",
+                Math.Round(GC.GetTotalMemory(false) / 1024.0 / 1024.0));
+
+            sb.AppendFormat(
+                "OpenSim object memory churn : {0} KB/s\n",
+                Math.Round((MemoryWatchdog.AverageMemoryChurn * 1000) / 1024.0 / 1024, 3));
+
+            sb.AppendFormat(
+                "Process memory              : {0} MB\n",
+                Math.Round(Process.GetCurrentProcess().WorkingSet64 / 1024.0 / 1024.0));
 
             return sb.ToString();
         }
diff --git a/OpenSim/Framework/Watchdog.cs b/OpenSim/Framework/Watchdog.cs
index 8a74f53..54e3d1a 100644
--- a/OpenSim/Framework/Watchdog.cs
+++ b/OpenSim/Framework/Watchdog.cs
@@ -325,6 +325,9 @@ namespace OpenSim.Framework
                         callback(callbackInfo);
             }
 
+            if (MemoryWatchdog.Enabled)
+                MemoryWatchdog.Update();
+
             m_watchdogTimer.Start();
         }
     }
diff --git a/OpenSim/Region/Application/OpenSim.cs b/OpenSim/Region/Application/OpenSim.cs
index a4d85ac..f68974c 100644
--- a/OpenSim/Region/Application/OpenSim.cs
+++ b/OpenSim/Region/Application/OpenSim.cs
@@ -200,9 +200,9 @@ namespace OpenSim
             PrintFileToConsole("startuplogo.txt");
 
             // For now, start at the 'root' level by default
-            if (m_sceneManager.Scenes.Count == 1) // If there is only one region, select it
+            if (SceneManager.Scenes.Count == 1) // If there is only one region, select it
                 ChangeSelectedRegion("region",
-                                     new string[] {"change", "region", m_sceneManager.Scenes[0].RegionInfo.RegionName});
+                                     new string[] {"change", "region", SceneManager.Scenes[0].RegionInfo.RegionName});
             else
                 ChangeSelectedRegion("region", new string[] {"change", "region", "root"});
 
@@ -461,7 +461,7 @@ namespace OpenSim
             if (cmdparams.Length > 4)
                 alert = String.Format("\n{0}\n", String.Join(" ", cmdparams, 4, cmdparams.Length - 4));
 
-            IList agents = m_sceneManager.GetCurrentSceneAvatars();
+            IList agents = SceneManager.GetCurrentSceneAvatars();
 
             foreach (ScenePresence presence in agents)
             {
@@ -542,7 +542,7 @@ namespace OpenSim
         private void HandleForceUpdate(string module, string[] args)
         {
             MainConsole.Instance.Output("Updating all clients");
-            m_sceneManager.ForceCurrentSceneClientUpdate();
+            SceneManager.ForceCurrentSceneClientUpdate();
         }
 
         /// <summary>
@@ -554,7 +554,7 @@ namespace OpenSim
         {
             if (args.Length == 6)
             {
-                m_sceneManager.HandleEditCommandOnCurrentScene(args);
+                SceneManager.HandleEditCommandOnCurrentScene(args);
             }
             else
             {
@@ -765,7 +765,7 @@ namespace OpenSim
                     case "load":
                         if (cmdparams.Length > 1)
                         {
-                            foreach (Scene s in new ArrayList(m_sceneManager.Scenes))
+                            foreach (Scene s in new ArrayList(SceneManager.Scenes))
                             {
                                 MainConsole.Instance.Output(String.Format("Loading module: {0}", cmdparams[1]));
                                 m_moduleLoader.LoadRegionModules(cmdparams[1], s);
@@ -803,14 +803,14 @@ namespace OpenSim
 
                 case "backup":
                     MainConsole.Instance.Output("Triggering save of pending object updates to persistent store");
-                    m_sceneManager.BackupCurrentScene();
+                    SceneManager.BackupCurrentScene();
                     break;
 
                 case "remove-region":
                     string regRemoveName = CombineParams(cmdparams, 0);
 
                     Scene removeScene;
-                    if (m_sceneManager.TryGetScene(regRemoveName, out removeScene))
+                    if (SceneManager.TryGetScene(regRemoveName, out removeScene))
                         RemoveRegion(removeScene, false);
                     else
                         MainConsole.Instance.Output("No region with that name");
@@ -820,14 +820,14 @@ namespace OpenSim
                     string regDeleteName = CombineParams(cmdparams, 0);
 
                     Scene killScene;
-                    if (m_sceneManager.TryGetScene(regDeleteName, out killScene))
+                    if (SceneManager.TryGetScene(regDeleteName, out killScene))
                         RemoveRegion(killScene, true);
                     else
                         MainConsole.Instance.Output("no region with that name");
                     break;
 
                 case "restart":
-                    m_sceneManager.RestartCurrentScene();
+                    SceneManager.RestartCurrentScene();
                     break;
             }
         }
@@ -842,7 +842,7 @@ namespace OpenSim
             {
                 string newRegionName = CombineParams(cmdparams, 2);
 
-                if (!m_sceneManager.TrySetCurrentScene(newRegionName))
+                if (!SceneManager.TrySetCurrentScene(newRegionName))
                     MainConsole.Instance.Output(String.Format("Couldn't select region {0}", newRegionName));
             }
             else
@@ -850,7 +850,7 @@ namespace OpenSim
                 MainConsole.Instance.Output("Usage: change region <region name>");
             }
 
-            string regionName = (m_sceneManager.CurrentScene == null ? "root" : m_sceneManager.CurrentScene.RegionInfo.RegionName);
+            string regionName = (SceneManager.CurrentScene == null ? "root" : SceneManager.CurrentScene.RegionInfo.RegionName);
             MainConsole.Instance.Output(String.Format("Currently selected region is {0}", regionName));
 
 //            m_log.DebugFormat("Original prompt is {0}", m_consolePrompt);
@@ -868,7 +868,7 @@ namespace OpenSim
             });
 
             m_console.DefaultPrompt = prompt;
-            m_console.ConsoleScene = m_sceneManager.CurrentScene;
+            m_console.ConsoleScene = SceneManager.CurrentScene;
         }
 
         /// <summary>
@@ -892,7 +892,7 @@ namespace OpenSim
                         int newDebug;
                         if (int.TryParse(args[2], out newDebug))
                         {
-                            m_sceneManager.SetDebugPacketLevelOnCurrentScene(newDebug, name);
+                            SceneManager.SetDebugPacketLevelOnCurrentScene(newDebug, name);
                             // We provide user information elsewhere if any clients had their debug level set.
 //                            MainConsole.Instance.OutputFormat("Debug packet level set to {0}", newDebug);
                         }
@@ -907,7 +907,7 @@ namespace OpenSim
                 case "scene":
                     if (args.Length == 4)
                     {
-                        if (m_sceneManager.CurrentScene == null)
+                        if (SceneManager.CurrentScene == null)
                         {
                             MainConsole.Instance.Output("Please use 'change region <regioname>' first");
                         }
@@ -915,7 +915,7 @@ namespace OpenSim
                         {
                             string key = args[2];
                             string value = args[3];
-                            m_sceneManager.CurrentScene.SetSceneCoreDebug(
+                            SceneManager.CurrentScene.SetSceneCoreDebug(
                                 new Dictionary<string, string>() { { key, value } });
 
                             MainConsole.Instance.OutputFormat("Set debug scene {0} = {1}", key, value);
@@ -954,10 +954,10 @@ namespace OpenSim
                     IList agents;
                     if (showParams.Length > 1 && showParams[1] == "full")
                     {
-                        agents = m_sceneManager.GetCurrentScenePresences();
+                        agents = SceneManager.GetCurrentScenePresences();
                     } else
                     {
-                        agents = m_sceneManager.GetCurrentSceneAvatars();
+                        agents = SceneManager.GetCurrentSceneAvatars();
                     }
                 
                     MainConsole.Instance.Output(String.Format("\nAgents connected: {0}\n", agents.Count));
@@ -1037,7 +1037,7 @@ namespace OpenSim
                         MainConsole.Instance.Output("Shared Module: " + module.Name);
                     }
 
-                    m_sceneManager.ForEachScene(
+                    SceneManager.ForEachScene(
                         delegate(Scene scene) {
                         m_log.Error("The currently loaded modules in " + scene.RegionInfo.RegionName + " are:");
                         foreach (IRegionModule module in scene.Modules.Values)
@@ -1050,7 +1050,7 @@ namespace OpenSim
                     }
                     );
 
-                    m_sceneManager.ForEachScene(
+                    SceneManager.ForEachScene(
                         delegate(Scene scene) {
                         MainConsole.Instance.Output("Loaded new region modules in" + scene.RegionInfo.RegionName + " are:");
                         foreach (IRegionModuleBase module in scene.RegionModules.Values)
@@ -1066,7 +1066,7 @@ namespace OpenSim
                     break;
 
                 case "regions":
-                    m_sceneManager.ForEachScene(
+                    SceneManager.ForEachScene(
                         delegate(Scene scene)
                             {
                                 MainConsole.Instance.Output(String.Format(
@@ -1080,7 +1080,7 @@ namespace OpenSim
                     break;
 
                 case "ratings":
-                    m_sceneManager.ForEachScene(
+                    SceneManager.ForEachScene(
                     delegate(Scene scene)
                     {
                         string rating = "";
@@ -1115,7 +1115,7 @@ namespace OpenSim
             cdt.AddColumn("IP", 16);
             cdt.AddColumn("Viewer Name", 24);
 
-            m_sceneManager.ForEachScene(
+            SceneManager.ForEachScene(
                 s =>
                 {
                     foreach (AgentCircuitData aCircuit in s.AuthenticateHandler.GetAgentCircuits().Values)
@@ -1140,7 +1140,7 @@ namespace OpenSim
             cdt.AddColumn("Endpoint", 23);
             cdt.AddColumn("Active?", 7);
 
-            m_sceneManager.ForEachScene(
+            SceneManager.ForEachScene(
                 s => s.ForEachClient(
                     c => cdt.AddRow(
                         s.Name,
@@ -1161,11 +1161,11 @@ namespace OpenSim
         {
             if (cmdparams.Length > 5)
             {
-                m_sceneManager.SaveNamedPrimsToXml2(cmdparams[3], cmdparams[4]);
+                SceneManager.SaveNamedPrimsToXml2(cmdparams[3], cmdparams[4]);
             }
             else
             {
-                m_sceneManager.SaveNamedPrimsToXml2("Primitive", DEFAULT_PRIM_BACKUP_FILENAME);
+                SceneManager.SaveNamedPrimsToXml2("Primitive", DEFAULT_PRIM_BACKUP_FILENAME);
             }
         }
 
@@ -1180,11 +1180,11 @@ namespace OpenSim
 
             if (cmdparams.Length > 0)
             {
-                m_sceneManager.SaveCurrentSceneToXml(cmdparams[2]);
+                SceneManager.SaveCurrentSceneToXml(cmdparams[2]);
             }
             else
             {
-                m_sceneManager.SaveCurrentSceneToXml(DEFAULT_PRIM_BACKUP_FILENAME);
+                SceneManager.SaveCurrentSceneToXml(DEFAULT_PRIM_BACKUP_FILENAME);
             }
         }
 
@@ -1221,13 +1221,13 @@ namespace OpenSim
                         MainConsole.Instance.Output(String.Format("loadOffsets <X,Y,Z> = <{0},{1},{2}>",loadOffset.X,loadOffset.Y,loadOffset.Z));
                     }
                 }
-                m_sceneManager.LoadCurrentSceneFromXml(cmdparams[2], generateNewIDS, loadOffset);
+                SceneManager.LoadCurrentSceneFromXml(cmdparams[2], generateNewIDS, loadOffset);
             }
             else
             {
                 try
                 {
-                    m_sceneManager.LoadCurrentSceneFromXml(DEFAULT_PRIM_BACKUP_FILENAME, false, loadOffset);
+                    SceneManager.LoadCurrentSceneFromXml(DEFAULT_PRIM_BACKUP_FILENAME, false, loadOffset);
                 }
                 catch (FileNotFoundException)
                 {
@@ -1244,11 +1244,11 @@ namespace OpenSim
         {
             if (cmdparams.Length > 2)
             {
-                m_sceneManager.SaveCurrentSceneToXml2(cmdparams[2]);
+                SceneManager.SaveCurrentSceneToXml2(cmdparams[2]);
             }
             else
             {
-                m_sceneManager.SaveCurrentSceneToXml2(DEFAULT_PRIM_BACKUP_FILENAME);
+                SceneManager.SaveCurrentSceneToXml2(DEFAULT_PRIM_BACKUP_FILENAME);
             }
         }
 
@@ -1263,7 +1263,7 @@ namespace OpenSim
             {
                 try
                 {
-                    m_sceneManager.LoadCurrentSceneFromXml2(cmdparams[2]);
+                    SceneManager.LoadCurrentSceneFromXml2(cmdparams[2]);
                 }
                 catch (FileNotFoundException)
                 {
@@ -1274,7 +1274,7 @@ namespace OpenSim
             {
                 try
                 {
-                    m_sceneManager.LoadCurrentSceneFromXml2(DEFAULT_PRIM_BACKUP_FILENAME);
+                    SceneManager.LoadCurrentSceneFromXml2(DEFAULT_PRIM_BACKUP_FILENAME);
                 }
                 catch (FileNotFoundException)
                 {
@@ -1291,7 +1291,7 @@ namespace OpenSim
         {
             try
             {
-                m_sceneManager.LoadArchiveToCurrentScene(cmdparams);
+                SceneManager.LoadArchiveToCurrentScene(cmdparams);
             }
             catch (Exception e)
             {
@@ -1305,7 +1305,7 @@ namespace OpenSim
         /// <param name="cmdparams"></param>
         protected void SaveOar(string module, string[] cmdparams)
         {
-            m_sceneManager.SaveCurrentSceneToArchive(cmdparams);
+            SceneManager.SaveCurrentSceneToArchive(cmdparams);
         }
 
         private static string CombineParams(string[] commandParams, int pos)
diff --git a/OpenSim/Region/Application/OpenSimBase.cs b/OpenSim/Region/Application/OpenSimBase.cs
index 3271555..825c4c4 100644
--- a/OpenSim/Region/Application/OpenSimBase.cs
+++ b/OpenSim/Region/Application/OpenSimBase.cs
@@ -285,7 +285,7 @@ namespace OpenSim
 
         private void HandleCommanderCommand(string module, string[] cmd)
         {
-            m_sceneManager.SendCommandToPluginModules(cmd);
+            SceneManager.SendCommandToPluginModules(cmd);
         }
 
         private void HandleCommanderHelp(string module, string[] cmd)
@@ -303,7 +303,10 @@ namespace OpenSim
             // Called from base.StartUp()
 
             m_httpServerPort = m_networkServersInfo.HttpListenerPort;
-            m_sceneManager.OnRestartSim += handleRestartRegion;
+            SceneManager.OnRestartSim += handleRestartRegion;
+
+            // Only start the memory watchdog once all regions are ready
+            SceneManager.OnRegionsReadyStatusChange += sm => MemoryWatchdog.Enabled = sm.AllRegionsReady;
         }
 
         /// <summary>
@@ -412,7 +415,7 @@ namespace OpenSim
             // scripting engines.
             scene.CreateScriptInstances();
 
-            m_sceneManager.Add(scene);
+            SceneManager.Add(scene);
 
             if (m_autoCreateClientStack)
             {
@@ -432,7 +435,6 @@ namespace OpenSim
             mscene = scene;
 
             scene.Start();
-
             scene.StartScripts();
 
             return clientServer;
@@ -561,14 +563,14 @@ namespace OpenSim
         {
             // only need to check this if we are not at the
             // root level
-            if ((m_sceneManager.CurrentScene != null) &&
-                (m_sceneManager.CurrentScene.RegionInfo.RegionID == scene.RegionInfo.RegionID))
+            if ((SceneManager.CurrentScene != null) &&
+                (SceneManager.CurrentScene.RegionInfo.RegionID == scene.RegionInfo.RegionID))
             {
-                m_sceneManager.TrySetCurrentScene("..");
+                SceneManager.TrySetCurrentScene("..");
             }
 
             scene.DeleteAllSceneObjects();
-            m_sceneManager.CloseScene(scene);
+            SceneManager.CloseScene(scene);
             ShutdownClientServer(scene.RegionInfo);
             
             if (!cleanup)
@@ -610,7 +612,7 @@ namespace OpenSim
         public void RemoveRegion(string name, bool cleanUp)
         {
             Scene target;
-            if (m_sceneManager.TryGetScene(name, out target))
+            if (SceneManager.TryGetScene(name, out target))
                 RemoveRegion(target, cleanUp);
         }
 
@@ -623,13 +625,13 @@ namespace OpenSim
         {
             // only need to check this if we are not at the
             // root level
-            if ((m_sceneManager.CurrentScene != null) &&
-                (m_sceneManager.CurrentScene.RegionInfo.RegionID == scene.RegionInfo.RegionID))
+            if ((SceneManager.CurrentScene != null) &&
+                (SceneManager.CurrentScene.RegionInfo.RegionID == scene.RegionInfo.RegionID))
             {
-                m_sceneManager.TrySetCurrentScene("..");
+                SceneManager.TrySetCurrentScene("..");
             }
 
-            m_sceneManager.CloseScene(scene);
+            SceneManager.CloseScene(scene);
             ShutdownClientServer(scene.RegionInfo);
         }
         
@@ -641,7 +643,7 @@ namespace OpenSim
         public void CloseRegion(string name)
         {
             Scene target;
-            if (m_sceneManager.TryGetScene(name, out target))
+            if (SceneManager.TryGetScene(name, out target))
                 CloseRegion(target);
         }
         
@@ -897,7 +899,7 @@ namespace OpenSim
 
             try
             {
-                m_sceneManager.Close();
+                SceneManager.Close();
             }
             catch (Exception e)
             {
@@ -922,7 +924,7 @@ namespace OpenSim
         /// <param name="usernum">The first out parameter describing the number of all the avatars in the Region server</param>
         public void GetAvatarNumber(out int usernum)
         {
-            usernum = m_sceneManager.GetCurrentSceneAvatars().Count;
+            usernum = SceneManager.GetCurrentSceneAvatars().Count;
         }
 
         /// <summary>
@@ -931,7 +933,7 @@ namespace OpenSim
         /// <param name="regionnum">The first out parameter describing the number of regions</param>
         public void GetRegionNumber(out int regionnum)
         {
-            regionnum = m_sceneManager.Scenes.Count;
+            regionnum = SceneManager.Scenes.Count;
         }
         
         /// <summary>
diff --git a/OpenSim/Region/ClientStack/RegionApplicationBase.cs b/OpenSim/Region/ClientStack/RegionApplicationBase.cs
index c4324e8..4672f8a 100644
--- a/OpenSim/Region/ClientStack/RegionApplicationBase.cs
+++ b/OpenSim/Region/ClientStack/RegionApplicationBase.cs
@@ -53,9 +53,8 @@ namespace OpenSim.Region.ClientStack
         protected ISimulationDataService m_simulationDataService;
         protected IEstateDataService m_estateDataService;
         protected ClientStackManager m_clientStackManager;
-        protected SceneManager m_sceneManager = new SceneManager();
 
-        public SceneManager SceneManager { get { return m_sceneManager; } }
+        public SceneManager SceneManager { get; protected set; }
         public NetworkServersInfo NetServersInfo { get { return m_networkServersInfo; } }
         public ISimulationDataService SimulationDataService { get { return m_simulationDataService; } }
         public IEstateDataService EstateDataService { get { return m_estateDataService; } }
@@ -77,6 +76,7 @@ namespace OpenSim.Region.ClientStack
 
         protected override void StartupSpecific()
         {
+            SceneManager = new SceneManager();
             m_clientStackManager = CreateClientStackManager();
 
             Initialize();