* Restaring the sim works fine in grid mode now. Sims announce themselves to their neighbors when they start up. Neighbors get this message and tell their agents that there's a new sim up.

* Certain unrecoverable physics based crashes in ODE are now hooked up to the 'restart the sim' routine.

10 files changed, 147 insertions, 43 deletions

diff --git a/OpenSim/Region/Application/OpenSimMain.cs b/OpenSim/Region/Application/OpenSimMain.cs
index eab4827..5877aa6 100644
--- a/OpenSim/Region/Application/OpenSimMain.cs
+++ b/OpenSim/Region/Application/OpenSimMain.cs
@@ -601,6 +601,8 @@ namespace OpenSim
                     m_log.Error("show uptime - show simulator startup and uptime.");
                     m_log.Error("show users - show info about connected users.");
                     m_log.Error("show modules - shows info aboutloaded modules.");
+                    m_log.Error("change-region - sets the region that many of these commands affect.");
+                    m_log.Error("restart - disconnects all clients and restarts the sims in the instance.");
                     m_log.Error("shutdown - disconnect all clients and shutdown.");
                     m_log.Error("terrain help - show help for terrain commands.");
                     m_log.Error("quit - equivalent to shutdown.");
diff --git a/OpenSim/Region/Communications/Local/LocalBackEndServices.cs b/OpenSim/Region/Communications/Local/LocalBackEndServices.cs
index 6e807ec..0f3ec73 100644
--- a/OpenSim/Region/Communications/Local/LocalBackEndServices.cs
+++ b/OpenSim/Region/Communications/Local/LocalBackEndServices.cs
@@ -106,7 +106,6 @@ namespace OpenSim.Region.Communications.Local
 
                 return regionHost;
             }
-            return null;
         }
 
         public bool DeregisterRegion(RegionInfo regionInfo)
@@ -201,8 +200,9 @@ namespace OpenSim.Region.Communications.Local
         /// <param name="agentData"></param>
         /// <returns></returns>
         /// 
-        public bool RegionUp(RegionInfo region)
+        public bool RegionUp(SearializableRegionInfo sregion)
         {
+            RegionInfo region = new RegionInfo(sregion);
             foreach (RegionCommsListener listener in m_regionListeners.Values)
             {
                 listener.TriggerRegionUp(region);
@@ -213,8 +213,13 @@ namespace OpenSim.Region.Communications.Local
 
         public bool TriggerRegionUp(RegionInfo region)
         {
+            
+            foreach (RegionCommsListener listener in m_regionListeners.Values)
+            {
+                listener.TriggerRegionUp(region);
+            }
 
-            return false;
+            return true;
         }
 
         public bool InformRegionOfChildAgent(ulong regionHandle, AgentCircuitData agentData)
diff --git a/OpenSim/Region/Communications/OGS1/OGS1GridServices.cs b/OpenSim/Region/Communications/OGS1/OGS1GridServices.cs
index 282b720..b0442d2 100644
--- a/OpenSim/Region/Communications/OGS1/OGS1GridServices.cs
+++ b/OpenSim/Region/Communications/OGS1/OGS1GridServices.cs
@@ -133,14 +133,10 @@ namespace OpenSim.Region.Communications.OGS1
             }
             else
             {
-                //m_knownRegions = RequestNeighbours(regionInfo.RegionLocX, regionInfo.RegionLocY);
+                m_knownRegions = RequestNeighbours(regionInfo.RegionLocX, regionInfo.RegionLocY);
                 
                 
             }
-            //SimpleRegionInfo regiondata = new SimpleRegionInfo();
-            //regiondata.RegionID = griddatahash["UUID"];
-            //regiondata.RemotingAddress = 
-
             return m_localBackend.RegisterRegion(regionInfo);
         }
 
@@ -458,7 +454,7 @@ namespace OpenSim.Region.Communications.OGS1
             InterRegionSingleton.Instance.OnPrimGroupArrival += IncomingPrim;
             InterRegionSingleton.Instance.OnPrimGroupNear += TriggerExpectPrimCrossing;
             InterRegionSingleton.Instance.OnRegionUp += TriggerRegionUp;
-            InterRegionSingleton.Instance.OnRegionUp += RegionUp;
+            //InterRegionSingleton.Instance.OnRegionUp += RegionUp;
         }
 
         #region Methods called by regions in this instance
@@ -541,11 +537,10 @@ namespace OpenSim.Region.Communications.OGS1
                 MainLog.Instance.Debug(e.ToString());
                 return false;
             }
-            return true;
         }
 
         // UGLY!
-        public bool RegionUp(RegionInfo region)
+        public bool RegionUp(SearializableRegionInfo region)
         {
             
             // This is stupid.  For this to work, when the region registers it must request nearby map blocks.
@@ -558,11 +553,11 @@ namespace OpenSim.Region.Communications.OGS1
             {
                 return true;
             }
-            return true;
+            
             foreach (SimpleRegionInfo knownregion in m_knownRegions)
              {
-                 // Wha?
-                 RegionInfo regInfo = RequestNeighbourInfo(knownregion.RegionID);
+                 
+                 SearializableRegionInfo regInfo = new SearializableRegionInfo(RequestNeighbourInfo(knownregion.RegionID));
 
                   try
                   {
@@ -572,7 +567,7 @@ namespace OpenSim.Region.Communications.OGS1
                         //don't want to be creating a new link to the remote instance every time like we are here
                         bool retValue = false;
 
-
+                        
                         OGS1InterRegionRemoting remObject = (OGS1InterRegionRemoting)Activator.GetObject(
                                                                                           typeof(OGS1InterRegionRemoting),
                                                                                           "tcp://" + regInfo.RemotingAddress +
@@ -588,38 +583,38 @@ namespace OpenSim.Region.Communications.OGS1
                             Console.WriteLine("remoting object not found");
                         }
                         remObject = null;
-                        //MainLog.Instance.Verbose("INTER", gdebugRegionName + ": OGS1 tried to NotifyRegionUp for " + region.RegionName + " and got " + retValue.ToString());
+                        MainLog.Instance.Verbose("INTER", gdebugRegionName + ": OGS1 tried to NotifyRegionUp for " + regInfo.RegionLocX + "," + regInfo.RegionLocY);
 
                     }
 
                 }
                 catch (RemotingException e)
                 {
-                    MainLog.Instance.Warn("Remoting Error: Unable to connect to adjacent region: " + regInfo.RegionName + " " + regInfo.RegionLocX + "," + regInfo.RegionLocY);
+                    MainLog.Instance.Warn("Remoting Error: Unable to connect to adjacent region:  " + regInfo.RegionLocX + "," + regInfo.RegionLocY);
                     MainLog.Instance.Debug(e.ToString());
                     //return false;
                 }
                 catch (SocketException e)
                 {
-                    MainLog.Instance.Warn("Socket Error: Unable to connect to adjacent region: " + regInfo.RegionName + " " + regInfo.RegionLocX + "," + regInfo.RegionLocY);
+                    MainLog.Instance.Warn("Socket Error: Unable to connect to adjacent region:  " + regInfo.RegionLocX + "," + regInfo.RegionLocY);
                     MainLog.Instance.Debug(e.ToString());
                     //return false;
                 }
                 catch (InvalidCredentialException e)
                 {
-                    MainLog.Instance.Warn("Invalid Credentials: Unable to connect to adjacent region: " + regInfo.RegionName + " " + regInfo.RegionLocX + "," + regInfo.RegionLocY);
+                    MainLog.Instance.Warn("Invalid Credentials: Unable to connect to adjacent region:  " + regInfo.RegionLocX + "," + regInfo.RegionLocY);
                     MainLog.Instance.Debug(e.ToString());
                     //return false;
                 }
                 catch (AuthenticationException e)
                 {
-                    MainLog.Instance.Warn("Authentication exception: Unable to connect to adjacent region: " + regInfo.RegionName + " " + regInfo.RegionLocX + "," + regInfo.RegionLocY);
+                    MainLog.Instance.Warn("Authentication exception: Unable to connect to adjacent region:  " + regInfo.RegionLocX + "," + regInfo.RegionLocY);
                     MainLog.Instance.Debug(e.ToString());
                     //return false;
                 }
                 catch (Exception e)
                 {
-                    MainLog.Instance.Warn("Unknown exception: Unable to connect to adjacent region: " + regInfo.RegionName + " " + regInfo.RegionLocX + "," + regInfo.RegionLocY);
+                    MainLog.Instance.Warn("Unknown exception: Unable to connect to adjacent region:  " + regInfo.RegionLocX + "," + regInfo.RegionLocY);
                     MainLog.Instance.Debug(e.ToString());
                     //return false;
                 }
@@ -702,7 +697,6 @@ namespace OpenSim.Region.Communications.OGS1
                 MainLog.Instance.Debug(e.ToString());
                 return false;
             }
-            return true;
         }
         /// <summary>
         /// 
@@ -843,13 +837,13 @@ namespace OpenSim.Region.Communications.OGS1
             }
         }
 
-        public bool TriggerRegionUp(RegionInfo regionData)
+        public bool TriggerRegionUp(SearializableRegionInfo regionData)
         {
-            //MainLog.Instance.Verbose("INTER", gdebugRegionName + ": Incoming OGS1 RegionUpReport " + regionData.RegionName);
+            MainLog.Instance.Verbose("INTER", gdebugRegionName + ": Incoming OGS1 RegionUpReport:  " + regionData.RegionLocX + "," + regionData.RegionLocY);
 
             try
             {
-                return m_localBackend.TriggerRegionUp(regionData);
+                return m_localBackend.TriggerRegionUp(new RegionInfo(regionData));
             }
 
             catch (RemotingException e)
diff --git a/OpenSim/Region/Communications/OGS1/OGS1InterSimComms.cs b/OpenSim/Region/Communications/OGS1/OGS1InterSimComms.cs
index aa6f59a..2f7e591 100644
--- a/OpenSim/Region/Communications/OGS1/OGS1InterSimComms.cs
+++ b/OpenSim/Region/Communications/OGS1/OGS1InterSimComms.cs
@@ -41,7 +41,7 @@ namespace OpenSim.Region.Communications.OGS1
 
     public delegate bool PrimGroupArrival(ulong regionHandle, LLUUID primID, string objData);
 
-    public delegate bool RegionUP (RegionInfo region);
+    public delegate bool RegionUp (SearializableRegionInfo region);
 
     public sealed class InterRegionSingleton
     {
@@ -76,11 +76,12 @@ namespace OpenSim.Region.Communications.OGS1
             return false;
         }
 
-        public bool RegionUp(RegionInfo region)
+        public bool RegionUp(SearializableRegionInfo sregion)
         {
+            
             if (OnRegionUp != null)
             {
-                return OnRegionUp(region);
+                return OnRegionUp(sregion);
             }
             return false;
         }
@@ -129,7 +130,7 @@ namespace OpenSim.Region.Communications.OGS1
                 return false;
             }
         }
-        public bool RegionUp(RegionInfo region)
+        public bool RegionUp(SearializableRegionInfo  region)
         {
             try
             {
@@ -141,6 +142,7 @@ namespace OpenSim.Region.Communications.OGS1
                 return false;
             }
         }
+
         public bool ExpectAvatarCrossing(ulong regionHandle, LLUUID agentID, LLVector3 position, bool isFlying)
         {
             try
diff --git a/OpenSim/Region/Environment/Scenes/InnerScene.cs b/OpenSim/Region/Environment/Scenes/InnerScene.cs
index 8f7cbee..c1acde4 100644
--- a/OpenSim/Region/Environment/Scenes/InnerScene.cs
+++ b/OpenSim/Region/Environment/Scenes/InnerScene.cs
@@ -11,8 +11,14 @@ using OpenSim.Region.Physics.Manager;
 
 namespace OpenSim.Region.Environment.Scenes
 {
+    public delegate void PhysicsCrash();
+
     public class InnerScene
     {
+        #region Events
+        public event PhysicsCrash UnRecoverableError;
+        #endregion
+
         #region Fields
         public Dictionary<LLUUID, ScenePresence> ScenePresences;
         public Dictionary<LLUUID, SceneObjectGroup> SceneObjects;
@@ -26,17 +32,47 @@ namespace OpenSim.Region.Environment.Scenes
 
         internal object m_syncRoot = new object();
 
-        public PhysicsScene PhyScene;
+        public PhysicsScene _PhyScene;
         #endregion
 
         public InnerScene(Scene parent, RegionInfo regInfo, PermissionManager permissionsMngr)
         {
+
             m_parentScene = parent;
             m_regInfo = regInfo;
             PermissionsMngr = permissionsMngr;
             QuadTree = new BasicQuadTreeNode(null, "/0/", 0, 0, 256, 256);
             QuadTree.Subdivide();
             QuadTree.Subdivide();
+            
+
+        }
+        public PhysicsScene PhyScene
+        {
+            get
+            { return _PhyScene; }
+            set
+            {
+                // If we're not doing the initial set
+                // Then we've got to remove the previous 
+                // event handler
+                    try
+                    {
+                        _PhyScene.OnPhysicsCrash -= physicsBasedCrash;
+                    }
+                    catch (System.NullReferenceException)
+                    {
+                        // This occurs when storing to _PhyScene the first time.
+                        // Is there a better way to check the event handler before
+                        // getting here
+                        // This can be safely ignored.  We're setting the first inital 
+                        // there are no event handler's registered.
+                    }
+                
+                _PhyScene = value;
+                
+                _PhyScene.OnPhysicsCrash += physicsBasedCrash;
+            }
         }
 
         public void Close()
@@ -55,9 +91,9 @@ namespace OpenSim.Region.Environment.Scenes
 
             // PhysX does this (runs in the background).
 
-            if (PhyScene.IsThreaded)
+            if (_PhyScene.IsThreaded)
             {
-                PhyScene.GetResults();
+                _PhyScene.GetResults();
             }
         }
 
@@ -75,7 +111,7 @@ namespace OpenSim.Region.Environment.Scenes
         {
             lock (m_syncRoot)
             {
-                PhyScene.Simulate((float)elapsed);
+                _PhyScene.Simulate((float)elapsed);
             }
         }
 
@@ -338,6 +374,15 @@ namespace OpenSim.Region.Environment.Scenes
 
         #region Other Methods
 
+
+        public void physicsBasedCrash()
+        {
+            if (UnRecoverableError != null)
+            {
+                UnRecoverableError();
+            }
+        }
+
         public LLUUID ConvertLocalIDToFullID(uint localID)
         {
             SceneObjectGroup group = GetGroupByPrim(localID);
diff --git a/OpenSim/Region/Environment/Scenes/Scene.cs b/OpenSim/Region/Environment/Scenes/Scene.cs
index bdafce8..1359bd2 100644
--- a/OpenSim/Region/Environment/Scenes/Scene.cs
+++ b/OpenSim/Region/Environment/Scenes/Scene.cs
@@ -232,6 +232,15 @@ namespace OpenSim.Region.Environment.Scenes
             m_permissionManager.Initialise(this);
 
             m_innerScene = new InnerScene(this, m_regInfo, m_permissionManager);
+            
+            // If the Inner scene has an Unrecoverable error, restart this sim.
+            // Currently the only thing that causes it to happen is two kinds of specific
+            // Physics based crashes.
+            //
+            // Out of memory
+            // Operating system has killed the plugin
+            m_innerScene.UnRecoverableError += restartNOW;
+
             m_sceneXmlLoader = new SceneXmlLoader(this, m_innerScene, m_regInfo);
 
             RegisterDefaultSceneEvents();
@@ -315,13 +324,17 @@ namespace OpenSim.Region.Environment.Scenes
             {
                 t_restartTimer.Stop();
                 t_restartTimer.AutoReset = false;
-                MainLog.Instance.Error("REGION", "Closing");
-                Close();
-                MainLog.Instance.Error("REGION", "Firing Region Restart Message");
-                base.Restart(0);
+                restartNOW();
             }
             
         }
+        public void restartNOW()
+        {
+            MainLog.Instance.Error("REGION", "Closing");
+            Close();
+            MainLog.Instance.Error("REGION", "Firing Region Restart Message");
+            base.Restart(0);
+        }
         public void restart_Notify_Wait_Elapsed(object sender, ElapsedEventArgs e)
         { 
             m_restartWaitTimer.Stop();
diff --git a/OpenSim/Region/Environment/Scenes/SceneCommunicationService.cs b/OpenSim/Region/Environment/Scenes/SceneCommunicationService.cs
index 9a83710..892bf81 100644
--- a/OpenSim/Region/Environment/Scenes/SceneCommunicationService.cs
+++ b/OpenSim/Region/Environment/Scenes/SceneCommunicationService.cs
@@ -171,6 +171,18 @@ namespace OpenSim.Region.Environment.Scenes
             }
         }
 
+        public void RequestNeighbors(RegionInfo region)
+        {
+            List<SimpleRegionInfo> neighbours =
+                m_commsProvider.GridService.RequestNeighbours(m_regionInfo.RegionLocX, m_regionInfo.RegionLocY);
+            //IPEndPoint blah = new IPEndPoint();
+            
+            //blah.Address = region.RemotingAddress;
+            //blah.Port = region.RemotingPort;
+           
+
+        }
+
         /// <summary>
         /// 
         /// </summary>
@@ -311,7 +323,7 @@ namespace OpenSim.Region.Environment.Scenes
         public void InformNeighborsThatRegionisUp(RegionInfo region)
         {
             //MainLog.Instance.Verbose("INTER", debugRegionName + ": SceneCommunicationService: Sending InterRegion Notification that region is up " + region.RegionName);
-            bool val = m_commsProvider.InterRegion.RegionUp(region);
+            bool val = m_commsProvider.InterRegion.RegionUp(new SearializableRegionInfo(region));
         }
 
         public bool PrimCrossToNeighboringRegion(ulong regionhandle, LLUUID primID, LLVector3 position, bool isPhysical)
diff --git a/OpenSim/Region/Environment/Scenes/SceneManager.cs b/OpenSim/Region/Environment/Scenes/SceneManager.cs
index eba45fc..ed33bf7 100644
--- a/OpenSim/Region/Environment/Scenes/SceneManager.cs
+++ b/OpenSim/Region/Environment/Scenes/SceneManager.cs
@@ -216,7 +216,7 @@ namespace OpenSim.Region.Environment.Scenes
 
         public void RestartCurrentScene()
         {
-            ForEachCurrentScene(delegate(Scene scene) { scene.Restart(15); });
+            ForEachCurrentScene(delegate(Scene scene) { scene.restartNOW(); });
 
         }
 
diff --git a/OpenSim/Region/Physics/Manager/PhysicsScene.cs b/OpenSim/Region/Physics/Manager/PhysicsScene.cs
index 651992c..635b208 100644
--- a/OpenSim/Region/Physics/Manager/PhysicsScene.cs
+++ b/OpenSim/Region/Physics/Manager/PhysicsScene.cs
@@ -32,12 +32,29 @@ using OpenSim.Region.Physics.Manager;
 
 namespace OpenSim.Region.Physics.Manager
 {
+    public delegate void physicsCrash();
+
     public abstract class PhysicsScene
     {
+        // The only thing that should register for this event is the InnerScene
+        // Anything else could cause problems.
+
+        public event physicsCrash OnPhysicsCrash;
+
         public static PhysicsScene Null
         {
             get { return new NullPhysicsScene(); }
         }
+        public virtual void TriggerPhysicsBasedRestart()
+        {
+            physicsCrash handler = OnPhysicsCrash;
+            if (handler != null)
+            {
+                OnPhysicsCrash();
+            }
+            
+        }
+        
 
         public abstract void Initialise(IMesher meshmerizer);
 
diff --git a/OpenSim/Region/Physics/OdePlugin/OdePlugin.cs b/OpenSim/Region/Physics/OdePlugin/OdePlugin.cs
index ec7d04d..7193886 100644
--- a/OpenSim/Region/Physics/OdePlugin/OdePlugin.cs
+++ b/OpenSim/Region/Physics/OdePlugin/OdePlugin.cs
@@ -32,6 +32,7 @@ using Axiom.Math;
 using Ode.NET;
 using OpenSim.Framework;
 using OpenSim.Region.Physics.Manager;
+
 //using OpenSim.Region.Physics.OdePlugin.Meshing;
 
 namespace OpenSim.Region.Physics.OdePlugin
@@ -234,9 +235,16 @@ namespace OpenSim.Region.Physics.OdePlugin
                     //System.Console.WriteLine("near: A collision was detected between {1} and {2}", 0, name1, name2);
                 }
                 
-                int count;
-                
+                int count = 0;
+                try
+                {
                     count = d.Collide(g1, g2, contacts.GetLength(0), contacts, d.ContactGeom.SizeOf);
+                }
+                catch (System.Runtime.InteropServices.SEHException)
+                {
+                    OpenSim.Framework.Console.MainLog.Instance.Error("PHYSICS", "The Operating system shut down ODE because of corrupt memory.  This could be a result of really irregular terrain.  If this repeats continuously, restart using Basic Physics and terrain fill your terrain.  Restarting the sim.");
+                    base.TriggerPhysicsBasedRestart();
+                }
              
                 for (int i = 0; i < count; i++)
                 {
@@ -805,8 +813,14 @@ namespace OpenSim.Region.Physics.OdePlugin
                 {
                 // Process 10 frames if the sim is running normal..  
                 // process 5 frames if the sim is running slow
-                d.WorldSetQuickStepNumIterations(world, m_physicsiterations);
-
+                    try{
+                        d.WorldSetQuickStepNumIterations(world, m_physicsiterations);
+                    }
+                    catch (System.StackOverflowException)
+                    {
+                        OpenSim.Framework.Console.MainLog.Instance.Error("PHYSICS", "The operating system wasn't able to allocate enough memory for the simulation.  Restarting the sim.");
+                        base.TriggerPhysicsBasedRestart();
+                    }
 
                 int i = 0;
                 while (step_time > 0.0f)