Serialize calls to ODE Collide() function across OdeScene instances to prevent ODE crashes on simulators running more than one region.

It turns out that calls to Collide() are not thread-safe even for objects in different ODE physics worlds due to ODE static caches.
For simulators running multiple regions, not serializing calls from different scene loops will sooner or later cause OpenSim to crash with a native stack trace referencing OBBCollider.
This affects the default OPCODE collider but not GIMPACT.  However, GIMPACT fails for other reasons under some current simulator loads.
ODE provides a thread local storage option, but as of ODE r1755 (and r1840) DLLs compiled with this crash OpenSim immediately.

2 files changed, 32 insertions, 1 deletions

diff --git a/OpenSim/Region/Physics/OdePlugin/ODECharacter.cs b/OpenSim/Region/Physics/OdePlugin/ODECharacter.cs
index c6e8286..7c1c046 100644
--- a/OpenSim/Region/Physics/OdePlugin/ODECharacter.cs
+++ b/OpenSim/Region/Physics/OdePlugin/ODECharacter.cs
@@ -1048,6 +1048,7 @@ namespace OpenSim.Region.Physics.OdePlugin
                 CAPSULE_RADIUS = 0.01f;
             }
 
+//          lock (OdeScene.UniversalColliderSyncObject)
             Shell = d.CreateCapsule(_parent_scene.space, CAPSULE_RADIUS, CAPSULE_LENGTH);
 
             d.GeomSetCategoryBits(Shell, (int)m_collisionCategories);
@@ -1179,7 +1180,9 @@ namespace OpenSim.Region.Physics.OdePlugin
 
             if (Shell != IntPtr.Zero)
             {
+//              lock (OdeScene.UniversalColliderSyncObject)
                 d.GeomDestroy(Shell);
+
                 _parent_scene.geom_name_map.Remove(Shell);
                 _parent_scene.actor_name_map.Remove(Shell);
 
diff --git a/OpenSim/Region/Physics/OdePlugin/OdeScene.cs b/OpenSim/Region/Physics/OdePlugin/OdeScene.cs
index 37daf46..4530c09 100644
--- a/OpenSim/Region/Physics/OdePlugin/OdeScene.cs
+++ b/OpenSim/Region/Physics/OdePlugin/OdeScene.cs
@@ -105,6 +105,32 @@ namespace OpenSim.Region.Physics.OdePlugin
         private readonly ILog m_log;
         // private Dictionary<string, sCollisionData> m_storedCollisions = new Dictionary<string, sCollisionData>();
 
+        /// <summary>
+        /// Provide a sync object so that only one thread calls d.Collide() at a time across all OdeScene instances.
+        /// </summary>
+        /// <remarks>
+        /// With ODE as of r1755 (though also tested on r1860), only one thread can call d.Collide() at a
+        /// time, even where physics objects are in entirely different ODE worlds.  This is because generating contacts
+        /// uses a static cache at the ODE level.
+        ///
+        /// Without locking, simulators running multiple regions will eventually crash with a native stack trace similar
+        /// to
+        ///
+        /// mono() [0x489171]
+        /// mono() [0x4d154f]
+        /// /lib/x86_64-linux-gnu/libpthread.so.0(+0xfc60) [0x7f6ded592c60]
+        /// .../opensim/bin/libode-x86_64.so(_ZN6Opcode11OBBCollider8_CollideEPKNS_14AABBNoLeafNodeE+0xd7a) [0x7f6dd822628a]
+        ///
+        /// ODE provides an experimental option to cache in thread local storage but compiling ODE with this option
+        /// causes OpenSimulator to immediately crash with a native stack trace similar to
+        ///
+        /// mono() [0x489171]
+        /// mono() [0x4d154f]
+        /// /lib/x86_64-linux-gnu/libpthread.so.0(+0xfc60) [0x7f03c9849c60]
+        /// .../opensim/bin/libode-x86_64.so(_Z12dCollideCCTLP6dxGeomS0_iP12dContactGeomi+0x92) [0x7f03b44bcf82]
+        /// </remarks>
+        internal static Object UniversalColliderSyncObject = new Object();
+
         private Random fluidRandomizer = new Random(Environment.TickCount);
 
         private const uint m_regionWidth = Constants.RegionSize;
@@ -799,7 +825,9 @@ namespace OpenSim.Region.Physics.OdePlugin
                 if (b1 != IntPtr.Zero && b2 != IntPtr.Zero && d.AreConnectedExcluding(b1, b2, d.JointType.Contact))
                     return;
 
-                count = d.Collide(g1, g2, contacts.Length, contacts, d.ContactGeom.SizeOf);
+                lock (OdeScene.UniversalColliderSyncObject)
+                    count = d.Collide(g1, g2, contacts.Length, contacts, d.ContactGeom.SizeOf);
+
                 if (count > contacts.Length)
                     m_log.Error("[ODE SCENE]: Got " + count + " contacts when we asked for a maximum of " + contacts.Length);
             }