Implement co-operative script termination if termination comes during a script wait event (llSleep(), etc.)

This makes use of EventWaitHandles since various web references indicate that Thread.Interrupt() can also cause runtime instability.
If co-op termination is enabled, then termination sets the wait handle instead of waiting for a timeout before possibly aborting the thread.
This allows the script to cleanly terminate if it's in a llSleep/LL function delay or the next time it enters such a wait without any timeout period.
Co-op termination is not yet testable since checking for termination request within loops that never trigger a wait is not yet implemented.

8 files changed, 293 insertions, 14 deletions

diff --git a/OpenSim/Region/Framework/Scenes/Scene.Inventory.cs b/OpenSim/Region/Framework/Scenes/Scene.Inventory.cs
index 5c8b097..92bf85a 100644
--- a/OpenSim/Region/Framework/Scenes/Scene.Inventory.cs
+++ b/OpenSim/Region/Framework/Scenes/Scene.Inventory.cs
@@ -1738,6 +1738,21 @@ namespace OpenSim.Region.Framework.Scenes
         /// <returns>The part where the script was rezzed if successful.  False otherwise.</returns>
         public SceneObjectPart RezNewScript(UUID agentID, InventoryItemBase itemBase)
         {
+            return RezNewScript(
+                agentID,
+                itemBase,
+                "default\n{\n    state_entry()\n    {\n        llSay(0, \"Script running\");\n    }\n}");
+        }
+
+        /// <summary>
+        /// Rez a new script from nothing with given script text.
+        /// </summary>
+        /// <param name="remoteClient"></param>
+        /// <param name="itemBase">Template item.</param>
+        /// <param name="scriptText"></param>
+        /// <returns>The part where the script was rezzed if successful.  False otherwise.</returns>
+        public SceneObjectPart RezNewScript(UUID agentID, InventoryItemBase itemBase, string scriptText)
+        {
             // The part ID is the folder ID!
             SceneObjectPart part = GetSceneObjectPart(itemBase.Folder);
             if (part == null)
@@ -1757,9 +1772,14 @@ namespace OpenSim.Region.Framework.Scenes
                 return null;
             }
 
-            AssetBase asset = CreateAsset(itemBase.Name, itemBase.Description, (sbyte)itemBase.AssetType,
-                Encoding.ASCII.GetBytes("default\n{\n    state_entry()\n    {\n        llSay(0, \"Script running\");\n    }\n}"),
-                agentID);
+            AssetBase asset 
+                = CreateAsset(
+                    itemBase.Name, 
+                    itemBase.Description, 
+                    (sbyte)itemBase.AssetType,
+                    Encoding.ASCII.GetBytes(scriptText), 
+                    agentID);
+
             AssetService.Store(asset);
 
             TaskInventoryItem taskItem = new TaskInventoryItem();
diff --git a/OpenSim/Region/ScriptEngine/Interfaces/IScriptInstance.cs b/OpenSim/Region/ScriptEngine/Interfaces/IScriptInstance.cs
index 9de2d72..38fff52 100644
--- a/OpenSim/Region/ScriptEngine/Interfaces/IScriptInstance.cs
+++ b/OpenSim/Region/ScriptEngine/Interfaces/IScriptInstance.cs
@@ -28,6 +28,7 @@
 using System;
 using System.Collections;
 using System.Collections.Generic;
+using System.Threading;
 using OpenMetaverse;
 using log4net;
 using OpenSim.Framework;
@@ -181,6 +182,18 @@ namespace OpenSim.Region.ScriptEngine.Interfaces
         void Resume();
 
         /// <summary>
+        /// If true then scripts should look to terminate their threads in co-operation with the script engine rather
+        /// than through Thread.Abort()
+        /// </summary>
+        bool CoopTermination { get; }
+
+        /// <summary>
+        /// Used for script sleeps when we are using co-operative script termination.
+        /// </summary>
+        /// <remarks>null if CoopTermination is not active</remarks>  
+        EventWaitHandle CoopSleepHandle { get; }
+
+        /// <summary>
         /// Process the next event queued for this script instance.
         /// </summary>
         /// <returns></returns>
diff --git a/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/LSL_Api.cs b/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/LSL_Api.cs
index 44072c6..b992efa 100644
--- a/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/LSL_Api.cs
+++ b/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/LSL_Api.cs
@@ -83,6 +83,12 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api
     public class LSL_Api : MarshalByRefObject, ILSL_Api, IScriptApi
     {
         private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
+
+        /// <summary>
+        /// Instance of this script.
+        /// </summary>
+        protected IScriptInstance m_scriptInstance;
+
         protected IScriptEngine m_ScriptEngine;
         protected SceneObjectPart m_host;
 
@@ -112,11 +118,12 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api
 
         public void Initialize(IScriptInstance scriptInstance)
         {
-            m_ScriptEngine = scriptInstance.Engine;
-            m_host = scriptInstance.Part;
-            m_item = scriptInstance.ScriptTask;
+            m_scriptInstance = scriptInstance;
+            m_ScriptEngine = m_scriptInstance.Engine;
+            m_host = m_scriptInstance.Part;
+            m_item = m_scriptInstance.ScriptTask;
 
-            LoadLimits();  // read script limits from config.
+            LoadConfig();
 
             m_TransferModule =
                     m_ScriptEngine.World.RequestModuleInterface<IMessageTransferModule>();
@@ -129,7 +136,7 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api
         /// <summary>
         /// Load configuration items that affect script, object and run-time behavior. */
         /// </summary>
-        private void LoadLimits()
+        private void LoadConfig()
         {
             m_ScriptDelayFactor =
                 m_ScriptEngine.Config.GetFloat("ScriptDelayFactor", 1.0f);
@@ -175,7 +182,16 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api
             delay = (int)((float)delay * m_ScriptDelayFactor);
             if (delay == 0)
                 return;
-            System.Threading.Thread.Sleep(delay);
+
+            Sleep(delay);
+        }
+
+        protected virtual void Sleep(int delay)
+        {
+            if (!m_scriptInstance.CoopTermination)
+                System.Threading.Thread.Sleep(delay);
+            else if (m_scriptInstance.CoopSleepHandle.WaitOne(delay))
+                throw new ScriptCoopStopException();
         }
 
         public Scene World
@@ -2914,7 +2930,8 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api
         {
 //            m_log.Info("llSleep snoozing " + sec + "s.");
             m_host.AddScriptLPS(1);
-            Thread.Sleep((int)(sec * 1000));
+
+            Sleep((int)(sec * 1000));
         }
 
         public LSL_Float llGetMass()
diff --git a/OpenSim/Region/ScriptEngine/Shared/Helpers.cs b/OpenSim/Region/ScriptEngine/Shared/Helpers.cs
index 5a58f73..e02d35e 100644
--- a/OpenSim/Region/ScriptEngine/Shared/Helpers.cs
+++ b/OpenSim/Region/ScriptEngine/Shared/Helpers.cs
@@ -81,6 +81,24 @@ namespace OpenSim.Region.ScriptEngine.Shared
         }
     }
 
+    /// <summary>
+    /// Used to signal when the script is stopping in co-operation with the script engine 
+    /// (instead of through Thread.Abort()).
+    /// </summary>
+    [Serializable]
+    public class ScriptCoopStopException : Exception
+    {
+        public ScriptCoopStopException()
+        {
+        }
+
+        protected ScriptCoopStopException(
+                SerializationInfo info, 
+                StreamingContext context)
+        {
+        }
+    }
+
     public class DetectParams
     {
         public const int AGENT = 1;
diff --git a/OpenSim/Region/ScriptEngine/Shared/Instance/ScriptInstance.cs b/OpenSim/Region/ScriptEngine/Shared/Instance/ScriptInstance.cs
index a2ff51b..00048a1 100644
--- a/OpenSim/Region/ScriptEngine/Shared/Instance/ScriptInstance.cs
+++ b/OpenSim/Region/ScriptEngine/Shared/Instance/ScriptInstance.cs
@@ -200,6 +200,10 @@ namespace OpenSim.Region.ScriptEngine.Shared.Instance
 
         public static readonly long MaxMeasurementPeriod = 30 * TimeSpan.TicksPerMinute;
 
+        public bool CoopTermination { get; private set; }
+ 
+        public EventWaitHandle CoopSleepHandle { get; private set; }
+
         public void ClearQueue()
         {
             m_TimerQueued = false;
@@ -233,6 +237,12 @@ namespace OpenSim.Region.ScriptEngine.Shared.Instance
             m_postOnRez = postOnRez;
             m_AttachedAvatar = Part.ParentGroup.AttachedAvatar;
             m_RegionID = Part.ParentGroup.Scene.RegionInfo.RegionID;
+
+            if (Engine.Config.GetString("ScriptStopStrategy", "abort") == "co-op")
+            {
+                CoopTermination = true;
+                CoopSleepHandle = new AutoResetEvent(false);
+            }
         }
 
         /// <summary>
@@ -532,9 +542,34 @@ namespace OpenSim.Region.ScriptEngine.Shared.Instance
             }
 
             // Wait for the current event to complete.
-            if (!m_InSelfDelete && workItem.Wait(new TimeSpan((long)timeout * 100000)))
+            if (!m_InSelfDelete)
             {
-                return true;
+                if (!CoopTermination)
+                {
+                    // If we're not co-operative terminating then try and wait for the event to complete before stopping
+                    if (workItem.Wait(new TimeSpan((long)timeout * 100000)))
+                        return true;
+                }
+                else
+                {
+                    m_log.DebugFormat(
+                        "[SCRIPT INSTANCE]: Co-operatively stopping script {0} {1} in {2} {3}",
+                        ScriptName, ItemID, PrimName, ObjectID);
+
+                    // This will terminate the event on next handle check by the script.
+                    CoopSleepHandle.Set();
+
+                    // For now, we will wait forever since the event should always cleanly terminate once LSL loop
+                    // checking is implemented.  May want to allow a shorter timeout option later.
+                    if (workItem.Wait(TimeSpan.MaxValue))
+                    {
+                        m_log.DebugFormat(
+                            "[SCRIPT INSTANCE]: Co-operatively stopped script {0} {1} in {2} {3}",
+                            ScriptName, ItemID, PrimName, ObjectID);
+
+                        return true;
+                    }
+                }
             }
 
             lock (EventQueue)
@@ -547,6 +582,7 @@ namespace OpenSim.Region.ScriptEngine.Shared.Instance
 
             // If the event still hasn't stopped and we the stop isn't the result of script or object removal, then
             // forcibly abort the work item (this aborts the underlying thread).
+            // Co-operative termination should never reach this point.
             if (!m_InSelfDelete)
             {
                 m_log.DebugFormat(
@@ -786,7 +822,11 @@ namespace OpenSim.Region.ScriptEngine.Shared.Instance
                             m_InEvent = false;
                             m_CurrentEvent = String.Empty;
 
-                            if ((!(e is TargetInvocationException) || (!(e.InnerException is SelfDeleteException) && !(e.InnerException is ScriptDeleteException))) && !(e is ThreadAbortException))
+                            if ((!(e is TargetInvocationException) 
+                                || (!(e.InnerException is SelfDeleteException) 
+                                    && !(e.InnerException is ScriptDeleteException)
+                                    && !(e.InnerException is ScriptCoopStopException))) 
+                                && !(e is ThreadAbortException))
                             {
                                 try
                                 {
@@ -834,6 +874,12 @@ namespace OpenSim.Region.ScriptEngine.Shared.Instance
                                 m_InSelfDelete = true;
                                 Part.Inventory.RemoveInventoryItem(ItemID);
                             }
+                            else if ((e is TargetInvocationException) && (e.InnerException is ScriptCoopStopException))
+                            {
+                                m_log.DebugFormat(
+                                    "[SCRIPT INSTANCE]: Script {0}.{1} in event {2}, state {3} stopped co-operatively.",
+                                    PrimName, ScriptName, data.EventName, State);
+                            }
                         }
                     }
                 }
diff --git a/OpenSim/Region/ScriptEngine/Shared/Instance/Tests/CoopTerminationTests.cs b/OpenSim/Region/ScriptEngine/Shared/Instance/Tests/CoopTerminationTests.cs
new file mode 100644
index 0000000..f3a6cc9
--- /dev/null
+++ b/OpenSim/Region/ScriptEngine/Shared/Instance/Tests/CoopTerminationTests.cs
@@ -0,0 +1,157 @@
+/*
+ * Copyright (c) Contributors, http://opensimulator.org/
+ * See CONTRIBUTORS.TXT for a full list of copyright holders.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the OpenSimulator Project nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using Nini.Config;
+using NUnit.Framework;
+using OpenMetaverse;
+using OpenSim.Framework;
+using OpenSim.Region.CoreModules.Scripting.WorldComm;
+using OpenSim.Region.Framework.Scenes;
+using OpenSim.Region.Framework.Interfaces;
+using OpenSim.Region.ScriptEngine.XEngine;
+using OpenSim.Tests.Common;
+using OpenSim.Tests.Common.Mock;
+
+namespace OpenSim.Region.ScriptEngine.Shared.Instance.Tests
+{
+    /// <summary>
+    /// Test that co-operative script thread termination is working correctly.
+    /// </summary>
+    [TestFixture]
+    public class CoopTerminationTests : OpenSimTestCase
+    {
+        private TestScene m_scene;
+        private OpenSim.Region.ScriptEngine.XEngine.XEngine m_xEngine;
+
+        private AutoResetEvent m_chatEvent = new AutoResetEvent(false);
+        private AutoResetEvent m_stoppedEvent = new AutoResetEvent(false);
+
+        private OSChatMessage m_osChatMessageReceived;
+
+        [TestFixtureSetUp]
+        public void Init()
+        {
+            //AppDomain.CurrentDomain.SetData("APPBASE", Environment.CurrentDirectory + "/bin");
+//            Console.WriteLine(AppDomain.CurrentDomain.BaseDirectory);
+            m_xEngine = new OpenSim.Region.ScriptEngine.XEngine.XEngine();
+
+            IniConfigSource configSource = new IniConfigSource();
+            
+            IConfig startupConfig = configSource.AddConfig("Startup");
+            startupConfig.Set("DefaultScriptEngine", "XEngine");
+
+            IConfig xEngineConfig = configSource.AddConfig("XEngine");
+            xEngineConfig.Set("Enabled", "true");
+            xEngineConfig.Set("StartDelay", "0");
+
+            // These tests will not run with AppDomainLoading = true, at least on mono.  For unknown reasons, the call
+            // to AssemblyResolver.OnAssemblyResolve fails.
+            xEngineConfig.Set("AppDomainLoading", "false");
+
+            xEngineConfig.Set("ScriptStopStrategy", "co-op");
+
+            m_scene = new SceneHelpers().SetupScene("My Test", UUID.Random(), 1000, 1000, configSource);
+            SceneHelpers.SetupSceneModules(m_scene, configSource, m_xEngine);
+            m_scene.StartScripts();
+        }
+
+        /// <summary>
+        /// Test co-operative termination on derez of an object containing a script with a long-running event.
+        /// </summary>
+        /// <remarks>
+        /// TODO: Actually compiling the script is incidental to this test.  Really want a way to compile test scripts
+        /// within the build itself.
+        /// </remarks>
+        [Test]
+        public void TestStopOnLongSleep()
+        {
+            TestHelpers.InMethod();
+            TestHelpers.EnableLogging();
+
+            UUID userId = TestHelpers.ParseTail(0x1);
+//            UUID objectId = TestHelpers.ParseTail(0x100);
+//            UUID itemId = TestHelpers.ParseTail(0x3);
+            string itemName = "TestStopOnObjectDerezLongSleep() Item";
+
+            SceneObjectGroup so = SceneHelpers.CreateSceneObject(1, userId, "TestStopOnObjectDerezLongSleep", 0x100);
+            m_scene.AddNewSceneObject(so, true);
+
+            InventoryItemBase itemTemplate = new InventoryItemBase();
+//            itemTemplate.ID = itemId;
+            itemTemplate.Name = itemName;
+            itemTemplate.Folder = so.UUID;
+            itemTemplate.InvType = (int)InventoryType.LSL;
+
+            m_scene.EventManager.OnChatFromWorld += OnChatFromWorld;
+
+            SceneObjectPart partWhereRezzed = m_scene.RezNewScript(userId, itemTemplate, 
+@"default
+{    
+    state_entry()
+    {
+        llSay(0, ""Thin Lizzy"");
+        llSleep(60);
+    }
+}");
+
+            TaskInventoryItem rezzedItem = partWhereRezzed.Inventory.GetInventoryItem(itemName);
+
+            // Wait for the script to start the event before we try stopping it.
+            m_chatEvent.WaitOne(60000);
+
+            Console.WriteLine("Script started with message [{0}]", m_osChatMessageReceived.Message);
+
+            // FIXME: This is a very poor way of trying to avoid a low-probability race condition where the script
+            // executes llSay() but has not started the sleep before we try to stop it.
+            Thread.Sleep(1000);
+
+            // We need a way of carrying on if StopScript() fail, since it won't return if the script isn't actually
+            // stopped.  This kind of multi-threading is far from ideal in a regression test.
+            new Thread(() => { m_xEngine.StopScript(rezzedItem.ItemID); m_stoppedEvent.Set(); }).Start();
+
+            if (!m_stoppedEvent.WaitOne(30000))
+                Assert.Fail("Script did not co-operatively stop.");
+
+            bool running;
+            TaskInventoryItem scriptItem = partWhereRezzed.Inventory.GetInventoryItem(itemName);
+            Assert.That(
+                SceneObjectPartInventory.TryGetScriptInstanceRunning(m_scene, scriptItem, out running), Is.True);
+            Assert.That(running, Is.False);
+        }
+
+        private void OnChatFromWorld(object sender, OSChatMessage oscm)
+        {
+//            Console.WriteLine("Got chat [{0}]", oscm.Message);
+
+            m_osChatMessageReceived = oscm;
+            m_chatEvent.Set();
+        }
+    }
+}
\ No newline at end of file
diff --git a/OpenSim/Region/ScriptEngine/XEngine/XEngine.cs b/OpenSim/Region/ScriptEngine/XEngine/XEngine.cs
index 186ae04..a17a018 100644
--- a/OpenSim/Region/ScriptEngine/XEngine/XEngine.cs
+++ b/OpenSim/Region/ScriptEngine/XEngine/XEngine.cs
@@ -1716,9 +1716,14 @@ namespace OpenSim.Region.ScriptEngine.XEngine
             IScriptInstance instance = GetInstance(itemID);
 
             if (instance != null)
+            {
                 instance.Stop(m_WaitForEventCompletionOnScriptStop);
+            }
             else
+            {
+//                m_log.DebugFormat("[XENGINE]: Could not find script with ID {0} to stop in {1}", itemID, World.Name);
                 m_runFlags.AddOrUpdate(itemID, false, 240);
+            }
         }
 
         public DetectParams GetDetectParams(UUID itemID, int idx)
diff --git a/prebuild.xml b/prebuild.xml
index c6bef5c..abf8f36 100644
--- a/prebuild.xml
+++ b/prebuild.xml
@@ -2417,7 +2417,9 @@
       <Reference name="log4net" path="../../../../../bin/"/>
 
       <Files>
-        <Match pattern="*.cs" recurse="true"/>
+        <Match pattern="*.cs" recurse="true">
+          <Exclude name="Tests" pattern="Tests"/>
+        </Match>
       </Files>
     </Project>
 
@@ -3309,6 +3311,7 @@
         <!-- SADLY the way this works means you need to keep adding these paths -->
         <Match path="Shared/Tests" pattern="*.cs" recurse="true"/>
         <Match path="Shared/CodeTools/Tests" pattern="*.cs" recurse="true"/>
+        <Match path="Shared/Instance/Tests" pattern="*.cs" recurse="true"/>
         <Match path="XEngine/Tests" pattern="*.cs" recurse="true"/>
       </Files>
     </Project>