corrected script notecard parser. It now handles notecards with inventory as well.

Signed-off-by: BlueWall <jamesh@bluewallgroup.com>

4 files changed, 262 insertions, 94 deletions

diff --git a/OpenSim/Framework/SLUtil.cs b/OpenSim/Framework/SLUtil.cs
index 9249105..027fc0e 100644
--- a/OpenSim/Framework/SLUtil.cs
+++ b/OpenSim/Framework/SLUtil.cs
@@ -25,13 +25,9 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+using OpenMetaverse;
 using System;
 using System.Collections.Generic;
-using System.IO;
-using System.Reflection;
-using System.Xml;
-using log4net;
-using OpenMetaverse;
 
 namespace OpenSim.Framework
 {
@@ -251,106 +247,270 @@ namespace OpenSim.Framework
 
         #endregion SL / file extension / content-type conversions
 
-        /// <summary>
-        /// Parse a notecard in Linden format to a string of ordinary text.
-        /// </summary>
-        /// <param name="rawInput"></param>
-        /// <returns></returns>
-        public static string ParseNotecardToString(string rawInput)
+        private class NotecardReader
         {
-            string[] output = ParseNotecardToList(rawInput).ToArray();
+            private string rawInput;
+            private int lineNumber;
 
-//            foreach (string line in output)
-//                m_log.DebugFormat("[PARSE NOTECARD]: ParseNotecardToString got line {0}", line);
-            
-            return string.Join("\n", output);
+            public int LineNumber
+            {
+                get
+                {
+                    return lineNumber;
+                }
+            }
+
+            public NotecardReader(string _rawInput)
+            {
+                rawInput = (string)_rawInput.Clone();
+                lineNumber = 0;
+            }
+
+            public string getLine()
+            {
+                if(rawInput.Length == 0)
+                {
+                    throw new NotANotecardFormatException(lineNumber + 1);
+                }
+
+                int pos = rawInput.IndexOf('\n');
+                if(pos < 0)
+                {
+                    pos = rawInput.Length;
+                }
+
+                /* cut line from rest */
+                ++lineNumber;
+                string line = rawInput.Substring(0, pos);
+                if (pos + 1 >= rawInput.Length)
+                {
+                    rawInput = string.Empty;
+                }
+                else
+                {
+                    rawInput = rawInput.Substring(pos + 1);
+                }
+                /* clean up line from double spaces and tabs */
+                line = line.Replace("\t", " ");
+                while(line.IndexOf("  ") >= 0)
+                {
+                    line = line.Replace("  ", " ");
+                }
+                return line.Replace("\r", "").Trim();
+            }
+
+            public string getBlock(int length)
+            {
+                /* cut line from rest */
+                if(length > rawInput.Length)
+                {
+                    throw new NotANotecardFormatException(lineNumber);
+                }
+                string line = rawInput.Substring(0, length);
+                rawInput = rawInput.Substring(length);
+                return line;
+            }
         }
-                
-        /// <summary>
-        /// Parse a notecard in Linden format to a list of ordinary lines.
-        /// </summary>
-        /// <param name="rawInput"></param>
-        /// <returns></returns>
-        public static List<string> ParseNotecardToList(string rawInput)
+
+        public class NotANotecardFormatException : Exception
         {
-            string[] input;
-            int idx = 0;
-            int level = 0;
-            List<string> output = new List<string>();
-            string[] words;
-
-            //The Linden format always ends with a } after the input data.
-            //Strip off trailing } so there is nothing after the input data.
-            int i = rawInput.LastIndexOf("}");
-            rawInput = rawInput.Remove(i, rawInput.Length-i);
-            input = rawInput.Replace("\r", "").Split('\n');
-
-            while (idx < input.Length)
+            public int lineNumber;
+            public NotANotecardFormatException(int _lineNumber)
+                : base()
             {
-                if (input[idx] == "{")
+               lineNumber = _lineNumber;
+            }
+        }
+
+        private static void skipSection(NotecardReader reader)
+        {
+            if (reader.getLine() != "{")
+                throw new NotANotecardFormatException(reader.LineNumber);
+
+            string line;
+            while ((line = reader.getLine()) != "}")
+            {
+                if(line.IndexOf('{')>=0)
+                {
+                    throw new NotANotecardFormatException(reader.LineNumber);
+                }
+            }
+        }
+
+        private static void skipInventoryItem(NotecardReader reader)
+        {
+            if (reader.getLine() != "{")
+                throw new NotANotecardFormatException(reader.LineNumber);
+
+            string line;
+            while((line = reader.getLine()) != "}")
+            {
+                string[] data = line.Split(' ');
+                if(data.Length == 0)
                 {
-                    level++;
-                    idx++;
                     continue;
                 }
+                if(data[0] == "permissions")
+                {
+                    skipSection(reader);
+                }
+                else if(data[0] == "sale_info")
+                {
+                    skipSection(reader);
+                }
+                else if (line.IndexOf('{') >= 0)
+                {
+                    throw new NotANotecardFormatException(reader.LineNumber);
+                }
+            }
+        }
+
+        private static void skipInventoryItems(NotecardReader reader)
+        {
+            if(reader.getLine() != "{")
+            {
+                throw new NotANotecardFormatException(reader.LineNumber);
+            }
 
-                if (input[idx]== "}")
+            string line;
+            while((line = reader.getLine()) != "}")
+            {
+                string[] data = line.Split(' ');
+                if(data.Length == 0)
                 {
-                    level--;
-                    idx++;
                     continue;
                 }
 
-                switch (level)
+                if(data[0] == "inv_item")
                 {
-                case 0:
-                    words = input[idx].Split(' '); // Linden text ver
-                    // Notecards are created *really* empty. Treat that as "no text" (just like after saving an empty notecard)
-                    if (words.Length < 3)
-                        return output;
-
-                    int version = int.Parse(words[3]);
-                    if (version != 2)
-                        return output;
-                    break;
-                case 1:
-                    words = input[idx].Split(' ');
-                    if (words[0] == "LLEmbeddedItems")
-                        break;
-                    if (words[0] == "Text")
-                    {
-                        idx++;  //Now points to first line of notecard text
+                    skipInventoryItem(reader);
+                } 
+                else if (line.IndexOf('{') >= 0)
+                {
+                    throw new NotANotecardFormatException(reader.LineNumber);
+                }
 
-                        //Number of lines in notecard.
-                        int lines = input.Length - idx;
-                        int line = 0;
+            }
+        }
 
-                        while (line < lines)
-                        {
-//                            m_log.DebugFormat("[PARSE NOTECARD]: Adding line {0}", input[idx]);
-                            output.Add(input[idx]);
-                            idx++;
-                            line++;
-                        }
+        private static void skipInventory(NotecardReader reader)
+        {
+            if (reader.getLine() != "{")
+                throw new NotANotecardFormatException(reader.LineNumber);
 
-                        return output;
-                    }
-                    break;
-                case 2:
-                    words = input[idx].Split(' '); // count
-                    if (words[0] == "count")
+            string line;
+            while((line = reader.getLine()) != "}")
+            {
+                string[] data = line.Split(' ');
+                if(data[0] == "count")
+                {
+                    int count = Int32.Parse(data[1]);
+                    for(int i = 0; i < count; ++i)
                     {
-                        int c = int.Parse(words[1]);
-                        if (c > 0)
-                            return output;
-                        break;
+                        skipInventoryItems(reader);
                     }
-                    break;
                 }
-                idx++;
+                else if (line.IndexOf('{') >= 0)
+                {
+                    throw new NotANotecardFormatException(reader.LineNumber);
+                }
             }
-            
-            return output;
+        }
+
+        private static string readNotecardText(NotecardReader reader)
+        {
+            if (reader.getLine() != "{")
+                throw new NotANotecardFormatException(reader.LineNumber);
+
+            string notecardString = string.Empty;
+            string line;
+            while((line = reader.getLine()) != "}")
+            {
+                string[] data = line.Split(' ');
+                if (data.Length == 0)
+                {
+                    continue;
+                }
+
+                if (data[0] == "LLEmbeddedItems")
+                {
+                    skipInventory(reader);
+                }
+                else if(data[0] == "Text" && data.Length == 3)
+                {
+                    int length = Int32.Parse(data[2]);
+                    notecardString = reader.getBlock(length);
+                } 
+                else if (line.IndexOf('{') >= 0)
+                {
+                    throw new NotANotecardFormatException(reader.LineNumber);
+                }
+
+            }
+            return notecardString;
+        }
+
+        private static string readNotecard(byte[] rawInput)
+        {
+            string rawIntermedInput = string.Empty;
+
+            /* make up a Raw Encoding here */
+            foreach(byte c in rawInput)
+            {
+                char d = (char)c;
+                rawIntermedInput += d;
+            }
+
+            NotecardReader reader = new NotecardReader(rawIntermedInput);
+            string line;
+            try
+            {
+                line = reader.getLine();
+            }
+            catch(Exception)
+            {
+                return System.Text.Encoding.UTF8.GetString(rawInput);
+            }
+            string[] versioninfo = line.Split(' ');
+            if(versioninfo.Length < 3)
+            {
+                return System.Text.Encoding.UTF8.GetString(rawInput);
+            }
+            else if(versioninfo[0] != "Linden" || versioninfo[1] != "text")
+            {
+                return System.Text.Encoding.UTF8.GetString(rawInput);
+            }
+            else
+            {
+                /* now we actually decode the Encoding, before we needed it in raw */
+                string o = readNotecardText(reader);
+                byte[] a = new byte[o.Length];
+                for(int i = 0; i < o.Length; ++i)
+                {
+                    a[i] = (byte)o[i];
+                }
+                return System.Text.Encoding.UTF8.GetString(a);
+            }
+        }
+
+        /// <summary>
+        /// Parse a notecard in Linden format to a string of ordinary text.
+        /// </summary>
+        /// <param name="rawInput"></param>
+        /// <returns></returns>
+        public static string ParseNotecardToString(byte[] rawInput)
+        {
+            return readNotecard(rawInput);
+        }
+
+        /// <summary>
+        /// Parse a notecard in Linden format to a list of ordinary lines.
+        /// </summary>
+        /// <param name="rawInput"></param>
+        /// <returns></returns>
+        public static string[] ParseNotecardToArray(byte[] rawInput)
+        {
+            return readNotecard(rawInput).Replace("\r", "").Split('\n');
         }
     }
 }
diff --git a/OpenSim/Region/OptionalModules/Scripting/JsonStore/JsonStoreScriptModule.cs b/OpenSim/Region/OptionalModules/Scripting/JsonStore/JsonStoreScriptModule.cs
index b69676b..edf51a2 100644
--- a/OpenSim/Region/OptionalModules/Scripting/JsonStore/JsonStoreScriptModule.cs
+++ b/OpenSim/Region/OptionalModules/Scripting/JsonStore/JsonStoreScriptModule.cs
@@ -595,11 +595,15 @@ namespace OpenSim.Region.OptionalModules.Scripting.JsonStore
 
             try 
             {
-                string jsondata = SLUtil.ParseNotecardToString(Encoding.UTF8.GetString(a.Data));
+                string jsondata = SLUtil.ParseNotecardToString(a.Data);
                 int result = m_store.SetValue(storeID, path, jsondata,true) ? 1 : 0;
                 m_comms.DispatchReply(scriptID, result, "", reqID.ToString());
                 return;
             }
+            catch(SLUtil.NotANotecardFormatException e)
+            {
+                m_log.WarnFormat("[JsonStoreScripts]: Notecard parsing failed; assetId {0} at line number {1}", assetID.ToString(), e.lineNumber);
+            }
             catch (Exception e)
             {
                 m_log.WarnFormat("[JsonStoreScripts]: Json parsing failed; {0}", e.Message);
diff --git a/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/LSL_Api.cs b/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/LSL_Api.cs
index bb8ec9c..6a30da2 100644
--- a/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/LSL_Api.cs
+++ b/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/LSL_Api.cs
@@ -11632,9 +11632,7 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api
                     return;
                 }
 
-                string data = Encoding.UTF8.GetString(a.Data);
-                //m_log.Debug(data);
-                NotecardCache.Cache(id, data);
+                NotecardCache.Cache(id, a.Data);
                 AsyncCommands.DataserverPlugin.DataserverReply(reqIdentifier, NotecardCache.GetLines(id).ToString());
             });
 
@@ -11688,7 +11686,7 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api
 
                              string data = Encoding.UTF8.GetString(a.Data);
                              //m_log.Debug(data);
-                             NotecardCache.Cache(id, data);
+                             NotecardCache.Cache(id, a.Data);
                              AsyncCommands.DataserverPlugin.DataserverReply(
                                 reqIdentifier, NotecardCache.GetLine(assetID, line, m_notecardLineReadCharsMax));
                          });
@@ -12461,10 +12459,10 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api
             public DateTime lastRef;
         }
 
-        protected static Dictionary<UUID, Notecard> m_Notecards =
+        private static Dictionary<UUID, Notecard> m_Notecards =
             new Dictionary<UUID, Notecard>();
 
-        public static void Cache(UUID assetID, string text)
+        public static void Cache(UUID assetID, byte[] text)
         {
             CheckCache();
 
@@ -12475,7 +12473,14 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api
 
                 Notecard nc = new Notecard();
                 nc.lastRef = DateTime.Now;
-                nc.text = SLUtil.ParseNotecardToList(text).ToArray();
+                try
+                {
+                    nc.text = SLUtil.ParseNotecardToArray(text);
+                }
+                catch(SLUtil.NotANotecardFormatException)
+                {
+                    nc.text = new string[0];
+                }
                 m_Notecards[assetID] = nc;
             }
         }
diff --git a/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/OSSL_Api.cs b/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/OSSL_Api.cs
index 10ddf14..3afebe2 100644
--- a/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/OSSL_Api.cs
+++ b/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/OSSL_Api.cs
@@ -1926,8 +1926,7 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api
                 if (a == null)
                     return UUID.Zero;
 
-                string data = Encoding.UTF8.GetString(a.Data);
-                NotecardCache.Cache(assetID, data);
+                NotecardCache.Cache(assetID, a.Data);
             };
 
             return assetID;