Numerous packet improvements.

Don't allow packets to be resent before they have actually been sent for the
first time. Switch from serializing a packet to get it's length to the LibOMV
provided Length property. Fix resend timing. Fix the use of dangling references
to Acked packets. Fix the packet handler to play nice with the packet pool.
Fix the packet pool. Add data block recycling to the packet pool. Packet pool
is now ENABLED by default. Add config option to disable packet and data block
reuse. Add ObjectUpdate and ImprovedTerseObjectUpdate to the packets being
recycled.

7 files changed, 111 insertions, 53 deletions

diff --git a/OpenSim/Framework/PacketPool.cs b/OpenSim/Framework/PacketPool.cs
index f17d654..37924d7 100644
--- a/OpenSim/Framework/PacketPool.cs
+++ b/OpenSim/Framework/PacketPool.cs
@@ -41,7 +41,8 @@ namespace OpenSim.Framework
 
         private static readonly PacketPool instance = new PacketPool();
 
-        private bool packetPoolEnabled = false;
+        private bool packetPoolEnabled = true;
+        private bool dataBlockPoolEnabled = true;
 
         private readonly Dictionary<PacketType, Stack<Packet>> pool = new Dictionary<PacketType, Stack<Packet>>();
 
@@ -57,6 +58,18 @@ namespace OpenSim.Framework
             get { return instance; }
         }
 
+        public bool RecyclePackets
+        {
+            set { packetPoolEnabled = value; }
+            get { return packetPoolEnabled; }
+        }
+
+        public bool RecycleDataBlocks
+        {
+            set { dataBlockPoolEnabled = value; }
+            get { return dataBlockPoolEnabled; }
+        }
+
         public Packet GetPacket(PacketType type)
         {
             Packet packet;
@@ -140,55 +153,58 @@ namespace OpenSim.Framework
         /// <param name="packet"></param>
         public void ReturnPacket(Packet packet)
         {
-            switch (packet.Type)
+            if (dataBlockPoolEnabled)
             {
-                case PacketType.ObjectUpdate:
-                    ObjectUpdatePacket oup = (ObjectUpdatePacket)packet;
-
-                    foreach (ObjectUpdatePacket.ObjectDataBlock oupod in
-                            oup.ObjectData)
-                        ReturnDataBlock<ObjectUpdatePacket.ObjectDataBlock>(oupod);
-                    oup.ObjectData = null;
-                    break;
-
-                case PacketType.ImprovedTerseObjectUpdate:
-                    ImprovedTerseObjectUpdatePacket itoup =
-                            (ImprovedTerseObjectUpdatePacket)packet;
-
-                    foreach (ImprovedTerseObjectUpdatePacket.ObjectDataBlock
-                            itoupod in itoup.ObjectData)
-                        ReturnDataBlock<ImprovedTerseObjectUpdatePacket.ObjectDataBlock>(itoupod);
-                    itoup.ObjectData = null;
-                    break;
+                switch (packet.Type)
+                {
+                    case PacketType.ObjectUpdate:
+                        ObjectUpdatePacket oup = (ObjectUpdatePacket)packet;
+
+                        foreach (ObjectUpdatePacket.ObjectDataBlock oupod in
+                                oup.ObjectData)
+                            ReturnDataBlock<ObjectUpdatePacket.ObjectDataBlock>(oupod);
+                        oup.ObjectData = null;
+                        break;
+
+                    case PacketType.ImprovedTerseObjectUpdate:
+                        ImprovedTerseObjectUpdatePacket itoup =
+                                (ImprovedTerseObjectUpdatePacket)packet;
+
+                        foreach (ImprovedTerseObjectUpdatePacket.ObjectDataBlock
+                                itoupod in itoup.ObjectData)
+                            ReturnDataBlock<ImprovedTerseObjectUpdatePacket.ObjectDataBlock>(itoupod);
+                        itoup.ObjectData = null;
+                        break;
+                }
             }
 
-            if (!packetPoolEnabled)
-                return;
-
-            switch (packet.Type)
+            if (packetPoolEnabled)
             {
-                // List pooling packets here
-                case PacketType.PacketAck:
-                case PacketType.ObjectUpdate:
-                case PacketType.ImprovedTerseObjectUpdate:
-                    lock (pool)
-                    {
-                        PacketType type = packet.Type;
-
-                        if (!pool.ContainsKey(type))
-                        {
-                            pool[type] = new Stack<Packet>();
-                        }
-                        if ((pool[type]).Count < 50)
+                switch (packet.Type)
+                {
+                    // List pooling packets here
+                    case PacketType.PacketAck:
+                    case PacketType.ObjectUpdate:
+                    case PacketType.ImprovedTerseObjectUpdate:
+                        lock (pool)
                         {
-                            (pool[type]).Push(packet);
+                            PacketType type = packet.Type;
+
+                            if (!pool.ContainsKey(type))
+                            {
+                                pool[type] = new Stack<Packet>();
+                            }
+                            if ((pool[type]).Count < 50)
+                            {
+                                (pool[type]).Push(packet);
+                            }
                         }
-                    }
-                    break;
-                
-                // Other packets wont pool
-                default:
-                    return;
+                        break;
+                    
+                    // Other packets wont pool
+                    default:
+                        return;
+                }
             }
         }
 
diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs b/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs
index 4acc6be..4bc568c 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs
@@ -936,6 +936,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
                     LLQueItem item = new LLQueItem();
                     item.Packet = packet;
+                    item.Sequence = packet.Header.Sequence;
 
                     m_PacketHandler.ProcessOutPacket(item);
 
diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLPacketHandler.cs b/OpenSim/Region/ClientStack/LindenUDP/LLPacketHandler.cs
index 2191ca7..6229237 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLPacketHandler.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLPacketHandler.cs
@@ -253,7 +253,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             item.TickCount = Environment.TickCount;
             item.Identifier = id;
             item.Resends = 0;
-            item.Length = packet.ToBytes().Length;
+            item.Length = packet.Length;
+            item.Sequence = packet.Header.Sequence;
 
             m_PacketQueue.Enqueue(item);
             m_PacketsSent++;
@@ -310,7 +311,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
                     // Packets this old get resent
                     //
-                    if ((now - data.TickCount) > m_ResendTimeout)
+                    if ((now - data.TickCount) > m_ResendTimeout && data.Sequence != 0)
                     {
                         if (resent < 20)
                         {
@@ -325,6 +326,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                             {
                                 m_NeedAck.Remove(packet.Header.Sequence);
                                 TriggerOnPacketDrop(packet, data.Identifier);
+                                m_PacketQueue.Cancel(packet.Header.Sequence);
                                 PacketPool.Instance.ReturnPacket(packet);
                                 continue;
                             }
@@ -586,11 +588,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     return;
 
                 m_NeedAck.Remove(id);
-                // We can't return this packet, it will just have to be GC'd
-                // Reason for that is that the packet may still be in the
-                // send queue, and if it gets reused things get messy!
-                //
-                // PacketPool.Instance.ReturnPacket(data.Packet);
+                m_PacketQueue.Cancel(data.Sequence);
+                PacketPool.Instance.ReturnPacket(data.Packet);
                 m_UnackedBytes -= data.Length;
             }
         }
@@ -680,7 +679,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                 item.TickCount = Environment.TickCount;
                 item.Identifier = 0;
                 item.Resends = 0;
-                item.Length = packet.ToBytes().Length;
+                item.Length = packet.Length;
+                item.Sequence = packet.Header.Sequence;
                 m_NeedAck.Add(key, item);
             }
 
@@ -719,6 +719,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                 if (data.Identifier != null && data.Identifier == id)
                 {
                     m_NeedAck.Remove(data.Packet.Header.Sequence);
+                    m_PacketQueue.Cancel(data.Sequence);
                     PacketPool.Instance.ReturnPacket(data.Packet);
                     return;
                 }
@@ -745,6 +746,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             if (packet.Header.Sequence == 0)
             {
                 packet.Header.Sequence = NextPacketSequenceNumber();
+                item.Sequence = packet.Header.Sequence;
+                item.TickCount = Environment.TickCount;
 
                 lock (m_NeedAck)
                 {
@@ -793,7 +796,10 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             // Dont't return in that case
             //
             if (!packet.Header.Reliable)
+            {
+                m_PacketQueue.Cancel(item.Sequence);
                 PacketPool.Instance.ReturnPacket(packet);
+            }
         }
 
         private void Abort()
diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLPacketQueue.cs b/OpenSim/Region/ClientStack/LindenUDP/LLPacketQueue.cs
index ef1f34a..46d5610 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLPacketQueue.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLPacketQueue.cs
@@ -83,6 +83,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         internal LLPacketThrottle TextureThrottle;
         internal LLPacketThrottle TotalThrottle;
         
+        private List<uint> contents = new List<uint>();
+
         /// <summary>
         /// The number of packets in the OutgoingPacketQueue
         /// 
@@ -186,6 +188,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                 return;
             }
 
+            if (item.Sequence != 0)
+                contents.Add(item.Sequence);
+
             lock (this)
             {
                 switch (item.throttleType & ThrottleOutPacketType.TypeMask)
@@ -226,7 +231,24 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
         public LLQueItem Dequeue()
         {
-            return SendQueue.Dequeue();
+            while (true)
+            {
+                LLQueItem item = SendQueue.Dequeue();
+                if (item == null)
+                    return null;
+                if (item.Incoming)
+                    return item;
+                if (item.Sequence == 0)
+                    return item;
+                if (contents.Remove(item.Sequence))
+                    return item;
+            }
+        }
+
+        public void Cancel(uint sequence)
+        {
+            while(contents.Remove(sequence))
+                ;
         }
 
         public void Flush()
@@ -286,6 +308,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                 TextureOutgoingPacketQueue.Clear();
                 AssetOutgoingPacketQueue.Clear();
                 SendQueue.Clear();
+                contents.Clear();
             }
         }
 
diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLQueItem.cs b/OpenSim/Region/ClientStack/LindenUDP/LLQueItem.cs
index a5dcf55..3a08e9b 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLQueItem.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLQueItem.cs
@@ -44,5 +44,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         public Object Identifier;
         public int Resends;
         public int Length;
+        public uint Sequence;
     }
 }
diff --git a/OpenSim/Region/Framework/Scenes/Scene.cs b/OpenSim/Region/Framework/Scenes/Scene.cs
index 638187b..7872a6e 100644
--- a/OpenSim/Region/Framework/Scenes/Scene.cs
+++ b/OpenSim/Region/Framework/Scenes/Scene.cs
@@ -381,6 +381,12 @@ namespace OpenSim.Region.Framework.Scenes
                 m_defaultScriptEngine = startupConfig.GetString("DefaultScriptEngine", "DotNetEngine");
 
                 m_maxPrimsPerFrame = startupConfig.GetInt("MaxPrimsPerFrame", 200);
+                IConfig packetConfig = m_config.Configs["PacketPool"];
+                if (packetConfig != null)
+                {
+                    PacketPool.Instance.RecyclePackets = packetConfig.GetBoolean("RecyclePackets", true);
+                    PacketPool.Instance.RecycleDataBlocks = packetConfig.GetBoolean("RecycleDataBlocks", true);
+                }
             }
             catch
             {
diff --git a/bin/OpenSim.ini.example b/bin/OpenSim.ini.example
index 7f7bcf4..2ae7fb7 100644
--- a/bin/OpenSim.ini.example
+++ b/bin/OpenSim.ini.example
@@ -1269,3 +1269,8 @@
     ; System.Net.WebException: The request was aborted: The request was canceled.
         ;
         ; XmlRpcDisableKeepAlive = false
+
+[PacketPool]
+    ; Enables the experimental packet pool. Yes, we've been here before.
+    ;RecyclePackets = true;
+    ;RecycleDataBlocks = true;