Requeue unacknowledged entity updates rather than resend then "as is".

Often, by the time the UDPServer realizes that an entity update packet
has not been acknowledged, there is a newer update for the same entity
already queued up or there is a higher priority update that should be
sent first. This patch eliminates 1:1 packet resends for unacked entity
update packets. Insteawd, unacked update packets are decomposed into the
original entity updates and those updates are placed back into the
priority queues based on their new priority but the original update
timestamp. This will generally place them at the head of the line to be
put back on the wire as a new outgoing packet but prevents the resend
queue from filling up with multiple stale updates for the same entity.
This new approach takes advantage of the UDP nature of the Linden protocol
in that the intent of a reliable update packet is that if it goes
unacknowledge, SOMETHING has to happen to get the update to the client.
We are simply making sure that we are resending current object state
rather than stale object state.

Additionally, this patch includes a generalized callback mechanism so
that any caller can specify their own method to call when a packet
expires without being acknowledged. We use this mechanism to requeue
update packets and otherwise use the UDPServer default method of just
putting expired packets in the resend queue.

1 files changed, 75 insertions, 21 deletions

diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs b/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs
index 1f7e66d..14c5d6c 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs
@@ -3561,6 +3561,34 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                 m_entityUpdates.Enqueue(priority, new EntityUpdate(entity, updateFlags, m_scene.TimeDilation));
         }
 
+        /// <summary>
+        /// Requeue an EntityUpdate when it was not acknowledged by the client. 
+        /// We will update the priority and put it in the correct queue, merging update flags 
+        /// with any other updates that may be queued for the same entity. 
+        /// The original update time is used for the merged update.
+        /// </summary>
+        public void ResendPrimUpdate(EntityUpdate update)
+        {
+            // If the update exists in priority queue, it will be updated.
+            // If it does not exist then it will be added with the current (rather than its original) priority
+            uint priority = m_prioritizer.GetUpdatePriority(this, update.Entity);
+
+            lock (m_entityUpdates.SyncRoot)
+                m_entityUpdates.Enqueue(priority, update);
+        }
+
+        /// <summary>
+        /// Requeue a list of EntityUpdates when they were not acknowledged by the client. 
+        /// We will update the priority and put it in the correct queue, merging update flags 
+        /// with any other updates that may be queued for the same entity. 
+        /// The original update time is used for the merged update.
+        /// </summary>
+        void ResendPrimUpdates(List<EntityUpdate> updates)
+        {
+            foreach (EntityUpdate update in updates)
+                ResendPrimUpdate(update);
+        }
+
         private void ProcessEntityUpdates(int maxUpdates)
         {
             OpenSim.Framework.Lazy<List<ObjectUpdatePacket.ObjectDataBlock>> objectUpdateBlocks = new OpenSim.Framework.Lazy<List<ObjectUpdatePacket.ObjectDataBlock>>();
@@ -3568,6 +3596,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             OpenSim.Framework.Lazy<List<ImprovedTerseObjectUpdatePacket.ObjectDataBlock>> terseUpdateBlocks = new OpenSim.Framework.Lazy<List<ImprovedTerseObjectUpdatePacket.ObjectDataBlock>>();
             OpenSim.Framework.Lazy<List<ImprovedTerseObjectUpdatePacket.ObjectDataBlock>> terseAgentUpdateBlocks = new OpenSim.Framework.Lazy<List<ImprovedTerseObjectUpdatePacket.ObjectDataBlock>>();
 
+            OpenSim.Framework.Lazy<List<EntityUpdate>> objectUpdates = new OpenSim.Framework.Lazy<List<EntityUpdate>>();
+            OpenSim.Framework.Lazy<List<EntityUpdate>> compressedUpdates = new OpenSim.Framework.Lazy<List<EntityUpdate>>();
+            OpenSim.Framework.Lazy<List<EntityUpdate>> terseUpdates = new OpenSim.Framework.Lazy<List<EntityUpdate>>();
+            OpenSim.Framework.Lazy<List<EntityUpdate>> terseAgentUpdates = new OpenSim.Framework.Lazy<List<EntityUpdate>>();
+
             // Check to see if this is a flush
             if (maxUpdates <= 0)
             {
@@ -3583,7 +3616,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                 float avgTimeDilation = 1.0f;
                 IEntityUpdate iupdate;
                 Int32 timeinqueue; // this is just debugging code & can be dropped later
-                
+
                 while (updatesThisCall < maxUpdates)
                 {
                     lock (m_entityUpdates.SyncRoot)
@@ -3688,24 +3721,33 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                         if (update.Entity is ScenePresence)
                         {
                             objectUpdateBlocks.Value.Add(CreateAvatarUpdateBlock((ScenePresence)update.Entity));
+                            objectUpdates.Value.Add(update);
                         }
                         else
                         {
                             objectUpdateBlocks.Value.Add(CreatePrimUpdateBlock((SceneObjectPart)update.Entity, this.m_agentId));
+                            objectUpdates.Value.Add(update);
                         }
                     }
                     else if (!canUseImproved)
                     {
                         compressedUpdateBlocks.Value.Add(CreateCompressedUpdateBlock((SceneObjectPart)update.Entity, updateFlags));
+                        compressedUpdates.Value.Add(update);
                     }
                     else
                     {
                         if (update.Entity is ScenePresence && ((ScenePresence)update.Entity).UUID == AgentId)
+                        {
                             // Self updates go into a special list
                             terseAgentUpdateBlocks.Value.Add(CreateImprovedTerseBlock(update.Entity, updateFlags.HasFlag(PrimUpdateFlags.Textures)));
+                            terseAgentUpdates.Value.Add(update);
+                        }
                         else
+                        {
                             // Everything else goes here
                             terseUpdateBlocks.Value.Add(CreateImprovedTerseBlock(update.Entity, updateFlags.HasFlag(PrimUpdateFlags.Textures)));
+                            terseUpdates.Value.Add(update);
+                        }
                     }
     
                     #endregion Block Construction
@@ -3713,28 +3755,23 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                 
     
                 #region Packet Sending
-        
-                //const float TIME_DILATION = 1.0f;
-
-
                 ushort timeDilation = Utils.FloatToUInt16(avgTimeDilation, 0.0f, 1.0f);
-    
+
                 if (terseAgentUpdateBlocks.IsValueCreated)
                 {
                     List<ImprovedTerseObjectUpdatePacket.ObjectDataBlock> blocks = terseAgentUpdateBlocks.Value;
-    
+
                     ImprovedTerseObjectUpdatePacket packet = new ImprovedTerseObjectUpdatePacket();
                     packet.RegionData.RegionHandle = m_scene.RegionInfo.RegionHandle;
                     packet.RegionData.TimeDilation = timeDilation;
                     packet.ObjectData = new ImprovedTerseObjectUpdatePacket.ObjectDataBlock[blocks.Count];
-    
+
                     for (int i = 0; i < blocks.Count; i++)
                         packet.ObjectData[i] = blocks[i];
-
-                    
-                    OutPacket(packet, ThrottleOutPacketType.Unknown, true);
+                    // If any of the packets created from this call go unacknowledged, all of the updates will be resent
+                    OutPacket(packet, ThrottleOutPacketType.Unknown, true, delegate() { ResendPrimUpdates(terseAgentUpdates.Value); });
                 }
-    
+
                 if (objectUpdateBlocks.IsValueCreated)
                 {
                     List<ObjectUpdatePacket.ObjectDataBlock> blocks = objectUpdateBlocks.Value;
@@ -3746,8 +3783,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         
                     for (int i = 0; i < blocks.Count; i++)
                         packet.ObjectData[i] = blocks[i];
-        
-                    OutPacket(packet, ThrottleOutPacketType.Task, true);
+                    // If any of the packets created from this call go unacknowledged, all of the updates will be resent
+                    OutPacket(packet, ThrottleOutPacketType.Task, true, delegate() { ResendPrimUpdates(objectUpdates.Value); });
                 }
         
                 if (compressedUpdateBlocks.IsValueCreated)
@@ -3761,10 +3798,10 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         
                     for (int i = 0; i < blocks.Count; i++)
                         packet.ObjectData[i] = blocks[i];
-        
-                    OutPacket(packet, ThrottleOutPacketType.Task, true);
+                    // If any of the packets created from this call go unacknowledged, all of the updates will be resent
+                    OutPacket(packet, ThrottleOutPacketType.Task, true, delegate() { ResendPrimUpdates(compressedUpdates.Value); });
                 }
-        
+
                 if (terseUpdateBlocks.IsValueCreated)
                 {
                     List<ImprovedTerseObjectUpdatePacket.ObjectDataBlock> blocks = terseUpdateBlocks.Value;
@@ -3776,8 +3813,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         
                     for (int i = 0; i < blocks.Count; i++)
                         packet.ObjectData[i] = blocks[i];
-        
-                    OutPacket(packet, ThrottleOutPacketType.Task, true);
+                    // If any of the packets created from this call go unacknowledged, all of the updates will be resent
+                    OutPacket(packet, ThrottleOutPacketType.Task, true, delegate() { ResendPrimUpdates(terseUpdates.Value); });
                 }
             }
 
@@ -3969,7 +4006,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             {
                 SendFamilyProps = SendFamilyProps || update.SendFamilyProps;
                 SendObjectProps = SendObjectProps || update.SendObjectProps;
-                Flags |= update.Flags;
+                // other properties may need to be updated by base class
+                base.Update(update);
             }
         }
         
@@ -11363,6 +11401,22 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         /// handles splitting manually</param>
         protected void OutPacket(Packet packet, ThrottleOutPacketType throttlePacketType, bool doAutomaticSplitting)
         {
+            OutPacket(packet, throttlePacketType, doAutomaticSplitting, null);
+        }
+
+        /// <summary>
+        /// This is the starting point for sending a simulator packet out to the client
+        /// </summary>
+        /// <param name="packet">Packet to send</param>
+        /// <param name="throttlePacketType">Throttling category for the packet</param>
+        /// <param name="doAutomaticSplitting">True to automatically split oversized
+        /// packets (the default), or false to disable splitting if the calling code
+        /// handles splitting manually</param>
+        /// <param name="method">The method to be called in the event this packet is reliable
+        /// and unacknowledged. The server will provide normal resend capability if you do not
+        /// provide your own method.</param>
+        protected void OutPacket(Packet packet, ThrottleOutPacketType throttlePacketType, bool doAutomaticSplitting, UnackedPacketMethod method)
+        {
             if (m_debugPacketLevel > 0)
             {
                 bool logPacket = true;
@@ -11388,7 +11442,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     m_log.DebugFormat("[CLIENT]: Packet OUT {0}", packet.Type);
             }
             
-            m_udpServer.SendPacket(m_udpClient, packet, throttlePacketType, doAutomaticSplitting);
+            m_udpServer.SendPacket(m_udpClient, packet, throttlePacketType, doAutomaticSplitting, method);
         }
 
         public bool AddMoney(int debit)