From 6e4d970dae2e292e4155174f5de13f7e774384a3 Mon Sep 17 00:00:00 2001 From: Dan Lake Date: Mon, 18 Apr 2011 16:48:49 -0700 Subject: Requeue unacknowledged entity updates rather than resend then "as is". Often, by the time the UDPServer realizes that an entity update packet has not been acknowledged, there is a newer update for the same entity already queued up or there is a higher priority update that should be sent first. This patch eliminates 1:1 packet resends for unacked entity update packets. Insteawd, unacked update packets are decomposed into the original entity updates and those updates are placed back into the priority queues based on their new priority but the original update timestamp. This will generally place them at the head of the line to be put back on the wire as a new outgoing packet but prevents the resend queue from filling up with multiple stale updates for the same entity. This new approach takes advantage of the UDP nature of the Linden protocol in that the intent of a reliable update packet is that if it goes unacknowledge, SOMETHING has to happen to get the update to the client. We are simply making sure that we are resending current object state rather than stale object state. Additionally, this patch includes a generalized callback mechanism so that any caller can specify their own method to call when a packet expires without being acknowledged. We use this mechanism to requeue update packets and otherwise use the UDPServer default method of just putting expired packets in the resend queue. --- OpenSim/Framework/IClientAPI.cs | 59 ++++++++++--- OpenSim/Framework/Util.cs | 17 ++++ .../Region/ClientStack/LindenUDP/LLClientView.cs | 96 +++++++++++++++++----- .../Region/ClientStack/LindenUDP/LLUDPServer.cs | 64 +++++++-------- .../Region/ClientStack/LindenUDP/OutgoingPacket.cs | 6 +- 5 files changed, 176 insertions(+), 66 deletions(-) diff --git a/OpenSim/Framework/IClientAPI.cs b/OpenSim/Framework/IClientAPI.cs index f573c32..069987b 100644 --- a/OpenSim/Framework/IClientAPI.cs +++ b/OpenSim/Framework/IClientAPI.cs @@ -572,34 +572,69 @@ namespace OpenSim.Framework public class IEntityUpdate { - public ISceneEntity Entity; - public uint Flags; + private ISceneEntity m_entity; + private uint m_flags; + private int m_updateTime; + + public ISceneEntity Entity + { + get { return m_entity; } + } + + public uint Flags + { + get { return m_flags; } + } + + public int UpdateTime + { + get { return m_updateTime; } + } public virtual void Update(IEntityUpdate update) { - this.Flags |= update.Flags; + m_flags |= update.Flags; + + // Use the older of the updates as the updateTime + if (Util.EnvironmentTickCountCompare(UpdateTime, update.UpdateTime) > 0) + m_updateTime = update.UpdateTime; } public IEntityUpdate(ISceneEntity entity, uint flags) { - Entity = entity; - Flags = flags; + m_entity = entity; + m_flags = flags; + m_updateTime = Util.EnvironmentTickCount(); + } + + public IEntityUpdate(ISceneEntity entity, uint flags, Int32 updateTime) + { + m_entity = entity; + m_flags = flags; + m_updateTime = updateTime; } } - public class EntityUpdate : IEntityUpdate { - // public ISceneEntity Entity; - // public PrimUpdateFlags Flags; - public float TimeDilation; + private float m_timeDilation; + + public float TimeDilation + { + get { return m_timeDilation; } + } public EntityUpdate(ISceneEntity entity, PrimUpdateFlags flags, float timedilation) - : base(entity,(uint)flags) + : base(entity, (uint)flags) { - //Entity = entity; // Flags = flags; - TimeDilation = timedilation; + m_timeDilation = timedilation; + } + + public EntityUpdate(ISceneEntity entity, PrimUpdateFlags flags, float timedilation, Int32 updateTime) + : base(entity,(uint)flags,updateTime) + { + m_timeDilation = timedilation; } } diff --git a/OpenSim/Framework/Util.cs b/OpenSim/Framework/Util.cs index 5a5046e..aaa2724 100644 --- a/OpenSim/Framework/Util.cs +++ b/OpenSim/Framework/Util.cs @@ -1537,6 +1537,23 @@ namespace OpenSim.Framework return (diff >= 0) ? diff : (diff + EnvironmentTickCountMask + 1); } + // Returns value of Tick Count A - TickCount B accounting for wrapping of TickCount + // Assumes both tcA and tcB came from previous calls to Util.EnvironmentTickCount(). + // A positive return value indicates A occured later than B + public static Int32 EnvironmentTickCountCompare(Int32 tcA, Int32 tcB) + { + // A, B and TC are all between 0 and 0x3fffffff + int tc = EnvironmentTickCount(); + + if (tc - tcA >= 0) + tcA += EnvironmentTickCountMask + 1; + + if (tc - tcB >= 0) + tcB += EnvironmentTickCountMask + 1; + + return tcA - tcB; + } + /// /// Prints the call stack at any given point. Useful for debugging. /// diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs b/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs index 1f7e66d..14c5d6c 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs @@ -3561,6 +3561,34 @@ namespace OpenSim.Region.ClientStack.LindenUDP m_entityUpdates.Enqueue(priority, new EntityUpdate(entity, updateFlags, m_scene.TimeDilation)); } + /// + /// Requeue an EntityUpdate when it was not acknowledged by the client. + /// We will update the priority and put it in the correct queue, merging update flags + /// with any other updates that may be queued for the same entity. + /// The original update time is used for the merged update. + /// + public void ResendPrimUpdate(EntityUpdate update) + { + // If the update exists in priority queue, it will be updated. + // If it does not exist then it will be added with the current (rather than its original) priority + uint priority = m_prioritizer.GetUpdatePriority(this, update.Entity); + + lock (m_entityUpdates.SyncRoot) + m_entityUpdates.Enqueue(priority, update); + } + + /// + /// Requeue a list of EntityUpdates when they were not acknowledged by the client. + /// We will update the priority and put it in the correct queue, merging update flags + /// with any other updates that may be queued for the same entity. + /// The original update time is used for the merged update. + /// + void ResendPrimUpdates(List updates) + { + foreach (EntityUpdate update in updates) + ResendPrimUpdate(update); + } + private void ProcessEntityUpdates(int maxUpdates) { OpenSim.Framework.Lazy> objectUpdateBlocks = new OpenSim.Framework.Lazy>(); @@ -3568,6 +3596,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP OpenSim.Framework.Lazy> terseUpdateBlocks = new OpenSim.Framework.Lazy>(); OpenSim.Framework.Lazy> terseAgentUpdateBlocks = new OpenSim.Framework.Lazy>(); + OpenSim.Framework.Lazy> objectUpdates = new OpenSim.Framework.Lazy>(); + OpenSim.Framework.Lazy> compressedUpdates = new OpenSim.Framework.Lazy>(); + OpenSim.Framework.Lazy> terseUpdates = new OpenSim.Framework.Lazy>(); + OpenSim.Framework.Lazy> terseAgentUpdates = new OpenSim.Framework.Lazy>(); + // Check to see if this is a flush if (maxUpdates <= 0) { @@ -3583,7 +3616,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP float avgTimeDilation = 1.0f; IEntityUpdate iupdate; Int32 timeinqueue; // this is just debugging code & can be dropped later - + while (updatesThisCall < maxUpdates) { lock (m_entityUpdates.SyncRoot) @@ -3688,24 +3721,33 @@ namespace OpenSim.Region.ClientStack.LindenUDP if (update.Entity is ScenePresence) { objectUpdateBlocks.Value.Add(CreateAvatarUpdateBlock((ScenePresence)update.Entity)); + objectUpdates.Value.Add(update); } else { objectUpdateBlocks.Value.Add(CreatePrimUpdateBlock((SceneObjectPart)update.Entity, this.m_agentId)); + objectUpdates.Value.Add(update); } } else if (!canUseImproved) { compressedUpdateBlocks.Value.Add(CreateCompressedUpdateBlock((SceneObjectPart)update.Entity, updateFlags)); + compressedUpdates.Value.Add(update); } else { if (update.Entity is ScenePresence && ((ScenePresence)update.Entity).UUID == AgentId) + { // Self updates go into a special list terseAgentUpdateBlocks.Value.Add(CreateImprovedTerseBlock(update.Entity, updateFlags.HasFlag(PrimUpdateFlags.Textures))); + terseAgentUpdates.Value.Add(update); + } else + { // Everything else goes here terseUpdateBlocks.Value.Add(CreateImprovedTerseBlock(update.Entity, updateFlags.HasFlag(PrimUpdateFlags.Textures))); + terseUpdates.Value.Add(update); + } } #endregion Block Construction @@ -3713,28 +3755,23 @@ namespace OpenSim.Region.ClientStack.LindenUDP #region Packet Sending - - //const float TIME_DILATION = 1.0f; - - ushort timeDilation = Utils.FloatToUInt16(avgTimeDilation, 0.0f, 1.0f); - + if (terseAgentUpdateBlocks.IsValueCreated) { List blocks = terseAgentUpdateBlocks.Value; - + ImprovedTerseObjectUpdatePacket packet = new ImprovedTerseObjectUpdatePacket(); packet.RegionData.RegionHandle = m_scene.RegionInfo.RegionHandle; packet.RegionData.TimeDilation = timeDilation; packet.ObjectData = new ImprovedTerseObjectUpdatePacket.ObjectDataBlock[blocks.Count]; - + for (int i = 0; i < blocks.Count; i++) packet.ObjectData[i] = blocks[i]; - - - OutPacket(packet, ThrottleOutPacketType.Unknown, true); + // If any of the packets created from this call go unacknowledged, all of the updates will be resent + OutPacket(packet, ThrottleOutPacketType.Unknown, true, delegate() { ResendPrimUpdates(terseAgentUpdates.Value); }); } - + if (objectUpdateBlocks.IsValueCreated) { List blocks = objectUpdateBlocks.Value; @@ -3746,8 +3783,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP for (int i = 0; i < blocks.Count; i++) packet.ObjectData[i] = blocks[i]; - - OutPacket(packet, ThrottleOutPacketType.Task, true); + // If any of the packets created from this call go unacknowledged, all of the updates will be resent + OutPacket(packet, ThrottleOutPacketType.Task, true, delegate() { ResendPrimUpdates(objectUpdates.Value); }); } if (compressedUpdateBlocks.IsValueCreated) @@ -3761,10 +3798,10 @@ namespace OpenSim.Region.ClientStack.LindenUDP for (int i = 0; i < blocks.Count; i++) packet.ObjectData[i] = blocks[i]; - - OutPacket(packet, ThrottleOutPacketType.Task, true); + // If any of the packets created from this call go unacknowledged, all of the updates will be resent + OutPacket(packet, ThrottleOutPacketType.Task, true, delegate() { ResendPrimUpdates(compressedUpdates.Value); }); } - + if (terseUpdateBlocks.IsValueCreated) { List blocks = terseUpdateBlocks.Value; @@ -3776,8 +3813,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP for (int i = 0; i < blocks.Count; i++) packet.ObjectData[i] = blocks[i]; - - OutPacket(packet, ThrottleOutPacketType.Task, true); + // If any of the packets created from this call go unacknowledged, all of the updates will be resent + OutPacket(packet, ThrottleOutPacketType.Task, true, delegate() { ResendPrimUpdates(terseUpdates.Value); }); } } @@ -3969,7 +4006,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP { SendFamilyProps = SendFamilyProps || update.SendFamilyProps; SendObjectProps = SendObjectProps || update.SendObjectProps; - Flags |= update.Flags; + // other properties may need to be updated by base class + base.Update(update); } } @@ -11363,6 +11401,22 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// handles splitting manually protected void OutPacket(Packet packet, ThrottleOutPacketType throttlePacketType, bool doAutomaticSplitting) { + OutPacket(packet, throttlePacketType, doAutomaticSplitting, null); + } + + /// + /// This is the starting point for sending a simulator packet out to the client + /// + /// Packet to send + /// Throttling category for the packet + /// True to automatically split oversized + /// packets (the default), or false to disable splitting if the calling code + /// handles splitting manually + /// The method to be called in the event this packet is reliable + /// and unacknowledged. The server will provide normal resend capability if you do not + /// provide your own method. + protected void OutPacket(Packet packet, ThrottleOutPacketType throttlePacketType, bool doAutomaticSplitting, UnackedPacketMethod method) + { if (m_debugPacketLevel > 0) { bool logPacket = true; @@ -11388,7 +11442,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP m_log.DebugFormat("[CLIENT]: Packet OUT {0}", packet.Type); } - m_udpServer.SendPacket(m_udpClient, packet, throttlePacketType, doAutomaticSplitting); + m_udpServer.SendPacket(m_udpClient, packet, throttlePacketType, doAutomaticSplitting, method); } public bool AddMoney(int debit) diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs index d08b25f..0848979 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs @@ -297,7 +297,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP delegate(IClientAPI client) { if (client is LLClientView) - SendPacketData(((LLClientView)client).UDPClient, data, packet.Type, category); + SendPacketData(((LLClientView)client).UDPClient, data, packet.Type, category, null); } ); } @@ -309,7 +309,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP delegate(IClientAPI client) { if (client is LLClientView) - SendPacketData(((LLClientView)client).UDPClient, data, packet.Type, category); + SendPacketData(((LLClientView)client).UDPClient, data, packet.Type, category, null); } ); } @@ -322,7 +322,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// /// /// - public void SendPacket(LLUDPClient udpClient, Packet packet, ThrottleOutPacketType category, bool allowSplitting) + public void SendPacket(LLUDPClient udpClient, Packet packet, ThrottleOutPacketType category, bool allowSplitting, UnackedPacketMethod method) { // CoarseLocationUpdate packets cannot be split in an automated way if (packet.Type == PacketType.CoarseLocationUpdate && allowSplitting) @@ -339,13 +339,13 @@ namespace OpenSim.Region.ClientStack.LindenUDP for (int i = 0; i < packetCount; i++) { byte[] data = datas[i]; - SendPacketData(udpClient, data, packet.Type, category); + SendPacketData(udpClient, data, packet.Type, category, method); } } else { byte[] data = packet.ToBytes(); - SendPacketData(udpClient, data, packet.Type, category); + SendPacketData(udpClient, data, packet.Type, category, method); } } @@ -356,7 +356,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// /// /// - public void SendPacketData(LLUDPClient udpClient, byte[] data, PacketType type, ThrottleOutPacketType category) + public void SendPacketData(LLUDPClient udpClient, byte[] data, PacketType type, ThrottleOutPacketType category, UnackedPacketMethod method) { int dataLength = data.Length; bool doZerocode = (data[0] & Helpers.MSG_ZEROCODED) != 0; @@ -411,7 +411,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP #region Queue or Send - OutgoingPacket outgoingPacket = new OutgoingPacket(udpClient, buffer, category); + OutgoingPacket outgoingPacket = new OutgoingPacket(udpClient, buffer, category, null); + // If we were not provided a method for handling unacked, use the UDPServer default method + outgoingPacket.UnackedMethod = ((method == null) ? delegate() { ResendUnacked(outgoingPacket); } : method); // If a Linden Lab 1.23.5 client receives an update packet after a kill packet for an object, it will // continue to display the deleted object until relog. Therefore, we need to always queue a kill object @@ -445,7 +447,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP packet.Header.Reliable = false; packet.Packets = blocks.ToArray(); - SendPacket(udpClient, packet, ThrottleOutPacketType.Unknown, true); + SendPacket(udpClient, packet, ThrottleOutPacketType.Unknown, true, null); } } @@ -458,17 +460,17 @@ namespace OpenSim.Region.ClientStack.LindenUDP // We *could* get OldestUnacked, but it would hurt performance and not provide any benefit pc.PingID.OldestUnacked = 0; - SendPacket(udpClient, pc, ThrottleOutPacketType.Unknown, false); + SendPacket(udpClient, pc, ThrottleOutPacketType.Unknown, false, null); } public void CompletePing(LLUDPClient udpClient, byte pingID) { CompletePingCheckPacket completePing = new CompletePingCheckPacket(); completePing.PingID.PingID = pingID; - SendPacket(udpClient, completePing, ThrottleOutPacketType.Unknown, false); + SendPacket(udpClient, completePing, ThrottleOutPacketType.Unknown, false, null); } - public void ResendUnacked(LLUDPClient udpClient) + public void HandleUnacked(LLUDPClient udpClient) { if (!udpClient.IsConnected) return; @@ -488,31 +490,29 @@ namespace OpenSim.Region.ClientStack.LindenUDP if (expiredPackets != null) { - //m_log.Debug("[LLUDPSERVER]: Resending " + expiredPackets.Count + " packets to " + udpClient.AgentID + ", RTO=" + udpClient.RTO); - + //m_log.Debug("[LLUDPSERVER]: Handling " + expiredPackets.Count + " packets to " + udpClient.AgentID + ", RTO=" + udpClient.RTO); // Exponential backoff of the retransmission timeout udpClient.BackoffRTO(); + for (int i = 0; i < expiredPackets.Count; ++i) + expiredPackets[i].UnackedMethod(); + } + } - // Resend packets - for (int i = 0; i < expiredPackets.Count; i++) - { - OutgoingPacket outgoingPacket = expiredPackets[i]; - - //m_log.DebugFormat("[LLUDPSERVER]: Resending packet #{0} (attempt {1}), {2}ms have passed", - // outgoingPacket.SequenceNumber, outgoingPacket.ResendCount, Environment.TickCount - outgoingPacket.TickCount); + public void ResendUnacked(OutgoingPacket outgoingPacket) + { + //m_log.DebugFormat("[LLUDPSERVER]: Resending packet #{0} (attempt {1}), {2}ms have passed", + // outgoingPacket.SequenceNumber, outgoingPacket.ResendCount, Environment.TickCount - outgoingPacket.TickCount); - // Set the resent flag - outgoingPacket.Buffer.Data[0] = (byte)(outgoingPacket.Buffer.Data[0] | Helpers.MSG_RESENT); - outgoingPacket.Category = ThrottleOutPacketType.Resend; + // Set the resent flag + outgoingPacket.Buffer.Data[0] = (byte)(outgoingPacket.Buffer.Data[0] | Helpers.MSG_RESENT); + outgoingPacket.Category = ThrottleOutPacketType.Resend; - // Bump up the resend count on this packet - Interlocked.Increment(ref outgoingPacket.ResendCount); + // Bump up the resend count on this packet + Interlocked.Increment(ref outgoingPacket.ResendCount); - // Requeue or resend the packet - if (!outgoingPacket.Client.EnqueueOutgoing(outgoingPacket, false)) - SendPacketFinal(outgoingPacket); - } - } + // Requeue or resend the packet + if (!outgoingPacket.Client.EnqueueOutgoing(outgoingPacket, false)) + SendPacketFinal(outgoingPacket); } public void Flush(LLUDPClient udpClient) @@ -1096,7 +1096,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP if (udpClient.IsConnected) { if (m_resendUnacked) - ResendUnacked(udpClient); + HandleUnacked(udpClient); if (m_sendAcks) SendAcks(udpClient); @@ -1152,7 +1152,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP nticksUnack++; watch2.Start(); - ResendUnacked(udpClient); + HandleUnacked(udpClient); watch2.Stop(); avgResendUnackedTicks = (nticksUnack - 1)/(float)nticksUnack * avgResendUnackedTicks + (watch2.ElapsedTicks / (float)nticksUnack); diff --git a/OpenSim/Region/ClientStack/LindenUDP/OutgoingPacket.cs b/OpenSim/Region/ClientStack/LindenUDP/OutgoingPacket.cs index 1a1a1cb..f4f024b 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/OutgoingPacket.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/OutgoingPacket.cs @@ -31,6 +31,7 @@ using OpenMetaverse; namespace OpenSim.Region.ClientStack.LindenUDP { + public delegate void UnackedPacketMethod(); /// /// Holds a reference to the this packet is /// destined for, along with the serialized packet data, sequence number @@ -52,6 +53,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP public int TickCount; /// Category this packet belongs to public ThrottleOutPacketType Category; + /// The delegate to be called if this packet is determined to be unacknowledged + public UnackedPacketMethod UnackedMethod; /// /// Default constructor @@ -60,11 +63,12 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// Serialized packet data. If the flags or sequence number /// need to be updated, they will be injected directly into this binary buffer /// Throttling category for this packet - public OutgoingPacket(LLUDPClient client, UDPPacketBuffer buffer, ThrottleOutPacketType category) + public OutgoingPacket(LLUDPClient client, UDPPacketBuffer buffer, ThrottleOutPacketType category, UnackedPacketMethod method) { Client = client; Buffer = buffer; Category = category; + UnackedMethod = method; } } } -- cgit v1.1