issues with udp buffers pool on heavy load

4 files changed, 53 insertions, 42 deletions

diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPClient.cs
index e0cca05..fca7943 100644
--- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPClient.cs
+++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPClient.cs
@@ -650,6 +650,12 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     // leaving a dequeued packet still waiting to be sent out. Try to
                     // send it again
                     OutgoingPacket nextPacket = m_nextPackets[i];
+                    if(nextPacket.Buffer == null)
+                    {
+                        if (m_packetOutboxes[i].Count < 5)
+                            emptyCategories |= CategoryToFlag(i);
+                        continue;
+                    }
                     if (bucket.RemoveTokens(nextPacket.Buffer.DataLength))
                     {
                         // Send the packet
@@ -681,21 +687,29 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                         {
                             // A packet was pulled off the queue. See if we have
                             // enough tokens in the bucket to send it out
-                            if (bucket.RemoveTokens(packet.Buffer.DataLength))
+                            if(packet.Buffer == null)
                             {
-                                // Send the packet
-                                m_udpServer.SendPacketFinal(packet);
-                                packetSent = true;
-
+                                // packet canceled elsewhere (by a ack for example)
                                 if (queue.Count < 5)
                                     emptyCategories |= CategoryToFlag(i);
                             }
                             else
                             {
-                                // Save the dequeued packet for the next iteration
-                                m_nextPackets[i] = packet;
+                                if (bucket.RemoveTokens(packet.Buffer.DataLength))
+                                {
+                                    // Send the packet
+                                    m_udpServer.SendPacketFinal(packet);
+                                    packetSent = true;
+
+                                    if (queue.Count < 5)
+                                        emptyCategories |= CategoryToFlag(i);
+                                }
+                                else
+                                {
+                                    // Save the dequeued packet for the next iteration
+                                    m_nextPackets[i] = packet;
+                                }
                             }
-
                         }
                         else
                         {
diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs
index ba5a2f3..d324623 100644
--- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs
+++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs
@@ -983,7 +983,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         {
             LLUDPClient udpClient = client.UDPClient;
 
-            if (!udpClient.IsConnected)
+            if (!client.IsActive || !udpClient.IsConnected)
                 return;
 
             // Disconnect an agent if no packets are received for some time
@@ -1053,14 +1053,16 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         internal void SendPacketFinal(OutgoingPacket outgoingPacket)
         {
             UDPPacketBuffer buffer = outgoingPacket.Buffer;
+            if(buffer == null) // canceled packet
+                return;
+            LLUDPClient udpClient = outgoingPacket.Client;
+            if (!udpClient.IsConnected)
+                return;
+
             byte flags = buffer.Data[0];
             bool isResend = (flags & Helpers.MSG_RESENT) != 0;
             bool isReliable = (flags & Helpers.MSG_RELIABLE) != 0;
             bool isZerocoded = (flags & Helpers.MSG_ZEROCODED) != 0;
-            LLUDPClient udpClient = outgoingPacket.Client;
-
-            if (!udpClient.IsConnected)
-                return;
 
             int dataLength = buffer.DataLength;
 
@@ -1916,7 +1918,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             {
                 Scene.ThreadAlive(2);
 
-
                 try
                 {
                     m_packetSent = false;
@@ -1971,7 +1972,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     }
                     else if (!m_packetSent)
 //                        Thread.Sleep((int)TickCountResolution);  outch this is bad on linux
-                        Thread.Sleep(15); // match the 16ms of windows7, dont ask 16 or win may decide to do 32ms.
+                        Thread.Sleep(15); // match the 16ms of windows, dont ask 16 or win may decide to do 32ms.
 
                     Watchdog.UpdateThread();
                 }
@@ -1995,14 +1996,17 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
                     if (udpClient.IsConnected)
                     {
-                        if (m_resendUnacked)
+                        if (client.IsActive && m_resendUnacked)
                             HandleUnacked(llClient);
 
-                        if (m_sendAcks)
-                            SendAcks(udpClient);
+                        if (client.IsActive)
+                        {
+                            if (m_sendAcks)
+                                SendAcks(udpClient);
 
-                        if (m_sendPing)
-                            SendPing(udpClient);
+                            if (m_sendPing)
+                                SendPing(udpClient);
+                        }
 
                         // Dequeue any outgoing packets that are within the throttle limits
                         if (udpClient.DequeueOutgoing())
@@ -2015,7 +2019,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                 m_log.Error(
                     string.Format("[LLUDPSERVER]: OutgoingPacketHandler iteration for {0} threw ", client.Name), ex);
             }
-            client = null;
         }
 
         #region Emergency Monitoring
diff --git a/OpenSim/Region/ClientStack/Linden/UDP/OpenSimUDPBase.cs b/OpenSim/Region/ClientStack/Linden/UDP/OpenSimUDPBase.cs
index 5fa4637..6f346d3 100644
--- a/OpenSim/Region/ClientStack/Linden/UDP/OpenSimUDPBase.cs
+++ b/OpenSim/Region/ClientStack/Linden/UDP/OpenSimUDPBase.cs
@@ -343,14 +343,12 @@ namespace OpenMetaverse
             {
                 // kick off an async read
                 m_udpSocket.BeginReceiveFrom(
-                    //wrappedBuffer.Instance.Data,
                     buf.Data,
                     0,
-                    UDPPacketBuffer.BUFFER_SIZE,
+                    buf.Data.Length,
                     SocketFlags.None,
                     ref buf.RemoteEndPoint,
                     AsyncEndReceive,
-                    //wrappedBuffer);
                     buf);
             }
             catch (SocketException e)
@@ -364,14 +362,12 @@ namespace OpenMetaverse
                         try
                         {
                             m_udpSocket.BeginReceiveFrom(
-                                //wrappedBuffer.Instance.Data,
                                 buf.Data,
                                 0,
-                                UDPPacketBuffer.BUFFER_SIZE,
+                                buf.Data.Length,
                                 SocketFlags.None,
                                 ref buf.RemoteEndPoint,
                                 AsyncEndReceive,
-                                //wrappedBuffer);
                                 buf);
                             salvaged = true;
                         }
@@ -382,11 +378,6 @@ namespace OpenMetaverse
                     m_log.Warn("[UDPBASE]: Salvaged the UDP listener on port " + m_udpPort);
                 }
             }
-            catch (ObjectDisposedException e)
-            {
-                m_log.Error(
-                    string.Format("[UDPBASE]: Error processing UDP begin receive {0}.  Exception  ", UdpReceives), e);
-            }
             catch (Exception e)
             {
                 m_log.Error(
@@ -443,11 +434,6 @@ namespace OpenMetaverse
                             UdpReceives, se.ErrorCode),
                         se);
                 }
-                catch (ObjectDisposedException e)
-                {
-                    m_log.Error(
-                        string.Format("[UDPBASE]: Error processing UDP end receive {0}.  Exception  ", UdpReceives), e);
-                }
                 catch (Exception e)
                 {
                     m_log.Error(
@@ -502,6 +488,8 @@ namespace OpenMetaverse
 */
         public void SyncSend(UDPPacketBuffer buf)
         {
+            if(buf.RemoteEndPoint == null)
+                return; // was already expired
             try
             {
                 m_udpSocket.SendTo(
@@ -515,7 +503,7 @@ namespace OpenMetaverse
             }
             catch (SocketException e)
             {
-                m_log.Warn("[UDPBASE]: sync send SocketException {0} " + e.Message);
+                m_log.WarnFormat("[UDPBASE]: sync send SocketException {0} {1}", buf.RemoteEndPoint, e.Message);
             }
             catch (ObjectDisposedException) { }
         }
diff --git a/OpenSim/Region/ClientStack/Linden/UDP/UnackedPacketCollection.cs b/OpenSim/Region/ClientStack/Linden/UDP/UnackedPacketCollection.cs
index e0eee53..1f978e1 100644
--- a/OpenSim/Region/ClientStack/Linden/UDP/UnackedPacketCollection.cs
+++ b/OpenSim/Region/ClientStack/Linden/UDP/UnackedPacketCollection.cs
@@ -189,8 +189,10 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             // Process all the pending adds
             OutgoingPacket pendingAdd;
             while (m_pendingAdds.TryDequeue(out pendingAdd))
+            {
                 if (pendingAdd != null)
                     m_packets[pendingAdd.SequenceNumber] = pendingAdd;
+            }
 
             // Process all the pending removes, including updating statistics and round-trip times
             PendingAck pendingAcknowledgement;
@@ -203,15 +205,17 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     if (ackedPacket != null)
                     {
                         m_packets.Remove(pendingAcknowledgement.SequenceNumber);
+
+                        // Update stats
+                        Interlocked.Add(ref ackedPacket.Client.UnackedBytes, -ackedPacket.Buffer.DataLength);
+
                         ackedPacket.Client.FreeUDPBuffer(ackedPacket.Buffer);
+                        ackedPacket.Buffer = null;
 
                         // As with other network applications, assume that an acknowledged packet is an
                         // indication that the network can handle a little more load, speed up the transmission
                         ackedPacket.Client.FlowThrottle.AcknowledgePackets(1);
 
-                        // Update stats
-                        Interlocked.Add(ref ackedPacket.Client.UnackedBytes, -ackedPacket.Buffer.DataLength);
-
                         if (!pendingAcknowledgement.FromResend)
                         {
                             // Calculate the round-trip time for this packet and its ACK
@@ -242,10 +246,12 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     if (removedPacket != null)
                     {
                         m_packets.Remove(pendingRemove);
-                        removedPacket.Client.FreeUDPBuffer(removedPacket.Buffer);
 
                         // Update stats
                         Interlocked.Add(ref removedPacket.Client.UnackedBytes, -removedPacket.Buffer.DataLength);
+
+                        removedPacket.Client.FreeUDPBuffer(removedPacket.Buffer);
+                        removedPacket.Buffer = null;
                     }
                 }
             }