* Copied LocklessQueue.cs into OpenSim.Framework and added the .Count property and .Clear() method

* Changed the way the QueueEmpty callback is fired. It will be fired asynchronously as soon as an empty queue is detected (this can happen immediately following a dequeue), and will not be fired again until at least one packet is dequeued from that queue. This will give callbacks advanced notice of an empty queue and prevent callbacks from stacking up while the queue is empty
* Added LLUDPClient.IsConnected checks in several places to prevent unwanted network activity after a client disconnects
* Prevent LLClientView.Close() from being called twice every disconnect
* Removed the packet resend limit and improved the client timeout check

5 files changed, 103 insertions, 80 deletions

diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs b/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs
index 0acf6e8..ac558ff 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs
@@ -433,13 +433,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             // Remove ourselves from the scene
             m_scene.RemoveClient(AgentId);
 
-            //m_log.InfoFormat("[CLIENTVIEW] Memory pre  GC {0}", System.GC.GetTotalMemory(false));
-            //GC.Collect();
-            //m_log.InfoFormat("[CLIENTVIEW] Memory post GC {0}", System.GC.GetTotalMemory(true));
-
-            // FIXME: Is this still necessary?
-            //Thread.Sleep(2000);
-
             // Shut down timers. Thread Context of this method is murky.   Lock all timers
             if (m_avatarTerseUpdateTimer.Enabled)
                 lock (m_avatarTerseUpdateTimer)
@@ -461,6 +454,10 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
             // Disable UDP handling for this client
             m_udpClient.Shutdown();
+
+            //m_log.InfoFormat("[CLIENTVIEW] Memory pre  GC {0}", System.GC.GetTotalMemory(false));
+            //GC.Collect();
+            //m_log.InfoFormat("[CLIENTVIEW] Memory post GC {0}", System.GC.GetTotalMemory(true));
         }
 
         public void Kick(string message)
diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLImageManager.cs b/OpenSim/Region/ClientStack/LindenUDP/LLImageManager.cs
index 8469ba6..8410ee9 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLImageManager.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLImageManager.cs
@@ -213,13 +213,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
             lock (m_syncRoot)
             {
-
                 if (m_priorityQueue.Count > 0)
                 {
-                    try
-                    {
-                        image = m_priorityQueue.FindMax();
-                    }
+                    try { image = m_priorityQueue.FindMax(); }
                     catch (Exception) { }
                 }
             }
diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs
index e5b2594..b27d8d6 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs
@@ -80,7 +80,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         /// <summary>Packets we have sent that need to be ACKed by the client</summary>
         public readonly UnackedPacketCollection NeedAcks = new UnackedPacketCollection();
         /// <summary>ACKs that are queued up, waiting to be sent to the client</summary>
-        public readonly LocklessQueue<uint> PendingAcks = new LocklessQueue<uint>();
+        public readonly OpenSim.Framework.LocklessQueue<uint> PendingAcks = new OpenSim.Framework.LocklessQueue<uint>();
 
         /// <summary>Current packet sequence number</summary>
         public int CurrentSequence;
@@ -127,13 +127,16 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         /// <summary>Throttle rate defaults and limits</summary>
         private readonly ThrottleRates defaultThrottleRates;
         /// <summary>Outgoing queues for throttled packets</summary>
-        private readonly LocklessQueue<OutgoingPacket>[] packetOutboxes = new LocklessQueue<OutgoingPacket>[THROTTLE_CATEGORY_COUNT];
+        private readonly OpenSim.Framework.LocklessQueue<OutgoingPacket>[] packetOutboxes = new OpenSim.Framework.LocklessQueue<OutgoingPacket>[THROTTLE_CATEGORY_COUNT];
         /// <summary>A container that can hold one packet for each outbox, used to store
         /// dequeued packets that are being held for throttling</summary>
         private readonly OutgoingPacket[] nextPackets = new OutgoingPacket[THROTTLE_CATEGORY_COUNT];
         /// <summary>An optimization to store the length of dequeued packets being held
         /// for throttling. This avoids expensive calls to Packet.Length</summary>
         private readonly int[] nextPacketLengths = new int[THROTTLE_CATEGORY_COUNT];
+        /// <summary>Flags to prevent queue empty callbacks from repeatedly firing
+        /// before the callbacks have a chance to put packets in the queue</summary>
+        private readonly bool[] queueEmptySent = new bool[THROTTLE_CATEGORY_COUNT];
         /// <summary>A reference to the LLUDPServer that is managing this client</summary>
         private readonly LLUDPServer udpServer;
 
@@ -156,7 +159,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             defaultThrottleRates = rates;
 
             for (int i = 0; i < THROTTLE_CATEGORY_COUNT; i++)
-                packetOutboxes[i] = new LocklessQueue<OutgoingPacket>();
+                packetOutboxes[i] = new OpenSim.Framework.LocklessQueue<OutgoingPacket>();
 
             throttle = new TokenBucket(parentThrottle, 0, 0);
             throttleCategories = new TokenBucket[THROTTLE_CATEGORY_COUNT];
@@ -182,6 +185,14 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         public void Shutdown()
         {
             IsConnected = false;
+            NeedAcks.Clear();
+            for (int i = 0; i < THROTTLE_CATEGORY_COUNT; i++)
+            {
+                packetOutboxes[i].Clear();
+                nextPackets[i] = null;
+            }
+            OnPacketStats = null;
+            OnQueueEmpty = null;
         }
 
         /// <summary>
@@ -322,7 +333,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
             if (category >= 0 && category < packetOutboxes.Length)
             {
-                LocklessQueue<OutgoingPacket> queue = packetOutboxes[category];
+                OpenSim.Framework.LocklessQueue<OutgoingPacket> queue = packetOutboxes[category];
                 TokenBucket bucket = throttleCategories[category];
 
                 if (throttleCategories[category].RemoveTokens(packet.Buffer.DataLength))
@@ -354,7 +365,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         public bool DequeueOutgoing()
         {
             OutgoingPacket packet;
-            LocklessQueue<OutgoingPacket> queue;
+            OpenSim.Framework.LocklessQueue<OutgoingPacket> queue;
             TokenBucket bucket;
             bool packetSent = false;
 
@@ -382,6 +393,10 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     queue = packetOutboxes[i];
                     if (queue.Dequeue(out packet))
                     {
+                        // Reset the flag for firing this queue's OnQueueEmpty callback
+                        // now that we have dequeued a packet
+                        queueEmptySent[i] = false;
+
                         // A packet was pulled off the queue. See if we have
                         // enough tokens in the bucket to send it out
                         if (bucket.RemoveTokens(packet.Buffer.DataLength))
@@ -397,13 +412,18 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                             nextPackets[i] = packet;
                             nextPacketLengths[i] = packet.Buffer.DataLength;
                         }
+
+                        // If the queue is empty after this dequeue, fire the queue
+                        // empty callback now so it has a chance to fill before we 
+                        // get back here
+                        if (queue.Count == 0)
+                            FireQueueEmpty(i);
                     }
                     else
                     {
                         // No packets in this queue. Fire the queue empty callback
-                        QueueEmpty callback = OnQueueEmpty;
-                        if (callback != null)
-                            callback((ThrottleOutPacketType)i);
+                        // if it has not been called recently
+                        FireQueueEmpty(i);
                     }
                 }
             }
@@ -432,8 +452,20 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
             // Always round retransmission timeout up to two seconds
             RTO = Math.Max(2000, (int)(SRTT + Math.Max(G, K * RTTVAR)));
-            //Logger.Debug("Setting agent " + this.Agent.FullName + "'s RTO to " + RTO + "ms with an RTTVAR of " +
+            //m_log.Debug("[LLUDPCLIENT]: Setting agent " + this.Agent.FullName + "'s RTO to " + RTO + "ms with an RTTVAR of " +
             //    RTTVAR + " based on new RTT of " + r + "ms");
         }
+
+        private void FireQueueEmpty(int queueIndex)
+        {
+            if (!queueEmptySent[queueIndex])
+            {
+                queueEmptySent[queueIndex] = true;
+
+                QueueEmpty callback = OnQueueEmpty;
+                if (callback != null)
+                    Util.FireAndForget(delegate(object o) { callback((ThrottleOutPacketType)(int)o); }, queueIndex);
+            }
+        }
     }
 }
diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
index 8689af3..57fee59 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
@@ -332,8 +332,19 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
         public void ResendUnacked(LLUDPClient udpClient)
         {
-            if (udpClient.NeedAcks.Count > 0)
+            if (udpClient.IsConnected && udpClient.NeedAcks.Count > 0)
             {
+                // Disconnect an agent if no packets are received for some time
+                //FIXME: Make 60 an .ini setting
+                if (Environment.TickCount - udpClient.TickLastPacketReceived > 1000 * 60)
+                {
+                    m_log.Warn("[LLUDPSERVER]: Ack timeout, disconnecting " + udpClient.AgentID);
+
+                    RemoveClient(udpClient);
+                    return;
+                }
+
+                // Get a list of all of the packets that have been sitting unacked longer than udpClient.RTO
                 List<OutgoingPacket> expiredPackets = udpClient.NeedAcks.GetExpiredPackets(udpClient.RTO);
 
                 if (expiredPackets != null)
@@ -343,48 +354,24 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     {
                         OutgoingPacket outgoingPacket = expiredPackets[i];
 
-                        // FIXME: Make this an .ini setting
-                        if (outgoingPacket.ResendCount < 3)
-                        {
-                            //Logger.Debug(String.Format("Resending packet #{0} (attempt {1}), {2}ms have passed",
-                            //    outgoingPacket.SequenceNumber, outgoingPacket.ResendCount, Environment.TickCount - outgoingPacket.TickCount));
+                        //m_log.DebugFormat("[LLUDPSERVER]: Resending packet #{0} (attempt {1}), {2}ms have passed",
+                        //    outgoingPacket.SequenceNumber, outgoingPacket.ResendCount, Environment.TickCount - outgoingPacket.TickCount);
 
-                            // Set the resent flag
-                            outgoingPacket.Buffer.Data[0] = (byte)(outgoingPacket.Buffer.Data[0] | Helpers.MSG_RESENT);
-                            outgoingPacket.Category = ThrottleOutPacketType.Resend;
+                        // Set the resent flag
+                        outgoingPacket.Buffer.Data[0] = (byte)(outgoingPacket.Buffer.Data[0] | Helpers.MSG_RESENT);
+                        outgoingPacket.Category = ThrottleOutPacketType.Resend;
 
-                            // The TickCount will be set to the current time when the packet
-                            // is actually sent out again
-                            outgoingPacket.TickCount = 0;
+                        // The TickCount will be set to the current time when the packet
+                        // is actually sent out again
+                        outgoingPacket.TickCount = 0;
 
-                            // Bump up the resend count on this packet
-                            Interlocked.Increment(ref outgoingPacket.ResendCount);
-                            //Interlocked.Increment(ref Stats.ResentPackets);
+                        // Bump up the resend count on this packet
+                        Interlocked.Increment(ref outgoingPacket.ResendCount);
+                        //Interlocked.Increment(ref Stats.ResentPackets);
 
-                            // Queue or (re)send the packet
-                            if (!outgoingPacket.Client.EnqueueOutgoing(outgoingPacket))
-                                SendPacketFinal(outgoingPacket);
-                        }
-                        else
-                        {
-                            m_log.DebugFormat("[LLUDPSERVER]: Dropping packet #{0} for agent {1} after {2} failed attempts",
-                                outgoingPacket.SequenceNumber, outgoingPacket.Client.RemoteEndPoint, outgoingPacket.ResendCount);
-
-                            lock (udpClient.NeedAcks.SyncRoot)
-                                udpClient.NeedAcks.RemoveUnsafe(outgoingPacket.SequenceNumber);
-
-                            //Interlocked.Increment(ref Stats.DroppedPackets);
-
-                            // Disconnect an agent if no packets are received for some time
-                            //FIXME: Make 60 an .ini setting
-                            if (Environment.TickCount - udpClient.TickLastPacketReceived > 1000 * 60)
-                            {
-                                m_log.Warn("[LLUDPSERVER]: Ack timeout, disconnecting " + udpClient.AgentID);
-
-                                RemoveClient(udpClient);
-                                return;
-                            }
-                        }
+                        // Requeue or resend the packet
+                        if (!outgoingPacket.Client.EnqueueOutgoing(outgoingPacket))
+                            SendPacketFinal(outgoingPacket);
                     }
                 }
             }
@@ -403,6 +390,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             bool isReliable = (flags & Helpers.MSG_RELIABLE) != 0;
             LLUDPClient udpClient = outgoingPacket.Client;
 
+            if (!udpClient.IsConnected)
+                return;
+
             // Keep track of when this packet was sent out (right now)
             outgoingPacket.TickCount = Environment.TickCount;
 
@@ -481,14 +471,15 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             }
             catch (MalformedDataException)
             {
-                m_log.ErrorFormat("[LLUDPSERVER]: Malformed data, cannot parse packet:\n{0}",
-                    Utils.BytesToHexString(buffer.Data, buffer.DataLength, null));
+                m_log.ErrorFormat("[LLUDPSERVER]: Malformed data, cannot parse packet from {0}:\n{1}",
+                    buffer.RemoteEndPoint, Utils.BytesToHexString(buffer.Data, buffer.DataLength, null));
             }
 
             // Fail-safe check
             if (packet == null)
             {
-                m_log.Warn("[LLUDPSERVER]: Couldn't build a message from the incoming data");
+                m_log.Warn("[LLUDPSERVER]: Couldn't build a message from incoming data " + buffer.DataLength +
+                    " bytes long from " + buffer.RemoteEndPoint);
                 return;
             }
 
@@ -513,6 +504,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
             udpClient = ((LLClientView)client).UDPClient;
 
+            if (!udpClient.IsConnected)
+                return;
+
             #endregion Packet to Client Mapping
 
             // Stats tracking
@@ -643,7 +637,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                 // Create the LLClientView
                 LLClientView client = new LLClientView(remoteEndPoint, m_scene, this, udpClient, sessionInfo, agentID, sessionID, circuitCode);
                 client.OnLogout += LogoutHandler;
-                client.OnConnectionClosed += ConnectionClosedHandler;
 
                 // Start the IClientAPI
                 client.Start();
@@ -745,17 +738,20 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                         {
                             LLUDPClient udpClient = ((LLClientView)client).UDPClient;
 
-                            if (udpClient.DequeueOutgoing())
-                                packetSent = true;
-                            if (resendUnacked)
-                                ResendUnacked(udpClient);
-                            if (sendAcks)
+                            if (udpClient.IsConnected)
                             {
-                                SendAcks(udpClient);
-                                udpClient.SendPacketStats();
+                                if (udpClient.DequeueOutgoing())
+                                    packetSent = true;
+                                if (resendUnacked)
+                                    ResendUnacked(udpClient);
+                                if (sendAcks)
+                                {
+                                    SendAcks(udpClient);
+                                    udpClient.SendPacketStats();
+                                }
+                                if (sendPings)
+                                    SendPing(udpClient);
                             }
-                            if (sendPings)
-                                SendPing(udpClient);
                         }
                     }
                 );
@@ -808,14 +804,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
         private void LogoutHandler(IClientAPI client)
         {
-            client.OnLogout -= LogoutHandler;
             client.SendLogoutPacket();
         }
-
-        private void ConnectionClosedHandler(IClientAPI client)
-        {
-            client.OnConnectionClosed -= ConnectionClosedHandler;
-            RemoveClient(((LLClientView)client).UDPClient);
-        }
     }
 }
diff --git a/OpenSim/Region/ClientStack/LindenUDP/UnackedPacketCollection.cs b/OpenSim/Region/ClientStack/LindenUDP/UnackedPacketCollection.cs
index 195ca57..f3242c1 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/UnackedPacketCollection.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/UnackedPacketCollection.cs
@@ -103,6 +103,15 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         }
 
         /// <summary>
+        /// Removes all elements from the collection
+        /// </summary>
+        public void Clear()
+        {
+            lock (SyncRoot)
+                packets.Clear();
+        }
+
+        /// <summary>
         /// Gets the packet with the lowest sequence number
         /// </summary>
         /// <returns>The packet with the lowest sequence number, or null if the