* Change the OnQueueEmpty signature to send the flags of the queues that are empty instead of firing once per empty queue

* Change the OnQueueEmpty firing to use a minimum time until next fire instead of a sleep
* Set OutgoingPacket.TickCount = 0 earlier to avoid extra resends when things are running slowly (inside a profiler, for example)

5 files changed, 120 insertions, 61 deletions

diff --git a/OpenSim/Framework/ThrottleOutPacketType.cs b/OpenSim/Framework/ThrottleOutPacketType.cs
index e21ff32..d56231a 100644
--- a/OpenSim/Framework/ThrottleOutPacketType.cs
+++ b/OpenSim/Framework/ThrottleOutPacketType.cs
@@ -51,4 +51,16 @@ namespace OpenSim.Framework
         /// <remarks>This is a sub-category of Task</remarks>
         State = 7,
     }
+
+    [Flags]
+    public enum ThrottleOutPacketTypeFlags
+    {
+        Land = 1 << 0,
+        Wind = 1 << 1,
+        Cloud = 1 << 2,
+        Task = 1 << 3,
+        Texture = 1 << 4,
+        Asset = 1 << 5,
+        State = 1 << 6,
+    }
 }
diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs b/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs
index 0bb7a71..432fee7 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs
@@ -3550,33 +3550,35 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             OutPacket(attach, ThrottleOutPacketType.Task);
         }
 
-        void HandleQueueEmpty(ThrottleOutPacketType queue)
+        void HandleQueueEmpty(ThrottleOutPacketTypeFlags categories)
         {
-            switch (queue)
+            if ((categories & ThrottleOutPacketTypeFlags.Task) != 0)
             {
-                case ThrottleOutPacketType.Texture:
-                    ProcessTextureRequests();
-                    break;
-                case ThrottleOutPacketType.Task:
-                    lock (m_avatarTerseUpdates.SyncRoot)
-                    {
-                        if (m_avatarTerseUpdates.Count > 0)
-                            ProcessAvatarTerseUpdates();
-                    }
-                    break;
-                case ThrottleOutPacketType.State:
-                    lock (m_primFullUpdates.SyncRoot)
-                    {
-                        if (m_primFullUpdates.Count > 0)
-                            ProcessPrimFullUpdates();
-                    }
+                lock (m_avatarTerseUpdates.SyncRoot)
+                {
+                    if (m_avatarTerseUpdates.Count > 0)
+                        ProcessAvatarTerseUpdates();
+                }
+            }
 
-                    lock (m_primTerseUpdates.SyncRoot)
-                    {
-                        if (m_primTerseUpdates.Count > 0)
-                            ProcessPrimTerseUpdates();
-                    }
-                    break;
+            if ((categories & ThrottleOutPacketTypeFlags.State) != 0)
+            {
+                lock (m_primFullUpdates.SyncRoot)
+                {
+                    if (m_primFullUpdates.Count > 0)
+                        ProcessPrimFullUpdates();
+                }
+
+                lock (m_primTerseUpdates.SyncRoot)
+                {
+                    if (m_primTerseUpdates.Count > 0)
+                        ProcessPrimTerseUpdates();
+                }
+            }
+
+            if ((categories & ThrottleOutPacketTypeFlags.Texture) != 0)
+            {
+                ProcessTextureRequests();
             }
         }
 
diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs
index 2d86a40..a9bc7d2 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs
@@ -50,11 +50,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP
     /// are waiting on ACKs for</param>
     public delegate void PacketStats(int inPackets, int outPackets, int unAckedBytes);
     /// <summary>
-    /// Fired when the queue for a packet category is empty. This event can be
-    /// hooked to put more data on the empty queue
+    /// Fired when the queue for one or more packet categories is empty. This 
+    /// event can be hooked to put more data on the empty queues
     /// </summary>
-    /// <param name="category">Category of the packet queue that is empty</param>
-    public delegate void QueueEmpty(ThrottleOutPacketType category);
+    /// <param name="category">Categories of the packet queues that are empty</param>
+    public delegate void QueueEmpty(ThrottleOutPacketTypeFlags categories);
 
     #endregion Delegates
 
@@ -128,6 +128,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         private int m_packetsReceivedReported;
         /// <summary>Total number of sent packets that we have reported to the OnPacketStats event(s)</summary>
         private int m_packetsSentReported;
+        /// <summary>Holds the Environment.TickCount value of when the next OnQueueEmpty can be fired</summary>
+        private int m_nextOnQueueEmpty = 1;
 
         /// <summary>Throttle bucket for this agent's connection</summary>
         private readonly TokenBucket m_throttle;
@@ -140,9 +142,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         /// <summary>A container that can hold one packet for each outbox, used to store
         /// dequeued packets that are being held for throttling</summary>
         private readonly OutgoingPacket[] m_nextPackets = new OutgoingPacket[THROTTLE_CATEGORY_COUNT];
-        /// <summary>Flags to prevent queue empty callbacks from stacking up on
-        /// top of each other</summary>
-        private readonly bool[] m_onQueueEmptyRunning = new bool[THROTTLE_CATEGORY_COUNT];
         /// <summary>A reference to the LLUDPServer that is managing this client</summary>
         private readonly LLUDPServer m_udpServer;
 
@@ -405,6 +404,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             OpenSim.Framework.LocklessQueue<OutgoingPacket> queue;
             TokenBucket bucket;
             bool packetSent = false;
+            ThrottleOutPacketTypeFlags emptyCategories = 0;
 
             //string queueDebugOutput = String.Empty; // Serious debug business
 
@@ -452,17 +452,20 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                         // empty callback now so it has a chance to fill before we 
                         // get back here
                         if (queue.Count == 0)
-                            BeginFireQueueEmpty(i);
+                            emptyCategories |= CategoryToFlag(i);
                     }
                     else
                     {
                         // No packets in this queue. Fire the queue empty callback
                         // if it has not been called recently
-                        BeginFireQueueEmpty(i);
+                        emptyCategories |= CategoryToFlag(i);
                     }
                 }
             }
 
+            if (emptyCategories != 0)
+                BeginFireQueueEmpty(emptyCategories);
+
             //m_log.Info("[LLUDPCLIENT]: Queues: " + queueDebugOutput); // Serious debug business
             return packetSent;
         }
@@ -509,49 +512,90 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         /// </summary>
         /// <param name="throttleIndex">Throttle category to fire the callback
         /// for</param>
-        private void BeginFireQueueEmpty(int throttleIndex)
+        private void BeginFireQueueEmpty(ThrottleOutPacketTypeFlags categories)
         {
-            // Unknown is -1 and Resend is 0. Make sure we are only firing the 
-            // callback for categories other than those
-            if (throttleIndex > 0)
+            if (m_nextOnQueueEmpty != 0 && (Environment.TickCount & Int32.MaxValue) >= m_nextOnQueueEmpty)
             {
-                if (!m_onQueueEmptyRunning[throttleIndex])
-                {
-                    m_onQueueEmptyRunning[throttleIndex] = true;
-                    Util.FireAndForget(FireQueueEmpty, throttleIndex);
-                }
+                // Use a value of 0 to signal that FireQueueEmpty is running
+                m_nextOnQueueEmpty = 0;
+                // Asynchronously run the callback
+                Util.FireAndForget(FireQueueEmpty, categories);
             }
         }
 
         /// <summary>
-        /// Checks to see if this queue empty callback is already running,
-        /// then firing the event
+        /// Fires the OnQueueEmpty callback and sets the minimum time that it
+        /// can be called again
         /// </summary>
-        /// <param name="o">Throttle category to fire the callback for, stored
-        /// as an object to match the WaitCallback delegate signature</param>
+        /// <param name="o">Throttle categories to fire the callback for,
+        /// stored as an object to match the WaitCallback delegate
+        /// signature</param>
         private void FireQueueEmpty(object o)
         {
             const int MIN_CALLBACK_MS = 30;
 
-            int i = (int)o;
-            ThrottleOutPacketType type = (ThrottleOutPacketType)i;
+            ThrottleOutPacketTypeFlags categories = (ThrottleOutPacketTypeFlags)o;
             QueueEmpty callback = OnQueueEmpty;
-
+            
             int start = Environment.TickCount & Int32.MaxValue;
 
             if (callback != null)
             {
-                try { callback(type); }
-                catch (Exception e) { m_log.Error("[LLUDPCLIENT]: OnQueueEmpty(" + type + ") threw an exception: " + e.Message, e); }
+                try { callback(categories); }
+                catch (Exception e) { m_log.Error("[LLUDPCLIENT]: OnQueueEmpty(" + categories + ") threw an exception: " + e.Message, e); }
             }
 
-            // Make sure all queue empty calls take at least some amount of time,
-            // otherwise we'll peg a CPU trying to fire these too fast
-            int elapsedMS = (Environment.TickCount & Int32.MaxValue) - start;
-            if (elapsedMS < MIN_CALLBACK_MS)
-                System.Threading.Thread.Sleep(MIN_CALLBACK_MS - elapsedMS);
+            m_nextOnQueueEmpty = start + MIN_CALLBACK_MS;
+            if (m_nextOnQueueEmpty == 0)
+                m_nextOnQueueEmpty = 1;
+        }
 
-            m_onQueueEmptyRunning[i] = false;
+        /// <summary>
+        /// Converts a <seealso cref="ThrottleOutPacketType"/> integer to a
+        /// flag value
+        /// </summary>
+        /// <param name="i">Throttle category to convert</param>
+        /// <returns>Flag representation of the throttle category</returns>
+        private static ThrottleOutPacketTypeFlags CategoryToFlag(int i)
+        {
+            ThrottleOutPacketType category = (ThrottleOutPacketType)i;
+
+            /*
+             * Land = 1,
+        /// <summary>Wind data</summary>
+        Wind = 2,
+        /// <summary>Cloud data</summary>
+        Cloud = 3,
+        /// <summary>Any packets that do not fit into the other throttles</summary>
+        Task = 4,
+        /// <summary>Texture assets</summary>
+        Texture = 5,
+        /// <summary>Non-texture assets</summary>
+        Asset = 6,
+        /// <summary>Avatar and primitive data</summary>
+        /// <remarks>This is a sub-category of Task</remarks>
+        State = 7,
+             */
+
+            switch (category)
+            {
+                case ThrottleOutPacketType.Land:
+                    return ThrottleOutPacketTypeFlags.Land;
+                case ThrottleOutPacketType.Wind:
+                    return ThrottleOutPacketTypeFlags.Wind;
+                case ThrottleOutPacketType.Cloud:
+                    return ThrottleOutPacketTypeFlags.Cloud;
+                case ThrottleOutPacketType.Task:
+                    return ThrottleOutPacketTypeFlags.Task;
+                case ThrottleOutPacketType.Texture:
+                    return ThrottleOutPacketTypeFlags.Texture;
+                case ThrottleOutPacketType.Asset:
+                    return ThrottleOutPacketTypeFlags.Asset;
+                case ThrottleOutPacketType.State:
+                    return ThrottleOutPacketTypeFlags.State;
+                default:
+                    return 0;
+            }
         }
     }
 }
diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
index 40d3771..a9f4b2c 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
@@ -424,10 +424,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     outgoingPacket.Buffer.Data[0] = (byte)(outgoingPacket.Buffer.Data[0] | Helpers.MSG_RESENT);
                     outgoingPacket.Category = ThrottleOutPacketType.Resend;
 
-                    // The TickCount will be set to the current time when the packet
-                    // is actually sent out again
-                    outgoingPacket.TickCount = 0;
-
                     // Bump up the resend count on this packet
                     Interlocked.Increment(ref outgoingPacket.ResendCount);
                     //Interlocked.Increment(ref Stats.ResentPackets);
diff --git a/OpenSim/Region/ClientStack/LindenUDP/UnackedPacketCollection.cs b/OpenSim/Region/ClientStack/LindenUDP/UnackedPacketCollection.cs
index 3e2e81c..e43f7cf 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/UnackedPacketCollection.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/UnackedPacketCollection.cs
@@ -123,6 +123,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     {
                         if (expiredPackets == null)
                             expiredPackets = new List<OutgoingPacket>();
+
+                        // The TickCount will be set to the current time when the packet
+                        // is actually sent out again
+                        packet.TickCount = 0;
+
                         expiredPackets.Add(packet);
                     }
                 }