* Minimized the number of times textures are pulled off the priority queue

* OnQueueEmpty is still called async, but will not be called for a given category if the previous callback for that category is still running. This is the most balanced behavior I could find, and seems to work well
* Added support for the old [ClientStack.LindenUDP] settings (including setting the receive buffer size) and added the new token bucket and global throttle settings
* Added the AssetLoaderEnabled config variable to optionally disable loading assets from XML every startup. This gives a dramatic improvement in startup times for those who don't need the functionality every startup

7 files changed, 141 insertions, 67 deletions

diff --git a/OpenSim/Region/ClientStack/LindenUDP/J2KImage.cs b/OpenSim/Region/ClientStack/LindenUDP/J2KImage.cs
index 5877779..9ded390 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/J2KImage.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/J2KImage.cs
@@ -72,14 +72,20 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             m_imageManager = imageManager;
         }
 
-        public bool SendPackets(LLClientView client, int maxpack)
+        /// <summary>
+        /// Sends packets for this texture to a client until packetsToSend is 
+        /// hit or the transfer completes
+        /// </summary>
+        /// <param name="client">Reference to the client that the packets are destined for</param>
+        /// <param name="packetsToSend">Maximum number of packets to send during this call</param>
+        /// <param name="packetsSent">Number of packets sent during this call</param>
+        /// <returns>True if the transfer completes at the current discard level, otherwise false</returns>
+        public bool SendPackets(LLClientView client, int packetsToSend, out int packetsSent)
         {
-            if (client == null)
-                return false;
+            packetsSent = 0;
 
             if (m_currentPacket <= m_stopPacket)
             {
-                int count = 0;
                 bool sendMore = true;
 
                 if (!m_sentInfo || (m_currentPacket == 0))
@@ -88,25 +94,22 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
                     m_sentInfo = true;
                     ++m_currentPacket;
-                    ++count;
+                    ++packetsSent;
                 }
                 if (m_currentPacket < 2)
                 {
                     m_currentPacket = 2;
                 }
                 
-                while (sendMore && count < maxpack && m_currentPacket <= m_stopPacket)
+                while (sendMore && packetsSent < packetsToSend && m_currentPacket <= m_stopPacket)
                 {
                     sendMore = SendPacket(client);
                     ++m_currentPacket;
-                    ++count;
+                    ++packetsSent;
                 }
-
-                if (m_currentPacket > m_stopPacket)
-                    return true;
             }
 
-            return false;
+            return (m_currentPacket > m_stopPacket);
         }
 
         public void RunUpdate()
diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs b/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs
index 3b1a0bd..9afff5a 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs
@@ -313,10 +313,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         protected int m_primFullUpdatesPerPacket = 14;
         protected int m_primTerseUpdateRate = 10;
         protected int m_primFullUpdateRate = 14;
-        protected int m_textureSendLimit   = 20;
-        protected int m_textureDataLimit   = 10;
         protected int m_avatarTerseUpdateRate = 50;
         protected int m_avatarTerseUpdatesPerPacket = 5;
+        /// <summary>Number of texture packets to put on the queue each time the
+        /// OnQueueEmpty event is triggered for the texture category</summary>
+        protected int m_textureSendLimit = 20;
         protected IAssetService m_assetService;
 
         #endregion Class Members
@@ -3453,7 +3454,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         void ProcessTextureRequests()
         {
             if (m_imageManager != null)
-                m_imageManager.ProcessImageQueue(m_textureSendLimit, m_textureDataLimit);
+                m_imageManager.ProcessImageQueue(m_textureSendLimit);
         }
 
         void ProcessPrimFullUpdates(object sender, ElapsedEventArgs e)
diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLImageManager.cs b/OpenSim/Region/ClientStack/LindenUDP/LLImageManager.cs
index 8410ee9..9d36cff 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLImageManager.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLImageManager.cs
@@ -162,32 +162,38 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             }
         }
 
-        public bool ProcessImageQueue(int count, int maxpack)
+        public bool ProcessImageQueue(int packetsToSend)
         {
-            J2KImage imagereq;
-            int numCollected = 0;
-
             m_lastloopprocessed = DateTime.Now.Ticks;
+            int packetsSent = 0;
 
-            // This can happen during Close()
-            if (m_client == null)
-                return false;
-
-            while ((imagereq = GetHighestPriorityImage()) != null)
+            while (packetsSent < packetsToSend)
             {
-                if (imagereq.IsDecoded == true)
+                J2KImage image = GetHighestPriorityImage();
+
+                // If null was returned, the texture priority queue is currently empty
+                if (image == null)
+                    return false;
+
+                if (image.IsDecoded)
                 {
-                    ++numCollected;
+                    int sent;
+                    bool imageDone = image.SendPackets(m_client, packetsToSend - packetsSent, out sent);
 
-                    if (imagereq.SendPackets(m_client, maxpack))
-                    {
-                        // Send complete. Destroy any knowledge of this transfer
-                        RemoveImageFromQueue(imagereq);
-                    }
-                }
+                    packetsSent += sent;
 
-                if (numCollected == count)
-                    break;
+                    // If the send is complete, destroy any knowledge of this transfer
+                    if (imageDone)
+                        RemoveImageFromQueue(image);
+                }
+                else
+                {
+                    // TODO: This is a limitation of how LLImageManager is currently
+                    // written. Undecoded textures should not be going into the priority
+                    // queue, because a high priority undecoded texture will clog up the
+                    // pipeline for a client
+                    return true;
+                }
             }
 
             return m_priorityQueue.Count > 0;
@@ -199,10 +205,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         public void Close()
         {
             m_shuttingdown = true;
-            m_priorityQueue = null;
-            m_j2kDecodeModule = null;
-            m_assetCache = null;
-            m_client = null;
         }
 
         #region Priority Queue Helpers
diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs
index e7707a9..39472cb 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs
@@ -28,6 +28,7 @@
 using System;
 using System.Collections.Generic;
 using System.Net;
+using log4net;
 using OpenSim.Framework;
 using OpenMetaverse;
 
@@ -59,6 +60,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
     /// </summary>
     public sealed class LLUDPClient
     {
+        private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
+
         // FIXME: Make this a config setting
         /// <summary>Percentage of the task throttle category that is allocated to avatar and prim
         /// state updates</summary>
@@ -136,9 +139,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         /// <summary>A container that can hold one packet for each outbox, used to store
         /// dequeued packets that are being held for throttling</summary>
         private readonly OutgoingPacket[] nextPackets = new OutgoingPacket[THROTTLE_CATEGORY_COUNT];
-        /// <summary>An optimization to store the length of dequeued packets being held
-        /// for throttling. This avoids expensive calls to Packet.Length</summary>
-        private readonly int[] nextPacketLengths = new int[THROTTLE_CATEGORY_COUNT];
+        /// <summary>Flags to prevent queue empty callbacks from stacking up on
+        /// top of each other</summary>
+        private readonly bool[] onQueueEmptyRunning = new bool[THROTTLE_CATEGORY_COUNT];
         /// <summary>A reference to the LLUDPServer that is managing this client</summary>
         private readonly LLUDPServer udpServer;
 
@@ -163,7 +166,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             for (int i = 0; i < THROTTLE_CATEGORY_COUNT; i++)
                 packetOutboxes[i] = new OpenSim.Framework.LocklessQueue<OutgoingPacket>();
 
-            throttle = new TokenBucket(parentThrottle, 0, 0);
+            throttle = new TokenBucket(parentThrottle, rates.TotalLimit, rates.Total);
             throttleCategories = new TokenBucket[THROTTLE_CATEGORY_COUNT];
             throttleCategories[(int)ThrottleOutPacketType.Resend] = new TokenBucket(throttle, rates.ResendLimit, rates.Resend);
             throttleCategories[(int)ThrottleOutPacketType.Land] = new TokenBucket(throttle, rates.LandLimit, rates.Land);
@@ -401,10 +404,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     // This bucket was empty the last time we tried to send a packet,
                     // leaving a dequeued packet still waiting to be sent out. Try to
                     // send it again
-                    if (bucket.RemoveTokens(nextPacketLengths[i]))
+                    OutgoingPacket nextPacket = nextPackets[i];
+                    if (bucket.RemoveTokens(nextPacket.Buffer.DataLength))
                     {
                         // Send the packet
-                        udpServer.SendPacketFinal(nextPackets[i]);
+                        udpServer.SendPacketFinal(nextPacket);
                         nextPackets[i] = null;
                         packetSent = true;
                     }
@@ -426,23 +430,21 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                         }
                         else
                         {
-                            // Save the dequeued packet and the length calculation for
-                            // the next iteration
+                            // Save the dequeued packet for the next iteration
                             nextPackets[i] = packet;
-                            nextPacketLengths[i] = packet.Buffer.DataLength;
                         }
 
                         // If the queue is empty after this dequeue, fire the queue
                         // empty callback now so it has a chance to fill before we 
                         // get back here
                         if (queue.Count == 0)
-                            FireQueueEmpty(i);
+                            BeginFireQueueEmpty(i);
                     }
                     else
                     {
                         // No packets in this queue. Fire the queue empty callback
                         // if it has not been called recently
-                        FireQueueEmpty(i);
+                        BeginFireQueueEmpty(i);
                     }
                 }
             }
@@ -450,6 +452,14 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             return packetSent;
         }
 
+        /// <summary>
+        /// Called when an ACK packet is received and a round-trip time for a
+        /// packet is calculated. This is used to calculate the smoothed
+        /// round-trip time, round trip time variance, and finally the
+        /// retransmission timeout
+        /// </summary>
+        /// <param name="r">Round-trip time of a single packet and its
+        /// acknowledgement</param>
         public void UpdateRoundTrip(float r)
         {
             const float ALPHA = 0.125f;
@@ -475,11 +485,40 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             //    RTTVAR + " based on new RTT of " + r + "ms");
         }
 
-        private void FireQueueEmpty(int queueIndex)
+        /// <summary>
+        /// Does an early check to see if this queue empty callback is already
+        /// running, then asynchronously firing the event
+        /// </summary>
+        /// <param name="throttleIndex">Throttle category to fire the callback
+        /// for</param>
+        private void BeginFireQueueEmpty(int throttleIndex)
         {
+            if (!onQueueEmptyRunning[throttleIndex])
+                Util.FireAndForget(FireQueueEmpty, throttleIndex);
+        }
+
+        /// <summary>
+        /// Checks to see if this queue empty callback is already running,
+        /// then firing the event
+        /// </summary>
+        /// <param name="o">Throttle category to fire the callback for, stored
+        /// as an object to match the WaitCallback delegate signature</param>
+        private void FireQueueEmpty(object o)
+        {
+            int i = (int)o;
+            ThrottleOutPacketType type = (ThrottleOutPacketType)i;
             QueueEmpty callback = OnQueueEmpty;
+
             if (callback != null)
-                Util.FireAndForget(delegate(object o) { callback((ThrottleOutPacketType)(int)o); }, queueIndex);
+            {
+                if (!onQueueEmptyRunning[i])
+                {
+                    onQueueEmptyRunning[i] = true;
+                    try { callback(type); }
+                    catch (Exception e) { m_log.Error("[LLUDPCLIENT]: OnQueueEmpty(" + type + ") threw an exception: " + e.Message, e); }
+                    onQueueEmptyRunning[i] = false;
+                }
+            }
         }
     }
 }
diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
index 57fee59..1cfde91 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
@@ -109,6 +109,10 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         private Location m_location;
         /// <summary>The measured resolution of Environment.TickCount</summary>
         private float m_tickCountResolution;
+        /// <summary>The size of the receive buffer for the UDP socket. This value
+        /// is passed up to the operating system and used in the system networking
+        /// stack. Use zero to leave this value as the default</summary>
+        private int m_recvBufferSize;
 
         /// <summary>The measured resolution of Environment.TickCount</summary>
         public float TickCountResolution { get { return m_tickCountResolution; } }
@@ -135,6 +139,12 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
             m_circuitManager = circuitManager;
 
+            IConfig config = configSource.Configs["ClientStack.LindenUDP"];
+            if (config != null)
+            {
+                m_recvBufferSize = config.GetInt("client_socket_rcvbuf_size", 0);
+            }
+
             // TODO: Config support for throttling the entire connection
             m_throttle = new TokenBucket(null, 0, 0);
             m_throttleRates = new ThrottleRates(configSource);
@@ -145,7 +155,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             if (m_scene == null)
                 throw new InvalidOperationException("[LLUDPSERVER]: Cannot LLUDPServer.Start() without an IScene reference");
 
-            base.Start();
+            base.Start(m_recvBufferSize);
 
             // Start the incoming packet processing thread
             Thread incomingThread = new Thread(IncomingPacketHandler);
diff --git a/OpenSim/Region/ClientStack/LindenUDP/OpenSimUDPBase.cs b/OpenSim/Region/ClientStack/LindenUDP/OpenSimUDPBase.cs
index fad2ea8..44a6ed6 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/OpenSimUDPBase.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/OpenSimUDPBase.cs
@@ -73,6 +73,7 @@ namespace OpenMetaverse
         /// </summary>
         /// <param name="bindAddress">Local IP address to bind the server to</param>
         /// <param name="port">Port to listening for incoming UDP packets on</param>
+        /// 
         public OpenSimUDPBase(IPAddress bindAddress, int port)
         {
             m_localBindAddress = bindAddress;
@@ -82,25 +83,31 @@ namespace OpenMetaverse
         /// <summary>
         /// Start the UDP server
         /// </summary>
+        /// <param name="recvBufferSize">The size of the receive buffer for 
+        /// the UDP socket. This value is passed up to the operating system 
+        /// and used in the system networking stack. Use zero to leave this
+        /// value as the default</param>
         /// <remarks>This method will attempt to set the SIO_UDP_CONNRESET flag
         /// on the socket to get newer versions of Windows to behave in a sane
         /// manner (not throwing an exception when the remote side resets the
         /// connection). This call is ignored on Mono where the flag is not
         /// necessary</remarks>
-        public void Start()
+        public void Start(int recvBufferSize)
         {
             if (m_shutdownFlag)
             {
                 const int SIO_UDP_CONNRESET = -1744830452;
 
                 IPEndPoint ipep = new IPEndPoint(m_localBindAddress, m_udpPort);
+
                 m_udpSocket = new Socket(
                     AddressFamily.InterNetwork,
                     SocketType.Dgram,
                     ProtocolType.Udp);
+
                 try
                 {
-                    // this udp socket flag is not supported under mono, 
+                    // This udp socket flag is not supported under mono, 
                     // so we'll catch the exception and continue
                     m_udpSocket.IOControl(SIO_UDP_CONNRESET, new byte[] { 0 }, null);
                     m_log.Debug("[UDPBASE]: SIO_UDP_CONNRESET flag set");
@@ -109,6 +116,10 @@ namespace OpenMetaverse
                 {
                     m_log.Debug("[UDPBASE]: SIO_UDP_CONNRESET flag not supported on this platform, ignoring");
                 }
+
+                if (recvBufferSize != 0)
+                    m_udpSocket.ReceiveBufferSize = recvBufferSize;
+
                 m_udpSocket.Bind(ipep);
 
                 // we're not shutting down, we're starting up
diff --git a/OpenSim/Region/ClientStack/LindenUDP/ThrottleRates.cs b/OpenSim/Region/ClientStack/LindenUDP/ThrottleRates.cs
index 858a03c..adad4c3 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/ThrottleRates.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/ThrottleRates.cs
@@ -51,6 +51,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         public int Texture;
         /// <summary>Drip rate for asset packets</summary>
         public int Asset;
+        /// <summary>Drip rate for the parent token bucket</summary>
+        public int Total;
 
         /// <summary>Maximum burst rate for resent packets</summary>
         public int ResendLimit;
@@ -66,6 +68,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         public int TextureLimit;
         /// <summary>Maximum burst rate for asset packets</summary>
         public int AssetLimit;
+        /// <summary>Burst rate for the parent token bucket</summary>
+        public int TotalLimit;
 
         /// <summary>
         /// Default constructor
@@ -77,21 +81,25 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             {
                 IConfig throttleConfig = config.Configs["ClientStack.LindenUDP"];
 
-                Resend = throttleConfig.GetInt("ResendDefault", 12500);
-                Land = throttleConfig.GetInt("LandDefault", 500);
-                Wind = throttleConfig.GetInt("WindDefault", 500);
-                Cloud = throttleConfig.GetInt("CloudDefault", 500);
-                Task = throttleConfig.GetInt("TaskDefault", 500);
-                Texture = throttleConfig.GetInt("TextureDefault", 500);
-                Asset = throttleConfig.GetInt("AssetDefault", 500);
+                Resend = throttleConfig.GetInt("resend_default", 12500);
+                Land = throttleConfig.GetInt("land_default", 500);
+                Wind = throttleConfig.GetInt("wind_default", 500);
+                Cloud = throttleConfig.GetInt("cloud_default", 500);
+                Task = throttleConfig.GetInt("task_default", 500);
+                Texture = throttleConfig.GetInt("texture_default", 500);
+                Asset = throttleConfig.GetInt("asset_default", 500);
 
-                ResendLimit = throttleConfig.GetInt("ResendLimit", 18750);
-                LandLimit = throttleConfig.GetInt("LandLimit", 29750);
-                WindLimit = throttleConfig.GetInt("WindLimit", 18750);
-                CloudLimit = throttleConfig.GetInt("CloudLimit", 18750);
-                TaskLimit = throttleConfig.GetInt("TaskLimit", 55750);
-                TextureLimit = throttleConfig.GetInt("TextureLimit", 55750);
-                AssetLimit = throttleConfig.GetInt("AssetLimit", 27500);
+                Total = throttleConfig.GetInt("client_throttle_max_bps", 0);
+
+                ResendLimit = throttleConfig.GetInt("resend_limit", 18750);
+                LandLimit = throttleConfig.GetInt("land_limit", 29750);
+                WindLimit = throttleConfig.GetInt("wind_limit", 18750);
+                CloudLimit = throttleConfig.GetInt("cloud_limit", 18750);
+                TaskLimit = throttleConfig.GetInt("task_limit", 55750);
+                TextureLimit = throttleConfig.GetInt("texture_limit", 55750);
+                AssetLimit = throttleConfig.GetInt("asset_limit", 27500);
+
+                TotalLimit = throttleConfig.GetInt("client_throttle_max_bps", 0);
             }
             catch (Exception) { }
         }