From a18489dc9badfccd775145f5a1a800a763d0c554 Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Fri, 16 Oct 2009 12:20:01 -0700 Subject: * Change appearance packets from State to Task. This will hopefully fix the cloud issues * Changed the throttling logic to obey the requested client bandwidth limit but also share bandwidth between some of the categories to improve throughput on high prim or heavily trafficked regions --- .../Region/ClientStack/LindenUDP/LLUDPClient.cs | 79 ++++++++++++---------- 1 file changed, 44 insertions(+), 35 deletions(-) (limited to 'OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs') diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs index 4eee6b6..8c42ca4 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs @@ -170,7 +170,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP { ThrottleOutPacketType type = (ThrottleOutPacketType)i; + // Initialize the packet outboxes, where packets sit while they are waiting for tokens m_packetOutboxes[i] = new OpenSim.Framework.LocklessQueue(); + // Initialize the token buckets that control the throttling for each category m_throttleCategories[i] = new TokenBucket(m_throttle, rates.GetLimit(type), rates.GetRate(type)); } @@ -293,36 +295,54 @@ namespace OpenSim.Region.ClientStack.LindenUDP int state = (int)((float)task * STATE_TASK_PERCENTAGE); task -= state; - int ceiling = Int32.MaxValue; - if (m_defaultThrottleRates.Total != 0) - { - ceiling = m_defaultThrottleRates.Total; - if (ceiling < Packet.MTU) ceiling = Packet.MTU; - } - - resend = Utils.Clamp(resend, Packet.MTU, ceiling); - land = Utils.Clamp(land, Packet.MTU, ceiling); - wind = Utils.Clamp(wind, Packet.MTU, ceiling); - cloud = Utils.Clamp(cloud, Packet.MTU, ceiling); - task = Utils.Clamp(task, Packet.MTU, ceiling); - texture = Utils.Clamp(texture, Packet.MTU, ceiling); - asset = Utils.Clamp(asset, Packet.MTU, ceiling); - state = Utils.Clamp(state, Packet.MTU, ceiling); + // Make sure none of the throttles are set below our packet MTU, + // otherwise a throttle could become permanently clogged + resend = Math.Max(resend, Packet.MTU); + land = Math.Max(land, Packet.MTU); + wind = Math.Max(wind, Packet.MTU); + cloud = Math.Max(cloud, Packet.MTU); + task = Math.Max(task, Packet.MTU); + texture = Math.Max(texture, Packet.MTU); + asset = Math.Max(asset, Packet.MTU); + state = Math.Max(state, Packet.MTU); int total = resend + land + wind + cloud + task + texture + asset + state; - int taskTotal = task + state; m_log.DebugFormat("[LLUDPCLIENT]: {0} is setting throttles. Resend={1}, Land={2}, Wind={3}, Cloud={4}, Task={5}, Texture={6}, Asset={7}, State={8}, Total={9}", AgentID, resend, land, wind, cloud, task, texture, asset, state, total); - SetThrottle(ThrottleOutPacketType.Resend, resend, resend); - SetThrottle(ThrottleOutPacketType.Land, land, land); - SetThrottle(ThrottleOutPacketType.Wind, wind, wind); - SetThrottle(ThrottleOutPacketType.Cloud, cloud, cloud); - SetThrottle(ThrottleOutPacketType.Task, task, taskTotal); - SetThrottle(ThrottleOutPacketType.Texture, texture, texture); - SetThrottle(ThrottleOutPacketType.Asset, asset, asset); - SetThrottle(ThrottleOutPacketType.State, state, taskTotal); + // Update the token buckets with new throttle values + TokenBucket bucket; + + bucket = m_throttle; + bucket.MaxBurst = total; + + bucket = m_throttleCategories[(int)ThrottleOutPacketType.Resend]; + bucket.DripRate = bucket.MaxBurst = resend; + + bucket = m_throttleCategories[(int)ThrottleOutPacketType.Land]; + bucket.DripRate = bucket.MaxBurst = land; + + bucket = m_throttleCategories[(int)ThrottleOutPacketType.Wind]; + bucket.DripRate = bucket.MaxBurst = wind; + + bucket = m_throttleCategories[(int)ThrottleOutPacketType.Cloud]; + bucket.DripRate = bucket.MaxBurst = cloud; + + bucket = m_throttleCategories[(int)ThrottleOutPacketType.Asset]; + bucket.DripRate = bucket.MaxBurst = asset; + + bucket = m_throttleCategories[(int)ThrottleOutPacketType.Task]; + bucket.DripRate = task + state + texture; + bucket.MaxBurst = task + state + texture; + + bucket = m_throttleCategories[(int)ThrottleOutPacketType.State]; + bucket.DripRate = state + texture; + bucket.MaxBurst = state + texture; + + bucket = m_throttleCategories[(int)ThrottleOutPacketType.Texture]; + bucket.DripRate = texture; + bucket.MaxBurst = texture; } public byte[] GetThrottlesPacked() @@ -342,17 +362,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP return data; } - public void SetThrottle(ThrottleOutPacketType category, int rate, int maxBurst) - { - int i = (int)category; - if (i >= 0 && i < m_throttleCategories.Length) - { - TokenBucket bucket = m_throttleCategories[(int)category]; - bucket.DripRate = rate; - bucket.MaxBurst = maxBurst; - } - } - public bool EnqueueOutgoing(OutgoingPacket packet) { int category = (int)packet.Category; -- cgit v1.1 From 1bd9202f2439ac73a70fa2a881f824797f61f589 Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Fri, 16 Oct 2009 14:17:13 -0700 Subject: * Simplified the prioritization packet creation code to reduce CPU usage and increase throughput. Apologies to Jim for hacking on your code while it's only halfway done, I'll take responsibility for the manual merge * Changed LLUDP to use its own MTU value of 1400 instead of the 1200 value pulled from the currently shipped libomv --- OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) (limited to 'OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs') diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs index 8c42ca4..9476eed 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs @@ -297,14 +297,14 @@ namespace OpenSim.Region.ClientStack.LindenUDP // Make sure none of the throttles are set below our packet MTU, // otherwise a throttle could become permanently clogged - resend = Math.Max(resend, Packet.MTU); - land = Math.Max(land, Packet.MTU); - wind = Math.Max(wind, Packet.MTU); - cloud = Math.Max(cloud, Packet.MTU); - task = Math.Max(task, Packet.MTU); - texture = Math.Max(texture, Packet.MTU); - asset = Math.Max(asset, Packet.MTU); - state = Math.Max(state, Packet.MTU); + resend = Math.Max(resend, LLUDPServer.MTU); + land = Math.Max(land, LLUDPServer.MTU); + wind = Math.Max(wind, LLUDPServer.MTU); + cloud = Math.Max(cloud, LLUDPServer.MTU); + task = Math.Max(task, LLUDPServer.MTU); + texture = Math.Max(texture, LLUDPServer.MTU); + asset = Math.Max(asset, LLUDPServer.MTU); + state = Math.Max(state, LLUDPServer.MTU); int total = resend + land + wind + cloud + task + texture + asset + state; @@ -404,9 +404,12 @@ namespace OpenSim.Region.ClientStack.LindenUDP TokenBucket bucket; bool packetSent = false; + //string queueDebugOutput = String.Empty; // Serious debug business + for (int i = 0; i < THROTTLE_CATEGORY_COUNT; i++) { bucket = m_throttleCategories[i]; + //queueDebugOutput += m_packetOutboxes[i].Count + " "; // Serious debug business if (m_nextPackets[i] != null) { @@ -458,6 +461,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP } } + //m_log.Info("[LLUDPCLIENT]: Queues: " + queueDebugOutput); // Serious debug business return packetSent; } -- cgit v1.1 From 142008121e2e9c5ca5fca5de07b8a14e37279800 Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Mon, 19 Oct 2009 15:19:09 -0700 Subject: * Change Util.FireAndForget to use ThreadPool.UnsafeQueueUserWorkItem(). This avoids .NET remoting and a managed->unmanaged->managed jump. Overall, a night and day performance difference * Initialize the LLClientView prim full update queue to the number of prims in the scene for a big performance boost * Reordered some comparisons on hot code paths for a minor speed boost * Removed an unnecessary call to the expensive DateTime.Now function (if you *have* to get the current time as opposed to Environment.TickCount, always use DateTime.UtcNow) * Don't fire the queue empty callback for the Resend category * Run the outgoing packet handler thread loop for each client synchronously. It seems like more time was being spent doing the execution asynchronously, and it made deadlocks very difficult to track down * Rewrote some expensive math in LandObject.cs * Optimized EntityManager to only lock on operations that need locking, and use TryGetValue() where possible * Only update the attachment database when an object is attached or detached * Other small misc. performance improvements --- OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs') diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs index 9476eed..4b6a358 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs @@ -506,8 +506,13 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// for private void BeginFireQueueEmpty(int throttleIndex) { - if (!m_onQueueEmptyRunning[throttleIndex]) - Util.FireAndForget(FireQueueEmpty, throttleIndex); + // Unknown is -1 and Resend is 0. Make sure we are only firing the + // callback for categories other than those + if (throttleIndex > 0) + { + if (!m_onQueueEmptyRunning[throttleIndex]) + Util.FireAndForget(FireQueueEmpty, throttleIndex); + } } /// -- cgit v1.1 From edd393ff308db2bf6802654dc37de6aa6a10f78a Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Tue, 20 Oct 2009 11:58:23 -0700 Subject: Reverting the deletion of files related to texture sending until we figure out exactly what is and isn't needed --- OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs | 2 ++ 1 file changed, 2 insertions(+) (limited to 'OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs') diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs index 4b6a358..4a3a04e 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs @@ -163,7 +163,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP CircuitCode = circuitCode; m_udpServer = server; m_defaultThrottleRates = rates; + // Create a token bucket throttle for this client that has the scene token bucket as a parent m_throttle = new TokenBucket(parentThrottle, rates.TotalLimit, rates.Total); + // Create an array of token buckets for this clients different throttle categories m_throttleCategories = new TokenBucket[THROTTLE_CATEGORY_COUNT]; for (int i = 0; i < THROTTLE_CATEGORY_COUNT; i++) -- cgit v1.1 From d38f33736c371cf8c09d78ee5c42b8cc943bb1d7 Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Tue, 20 Oct 2009 14:41:20 -0700 Subject: * Removed the throttle speed optimizations to see if it brings stability back * Changed the outgoing packet handler to use a real function instead of a closure and to track time on a per-client basis instead of a global basis --- .../Region/ClientStack/LindenUDP/LLUDPClient.cs | 25 ++++++++++++++-------- 1 file changed, 16 insertions(+), 9 deletions(-) (limited to 'OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs') diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs index 4a3a04e..ec74188 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs @@ -101,6 +101,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP public bool IsPaused = true; /// Environment.TickCount when the last packet was received for this client public int TickLastPacketReceived; + /// Environment.TickCount of the last time the outgoing packet handler executed for this client + public int TickLastOutgoingPacketHandler; /// Timer granularity. This is set to the measured resolution of Environment.TickCount public readonly float G; @@ -320,27 +322,32 @@ namespace OpenSim.Region.ClientStack.LindenUDP bucket.MaxBurst = total; bucket = m_throttleCategories[(int)ThrottleOutPacketType.Resend]; - bucket.DripRate = bucket.MaxBurst = resend; + bucket.DripRate = resend; + bucket.MaxBurst = resend; bucket = m_throttleCategories[(int)ThrottleOutPacketType.Land]; - bucket.DripRate = bucket.MaxBurst = land; + bucket.DripRate = land; + bucket.MaxBurst = land; bucket = m_throttleCategories[(int)ThrottleOutPacketType.Wind]; - bucket.DripRate = bucket.MaxBurst = wind; + bucket.DripRate = wind; + bucket.MaxBurst = wind; bucket = m_throttleCategories[(int)ThrottleOutPacketType.Cloud]; - bucket.DripRate = bucket.MaxBurst = cloud; + bucket.DripRate = cloud; + bucket.MaxBurst = cloud; bucket = m_throttleCategories[(int)ThrottleOutPacketType.Asset]; - bucket.DripRate = bucket.MaxBurst = asset; + bucket.DripRate = asset; + bucket.MaxBurst = asset; bucket = m_throttleCategories[(int)ThrottleOutPacketType.Task]; - bucket.DripRate = task + state + texture; - bucket.MaxBurst = task + state + texture; + bucket.DripRate = task + state; + bucket.MaxBurst = task + state; bucket = m_throttleCategories[(int)ThrottleOutPacketType.State]; - bucket.DripRate = state + texture; - bucket.MaxBurst = state + texture; + bucket.DripRate = state; + bucket.MaxBurst = state; bucket = m_throttleCategories[(int)ThrottleOutPacketType.Texture]; bucket.DripRate = texture; -- cgit v1.1 From 1833c6956892f1c8324ecbe0179103bff2079151 Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Tue, 20 Oct 2009 15:19:19 -0700 Subject: * Removed the unused m_agentUpdates collection and some extra work that was being done for AgentUpdate packets * Start LLUDPClients unpaused (this variable is not being used yet) --- OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs') diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs index ec74188..bf0fda3 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs @@ -98,7 +98,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// True when this connection is alive, otherwise false public bool IsConnected = true; /// True when this connection is paused, otherwise false - public bool IsPaused = true; + public bool IsPaused; /// Environment.TickCount when the last packet was received for this client public int TickLastPacketReceived; /// Environment.TickCount of the last time the outgoing packet handler executed for this client -- cgit v1.1 From 45dc4e0a5442d1d03f7387164070145386a9b4e1 Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Tue, 20 Oct 2009 18:19:17 -0700 Subject: * Added a sanity check to GetScriptAssemblies() and GetScriptStates() for the case where no scripting engine is enabled * Added TokenBucket.cs to OpenSim, with some fixes for setting a more accurate MaxBurst value and getting a more accurate Content value (by Drip()ing each get) --- OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs | 2 ++ 1 file changed, 2 insertions(+) (limited to 'OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs') diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs index bf0fda3..134cfe5 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs @@ -33,6 +33,8 @@ using OpenSim.Framework; using OpenMetaverse; using OpenMetaverse.Packets; +using TokenBucket = OpenSim.Region.ClientStack.LindenUDP.TokenBucket; + namespace OpenSim.Region.ClientStack.LindenUDP { #region Delegates -- cgit v1.1 From cde47c2b3d7089be556252246eb03365c1f39b54 Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Wed, 21 Oct 2009 00:18:35 -0700 Subject: Committing Jim's optimization to replace the 20ms sleep in outgoing packet handling with an interruptible wait handle --- .../Region/ClientStack/LindenUDP/LLUDPClient.cs | 51 +++++++++++++++------- 1 file changed, 35 insertions(+), 16 deletions(-) (limited to 'OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs') diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs index 134cfe5..b9d2c15 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs @@ -105,9 +105,13 @@ namespace OpenSim.Region.ClientStack.LindenUDP public int TickLastPacketReceived; /// Environment.TickCount of the last time the outgoing packet handler executed for this client public int TickLastOutgoingPacketHandler; + /// Keeps track of the number of elapsed milliseconds since the last time the outgoing packet handler executed for this client + public int ElapsedMSOutgoingPacketHandler; + /// Keeps track of the number of 100 millisecond periods elapsed in the outgoing packet handler executed for this client + public int Elapsed100MSOutgoingPacketHandler; + /// Keeps track of the number of 500 millisecond periods elapsed in the outgoing packet handler executed for this client + public int Elapsed500MSOutgoingPacketHandler; - /// Timer granularity. This is set to the measured resolution of Environment.TickCount - public readonly float G; /// Smoothed round-trip time. A smoothed average of the round-trip time for sending a /// reliable packet to the client and receiving an ACK public float SRTT; @@ -182,15 +186,14 @@ namespace OpenSim.Region.ClientStack.LindenUDP m_throttleCategories[i] = new TokenBucket(m_throttle, rates.GetLimit(type), rates.GetRate(type)); } - // Set the granularity variable used for retransmission calculations to - // the measured resolution of Environment.TickCount - G = server.TickCountResolution; - // Default the retransmission timeout to three seconds RTO = 3000; // Initialize this to a sane value to prevent early disconnects - TickLastPacketReceived = Environment.TickCount; + TickLastPacketReceived = Environment.TickCount & Int32.MaxValue; + ElapsedMSOutgoingPacketHandler = 0; + Elapsed100MSOutgoingPacketHandler = 0; + Elapsed500MSOutgoingPacketHandler = 0; } /// @@ -391,6 +394,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP { // Not enough tokens in the bucket, queue this packet queue.Enqueue(packet); + m_udpServer.SignalOutgoingPacketHandler(); return true; } } @@ -407,13 +411,15 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// /// This function is only called from a synchronous loop in the /// UDPServer so we don't need to bother making this thread safe - /// True if any packets were sent, otherwise false - public bool DequeueOutgoing() + /// The minimum amount of time before the next packet + /// can be sent to this client + public int DequeueOutgoing() { OutgoingPacket packet; OpenSim.Framework.LocklessQueue queue; TokenBucket bucket; - bool packetSent = false; + int dataLength; + int minTimeout = Int32.MaxValue; //string queueDebugOutput = String.Empty; // Serious debug business @@ -428,12 +434,18 @@ namespace OpenSim.Region.ClientStack.LindenUDP // leaving a dequeued packet still waiting to be sent out. Try to // send it again OutgoingPacket nextPacket = m_nextPackets[i]; - if (bucket.RemoveTokens(nextPacket.Buffer.DataLength)) + dataLength = nextPacket.Buffer.DataLength; + if (bucket.RemoveTokens(dataLength)) { // Send the packet m_udpServer.SendPacketFinal(nextPacket); m_nextPackets[i] = null; - packetSent = true; + minTimeout = 0; + } + else if (minTimeout != 0) + { + // Check the minimum amount of time we would have to wait before this packet can be sent out + minTimeout = Math.Min(minTimeout, ((dataLength - bucket.Content) / bucket.DripPerMS) + 1); } } else @@ -445,16 +457,23 @@ namespace OpenSim.Region.ClientStack.LindenUDP { // A packet was pulled off the queue. See if we have // enough tokens in the bucket to send it out - if (bucket.RemoveTokens(packet.Buffer.DataLength)) + dataLength = packet.Buffer.DataLength; + if (bucket.RemoveTokens(dataLength)) { // Send the packet m_udpServer.SendPacketFinal(packet); - packetSent = true; + minTimeout = 0; } else { // Save the dequeued packet for the next iteration m_nextPackets[i] = packet; + + if (minTimeout != 0) + { + // Check the minimum amount of time we would have to wait before this packet can be sent out + minTimeout = Math.Min(minTimeout, ((dataLength - bucket.Content) / bucket.DripPerMS) + 1); + } } // If the queue is empty after this dequeue, fire the queue @@ -473,7 +492,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP } //m_log.Info("[LLUDPCLIENT]: Queues: " + queueDebugOutput); // Serious debug business - return packetSent; + return minTimeout; } /// @@ -504,7 +523,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP } // Always round retransmission timeout up to two seconds - RTO = Math.Max(2000, (int)(SRTT + Math.Max(G, K * RTTVAR))); + RTO = Math.Max(2000, (int)(SRTT + Math.Max(m_udpServer.TickCountResolution, K * RTTVAR))); //m_log.Debug("[LLUDPCLIENT]: Setting agent " + this.Agent.FullName + "'s RTO to " + RTO + "ms with an RTTVAR of " + // RTTVAR + " based on new RTT of " + r + "ms"); } -- cgit v1.1 From c0c845aea462ea7b15580c72f5513e5e1ef02030 Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Wed, 21 Oct 2009 01:07:40 -0700 Subject: Fixed the way OnQueueEmpty is called to prevent simultaneous calls for the same category --- OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs') diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs index b9d2c15..ca9925c 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs @@ -541,7 +541,10 @@ namespace OpenSim.Region.ClientStack.LindenUDP if (throttleIndex > 0) { if (!m_onQueueEmptyRunning[throttleIndex]) + { + m_onQueueEmptyRunning[throttleIndex] = true; Util.FireAndForget(FireQueueEmpty, throttleIndex); + } } } @@ -559,14 +562,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP if (callback != null) { - if (!m_onQueueEmptyRunning[i]) - { - m_onQueueEmptyRunning[i] = true; - try { callback(type); } - catch (Exception e) { m_log.Error("[LLUDPCLIENT]: OnQueueEmpty(" + type + ") threw an exception: " + e.Message, e); } - m_onQueueEmptyRunning[i] = false; - } + try { callback(type); } + catch (Exception e) { m_log.Error("[LLUDPCLIENT]: OnQueueEmpty(" + type + ") threw an exception: " + e.Message, e); } } + + m_onQueueEmptyRunning[i] = false; } } } -- cgit v1.1 From 9178537e9414478f0a9bd84bb5e106b2f15640c3 Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Wed, 21 Oct 2009 11:59:48 -0700 Subject: * Replaced the UnackedPacketCollection with a lockless implementation. The tiny amount of time spent in the locks turned into a lot of time when the rest of the LLUDP implementation went lockless * Changed the timer tracking numbers for each client to not have "memory". It will no longer queue up calls to functions like ResendUnacked * Reverted Jim's WaitHandle code. Although it was technically more correct, it exhibited the exact same behavior as the old code but spent more cycles. The 20ms has been replaced with the minimum amount of time before a token bucket could receive a drip, and an else { sleep(0); } was added to make sure the outgoing packet handler always yields at least a minimum amount --- .../Region/ClientStack/LindenUDP/LLUDPClient.cs | 33 ++++++---------------- 1 file changed, 8 insertions(+), 25 deletions(-) (limited to 'OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs') diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs index ca9925c..458e78d 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs @@ -202,7 +202,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP public void Shutdown() { IsConnected = false; - NeedAcks.Clear(); for (int i = 0; i < THROTTLE_CATEGORY_COUNT; i++) { m_packetOutboxes[i].Clear(); @@ -394,7 +393,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP { // Not enough tokens in the bucket, queue this packet queue.Enqueue(packet); - m_udpServer.SignalOutgoingPacketHandler(); return true; } } @@ -411,15 +409,13 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// /// This function is only called from a synchronous loop in the /// UDPServer so we don't need to bother making this thread safe - /// The minimum amount of time before the next packet - /// can be sent to this client - public int DequeueOutgoing() + /// True if any packets were sent, otherwise false + public bool DequeueOutgoing() { OutgoingPacket packet; OpenSim.Framework.LocklessQueue queue; TokenBucket bucket; - int dataLength; - int minTimeout = Int32.MaxValue; + bool packetSent = false; //string queueDebugOutput = String.Empty; // Serious debug business @@ -434,18 +430,12 @@ namespace OpenSim.Region.ClientStack.LindenUDP // leaving a dequeued packet still waiting to be sent out. Try to // send it again OutgoingPacket nextPacket = m_nextPackets[i]; - dataLength = nextPacket.Buffer.DataLength; - if (bucket.RemoveTokens(dataLength)) + if (bucket.RemoveTokens(nextPacket.Buffer.DataLength)) { // Send the packet m_udpServer.SendPacketFinal(nextPacket); m_nextPackets[i] = null; - minTimeout = 0; - } - else if (minTimeout != 0) - { - // Check the minimum amount of time we would have to wait before this packet can be sent out - minTimeout = Math.Min(minTimeout, ((dataLength - bucket.Content) / bucket.DripPerMS) + 1); + packetSent = true; } } else @@ -457,23 +447,16 @@ namespace OpenSim.Region.ClientStack.LindenUDP { // A packet was pulled off the queue. See if we have // enough tokens in the bucket to send it out - dataLength = packet.Buffer.DataLength; - if (bucket.RemoveTokens(dataLength)) + if (bucket.RemoveTokens(packet.Buffer.DataLength)) { // Send the packet m_udpServer.SendPacketFinal(packet); - minTimeout = 0; + packetSent = true; } else { // Save the dequeued packet for the next iteration m_nextPackets[i] = packet; - - if (minTimeout != 0) - { - // Check the minimum amount of time we would have to wait before this packet can be sent out - minTimeout = Math.Min(minTimeout, ((dataLength - bucket.Content) / bucket.DripPerMS) + 1); - } } // If the queue is empty after this dequeue, fire the queue @@ -492,7 +475,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP } //m_log.Info("[LLUDPCLIENT]: Queues: " + queueDebugOutput); // Serious debug business - return minTimeout; + return packetSent; } /// -- cgit v1.1 From 7ee422a344ff22cf988aea2355628d2dee831983 Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Wed, 21 Oct 2009 13:47:16 -0700 Subject: * Handle UseCircuitCode packets asynchronously. Adding an agent to a scene can take several seconds, and was blocking up packet handling in the meantime * Clamp retransmission timeout values between three and 10 seconds * Log outgoing time for a packet right after it is sent instead of well before * Loop through the entire UnackedPacketCollection when looking for expired packets --- OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs') diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs index 458e78d..a43197d 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs @@ -505,8 +505,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP SRTT = (1.0f - ALPHA) * SRTT + ALPHA * r; } - // Always round retransmission timeout up to two seconds - RTO = Math.Max(2000, (int)(SRTT + Math.Max(m_udpServer.TickCountResolution, K * RTTVAR))); + RTO = (int)(SRTT + Math.Max(m_udpServer.TickCountResolution, K * RTTVAR)); + + // Clamp the retransmission timeout to manageable values + RTO = Utils.Clamp(RTO, 3000, 10000); + //m_log.Debug("[LLUDPCLIENT]: Setting agent " + this.Agent.FullName + "'s RTO to " + RTO + "ms with an RTTVAR of " + // RTTVAR + " based on new RTT of " + r + "ms"); } -- cgit v1.1 From 62f1bfd136df7a68e9f829acc1db22354772bad3 Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Wed, 21 Oct 2009 14:25:22 -0700 Subject: Testing out a hack to identify the source of the high cpu usage --- OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs | 3 +++ 1 file changed, 3 insertions(+) (limited to 'OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs') diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs index a43197d..de67ca7 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs @@ -552,6 +552,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP catch (Exception e) { m_log.Error("[LLUDPCLIENT]: OnQueueEmpty(" + type + ") threw an exception: " + e.Message, e); } } + // HACK: Try spending some extra time here to slow down OnQueueEmpty calls + System.Threading.Thread.Sleep(100); + m_onQueueEmptyRunning[i] = false; } } -- cgit v1.1 From 2752a3525c6c57470024d0dddfd69d593abc4594 Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Wed, 21 Oct 2009 15:22:23 -0700 Subject: * Changed the timing calculations for sending resends/acks/pings from per-client back to per-scene * Testing a fix from Jim to make the cpu usage fix cleaner --- OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) (limited to 'OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs') diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs index de67ca7..0a090b4 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs @@ -103,14 +103,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP public bool IsPaused; /// Environment.TickCount when the last packet was received for this client public int TickLastPacketReceived; - /// Environment.TickCount of the last time the outgoing packet handler executed for this client - public int TickLastOutgoingPacketHandler; - /// Keeps track of the number of elapsed milliseconds since the last time the outgoing packet handler executed for this client - public int ElapsedMSOutgoingPacketHandler; - /// Keeps track of the number of 100 millisecond periods elapsed in the outgoing packet handler executed for this client - public int Elapsed100MSOutgoingPacketHandler; - /// Keeps track of the number of 500 millisecond periods elapsed in the outgoing packet handler executed for this client - public int Elapsed500MSOutgoingPacketHandler; /// Smoothed round-trip time. A smoothed average of the round-trip time for sending a /// reliable packet to the client and receiving an ACK @@ -191,9 +183,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP // Initialize this to a sane value to prevent early disconnects TickLastPacketReceived = Environment.TickCount & Int32.MaxValue; - ElapsedMSOutgoingPacketHandler = 0; - Elapsed100MSOutgoingPacketHandler = 0; - Elapsed500MSOutgoingPacketHandler = 0; } /// @@ -553,7 +542,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP } // HACK: Try spending some extra time here to slow down OnQueueEmpty calls - System.Threading.Thread.Sleep(100); + //System.Threading.Thread.Sleep(100); m_onQueueEmptyRunning[i] = false; } -- cgit v1.1 From b06f258319f088bcf6658880ed371ef313bac3b6 Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Wed, 21 Oct 2009 16:21:08 -0700 Subject: * FireQueueEmpty now checks if a measurable amount of time has passed, and if not it sleeps for a small amount of time. This throttles OnQueueEmpty calls where there is no callback or the callback is doing very little work * Changed HandleQueueEmpty()'s Monitor.TryEnter() calls to locks. We want to take our time in this function and do all the work necessary, since returning too fast will induce a sleep anyways --- OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs') diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs index 0a090b4..71f4c47 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs @@ -535,14 +535,18 @@ namespace OpenSim.Region.ClientStack.LindenUDP ThrottleOutPacketType type = (ThrottleOutPacketType)i; QueueEmpty callback = OnQueueEmpty; + int start = Environment.TickCount; + if (callback != null) { try { callback(type); } catch (Exception e) { m_log.Error("[LLUDPCLIENT]: OnQueueEmpty(" + type + ") threw an exception: " + e.Message, e); } } - // HACK: Try spending some extra time here to slow down OnQueueEmpty calls - //System.Threading.Thread.Sleep(100); + // Make sure all queue empty calls take at least a measurable amount of time, + // otherwise we'll peg a CPU trying to fire these too fast + if (Environment.TickCount == start) + System.Threading.Thread.Sleep((int)m_udpServer.TickCountResolution); m_onQueueEmptyRunning[i] = false; } -- cgit v1.1 From 4e04f6b3a5a875c7d8820c679bcbcdcfba1227bf Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Wed, 21 Oct 2009 17:02:55 -0700 Subject: * Clarified what FireQueueEmpty is doing with a MIN_CALLBACK_MS constant and upped it to 30ms * Removed the unused PacketSent() function * Switched UnackedPacketCollection from a SortedDictionary to a Dictionary now that the sorting is no longer needed. Big performance improvement for ResendUnacked() --- OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs') diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs index 71f4c47..2d86a40 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs @@ -531,11 +531,13 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// as an object to match the WaitCallback delegate signature private void FireQueueEmpty(object o) { + const int MIN_CALLBACK_MS = 30; + int i = (int)o; ThrottleOutPacketType type = (ThrottleOutPacketType)i; QueueEmpty callback = OnQueueEmpty; - int start = Environment.TickCount; + int start = Environment.TickCount & Int32.MaxValue; if (callback != null) { @@ -543,10 +545,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP catch (Exception e) { m_log.Error("[LLUDPCLIENT]: OnQueueEmpty(" + type + ") threw an exception: " + e.Message, e); } } - // Make sure all queue empty calls take at least a measurable amount of time, + // Make sure all queue empty calls take at least some amount of time, // otherwise we'll peg a CPU trying to fire these too fast - if (Environment.TickCount == start) - System.Threading.Thread.Sleep((int)m_udpServer.TickCountResolution); + int elapsedMS = (Environment.TickCount & Int32.MaxValue) - start; + if (elapsedMS < MIN_CALLBACK_MS) + System.Threading.Thread.Sleep(MIN_CALLBACK_MS - elapsedMS); m_onQueueEmptyRunning[i] = false; } -- cgit v1.1 From 6492640e72776d9f0a015e6a719c8cef28ccb7e3 Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Wed, 21 Oct 2009 18:03:41 -0700 Subject: * Change the OnQueueEmpty signature to send the flags of the queues that are empty instead of firing once per empty queue * Change the OnQueueEmpty firing to use a minimum time until next fire instead of a sleep * Set OutgoingPacket.TickCount = 0 earlier to avoid extra resends when things are running slowly (inside a profiler, for example) --- .../Region/ClientStack/LindenUDP/LLUDPClient.cs | 110 ++++++++++++++------- 1 file changed, 77 insertions(+), 33 deletions(-) (limited to 'OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs') diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs index 2d86a40..a9bc7d2 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs @@ -50,11 +50,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// are waiting on ACKs for public delegate void PacketStats(int inPackets, int outPackets, int unAckedBytes); /// - /// Fired when the queue for a packet category is empty. This event can be - /// hooked to put more data on the empty queue + /// Fired when the queue for one or more packet categories is empty. This + /// event can be hooked to put more data on the empty queues /// - /// Category of the packet queue that is empty - public delegate void QueueEmpty(ThrottleOutPacketType category); + /// Categories of the packet queues that are empty + public delegate void QueueEmpty(ThrottleOutPacketTypeFlags categories); #endregion Delegates @@ -128,6 +128,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP private int m_packetsReceivedReported; /// Total number of sent packets that we have reported to the OnPacketStats event(s) private int m_packetsSentReported; + /// Holds the Environment.TickCount value of when the next OnQueueEmpty can be fired + private int m_nextOnQueueEmpty = 1; /// Throttle bucket for this agent's connection private readonly TokenBucket m_throttle; @@ -140,9 +142,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// A container that can hold one packet for each outbox, used to store /// dequeued packets that are being held for throttling private readonly OutgoingPacket[] m_nextPackets = new OutgoingPacket[THROTTLE_CATEGORY_COUNT]; - /// Flags to prevent queue empty callbacks from stacking up on - /// top of each other - private readonly bool[] m_onQueueEmptyRunning = new bool[THROTTLE_CATEGORY_COUNT]; /// A reference to the LLUDPServer that is managing this client private readonly LLUDPServer m_udpServer; @@ -405,6 +404,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP OpenSim.Framework.LocklessQueue queue; TokenBucket bucket; bool packetSent = false; + ThrottleOutPacketTypeFlags emptyCategories = 0; //string queueDebugOutput = String.Empty; // Serious debug business @@ -452,17 +452,20 @@ namespace OpenSim.Region.ClientStack.LindenUDP // empty callback now so it has a chance to fill before we // get back here if (queue.Count == 0) - BeginFireQueueEmpty(i); + emptyCategories |= CategoryToFlag(i); } else { // No packets in this queue. Fire the queue empty callback // if it has not been called recently - BeginFireQueueEmpty(i); + emptyCategories |= CategoryToFlag(i); } } } + if (emptyCategories != 0) + BeginFireQueueEmpty(emptyCategories); + //m_log.Info("[LLUDPCLIENT]: Queues: " + queueDebugOutput); // Serious debug business return packetSent; } @@ -509,49 +512,90 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// /// Throttle category to fire the callback /// for - private void BeginFireQueueEmpty(int throttleIndex) + private void BeginFireQueueEmpty(ThrottleOutPacketTypeFlags categories) { - // Unknown is -1 and Resend is 0. Make sure we are only firing the - // callback for categories other than those - if (throttleIndex > 0) + if (m_nextOnQueueEmpty != 0 && (Environment.TickCount & Int32.MaxValue) >= m_nextOnQueueEmpty) { - if (!m_onQueueEmptyRunning[throttleIndex]) - { - m_onQueueEmptyRunning[throttleIndex] = true; - Util.FireAndForget(FireQueueEmpty, throttleIndex); - } + // Use a value of 0 to signal that FireQueueEmpty is running + m_nextOnQueueEmpty = 0; + // Asynchronously run the callback + Util.FireAndForget(FireQueueEmpty, categories); } } /// - /// Checks to see if this queue empty callback is already running, - /// then firing the event + /// Fires the OnQueueEmpty callback and sets the minimum time that it + /// can be called again /// - /// Throttle category to fire the callback for, stored - /// as an object to match the WaitCallback delegate signature + /// Throttle categories to fire the callback for, + /// stored as an object to match the WaitCallback delegate + /// signature private void FireQueueEmpty(object o) { const int MIN_CALLBACK_MS = 30; - int i = (int)o; - ThrottleOutPacketType type = (ThrottleOutPacketType)i; + ThrottleOutPacketTypeFlags categories = (ThrottleOutPacketTypeFlags)o; QueueEmpty callback = OnQueueEmpty; - + int start = Environment.TickCount & Int32.MaxValue; if (callback != null) { - try { callback(type); } - catch (Exception e) { m_log.Error("[LLUDPCLIENT]: OnQueueEmpty(" + type + ") threw an exception: " + e.Message, e); } + try { callback(categories); } + catch (Exception e) { m_log.Error("[LLUDPCLIENT]: OnQueueEmpty(" + categories + ") threw an exception: " + e.Message, e); } } - // Make sure all queue empty calls take at least some amount of time, - // otherwise we'll peg a CPU trying to fire these too fast - int elapsedMS = (Environment.TickCount & Int32.MaxValue) - start; - if (elapsedMS < MIN_CALLBACK_MS) - System.Threading.Thread.Sleep(MIN_CALLBACK_MS - elapsedMS); + m_nextOnQueueEmpty = start + MIN_CALLBACK_MS; + if (m_nextOnQueueEmpty == 0) + m_nextOnQueueEmpty = 1; + } - m_onQueueEmptyRunning[i] = false; + /// + /// Converts a integer to a + /// flag value + /// + /// Throttle category to convert + /// Flag representation of the throttle category + private static ThrottleOutPacketTypeFlags CategoryToFlag(int i) + { + ThrottleOutPacketType category = (ThrottleOutPacketType)i; + + /* + * Land = 1, + /// Wind data + Wind = 2, + /// Cloud data + Cloud = 3, + /// Any packets that do not fit into the other throttles + Task = 4, + /// Texture assets + Texture = 5, + /// Non-texture assets + Asset = 6, + /// Avatar and primitive data + /// This is a sub-category of Task + State = 7, + */ + + switch (category) + { + case ThrottleOutPacketType.Land: + return ThrottleOutPacketTypeFlags.Land; + case ThrottleOutPacketType.Wind: + return ThrottleOutPacketTypeFlags.Wind; + case ThrottleOutPacketType.Cloud: + return ThrottleOutPacketTypeFlags.Cloud; + case ThrottleOutPacketType.Task: + return ThrottleOutPacketTypeFlags.Task; + case ThrottleOutPacketType.Texture: + return ThrottleOutPacketTypeFlags.Texture; + case ThrottleOutPacketType.Asset: + return ThrottleOutPacketTypeFlags.Asset; + case ThrottleOutPacketType.State: + return ThrottleOutPacketTypeFlags.State; + default: + return 0; + } } } } -- cgit v1.1