From cde47c2b3d7089be556252246eb03365c1f39b54 Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Wed, 21 Oct 2009 00:18:35 -0700 Subject: Committing Jim's optimization to replace the 20ms sleep in outgoing packet handling with an interruptible wait handle --- .../Region/ClientStack/LindenUDP/LLClientView.cs | 6 ++ .../Region/ClientStack/LindenUDP/LLUDPClient.cs | 51 +++++++---- .../Region/ClientStack/LindenUDP/LLUDPServer.cs | 98 +++++++++++++++------- .../Region/ClientStack/LindenUDP/TokenBucket.cs | 16 ++-- 4 files changed, 121 insertions(+), 50 deletions(-) (limited to 'OpenSim/Region') diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs b/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs index dce9469..38bbce0 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs @@ -3320,6 +3320,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP // If we received an update about our own avatar, process the avatar update priority queue immediately if (data.AgentID == m_agentId) ProcessAvatarTerseUpdates(); + else + m_udpServer.SignalOutgoingPacketHandler(); } private void ProcessAvatarTerseUpdates() @@ -3407,6 +3409,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP lock (m_primFullUpdates.SyncRoot) m_primFullUpdates.Enqueue(data.priority, objectData, data.localID); + + m_udpServer.SignalOutgoingPacketHandler(); } void ProcessPrimFullUpdates() @@ -3450,6 +3454,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP lock (m_primTerseUpdates.SyncRoot) m_primTerseUpdates.Enqueue(data.Priority, objectData, data.LocalID); + + m_udpServer.SignalOutgoingPacketHandler(); } void ProcessPrimTerseUpdates() diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs index 134cfe5..b9d2c15 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs @@ -105,9 +105,13 @@ namespace OpenSim.Region.ClientStack.LindenUDP public int TickLastPacketReceived; /// Environment.TickCount of the last time the outgoing packet handler executed for this client public int TickLastOutgoingPacketHandler; + /// Keeps track of the number of elapsed milliseconds since the last time the outgoing packet handler executed for this client + public int ElapsedMSOutgoingPacketHandler; + /// Keeps track of the number of 100 millisecond periods elapsed in the outgoing packet handler executed for this client + public int Elapsed100MSOutgoingPacketHandler; + /// Keeps track of the number of 500 millisecond periods elapsed in the outgoing packet handler executed for this client + public int Elapsed500MSOutgoingPacketHandler; - /// Timer granularity. This is set to the measured resolution of Environment.TickCount - public readonly float G; /// Smoothed round-trip time. A smoothed average of the round-trip time for sending a /// reliable packet to the client and receiving an ACK public float SRTT; @@ -182,15 +186,14 @@ namespace OpenSim.Region.ClientStack.LindenUDP m_throttleCategories[i] = new TokenBucket(m_throttle, rates.GetLimit(type), rates.GetRate(type)); } - // Set the granularity variable used for retransmission calculations to - // the measured resolution of Environment.TickCount - G = server.TickCountResolution; - // Default the retransmission timeout to three seconds RTO = 3000; // Initialize this to a sane value to prevent early disconnects - TickLastPacketReceived = Environment.TickCount; + TickLastPacketReceived = Environment.TickCount & Int32.MaxValue; + ElapsedMSOutgoingPacketHandler = 0; + Elapsed100MSOutgoingPacketHandler = 0; + Elapsed500MSOutgoingPacketHandler = 0; } /// @@ -391,6 +394,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP { // Not enough tokens in the bucket, queue this packet queue.Enqueue(packet); + m_udpServer.SignalOutgoingPacketHandler(); return true; } } @@ -407,13 +411,15 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// /// This function is only called from a synchronous loop in the /// UDPServer so we don't need to bother making this thread safe - /// True if any packets were sent, otherwise false - public bool DequeueOutgoing() + /// The minimum amount of time before the next packet + /// can be sent to this client + public int DequeueOutgoing() { OutgoingPacket packet; OpenSim.Framework.LocklessQueue queue; TokenBucket bucket; - bool packetSent = false; + int dataLength; + int minTimeout = Int32.MaxValue; //string queueDebugOutput = String.Empty; // Serious debug business @@ -428,12 +434,18 @@ namespace OpenSim.Region.ClientStack.LindenUDP // leaving a dequeued packet still waiting to be sent out. Try to // send it again OutgoingPacket nextPacket = m_nextPackets[i]; - if (bucket.RemoveTokens(nextPacket.Buffer.DataLength)) + dataLength = nextPacket.Buffer.DataLength; + if (bucket.RemoveTokens(dataLength)) { // Send the packet m_udpServer.SendPacketFinal(nextPacket); m_nextPackets[i] = null; - packetSent = true; + minTimeout = 0; + } + else if (minTimeout != 0) + { + // Check the minimum amount of time we would have to wait before this packet can be sent out + minTimeout = Math.Min(minTimeout, ((dataLength - bucket.Content) / bucket.DripPerMS) + 1); } } else @@ -445,16 +457,23 @@ namespace OpenSim.Region.ClientStack.LindenUDP { // A packet was pulled off the queue. See if we have // enough tokens in the bucket to send it out - if (bucket.RemoveTokens(packet.Buffer.DataLength)) + dataLength = packet.Buffer.DataLength; + if (bucket.RemoveTokens(dataLength)) { // Send the packet m_udpServer.SendPacketFinal(packet); - packetSent = true; + minTimeout = 0; } else { // Save the dequeued packet for the next iteration m_nextPackets[i] = packet; + + if (minTimeout != 0) + { + // Check the minimum amount of time we would have to wait before this packet can be sent out + minTimeout = Math.Min(minTimeout, ((dataLength - bucket.Content) / bucket.DripPerMS) + 1); + } } // If the queue is empty after this dequeue, fire the queue @@ -473,7 +492,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP } //m_log.Info("[LLUDPCLIENT]: Queues: " + queueDebugOutput); // Serious debug business - return packetSent; + return minTimeout; } /// @@ -504,7 +523,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP } // Always round retransmission timeout up to two seconds - RTO = Math.Max(2000, (int)(SRTT + Math.Max(G, K * RTTVAR))); + RTO = Math.Max(2000, (int)(SRTT + Math.Max(m_udpServer.TickCountResolution, K * RTTVAR))); //m_log.Debug("[LLUDPCLIENT]: Setting agent " + this.Agent.FullName + "'s RTO to " + RTO + "ms with an RTTVAR of " + // RTTVAR + " based on new RTT of " + r + "ms"); } diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs index 952d147..7d5c11e 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs @@ -96,6 +96,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + /// The measured resolution of Environment.TickCount + public readonly float TickCountResolution; + /// Handlers for incoming packets //PacketEventDictionary packetEvents = new PacketEventDictionary(); /// Incoming packets that are awaiting handling @@ -112,20 +115,19 @@ namespace OpenSim.Region.ClientStack.LindenUDP private Scene m_scene; /// The X/Y coordinates of the scene this UDP server is attached to private Location m_location; - /// The measured resolution of Environment.TickCount - private float m_tickCountResolution; /// The size of the receive buffer for the UDP socket. This value /// is passed up to the operating system and used in the system networking /// stack. Use zero to leave this value as the default private int m_recvBufferSize; /// Flag to process packets asynchronously or synchronously private bool m_asyncPacketHandling; - /// Track whether or not a packet was sent in the + /// Track the minimum amount of time needed to send the next packet in the /// OutgoingPacketHandler loop so we know when to sleep - private bool m_packetSentLastLoop; + private int m_minTimeout = Int32.MaxValue; + /// EventWaitHandle to signify the outgoing packet handler thread that + /// there is more work to do + private EventWaitHandle m_outgoingWaitHandle; - /// The measured resolution of Environment.TickCount - public float TickCountResolution { get { return m_tickCountResolution; } } public Socket Server { get { return null; } } public LLUDPServer(IPAddress listenIP, ref uint port, int proxyPortOffsetParm, bool allow_alternate_port, IConfigSource configSource, AgentCircuitManager circuitManager) @@ -134,16 +136,17 @@ namespace OpenSim.Region.ClientStack.LindenUDP #region Environment.TickCount Measurement // Measure the resolution of Environment.TickCount - m_tickCountResolution = 0f; + TickCountResolution = 0f; for (int i = 0; i < 5; i++) { int start = Environment.TickCount; int now = start; while (now == start) now = Environment.TickCount; - m_tickCountResolution += (float)(now - start) * 0.2f; + TickCountResolution += (float)(now - start) * 0.2f; } m_log.Info("[LLUDPSERVER]: Average Environment.TickCount resolution: " + TickCountResolution + "ms"); + TickCountResolution = (float)Math.Ceiling(TickCountResolution); #endregion Environment.TickCount Measurement @@ -171,6 +174,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP base.Start(m_recvBufferSize, m_asyncPacketHandling); + m_outgoingWaitHandle = new EventWaitHandle(false, EventResetMode.AutoReset); + // Start the incoming packet processing thread Thread incomingThread = new Thread(IncomingPacketHandler); incomingThread.Name = "Incoming Packets (" + m_scene.RegionInfo.RegionName + ")"; @@ -185,6 +190,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP { m_log.Info("[LLUDPSERVER]: Shutting down the LLUDP server for " + m_scene.RegionInfo.RegionName); base.Stop(); + + m_outgoingWaitHandle.Close(); } public void AddScene(IScene scene) @@ -768,6 +775,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP packetInbox.Clear(); } + public bool SignalOutgoingPacketHandler() + { + return m_outgoingWaitHandle.Set(); + } + private void OutgoingPacketHandler() { // Set this culture for the thread that outgoing packets are sent @@ -778,14 +790,28 @@ namespace OpenSim.Region.ClientStack.LindenUDP { try { - m_packetSentLastLoop = false; + m_minTimeout = Int32.MaxValue; + // Handle outgoing packets, resends, acknowledgements, and pings for each + // client. m_minTimeout will be set to 0 if more packets are waiting in the + // queues with bandwidth to spare, or the number of milliseconds we need to + // wait before at least one packet can be sent to a client m_scene.ClientManager.ForEachSync(ClientOutgoingPacketHandler); - // If no packets at all were sent, sleep to avoid chewing up CPU cycles - // when there is nothing to do - if (!m_packetSentLastLoop) - Thread.Sleep(20); + // Can't wait for a negative amount of time, and put a 100ms ceiling on our + // maximum wait time + m_minTimeout = Utils.Clamp(m_minTimeout, 0, 100); + + if (m_minTimeout > 0) + { + // Don't bother waiting for a shorter interval than our TickCountResolution + // since the token buckets wouldn't update anyways + m_minTimeout = Math.Max(m_minTimeout, (int)TickCountResolution); + + // Wait for someone to signal that packets are ready to be sent, or for our + // sleep interval to expire + m_outgoingWaitHandle.WaitOne(m_minTimeout); + } } catch (Exception ex) { @@ -802,32 +828,48 @@ namespace OpenSim.Region.ClientStack.LindenUDP { LLUDPClient udpClient = ((LLClientView)client).UDPClient; + // Update ElapsedMSOutgoingPacketHandler int thisTick = Environment.TickCount & Int32.MaxValue; - int elapsedMS = thisTick - udpClient.TickLastOutgoingPacketHandler; + if (udpClient.TickLastOutgoingPacketHandler > thisTick) + udpClient.ElapsedMSOutgoingPacketHandler += ((Int32.MaxValue - udpClient.TickLastOutgoingPacketHandler) + thisTick); + else + udpClient.ElapsedMSOutgoingPacketHandler += (thisTick - udpClient.TickLastOutgoingPacketHandler); if (udpClient.IsConnected) { // Check for pending outgoing resends every 100ms - if (elapsedMS >= 100) + if (udpClient.ElapsedMSOutgoingPacketHandler >= 100) { ResendUnacked(udpClient); + udpClient.ElapsedMSOutgoingPacketHandler -= 100; + udpClient.Elapsed100MSOutgoingPacketHandler += 1; + } - // Check for pending outgoing ACKs every 500ms - if (elapsedMS >= 500) - { - SendAcks(udpClient); - - // Send pings to clients every 5000ms - if (elapsedMS >= 5000) - { - SendPing(udpClient); - } - } + // Check for pending outgoing ACKs every 500ms + if (udpClient.Elapsed100MSOutgoingPacketHandler >= 5) + { + SendAcks(udpClient); + udpClient.Elapsed100MSOutgoingPacketHandler -= 5; + udpClient.Elapsed500MSOutgoingPacketHandler += 1; + } + + // Send pings to clients every 5000ms + if (udpClient.Elapsed500MSOutgoingPacketHandler >= 10) + { + SendPing(udpClient); + udpClient.Elapsed500MSOutgoingPacketHandler -= 10; } // Dequeue any outgoing packets that are within the throttle limits - if (udpClient.DequeueOutgoing()) - m_packetSentLastLoop = true; + // and get the minimum time we would have to sleep before this client + // could send a packet out + int minTimeoutThisLoop = udpClient.DequeueOutgoing(); + + // Although this is not thread safe, it is cheaper than locking and the + // worst that will happen is we sleep for slightly longer than the + // minimum necessary interval + if (minTimeoutThisLoop < m_minTimeout) + m_minTimeout = minTimeoutThisLoop; } udpClient.TickLastOutgoingPacketHandler = thisTick; diff --git a/OpenSim/Region/ClientStack/LindenUDP/TokenBucket.cs b/OpenSim/Region/ClientStack/LindenUDP/TokenBucket.cs index 0a64095..bdbd284 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/TokenBucket.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/TokenBucket.cs @@ -98,6 +98,14 @@ namespace OpenSim.Region.ClientStack.LindenUDP } /// + /// The speed limit of this bucket in bytes per millisecond + /// + public int DripPerMS + { + get { return tokensPerMS; } + } + + /// /// The number of bytes that can be sent at this moment. This is the /// current number of tokens in the bucket /// If this bucket has a parent bucket that does not have @@ -106,11 +114,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// public int Content { - get - { - Drip(); - return content; - } + get { return content; } } #endregion Properties @@ -182,7 +186,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// call to Drip /// /// True if tokens were added to the bucket, otherwise false - private bool Drip() + public bool Drip() { if (tokensPerMS == 0) { -- cgit v1.1