From 9178537e9414478f0a9bd84bb5e106b2f15640c3 Mon Sep 17 00:00:00 2001
From: John Hurliman
Date: Wed, 21 Oct 2009 11:59:48 -0700
Subject: * Replaced the UnackedPacketCollection with a lockless
implementation. The tiny amount of time spent in the locks turned into a lot
of time when the rest of the LLUDP implementation went lockless * Changed the
timer tracking numbers for each client to not have "memory". It will no
longer queue up calls to functions like ResendUnacked * Reverted Jim's
WaitHandle code. Although it was technically more correct, it exhibited the
exact same behavior as the old code but spent more cycles. The 20ms has been
replaced with the minimum amount of time before a token bucket could receive
a drip, and an else { sleep(0); } was added to make sure the outgoing packet
handler always yields at least a minimum amount
---
.../Region/ClientStack/LindenUDP/LLUDPServer.cs | 143 +++++++--------------
1 file changed, 45 insertions(+), 98 deletions(-)
(limited to 'OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs')
diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
index 7d5c11e..a8ce102 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
@@ -121,12 +121,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP
private int m_recvBufferSize;
/// Flag to process packets asynchronously or synchronously
private bool m_asyncPacketHandling;
- /// Track the minimum amount of time needed to send the next packet in the
- /// OutgoingPacketHandler loop so we know when to sleep
- private int m_minTimeout = Int32.MaxValue;
- /// EventWaitHandle to signify the outgoing packet handler thread that
- /// there is more work to do
- private EventWaitHandle m_outgoingWaitHandle;
+ /// Tracks whether or not a packet was sent each round so we know
+ /// whether or not to sleep
+ private bool m_packetSent;
public Socket Server { get { return null; } }
@@ -174,8 +171,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
base.Start(m_recvBufferSize, m_asyncPacketHandling);
- m_outgoingWaitHandle = new EventWaitHandle(false, EventResetMode.AutoReset);
-
// Start the incoming packet processing thread
Thread incomingThread = new Thread(IncomingPacketHandler);
incomingThread.Name = "Incoming Packets (" + m_scene.RegionInfo.RegionName + ")";
@@ -190,8 +185,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
{
m_log.Info("[LLUDPSERVER]: Shutting down the LLUDP server for " + m_scene.RegionInfo.RegionName);
base.Stop();
-
- m_outgoingWaitHandle.Close();
}
public void AddScene(IScene scene)
@@ -374,10 +367,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP
StartPingCheckPacket pc = (StartPingCheckPacket)PacketPool.Instance.GetPacket(PacketType.StartPingCheck);
pc.Header.Reliable = false;
- OutgoingPacket oldestPacket = udpClient.NeedAcks.GetOldest();
-
pc.PingID.PingID = (byte)udpClient.CurrentPingSequence++;
- pc.PingID.OldestUnacked = (oldestPacket != null) ? oldestPacket.SequenceNumber : 0;
+ // We *could* get OldestUnacked, but it would hurt performance and not provide any benefit
+ pc.PingID.OldestUnacked = 0;
SendPacket(udpClient, pc, ThrottleOutPacketType.Unknown, false);
}
@@ -397,39 +389,36 @@ namespace OpenSim.Region.ClientStack.LindenUDP
return;
}
- if (udpClient.NeedAcks.Count > 0)
+ // Get a list of all of the packets that have been sitting unacked longer than udpClient.RTO
+ List expiredPackets = udpClient.NeedAcks.GetExpiredPackets(udpClient.RTO);
+
+ if (expiredPackets != null)
{
- // Get a list of all of the packets that have been sitting unacked longer than udpClient.RTO
- List expiredPackets = udpClient.NeedAcks.GetExpiredPackets(udpClient.RTO);
+ m_log.Debug("[LLUDPSERVER]: Resending " + expiredPackets.Count + " packets to " + udpClient.AgentID + ", RTO=" + udpClient.RTO);
- if (expiredPackets != null)
+ // Resend packets
+ for (int i = 0; i < expiredPackets.Count; i++)
{
- m_log.Debug("[LLUDPSERVER]: Resending " + expiredPackets.Count + " packets to " + udpClient.AgentID);
+ OutgoingPacket outgoingPacket = expiredPackets[i];
- // Resend packets
- for (int i = 0; i < expiredPackets.Count; i++)
- {
- OutgoingPacket outgoingPacket = expiredPackets[i];
-
- //m_log.DebugFormat("[LLUDPSERVER]: Resending packet #{0} (attempt {1}), {2}ms have passed",
- // outgoingPacket.SequenceNumber, outgoingPacket.ResendCount, Environment.TickCount - outgoingPacket.TickCount);
+ //m_log.DebugFormat("[LLUDPSERVER]: Resending packet #{0} (attempt {1}), {2}ms have passed",
+ // outgoingPacket.SequenceNumber, outgoingPacket.ResendCount, Environment.TickCount - outgoingPacket.TickCount);
- // Set the resent flag
- outgoingPacket.Buffer.Data[0] = (byte)(outgoingPacket.Buffer.Data[0] | Helpers.MSG_RESENT);
- outgoingPacket.Category = ThrottleOutPacketType.Resend;
+ // Set the resent flag
+ outgoingPacket.Buffer.Data[0] = (byte)(outgoingPacket.Buffer.Data[0] | Helpers.MSG_RESENT);
+ outgoingPacket.Category = ThrottleOutPacketType.Resend;
- // The TickCount will be set to the current time when the packet
- // is actually sent out again
- outgoingPacket.TickCount = 0;
+ // The TickCount will be set to the current time when the packet
+ // is actually sent out again
+ outgoingPacket.TickCount = 0;
- // Bump up the resend count on this packet
- Interlocked.Increment(ref outgoingPacket.ResendCount);
- //Interlocked.Increment(ref Stats.ResentPackets);
+ // Bump up the resend count on this packet
+ Interlocked.Increment(ref outgoingPacket.ResendCount);
+ //Interlocked.Increment(ref Stats.ResentPackets);
- // Requeue or resend the packet
- if (!outgoingPacket.Client.EnqueueOutgoing(outgoingPacket))
- SendPacketFinal(outgoingPacket);
- }
+ // Requeue or resend the packet
+ if (!outgoingPacket.Client.EnqueueOutgoing(outgoingPacket))
+ SendPacketFinal(outgoingPacket);
}
}
}
@@ -577,11 +566,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
// Handle appended ACKs
if (packet.Header.AppendedAcks && packet.Header.AckList != null)
{
- lock (udpClient.NeedAcks.SyncRoot)
- {
- for (int i = 0; i < packet.Header.AckList.Length; i++)
- AcknowledgePacket(udpClient, packet.Header.AckList[i], now, packet.Header.Resent);
- }
+ for (int i = 0; i < packet.Header.AckList.Length; i++)
+ udpClient.NeedAcks.Remove(packet.Header.AckList[i], now, packet.Header.Resent);
}
// Handle PacketAck packets
@@ -589,11 +575,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
{
PacketAckPacket ackPacket = (PacketAckPacket)packet;
- lock (udpClient.NeedAcks.SyncRoot)
- {
- for (int i = 0; i < ackPacket.Packets.Length; i++)
- AcknowledgePacket(udpClient, ackPacket.Packets[i].ID, now, packet.Header.Resent);
- }
+ for (int i = 0; i < ackPacket.Packets.Length; i++)
+ udpClient.NeedAcks.Remove(ackPacket.Packets[i].ID, now, packet.Header.Resent);
// We don't need to do anything else with PacketAck packets
return;
@@ -734,21 +717,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
client.Close();
}
- private void AcknowledgePacket(LLUDPClient client, uint ack, int currentTime, bool fromResend)
- {
- OutgoingPacket ackedPacket;
- if (client.NeedAcks.RemoveUnsafe(ack, out ackedPacket) && !fromResend)
- {
- // Update stats
- Interlocked.Add(ref client.UnackedBytes, -ackedPacket.Buffer.DataLength);
-
- // Calculate the round-trip time for this packet and its ACK
- int rtt = currentTime - ackedPacket.TickCount;
- if (rtt > 0)
- client.UpdateRoundTrip(rtt);
- }
- }
-
private void IncomingPacketHandler()
{
// Set this culture for the thread that incoming packets are received
@@ -775,11 +743,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
packetInbox.Clear();
}
- public bool SignalOutgoingPacketHandler()
- {
- return m_outgoingWaitHandle.Set();
- }
-
private void OutgoingPacketHandler()
{
// Set this culture for the thread that outgoing packets are sent
@@ -790,28 +753,19 @@ namespace OpenSim.Region.ClientStack.LindenUDP
{
try
{
- m_minTimeout = Int32.MaxValue;
+ m_packetSent = false;
// Handle outgoing packets, resends, acknowledgements, and pings for each
- // client. m_minTimeout will be set to 0 if more packets are waiting in the
- // queues with bandwidth to spare, or the number of milliseconds we need to
- // wait before at least one packet can be sent to a client
+ // client. m_packetSent will be set to true if a packet is sent
m_scene.ClientManager.ForEachSync(ClientOutgoingPacketHandler);
- // Can't wait for a negative amount of time, and put a 100ms ceiling on our
- // maximum wait time
- m_minTimeout = Utils.Clamp(m_minTimeout, 0, 100);
-
- if (m_minTimeout > 0)
- {
- // Don't bother waiting for a shorter interval than our TickCountResolution
- // since the token buckets wouldn't update anyways
- m_minTimeout = Math.Max(m_minTimeout, (int)TickCountResolution);
-
- // Wait for someone to signal that packets are ready to be sent, or for our
- // sleep interval to expire
- m_outgoingWaitHandle.WaitOne(m_minTimeout);
- }
+ // If a packet was sent, only do a minimum length context switch to allow
+ // other parts of the code to do work. If nothing was sent, sleep for the
+ // minimum amount of time before a token bucket could get more tokens
+ if (m_packetSent)
+ Thread.Sleep(0);
+ else
+ Thread.Sleep((int)TickCountResolution);
}
catch (Exception ex)
{
@@ -841,7 +795,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
if (udpClient.ElapsedMSOutgoingPacketHandler >= 100)
{
ResendUnacked(udpClient);
- udpClient.ElapsedMSOutgoingPacketHandler -= 100;
+ udpClient.ElapsedMSOutgoingPacketHandler = 0;
udpClient.Elapsed100MSOutgoingPacketHandler += 1;
}
@@ -849,7 +803,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
if (udpClient.Elapsed100MSOutgoingPacketHandler >= 5)
{
SendAcks(udpClient);
- udpClient.Elapsed100MSOutgoingPacketHandler -= 5;
+ udpClient.Elapsed100MSOutgoingPacketHandler = 0;
udpClient.Elapsed500MSOutgoingPacketHandler += 1;
}
@@ -857,19 +811,12 @@ namespace OpenSim.Region.ClientStack.LindenUDP
if (udpClient.Elapsed500MSOutgoingPacketHandler >= 10)
{
SendPing(udpClient);
- udpClient.Elapsed500MSOutgoingPacketHandler -= 10;
+ udpClient.Elapsed500MSOutgoingPacketHandler = 0;
}
// Dequeue any outgoing packets that are within the throttle limits
- // and get the minimum time we would have to sleep before this client
- // could send a packet out
- int minTimeoutThisLoop = udpClient.DequeueOutgoing();
-
- // Although this is not thread safe, it is cheaper than locking and the
- // worst that will happen is we sleep for slightly longer than the
- // minimum necessary interval
- if (minTimeoutThisLoop < m_minTimeout)
- m_minTimeout = minTimeoutThisLoop;
+ if (udpClient.DequeueOutgoing())
+ m_packetSent = true;
}
udpClient.TickLastOutgoingPacketHandler = thisTick;
--
cgit v1.1