From 999eec603ea62056f599761b90c7a0510336cdd9 Mon Sep 17 00:00:00 2001 From: Teravus Ovares Date: Thu, 22 Nov 2007 19:01:53 +0000 Subject: Created a client driven packet throttler. The sim now respects the client's network throttle settings but does sanity checks to avoid too little(nothing gets sent) or too much(the sim crashes) data. * Consider this experimental.. however, it looks very promising. --- OpenSim/Framework/IClientAPI.cs | 4 +- .../Region/ClientStack/ClientView.PacketQueue.cs | 50 ++++- .../ClientStack/ClientView.ProcessPackets.cs | 247 +++++++++++++++++++-- OpenSim/Region/ClientStack/ClientView.cs | 158 ++++++++++++- 4 files changed, 437 insertions(+), 22 deletions(-) (limited to 'OpenSim') diff --git a/OpenSim/Framework/IClientAPI.cs b/OpenSim/Framework/IClientAPI.cs index 05adf22..2c82d97 100644 --- a/OpenSim/Framework/IClientAPI.cs +++ b/OpenSim/Framework/IClientAPI.cs @@ -62,7 +62,9 @@ namespace OpenSim.Framework Cloud = 3, Task = 4, Texture = 5, - Asset = 6 + Asset = 6, + Unknown = 7, + Back = 8 } /// diff --git a/OpenSim/Region/ClientStack/ClientView.PacketQueue.cs b/OpenSim/Region/ClientStack/ClientView.PacketQueue.cs index 179d02a..3ce3d8c 100644 --- a/OpenSim/Region/ClientStack/ClientView.PacketQueue.cs +++ b/OpenSim/Region/ClientStack/ClientView.PacketQueue.cs @@ -40,6 +40,17 @@ namespace OpenSim.Region.ClientStack public partial class ClientView { protected BlockingQueue PacketQueue; + + protected Queue IncomingPacketQueue; + protected Queue OutgoingPacketQueue; + protected Queue ResendOutgoingPacketQueue; + protected Queue LandOutgoingPacketQueue; + protected Queue WindOutgoingPacketQueue; + protected Queue CloudOutgoingPacketQueue; + protected Queue TaskOutgoingPacketQueue; + protected Queue TextureOutgoingPacketQueue; + protected Queue AssetOutgoingPacketQueue; + protected Dictionary PendingAcks = new Dictionary(); protected Dictionary NeedAck = new Dictionary(); @@ -213,7 +224,39 @@ namespace OpenSim.Region.ClientStack QueItem item = new QueItem(); item.Packet = NewPack; item.Incoming = false; - PacketQueue.Enqueue(item); + item.throttleType = throttlePacketType; // Packet throttle type + switch (throttlePacketType) + { + case ThrottleOutPacketType.Resend: + ResendOutgoingPacketQueue.Enqueue(item); + break; + case ThrottleOutPacketType.Texture: + TextureOutgoingPacketQueue.Enqueue(item); + break; + case ThrottleOutPacketType.Task: + TaskOutgoingPacketQueue.Enqueue(item); + break; + case ThrottleOutPacketType.Land: + LandOutgoingPacketQueue.Enqueue(item); + break; + case ThrottleOutPacketType.Asset: + AssetOutgoingPacketQueue.Enqueue(item); + break; + case ThrottleOutPacketType.Cloud: + CloudOutgoingPacketQueue.Enqueue(item); + break; + case ThrottleOutPacketType.Wind: + WindOutgoingPacketQueue.Enqueue(item); + break; + + default: + + // Acknowledgements and other such stuff should go directly to the blocking Queue + // Throttling them may and likely 'will' be problematic + PacketQueue.Enqueue(item); + break; + } + //OutgoingPacketQueue.Enqueue(item); } # region Low Level Packet Methods @@ -228,7 +271,7 @@ namespace OpenSim.Region.ClientStack ack_it.Packets[0].ID = Pack.Header.Sequence; ack_it.Header.Reliable = false; - OutPacket(ack_it, ThrottleOutPacketType.Task); + OutPacket(ack_it, ThrottleOutPacketType.Unknown); } /* if (Pack.Header.Reliable) @@ -289,7 +332,7 @@ namespace OpenSim.Region.ClientStack } acks.Header.Reliable = false; - OutPacket(acks, ThrottleOutPacketType.Task); + OutPacket(acks, ThrottleOutPacketType.Unknown); PendingAcks.Clear(); } @@ -314,6 +357,7 @@ namespace OpenSim.Region.ClientStack public Packet Packet; public bool Incoming; + public ThrottleOutPacketType throttleType; } #endregion diff --git a/OpenSim/Region/ClientStack/ClientView.ProcessPackets.cs b/OpenSim/Region/ClientStack/ClientView.ProcessPackets.cs index 7f762b6..93c12a6 100644 --- a/OpenSim/Region/ClientStack/ClientView.ProcessPackets.cs +++ b/OpenSim/Region/ClientStack/ClientView.ProcessPackets.cs @@ -781,12 +781,9 @@ namespace OpenSim.Region.ClientStack } break; - #endregion - - #region unimplemented handlers case PacketType.AgentThrottle: - - //OpenSim.Framework.Console.MainLog.Instance.Verbose("CLIENT", "unhandled packet " + Pack.ToString()); + + OpenSim.Framework.Console.MainLog.Instance.Verbose("CLIENT", "unhandled packet " + Pack.ToString()); AgentThrottlePacket atpack = (AgentThrottlePacket)Pack; @@ -803,7 +800,7 @@ namespace OpenSim.Region.ClientStack //Agent Throttle Block contains 7 single floatingpoint values. int j = 0; - + // Some Systems may be big endian... // it might be smart to do this check more often... if (!BitConverter.IsLittleEndian) @@ -813,22 +810,22 @@ namespace OpenSim.Region.ClientStack // values gotten from libsecondlife.org/wiki/Throttle. Thanks MW_ // bytes // Convert to integer, since.. the full fp space isn't used. - tResend = (int)BitConverter.ToSingle(throttle, j); + tResend = (int)BitConverter.ToSingle(throttle, j); j += singlefloat; - tLand = (int)BitConverter.ToSingle(throttle, j); + tLand = (int)BitConverter.ToSingle(throttle, j); j += singlefloat; - tWind = (int)BitConverter.ToSingle(throttle, j); + tWind = (int)BitConverter.ToSingle(throttle, j); j += singlefloat; - tCloud = (int)BitConverter.ToSingle(throttle, j); + tCloud = (int)BitConverter.ToSingle(throttle, j); j += singlefloat; - tTask = (int)BitConverter.ToSingle(throttle, j); + tTask = (int)BitConverter.ToSingle(throttle, j); j += singlefloat; - tTexture = (int)BitConverter.ToSingle(throttle, j); + tTexture = (int)BitConverter.ToSingle(throttle, j); j += singlefloat; tAsset = (int)BitConverter.ToSingle(throttle, j); tall = tResend + tLand + tWind + tCloud + tTask + tTexture + tAsset; - OpenSim.Framework.Console.MainLog.Instance.Verbose("CLIENT", "unhandled packet AgentThrottle - Got throttle:resendbytes=" + tResend + + OpenSim.Framework.Console.MainLog.Instance.Verbose("CLIENT", "Client AgentThrottle - Got throttle:resendbytes=" + tResend + " landbytes=" + tLand + " windbytes=" + tWind + " cloudbytes=" + tCloud + @@ -836,9 +833,233 @@ namespace OpenSim.Region.ClientStack " texturebytes=" + tTexture + " Assetbytes=" + tAsset + " Allbytes=" + tall); + // Total Sanity + // Make sure that the client sent sane total values. + + // If the client didn't send acceptable values.... + // Scale the clients values down until they are acceptable. + + if (tall <= throttleOutboundMax) + { + // Sanity + // Making sure the client sends sane values + // This gives us a measure of control of the comms + // Check Max of Type + // Then Check Min of type + + // Resend throttle + if (tResend <= ResendthrottleMAX) + ResendthrottleOutbound = tResend; + + if (tResend < ResendthrottleMin) + ResendthrottleOutbound = ResendthrottleMin; + + // Land throttle + if (tLand <= LandthrottleMax) + LandthrottleOutbound = tLand; + + if (tLand < LandthrottleMin) + LandthrottleOutbound = LandthrottleMin; + + // Wind throttle + if (tWind <= WindthrottleMax) + WindthrottleOutbound = tWind; + + if (tWind < WindthrottleMin) + WindthrottleOutbound = WindthrottleMin; + + // Cloud throttle + if (tCloud <= CloudthrottleMax) + CloudthrottleOutbound = tCloud; + + if (tCloud < CloudthrottleMin) + CloudthrottleOutbound = CloudthrottleMin; + + // Task throttle + if (tTask <= TaskthrottleMax) + TaskthrottleOutbound = tTask; + + if (tTask < TaskthrottleMin) + TaskthrottleOutbound = TaskthrottleMin; + + // Texture throttle + if (tTexture <= TexturethrottleMax) + TexturethrottleOutbound = tTexture; + + if (tTexture < TexturethrottleMin) + TexturethrottleOutbound = TexturethrottleMin; + + //Asset throttle + if (tAsset <= AssetthrottleMax) + AssetthrottleOutbound = tAsset; + + if (tAsset < AssetthrottleMin) + AssetthrottleOutbound = AssetthrottleMin; + + OpenSim.Framework.Console.MainLog.Instance.Verbose("THROTTLE", "Using:resendbytes=" + ResendthrottleOutbound + + " landbytes=" + LandthrottleOutbound + + " windbytes=" + WindthrottleOutbound + + " cloudbytes=" + CloudthrottleOutbound + + " taskbytes=" + TaskthrottleOutbound + + " texturebytes=" + TexturethrottleOutbound + + " Assetbytes=" + AssetthrottleOutbound + + " Allbytes=" + tall); + } + else + { + // The client didn't send acceptable values.. + // so it's our job now to turn them into acceptable values + // We're going to first scale the values down + // After that we're going to check if the scaled values are sane + + // We're going to be dividing by a user value.. so make sure + // we don't get a divide by zero error. + if (tall > 0) + { + // Find out the percentage of all communications + // the client requests for each type. We'll keep resend at + // it's client recommended level (won't scale it down) + // unless it's beyond sane values itself. + + if (tResend <= ResendthrottleMAX) + { + // This is nexted because we only want to re-set the values + // the packet throttler uses once. + + if (tResend >= ResendthrottleMin) + { + ResendthrottleOutbound = tResend; + } + else + { + ResendthrottleOutbound = ResendthrottleMin; + } + } + else + { + ResendthrottleOutbound = ResendthrottleMAX; + } + + + // Getting Percentages of communication for each type of data + float LandPercent = (float)(tLand / tall); + float WindPercent = (float)(tWind / tall); + float CloudPercent = (float)(tCloud / tall); + float TaskPercent = (float)(tTask / tall); + float TexturePercent = (float)(tTexture / tall); + float AssetPercent = (float)(tAsset / tall); + + // Okay.. now we've got the percentages of total communication. + // Apply them to a new max total + + int tLandResult = (int)(LandPercent * throttleOutboundMax); + int tWindResult = (int)(WindPercent * throttleOutboundMax); + int tCloudResult = (int)(CloudPercent * throttleOutboundMax); + int tTaskResult = (int)(TaskPercent * throttleOutboundMax); + int tTextureResult = (int)(TexturePercent * throttleOutboundMax); + int tAssetResult = (int)(AssetPercent * throttleOutboundMax); + + // Now we have to check our scaled values for sanity + + // Check Max of Type + // Then Check Min of type + + // Land throttle + if (tLandResult <= LandthrottleMax) + LandthrottleOutbound = tLandResult; + + if (tLandResult < LandthrottleMin) + LandthrottleOutbound = LandthrottleMin; + + // Wind throttle + if (tWindResult <= WindthrottleMax) + WindthrottleOutbound = tWindResult; + + if (tWindResult < WindthrottleMin) + WindthrottleOutbound = WindthrottleMin; + + // Cloud throttle + if (tCloudResult <= CloudthrottleMax) + CloudthrottleOutbound = tCloudResult; + + if (tCloudResult < CloudthrottleMin) + CloudthrottleOutbound = CloudthrottleMin; + + // Task throttle + if (tTaskResult <= TaskthrottleMax) + TaskthrottleOutbound = tTaskResult; + + if (tTaskResult < TaskthrottleMin) + TaskthrottleOutbound = TaskthrottleMin; + + // Texture throttle + if (tTextureResult <= TexturethrottleMax) + TexturethrottleOutbound = tTextureResult; + + if (tTextureResult < TexturethrottleMin) + TexturethrottleOutbound = TexturethrottleMin; + + //Asset throttle + if (tAssetResult <= AssetthrottleMax) + AssetthrottleOutbound = tAssetResult; + + if (tAssetResult < AssetthrottleMin) + AssetthrottleOutbound = AssetthrottleMin; + + OpenSim.Framework.Console.MainLog.Instance.Verbose("THROTTLE", "Using:resendbytes=" + ResendthrottleOutbound + + " landbytes=" + LandthrottleOutbound + + " windbytes=" + WindthrottleOutbound + + " cloudbytes=" + CloudthrottleOutbound + + " taskbytes=" + TaskthrottleOutbound + + " texturebytes=" + TexturethrottleOutbound + + " Assetbytes=" + AssetthrottleOutbound + + " Allbytes=" + tall); + + } + else + { + + // The client sent a stupid value.. + // We're going to set the throttles to the minimum possible + ResendthrottleOutbound = ResendthrottleMin; + LandthrottleOutbound = LandthrottleMin; + WindthrottleOutbound = WindthrottleMin; + CloudthrottleOutbound = CloudthrottleMin; + TaskthrottleOutbound = TaskthrottleMin; + TexturethrottleOutbound = TexturethrottleMin; + AssetthrottleOutbound = AssetthrottleMin; + OpenSim.Framework.Console.MainLog.Instance.Verbose("THROTTLE", "ClientSentBadThrottle Using:resendbytes=" + ResendthrottleOutbound + + " landbytes=" + LandthrottleOutbound + + " windbytes=" + WindthrottleOutbound + + " cloudbytes=" + CloudthrottleOutbound + + " taskbytes=" + TaskthrottleOutbound + + " texturebytes=" + TexturethrottleOutbound + + " Assetbytes=" + AssetthrottleOutbound + + " Allbytes=" + tall); + } + + } + // Reset Client Throttles + // This has the effect of 'wiggling the slider + // causes prim and stuck textures that didn't download to download + + ResendthrottleSentPeriod = 0; + LandthrottleSentPeriod = 0; + WindthrottleSentPeriod = 0; + CloudthrottleSentPeriod = 0; + TaskthrottleSentPeriod = 0; + AssetthrottleSentPeriod = 0; + TexturethrottleSentPeriod = 0; + + //Yay, we've finally handled the agent Throttle packet! + break; + + #endregion + + #region unimplemented handlers case PacketType.StartPingCheck: // Send the client the ping response back // Pass the same PingID in the matching packet diff --git a/OpenSim/Region/ClientStack/ClientView.cs b/OpenSim/Region/ClientStack/ClientView.cs index 10f880b..411883b 100644 --- a/OpenSim/Region/ClientStack/ClientView.cs +++ b/OpenSim/Region/ClientStack/ClientView.cs @@ -87,11 +87,53 @@ namespace OpenSim.Region.ClientStack private int probesWithNoIngressPackets = 0; private int lastPacketsReceived = 0; - - private int throttleOutbound = 262144; // Number of bytes allowed to go out per second. (256kbps per client) + // 1536000 + private int throttleOutboundMax = 1536000; // Number of bytes allowed to go out per second. (256kbps per client) // TODO: Make this variable. Lower throttle on un-ack. Raise over time? private int throttleSentPeriod = 0; // Number of bytes sent this period + private int throttleOutbound = 162144; // Number of bytes allowed to go out per second. (256kbps per client) + // TODO: Make this variable. Lower throttle on un-ack. Raise over time + + // All throttle times and number of bytes are calculated by dividing by this value + private int throttleTimeDivisor = 5; + + private int throttletimems = 1000; + + // Maximum -per type- throttle + private int ResendthrottleMAX = 100000; + private int LandthrottleMax = 100000; + private int WindthrottleMax = 100000; + private int CloudthrottleMax = 100000; + private int TaskthrottleMax = 800000; + private int AssetthrottleMax = 800000; + private int TexturethrottleMax = 800000; + + // Minimum -per type- throttle + private int ResendthrottleMin = 5000; // setting resendmin to 0 results in mostly dropped packets + private int LandthrottleMin = 1000; + private int WindthrottleMin = 1000; + private int CloudthrottleMin = 1000; + private int TaskthrottleMin = 1000; + private int AssetthrottleMin = 1000; + private int TexturethrottleMin = 1000; + + // Sim default per-client settings. + private int ResendthrottleOutbound = 50000; + private int ResendthrottleSentPeriod = 0; + private int LandthrottleOutbound = 100000; + private int LandthrottleSentPeriod = 0; + private int WindthrottleOutbound = 10000; + private int WindthrottleSentPeriod = 0; + private int CloudthrottleOutbound = 5000; + private int CloudthrottleSentPeriod = 0; + private int TaskthrottleOutbound = 100000; + private int TaskthrottleSentPeriod = 0; + private int AssetthrottleOutbound = 80000; + private int AssetthrottleSentPeriod = 0; + private int TexturethrottleOutbound = 100000; + private int TexturethrottleSentPeriod = 0; + private Timer throttleTimer; public ClientView(EndPoint remoteEP, UseCircuitCodePacket initialcirpack, ClientManager clientManager, @@ -114,14 +156,31 @@ namespace OpenSim.Region.ClientStack startpos = m_authenticateSessionsHandler.GetPosition(initialcirpack.CircuitCode.Code); + + // While working on this, the BlockingQueue had me fooled for a bit. + // The Blocking queue causes the thread to stop until there's something + // in it to process. it's an on-purpose threadlock though because + // without it, the clientloop will suck up all sim resources. + PacketQueue = new BlockingQueue(); + IncomingPacketQueue = new Queue(); + OutgoingPacketQueue = new Queue(); + ResendOutgoingPacketQueue = new Queue(); + LandOutgoingPacketQueue = new Queue(); + WindOutgoingPacketQueue = new Queue(); + CloudOutgoingPacketQueue = new Queue(); + TaskOutgoingPacketQueue = new Queue(); + TextureOutgoingPacketQueue = new Queue(); + AssetOutgoingPacketQueue = new Queue(); + + //this.UploadAssets = new AgentAssetUpload(this, m_assetCache, m_inventoryCache); AckTimer = new Timer(750); AckTimer.Elapsed += new ElapsedEventHandler(AckTimer_Elapsed); AckTimer.Start(); - throttleTimer = new Timer(1000); + throttleTimer = new Timer((int)(throttletimems/throttleTimeDivisor)); throttleTimer.Elapsed += new ElapsedEventHandler(throttleTimer_Elapsed); throttleTimer.Start(); @@ -133,8 +192,97 @@ namespace OpenSim.Region.ClientStack } void throttleTimer_Elapsed(object sender, ElapsedEventArgs e) - { + { throttleSentPeriod = 0; + ResendthrottleSentPeriod = 0; + LandthrottleSentPeriod = 0; + WindthrottleSentPeriod = 0; + CloudthrottleSentPeriod = 0; + TaskthrottleSentPeriod = 0; + AssetthrottleSentPeriod = 0; + TexturethrottleSentPeriod = 0; + + // I was considering this.. Will an event fire if the thread it's on is blocked? + + // Then I figured out.. it doesn't really matter.. because this thread won't be blocked for long + // The General overhead of the UDP protocol gets sent to the queue un-throttled by this + // so This'll pick up about around the right time. + + int MaxThrottleLoops = 5550; // 50*7 packets can be dequeued at once. + int throttleLoops = 0; + + // We're going to dequeue all of the saved up packets until + // we've hit the throttle limit or there's no more packets to send + while ((throttleSentPeriod <= ((int)(throttleOutbound/throttleTimeDivisor)) && + (ResendOutgoingPacketQueue.Count > 0 || + LandOutgoingPacketQueue.Count > 0 || + WindOutgoingPacketQueue.Count > 0 || + CloudOutgoingPacketQueue.Count > 0 || + TaskOutgoingPacketQueue.Count > 0 || + AssetOutgoingPacketQueue.Count > 0 || + TextureOutgoingPacketQueue.Count > 0)) && throttleLoops <= MaxThrottleLoops) + { + throttleLoops++; + //Now comes the fun part.. we dump all our elements into PacketQueue that we've saved up. + if (ResendthrottleSentPeriod <= ((int)(ResendthrottleOutbound/throttleTimeDivisor)) && ResendOutgoingPacketQueue.Count > 0) + { + QueItem qpack = ResendOutgoingPacketQueue.Dequeue(); + + PacketQueue.Enqueue(qpack); + throttleSentPeriod += qpack.Packet.ToBytes().Length; + ResendthrottleSentPeriod += qpack.Packet.ToBytes().Length; + } + if (LandthrottleSentPeriod <= ((int)(LandthrottleOutbound/throttleTimeDivisor)) && LandOutgoingPacketQueue.Count > 0) + { + QueItem qpack = LandOutgoingPacketQueue.Dequeue(); + + PacketQueue.Enqueue(qpack); + throttleSentPeriod += qpack.Packet.ToBytes().Length; + LandthrottleSentPeriod += qpack.Packet.ToBytes().Length; + } + if (WindthrottleSentPeriod <= ((int)(WindthrottleOutbound/throttleTimeDivisor)) && WindOutgoingPacketQueue.Count > 0) + { + QueItem qpack = WindOutgoingPacketQueue.Dequeue(); + + PacketQueue.Enqueue(qpack); + throttleSentPeriod += qpack.Packet.ToBytes().Length; + WindthrottleSentPeriod += qpack.Packet.ToBytes().Length; + } + if (CloudthrottleSentPeriod <= ((int)(CloudthrottleOutbound/throttleTimeDivisor)) && CloudOutgoingPacketQueue.Count > 0) + { + QueItem qpack = CloudOutgoingPacketQueue.Dequeue(); + + PacketQueue.Enqueue(qpack); + throttleSentPeriod += qpack.Packet.ToBytes().Length; + CloudthrottleSentPeriod += qpack.Packet.ToBytes().Length; + } + if (TaskthrottleSentPeriod <= ((int)(TaskthrottleOutbound/throttleTimeDivisor)) && TaskOutgoingPacketQueue.Count > 0) + { + QueItem qpack = TaskOutgoingPacketQueue.Dequeue(); + + PacketQueue.Enqueue(qpack); + throttleSentPeriod += qpack.Packet.ToBytes().Length; + TaskthrottleSentPeriod += qpack.Packet.ToBytes().Length; + } + if (TexturethrottleSentPeriod <= ((int)(TexturethrottleOutbound/throttleTimeDivisor)) && TextureOutgoingPacketQueue.Count > 0) + { + QueItem qpack = TextureOutgoingPacketQueue.Dequeue(); + + PacketQueue.Enqueue(qpack); + throttleSentPeriod += qpack.Packet.ToBytes().Length; + TexturethrottleSentPeriod += qpack.Packet.ToBytes().Length; + } + if (AssetthrottleSentPeriod <= ((int)(AssetthrottleOutbound/throttleTimeDivisor)) && AssetOutgoingPacketQueue.Count > 0) + { + QueItem qpack = AssetOutgoingPacketQueue.Dequeue(); + + PacketQueue.Enqueue(qpack); + throttleSentPeriod += qpack.Packet.ToBytes().Length; + AssetthrottleSentPeriod += qpack.Packet.ToBytes().Length; + } + + } + } public LLUUID SessionId @@ -277,7 +425,7 @@ namespace OpenSim.Region.ClientStack else { // Throw it back on the queue if it's going to cause us to flood the client - if (throttleSentPeriod > throttleOutbound) + if (throttleSentPeriod > throttleOutboundMax) { PacketQueue.Enqueue(nextPacket); MainLog.Instance.Verbose("Client over throttle limit, requeuing packet"); -- cgit v1.1