From c5de9840b05514eb659696239593e51e636398b8 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 16 Apr 2013 21:58:24 +0100 Subject: refactor: Remove IClientNetworkServer.NetworkStop() in favour of existing Stop(). This was an undocumented interface which I think was for long defunct region load balancing experiments. Also adds method doc for some IClientNetworkServer methods. --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 5 ----- 1 file changed, 5 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 72516cd..985aa4d 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -62,11 +62,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP m_udpServer = new LLUDPServer(listenIP, ref port, proxyPortOffsetParm, allow_alternate_port, configSource, circuitManager); } - public void NetworkStop() - { - m_udpServer.Stop(); - } - public void AddScene(IScene scene) { m_udpServer.AddScene(scene); -- cgit v1.1 From f32a21d96707f87ecbdaf42c0059f8494a119d31 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Tue, 14 May 2013 08:12:01 -0700 Subject: HGTP-mesh bug: the mesh download requests were going to the departing sims for a little while. This was also true for local TPs. BUt for local TPs the assets are on the same server, so it doesn't matter. For HGTPs, it matters. This potential fix moves sending the initial data to later, after the client has completed the movement into the region. Fingers crossed that it doesn't mess other things up! --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 3 --- 1 file changed, 3 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 985aa4d..0fa1c0e 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1371,9 +1371,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP // circuit code to the existing child agent. This is not particularly obvious. SendAckImmediate(endPoint, uccp.Header.Sequence); - // We only want to send initial data to new clients, not ones which are being converted from child to root. - if (client != null) - client.SceneAgent.SendInitialDataToMe(); } else { -- cgit v1.1 From 645da54f25bb034eb76151daa9723763eae13303 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Tue, 14 May 2013 08:47:18 -0700 Subject: Revert "HGTP-mesh bug: the mesh download requests were going to the departing sims for a little while. This was also true for local TPs. BUt for local TPs the assets are on the same server, so it doesn't matter. For HGTPs, it matters. This potential fix moves sending the initial data to later, after the client has completed the movement into the region. Fingers crossed that it doesn't mess other things up!" This reverts commit f32a21d96707f87ecbdaf42c0059f8494a119d31. --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 3 +++ 1 file changed, 3 insertions(+) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 0fa1c0e..985aa4d 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1371,6 +1371,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP // circuit code to the existing child agent. This is not particularly obvious. SendAckImmediate(endPoint, uccp.Header.Sequence); + // We only want to send initial data to new clients, not ones which are being converted from child to root. + if (client != null) + client.SceneAgent.SendInitialDataToMe(); } else { -- cgit v1.1 From 91091c3e5453088242e05a682283fc7f9f5c7ae3 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Tue, 14 May 2013 09:06:58 -0700 Subject: Second take at HGTP-mesh bug: delay sending the initial data only for agents that are coming via TP (root agents) --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 985aa4d..8eb2e06 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1373,7 +1373,13 @@ namespace OpenSim.Region.ClientStack.LindenUDP // We only want to send initial data to new clients, not ones which are being converted from child to root. if (client != null) - client.SceneAgent.SendInitialDataToMe(); + { + AgentCircuitData aCircuit = m_scene.AuthenticateHandler.GetAgentCircuitData(uccp.CircuitCode.Code); + bool tp = (aCircuit.teleportFlags > 0); + // Let's delay this for TP agents, otherwise the viewer doesn't know where to get meshes from + if (!tp) + client.SceneAgent.SendInitialDataToMe(); + } } else { -- cgit v1.1 From 25889b2d7ef08b27591aa61ab4950bdbc856d7a5 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 4 Jul 2013 00:02:53 +0100 Subject: change "debug packet" command to "debug lludp packet" to conform with other "debug lludp" options also moves the implementing code into LLUDPServer.cs along with other debug commands from OpenSim.cs gets all debug lludp commands to only activate for the set scene if not root --- .../Region/ClientStack/Linden/UDP/LLUDPServer.cs | 59 ++++++++++++++++++++++ 1 file changed, 59 insertions(+) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 8eb2e06..ff31ef5 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -513,6 +513,19 @@ namespace OpenSim.Region.ClientStack.LindenUDP EnablePoolStats(); MainConsole.Instance.Commands.AddCommand( + "Debug", false, "debug lludp packet", + "debug lludp packet [ ]", + "Turn on packet debugging", + "If level > 255 then all incoming and outgoing packets are logged.\n" + + "If level <= 255 then incoming AgentUpdate and outgoing SimStats and SimulatorViewerTimeMessage packets are not logged.\n" + + "If level <= 200 then incoming RequestImage and outgoing ImagePacket, ImageData, LayerData and CoarseLocationUpdate packets are not logged.\n" + + "If level <= 100 then incoming ViewerEffect and AgentAnimation and outgoing ViewerEffect and AvatarAnimation packets are not logged.\n" + + "If level <= 50 then outgoing ImprovedTerseObjectUpdate packets are not logged.\n" + + "If level <= 0 then no packets are logged.\n" + + "If an avatar name is given then only packets from that avatar are logged", + HandlePacketCommand); + + MainConsole.Instance.Commands.AddCommand( "Debug", false, "debug lludp start", @@ -553,8 +566,45 @@ namespace OpenSim.Region.ClientStack.LindenUDP HandleStatusCommand); } + private void HandlePacketCommand(string module, string[] args) + { + if (SceneManager.Instance.CurrentScene != null && SceneManager.Instance.CurrentScene != m_scene) + return; + + string name = null; + + if (args.Length == 6) + name = string.Format("{0} {1}", args[4], args[5]); + + if (args.Length > 3) + { + int newDebug; + if (int.TryParse(args[3], out newDebug)) + { + m_scene.ForEachScenePresence(sp => + { + if (name == null || sp.Name == name) + { + m_log.DebugFormat( + "Packet debug for {0} ({1}) set to {2} in {3}", + sp.Name, sp.IsChildAgent ? "child" : "root", newDebug, m_scene.Name); + + sp.ControllingClient.DebugPacketLevel = newDebug; + } + }); + } + else + { + MainConsole.Instance.Output("Usage: debug lludp packet 0..255 [ ]"); + } + } + } + private void HandleStartCommand(string module, string[] args) { + if (SceneManager.Instance.CurrentScene != null && SceneManager.Instance.CurrentScene != m_scene) + return; + if (args.Length != 4) { MainConsole.Instance.Output("Usage: debug lludp start "); @@ -572,6 +622,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP private void HandleStopCommand(string module, string[] args) { + if (SceneManager.Instance.CurrentScene != null && SceneManager.Instance.CurrentScene != m_scene) + return; + if (args.Length != 4) { MainConsole.Instance.Output("Usage: debug lludp stop "); @@ -589,6 +642,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP private void HandlePoolCommand(string module, string[] args) { + if (SceneManager.Instance.CurrentScene != null && SceneManager.Instance.CurrentScene != m_scene) + return; + if (args.Length != 4) { MainConsole.Instance.Output("Usage: debug lludp pool "); @@ -621,6 +677,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP private void HandleStatusCommand(string module, string[] args) { + if (SceneManager.Instance.CurrentScene != null && SceneManager.Instance.CurrentScene != m_scene) + return; + MainConsole.Instance.OutputFormat( "IN LLUDP packet processing for {0} is {1}", m_scene.Name, IsRunningInbound ? "enabled" : "disabled"); -- cgit v1.1 From 16f40c1a15454fd6b093ae901c307670f12602fe Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 4 Jul 2013 17:29:53 +0100 Subject: Add --default option to "debug lludp packet" command to allow packet logging to be performed immediately from client start --- .../Region/ClientStack/Linden/UDP/LLUDPServer.cs | 69 ++++++++++++++++------ 1 file changed, 52 insertions(+), 17 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index ff31ef5..82fad11 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -34,6 +34,7 @@ using System.Net.Sockets; using System.Reflection; using System.Threading; using log4net; +using NDesk.Options; using Nini.Config; using OpenMetaverse.Packets; using OpenSim.Framework; @@ -102,10 +103,15 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// public class LLUDPServer : OpenSimUDPBase { + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + /// Maximum transmission unit, or UDP packet size, for the LLUDP protocol public const int MTU = 1400; - private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + /// + /// Default packet debug level given to new clients + /// + public int DefaultClientPacketDebugLevel { get; set; } /// The measured resolution of Environment.TickCount public readonly float TickCountResolution; @@ -514,7 +520,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP MainConsole.Instance.Commands.AddCommand( "Debug", false, "debug lludp packet", - "debug lludp packet [ ]", + "debug lludp packet [--default] [ ]", "Turn on packet debugging", "If level > 255 then all incoming and outgoing packets are logged.\n" + "If level <= 255 then incoming AgentUpdate and outgoing SimStats and SimulatorViewerTimeMessage packets are not logged.\n" @@ -522,7 +528,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP + "If level <= 100 then incoming ViewerEffect and AgentAnimation and outgoing ViewerEffect and AvatarAnimation packets are not logged.\n" + "If level <= 50 then outgoing ImprovedTerseObjectUpdate packets are not logged.\n" + "If level <= 0 then no packets are logged.\n" - + "If an avatar name is given then only packets from that avatar are logged", + + "If --default is specified then the level becomes the default logging level for all subsequent agents.\n" + + "In this case, you cannot also specify an avatar name.\n" + + "If an avatar name is given then only packets from that avatar are logged.", HandlePacketCommand); MainConsole.Instance.Commands.AddCommand( @@ -571,31 +579,54 @@ namespace OpenSim.Region.ClientStack.LindenUDP if (SceneManager.Instance.CurrentScene != null && SceneManager.Instance.CurrentScene != m_scene) return; + bool setAsDefaultLevel = false; + OptionSet optionSet = new OptionSet().Add("default", o => setAsDefaultLevel = o != null); + List filteredArgs = optionSet.Parse(args); + string name = null; - if (args.Length == 6) - name = string.Format("{0} {1}", args[4], args[5]); + if (filteredArgs.Count == 6) + { + if (!setAsDefaultLevel) + { + name = string.Format("{0} {1}", filteredArgs[4], filteredArgs[5]); + } + else + { + MainConsole.Instance.OutputFormat("ERROR: Cannot specify a user name when setting default logging level"); + return; + } + } - if (args.Length > 3) + if (filteredArgs.Count > 3) { int newDebug; - if (int.TryParse(args[3], out newDebug)) + if (int.TryParse(filteredArgs[3], out newDebug)) { - m_scene.ForEachScenePresence(sp => + if (setAsDefaultLevel) + { + DefaultClientPacketDebugLevel = newDebug; + MainConsole.Instance.OutputFormat( + "Debug packet debug for new clients set to {0}", DefaultClientPacketDebugLevel); + } + else { - if (name == null || sp.Name == name) + m_scene.ForEachScenePresence(sp => { - m_log.DebugFormat( - "Packet debug for {0} ({1}) set to {2} in {3}", - sp.Name, sp.IsChildAgent ? "child" : "root", newDebug, m_scene.Name); - - sp.ControllingClient.DebugPacketLevel = newDebug; - } - }); + if (name == null || sp.Name == name) + { + MainConsole.Instance.OutputFormat( + "Packet debug for {0} ({1}) set to {2} in {3}", + sp.Name, sp.IsChildAgent ? "child" : "root", newDebug, m_scene.Name); + + sp.ControllingClient.DebugPacketLevel = newDebug; + } + }); + } } else { - MainConsole.Instance.Output("Usage: debug lludp packet 0..255 [ ]"); + MainConsole.Instance.Output("Usage: debug lludp packet [--default] 0..255 [ ]"); } } } @@ -687,6 +718,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP "OUT LLUDP packet processing for {0} is {1}", m_scene.Name, IsRunningOutbound ? "enabled" : "disabled"); MainConsole.Instance.OutputFormat("LLUDP pools in {0} are {1}", m_scene.Name, UsePools ? "on" : "off"); + + MainConsole.Instance.OutputFormat( + "Packet debug level for new clients is {0}", DefaultClientPacketDebugLevel); } public bool HandlesRegion(Location x) @@ -1533,6 +1567,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP client = new LLClientView(m_scene, this, udpClient, sessionInfo, agentID, sessionID, circuitCode); client.OnLogout += LogoutHandler; + client.DebugPacketLevel = DefaultClientPacketDebugLevel; ((LLClientView)client).DisableFacelights = m_disableFacelights; -- cgit v1.1 From 38e6da5522a53c7f65eac64ae7b0af929afb1ae6 Mon Sep 17 00:00:00 2001 From: Robert Adams Date: Tue, 9 Jul 2013 18:34:24 -0700 Subject: Comment out old inbound UDP throttling hack. This would cause the UDP reception thread to sleep for 30ms if the number of available user worker threads got low. It doesn't look like any of the UDP packet types are marked async so this check is 1) unnecessary and 2) really crazy since it stops up the reception thread under heavy load without any indication. --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 2 ++ 1 file changed, 2 insertions(+) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 82fad11..2aab4f9 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1615,6 +1615,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP { IncomingPacket incomingPacket = null; + /* // HACK: This is a test to try and rate limit packet handling on Mono. // If it works, a more elegant solution can be devised if (Util.FireAndForgetCount() < 2) @@ -1622,6 +1623,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP //m_log.Debug("[LLUDPSERVER]: Incoming packet handler is sleeping"); Thread.Sleep(30); } + */ if (packetInbox.Dequeue(100, ref incomingPacket)) { -- cgit v1.1 From a412b1d6823141129bef3ab906c6590eb6a81c72 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Sat, 13 Jul 2013 09:46:58 -0700 Subject: Moved SendInitialDataToMe to earlier in CompleteMovement. Moved TriggerOnMakeRootAgent to the end of CompleteMovement. Justin, if you read this, there's a long story here. Some time ago you placed SendInitialDataToMe at the very beginning of client creation (in LLUDPServer). That is problematic, as we discovered relatively recently: on TPs, as soon as the client starts getting data from child agents, it starts requesting resources back *from the simulator where its root agent is*. We found this to be the problem behind meshes missing on HG TPs (because the viewer was requesting the meshes of the receiving sim from the departing grid). But this affects much more than meshes and HG TPs. It may also explain cloud avatars after a local TP: baked textures are only stored in the simulator, so if a child agent receives a UUID of a baked texture in the destination sim and requests that texture from the departing sim where the root agent is, it will fail to get that texture. Bottom line: we need to delay sending the new simulator data to the viewer until we are absolutely sure that the viewer knows that its main agent is in a new sim. Hence, moving it to CompleteMovement. Now I am trying to tune the initial rez delay that we all experience in the CC. I think that when I fixed the issue described above, I may have moved SendInitialDataToMe to much later than it should be, so now I'm moving to earlier in CompleteMovement. --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 2aab4f9..85270a6 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1469,7 +1469,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP { AgentCircuitData aCircuit = m_scene.AuthenticateHandler.GetAgentCircuitData(uccp.CircuitCode.Code); bool tp = (aCircuit.teleportFlags > 0); - // Let's delay this for TP agents, otherwise the viewer doesn't know where to get meshes from + // Let's delay this for TP agents, otherwise the viewer doesn't know where to get resources from if (!tp) client.SceneAgent.SendInitialDataToMe(); } -- cgit v1.1 From ac73e702935dd4607c13aaec3095940fba7932ca Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Mon, 15 Jul 2013 11:27:49 -0700 Subject: Trying to hunt the CPU spikes recently experienced. Revert "Comment out old inbound UDP throttling hack. This would cause the UDP" This reverts commit 38e6da5522a53c7f65eac64ae7b0af929afb1ae6. --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 2 -- 1 file changed, 2 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 85270a6..e20a194 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1615,7 +1615,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP { IncomingPacket incomingPacket = null; - /* // HACK: This is a test to try and rate limit packet handling on Mono. // If it works, a more elegant solution can be devised if (Util.FireAndForgetCount() < 2) @@ -1623,7 +1622,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP //m_log.Debug("[LLUDPSERVER]: Incoming packet handler is sleeping"); Thread.Sleep(30); } - */ if (packetInbox.Dequeue(100, ref incomingPacket)) { -- cgit v1.1 From 864f15ce4dfd412df8442a26429b9dd7a2bf9e12 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Mon, 15 Jul 2013 11:52:26 -0700 Subject: Revert the revert Revert "Trying to hunt the CPU spikes recently experienced." This reverts commit ac73e702935dd4607c13aaec3095940fba7932ca. --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 2 ++ 1 file changed, 2 insertions(+) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index e20a194..85270a6 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1615,6 +1615,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP { IncomingPacket incomingPacket = null; + /* // HACK: This is a test to try and rate limit packet handling on Mono. // If it works, a more elegant solution can be devised if (Util.FireAndForgetCount() < 2) @@ -1622,6 +1623,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP //m_log.Debug("[LLUDPSERVER]: Incoming packet handler is sleeping"); Thread.Sleep(30); } + */ if (packetInbox.Dequeue(100, ref incomingPacket)) { -- cgit v1.1 From d9d995914c5fba00d4ccaf66b899384c8ea3d5eb Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 18 Jul 2013 01:17:46 +0100 Subject: try Hacking in an AutoResetEvent to control the outgoing UDP loop instead of a continuous loop with sleeps. Does appear to have a cpu impact but may need further tweaking --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 85270a6..54cafb2 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -806,8 +806,12 @@ namespace OpenSim.Region.ClientStack.LindenUDP } PacketPool.Instance.ReturnPacket(packet); + m_dataPresentEvent.Set(); + } + private AutoResetEvent m_dataPresentEvent = new AutoResetEvent(false); + /// /// Start the process of sending a packet to the client. /// @@ -1658,6 +1662,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP // Action generic every round Action clientPacketHandler = ClientOutgoingPacketHandler; +// while (true) while (base.IsRunningOutbound) { try @@ -1718,8 +1723,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP // If nothing was sent, sleep for the minimum amount of time before a // token bucket could get more tokens - if (!m_packetSent) - Thread.Sleep((int)TickCountResolution); + //if (!m_packetSent) + // Thread.Sleep((int)TickCountResolution); + m_dataPresentEvent.WaitOne(100); Watchdog.UpdateThread(); } -- cgit v1.1 From 63c42d66022ea7f1c2805b8f77980af5d4ba1fb4 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 18 Jul 2013 21:28:36 +0100 Subject: Do some simple queue empty checks in the main outgoing udp loop instead of always performing these on a separate fired thread. This appears to improve cpu usage since launching a new thread is more expensive than performing a small amount of inline logic. However, needs testing at scale. --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 54cafb2..e871ca2 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1662,8 +1662,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP // Action generic every round Action clientPacketHandler = ClientOutgoingPacketHandler; -// while (true) - while (base.IsRunningOutbound) + while (true) +// while (base.IsRunningOutbound) { try { -- cgit v1.1 From 3a476bf60ce6364f5fd562132625027b73606fea Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 18 Jul 2013 22:42:25 +0100 Subject: Fix up a temporary debugging change from last commit which stopped "lludp stop out" from actually doing anything --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index e871ca2..a1085fa 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1662,8 +1662,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP // Action generic every round Action clientPacketHandler = ClientOutgoingPacketHandler; - while (true) -// while (base.IsRunningOutbound) + while (base.IsRunningOutbound) { try { -- cgit v1.1 From 5a2d4d888c02bef984f42eecaec62164a1e58b72 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 18 Jul 2013 23:05:45 +0100 Subject: Hack in console command "debug lludp toggle agentupdate" to allow AgentUpdate in packets to be discarded at a very early stage. Enabling this will stop anybody from moving on a sim, though all other updates should be unaffected. Appears to make some cpu difference on very basic testing with a static standing avatar (though not all that much). Need to see the results with much higher av numbers. --- .../Region/ClientStack/Linden/UDP/LLUDPServer.cs | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index a1085fa..78d4165 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -572,6 +572,14 @@ namespace OpenSim.Region.ClientStack.LindenUDP "debug lludp status", "Return status of LLUDP packet processing.", HandleStatusCommand); + + MainConsole.Instance.Commands.AddCommand( + "Debug", + false, + "debug lludp toggle agentupdate", + "debug lludp toggle agentupdate", + "Toggle whether agentupdate packets are processed or simply discarded.", + HandleAgentUpdateCommand); } private void HandlePacketCommand(string module, string[] args) @@ -706,6 +714,19 @@ namespace OpenSim.Region.ClientStack.LindenUDP } } + bool m_discardAgentUpdates; + + private void HandleAgentUpdateCommand(string module, string[] args) + { + if (SceneManager.Instance.CurrentScene != null && SceneManager.Instance.CurrentScene != m_scene) + return; + + m_discardAgentUpdates = !m_discardAgentUpdates; + + MainConsole.Instance.OutputFormat( + "Discard AgentUpdates now {0} for {1}", m_discardAgentUpdates, m_scene.Name); + } + private void HandleStatusCommand(string module, string[] args) { if (SceneManager.Instance.CurrentScene != null && SceneManager.Instance.CurrentScene != m_scene) @@ -1286,6 +1307,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP LogPacketHeader(true, udpClient.CircuitCode, 0, packet.Type, (ushort)packet.Length); #endregion BinaryStats + if (m_discardAgentUpdates && packet.Type == PacketType.AgentUpdate) + return; + #region Ping Check Handling if (packet.Type == PacketType.StartPingCheck) -- cgit v1.1 From 61eda1f441092eb12936472de2dc73898e40aa16 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 19 Jul 2013 00:51:13 +0100 Subject: Make the check as to whether any particular inbound AgentUpdate packet is significant much earlier in UDP processing (i.e. before we pointlessly place such packets on internal queues, etc.) Appears to have some impact on cpu but needs testing. --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 78d4165..766c2fe 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1307,8 +1307,21 @@ namespace OpenSim.Region.ClientStack.LindenUDP LogPacketHeader(true, udpClient.CircuitCode, 0, packet.Type, (ushort)packet.Length); #endregion BinaryStats - if (m_discardAgentUpdates && packet.Type == PacketType.AgentUpdate) - return; + if (packet.Type == PacketType.AgentUpdate) + { + if (m_discardAgentUpdates) + return; + + AgentUpdatePacket agentUpdate = (AgentUpdatePacket)packet; + + if (agentUpdate.AgentData.SessionID != client.SessionId + || agentUpdate.AgentData.AgentID != client.AgentId + || !((LLClientView)client).CheckAgentUpdateSignificance(agentUpdate.AgentData)) + { + PacketPool.Instance.ReturnPacket(packet); + return; + } + } #region Ping Check Handling -- cgit v1.1 From 174105ad028c5ed318850238d97aa7c3b1d7f207 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Fri, 19 Jul 2013 22:11:32 -0700 Subject: Fixed the stats in show client stats. Also left some comments with observations about AgentUpdates. --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 2 ++ 1 file changed, 2 insertions(+) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 766c2fe..32282af 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1312,6 +1312,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP if (m_discardAgentUpdates) return; + ((LLClientView)client).TotalAgentUpdates++; + AgentUpdatePacket agentUpdate = (AgentUpdatePacket)packet; if (agentUpdate.AgentData.SessionID != client.SessionId -- cgit v1.1 From e6b6af62dd677077664fdc8801b3a7e894a2b8da Mon Sep 17 00:00:00 2001 From: Robert Adams Date: Mon, 22 Jul 2013 15:41:14 -0700 Subject: Added check for user movement specification before discarding an incoming AgentUpdate packet. This fixes the problem with vehicles not moving forward after the first up-arrow. Code to fix a potential exception when using different IClientAPIs. --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 32282af..f5c0b05 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1316,9 +1316,10 @@ namespace OpenSim.Region.ClientStack.LindenUDP AgentUpdatePacket agentUpdate = (AgentUpdatePacket)packet; + LLClientView llClient = client as LLClientView; if (agentUpdate.AgentData.SessionID != client.SessionId || agentUpdate.AgentData.AgentID != client.AgentId - || !((LLClientView)client).CheckAgentUpdateSignificance(agentUpdate.AgentData)) + || !(llClient == null || llClient.CheckAgentUpdateSignificance(agentUpdate.AgentData)) ) { PacketPool.Instance.ReturnPacket(packet); return; -- cgit v1.1 From bf517899a7a63278d4ed22d5485c37d61d47bb58 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Mon, 22 Jul 2013 23:30:09 +0100 Subject: Add AverageUDPProcessTime stat to try and get a handle on how long we're taking on the initial processing of a UDP packet. If we're not receiving packets with multiple threads (m_asyncPacketHandling) then this is critical since it will limit the number of incoming UDP requests that the region can handle and affects packet loss. If m_asyncPacketHandling then this is less critical though a long process will increase the scope for threads to race. This is an experimental stat which may be changed. --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index f5c0b05..d3823f3 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -70,8 +70,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP StatsManager.RegisterStat( new Stat( "IncomingPacketsProcessedCount", - "Number of inbound UDP packets processed", - "Number of inbound UDP packets processed", + "Number of inbound LL protocol packets processed", + "Number of inbound LL protocol packets processed", "", "clientstack", scene.Name, @@ -79,6 +79,21 @@ namespace OpenSim.Region.ClientStack.LindenUDP MeasuresOfInterest.AverageChangeOverTime, stat => stat.Value = m_udpServer.IncomingPacketsProcessed, StatVerbosity.Debug)); + + StatsManager.RegisterStat( + new Stat( + "AverageUDPProcessTime", + "Average number of milliseconds taken to process each incoming UDP packet in a sample.", + "This is for initial receive processing which is separate from the later client LL packet processing stage.", + "ms", + "clientstack", + scene.Name, + StatType.Pull, + MeasuresOfInterest.None, + stat => stat.Value = m_udpServer.AverageReceiveTicksForLastSamplePeriod / TimeSpan.TicksPerMillisecond, +// stat => +// stat.Value = Math.Round(m_udpServer.AverageReceiveTicksForLastSamplePeriod / TimeSpan.TicksPerMillisecond, 7), + StatVerbosity.Debug)); } public bool HandlesRegion(Location x) -- cgit v1.1 From 8396f1bd42cf42d412c09e769c491930aaa8bfea Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Mon, 22 Jul 2013 23:58:45 +0100 Subject: Record raw number of UDP receives as clientstack.IncomingUDPReceivesCount --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index d3823f3..5300b1e 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -69,6 +69,19 @@ namespace OpenSim.Region.ClientStack.LindenUDP StatsManager.RegisterStat( new Stat( + "IncomingUDPReceivesCount", + "Number of inbound LL protocol packets processed", + "Number of inbound LL protocol packets processed", + "", + "clientstack", + scene.Name, + StatType.Pull, + MeasuresOfInterest.AverageChangeOverTime, + stat => stat.Value = m_udpServer.UdpReceives, + StatVerbosity.Debug)); + + StatsManager.RegisterStat( + new Stat( "IncomingPacketsProcessedCount", "Number of inbound LL protocol packets processed", "Number of inbound LL protocol packets processed", -- cgit v1.1 From 60732c96efd149bbb0484b327b00463dc5b81aff Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 23 Jul 2013 00:15:58 +0100 Subject: Add clientstack.OutgoingUDPSendsCount stat to show number of outbound UDP packets sent by a region per second --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 5300b1e..cc4de10 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -70,8 +70,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP StatsManager.RegisterStat( new Stat( "IncomingUDPReceivesCount", - "Number of inbound LL protocol packets processed", - "Number of inbound LL protocol packets processed", + "Number of UDP receives performed", + "Number of UDP receives performed", "", "clientstack", scene.Name, @@ -95,6 +95,19 @@ namespace OpenSim.Region.ClientStack.LindenUDP StatsManager.RegisterStat( new Stat( + "OutgoingUDPSendsCount", + "Number of UDP sends performed", + "Number of UDP sends performed", + "", + "clientstack", + scene.Name, + StatType.Pull, + MeasuresOfInterest.AverageChangeOverTime, + stat => stat.Value = m_udpServer.UdpSends, + StatVerbosity.Debug)); + + StatsManager.RegisterStat( + new Stat( "AverageUDPProcessTime", "Average number of milliseconds taken to process each incoming UDP packet in a sample.", "This is for initial receive processing which is separate from the later client LL packet processing stage.", @@ -856,7 +869,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP PacketPool.Instance.ReturnPacket(packet); m_dataPresentEvent.Set(); - } private AutoResetEvent m_dataPresentEvent = new AutoResetEvent(false); -- cgit v1.1 From 9fb9da1b6c86e10b229706fe06f2875c8a63a52f Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 23 Jul 2013 00:31:57 +0100 Subject: Add clientstack.InboxPacketsCount stat. This records the number of packets waiting to be processed at the second stage (after initial UDP processing) If this consistently increases then this is a problem since it means the simulator is receiving more requests than it can distribute to other parts of the code. --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index cc4de10..0e9f1b7 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -500,6 +500,19 @@ namespace OpenSim.Region.ClientStack.LindenUDP m_scene = (Scene)scene; m_location = new Location(m_scene.RegionInfo.RegionHandle); + StatsManager.RegisterStat( + new Stat( + "InboxPacketsCount", + "Number of LL protocol packets waiting for the second stage of processing after initial receive.", + "Number of LL protocol packets waiting for the second stage of processing after initial receive.", + "", + "clientstack", + scene.Name, + StatType.Pull, + MeasuresOfInterest.AverageChangeOverTime, + stat => stat.Value = packetInbox.Count, + StatVerbosity.Debug)); + // XXX: These stats are also pool stats but we register them separately since they are currently not // turned on and off by EnablePools()/DisablePools() StatsManager.RegisterStat( -- cgit v1.1 From a57a472ab8edf70430a8391909e6078b9ae0f26d Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 23 Jul 2013 00:51:59 +0100 Subject: Add proper method doc and comments to m_dataPresentEvent (from d9d9959) --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 0e9f1b7..37fd252 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -223,6 +223,15 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// Flag to signal when clients should send pings protected bool m_sendPing; + /// + /// Event used to signal when queued packets are available for sending. + /// + /// + /// This allows the outbound loop to only operate when there is data to send rather than continuously polling. + /// Some data is sent immediately and not queued. That data would not trigger this event. + /// + private AutoResetEvent m_dataPresentEvent = new AutoResetEvent(false); + private Pool m_incomingPacketPool; /// @@ -881,11 +890,10 @@ namespace OpenSim.Region.ClientStack.LindenUDP } PacketPool.Instance.ReturnPacket(packet); + m_dataPresentEvent.Set(); } - private AutoResetEvent m_dataPresentEvent = new AutoResetEvent(false); - /// /// Start the process of sending a packet to the client. /// @@ -1817,6 +1825,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP // token bucket could get more tokens //if (!m_packetSent) // Thread.Sleep((int)TickCountResolution); + // + // Instead, now wait for data present to be explicitly signalled. Evidence so far is that with + // modern mono it reduces CPU base load since there is no more continuous polling. m_dataPresentEvent.WaitOne(100); Watchdog.UpdateThread(); -- cgit v1.1 From 3891a8946bb72c9256d8de4185bf4a72c90be859 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Mon, 22 Jul 2013 11:54:35 -0700 Subject: New Teleport protocol (V2), still compatible with V1 and older. (version of the destination is being checked) In this new protocol, and as committed before, the viewer is not sent EnableSimulator/EstablishChildCommunication for the destination. Instead, it is sent TeleportFinish directly. TeleportFinish, in turn, makes the viewer send a UserCircuitCode packet followed by CompleteMovementIntoRegion packet. These 2 packets tend to occur one after the other almost immediately to the point that when CMIR arrives the client is not even connected yet and that packet is ignored (there might have been some race conditions here before); then the viewer sends CMIR again within 5-8 secs. But the delay between them may be higher in busier regions, which may lead to race conditions. This commit improves the process so there are are no race conditions at the destination. CompleteMovement (triggered by the viewer) waits until Update has been sent from the origin. Update, in turn, waits until there is a *root* scene presence -- so making sure CompleteMovement has run MakeRoot. In other words, there are two threadlets at the destination, one from the viewer and one from the origin region, waiting for each other to do the right thing. That makes it safe to close the agent at the origin upon return of the Update call without having to wait for callback, because we are absolutely sure that the viewer knows it is in th new region. Note also that in the V1 protocol, the destination was getting UseCircuitCode from the viewer twice -- once on EstablishAgentCommunication and then again on TeleportFinish. The second UCC was being ignored, but it shows how we were not following the expected steps... --- .../Region/ClientStack/Linden/UDP/LLUDPServer.cs | 68 ++++++++++++++++++++++ 1 file changed, 68 insertions(+) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 37fd252..1b72b26 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1257,6 +1257,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP // UseCircuitCode handling if (packet.Type == PacketType.UseCircuitCode) { + m_log.DebugFormat("[ZZZ]: In the dungeon: UseCircuitCode"); // We need to copy the endpoint so that it doesn't get changed when another thread reuses the // buffer. object[] array = new object[] { new IPEndPoint(endPoint.Address, endPoint.Port), packet }; @@ -1265,6 +1266,20 @@ namespace OpenSim.Region.ClientStack.LindenUDP return; } + else if (packet.Type == PacketType.CompleteAgentMovement) + { + // Send ack straight away to let the viewer know that we got it. + SendAckImmediate(endPoint, packet.Header.Sequence); + + m_log.DebugFormat("[ZZZ]: In the dungeon: CompleteAgentMovement"); + // We need to copy the endpoint so that it doesn't get changed when another thread reuses the + // buffer. + object[] array = new object[] { new IPEndPoint(endPoint.Address, endPoint.Port), packet }; + + Util.FireAndForget(HandleCompleteMovementIntoRegion, array); + + return; + } // Determine which agent this packet came from IClientAPI client; @@ -1604,6 +1619,56 @@ namespace OpenSim.Region.ClientStack.LindenUDP } } + private void HandleCompleteMovementIntoRegion(object o) + { + IPEndPoint endPoint = null; + IClientAPI client = null; + + try + { + object[] array = (object[])o; + endPoint = (IPEndPoint)array[0]; + CompleteAgentMovementPacket packet = (CompleteAgentMovementPacket)array[1]; + + // Determine which agent this packet came from + int count = 10; + while (!m_scene.TryGetClient(endPoint, out client) && count-- > 0) + { + m_log.Debug("[LLUDPSERVER]: Received a CompleteMovementIntoRegion in " + m_scene.RegionInfo.RegionName + " (not ready yet)"); + Thread.Sleep(200); + } + + if (client == null) + return; + + IncomingPacket incomingPacket1; + + // Inbox insertion + if (UsePools) + { + incomingPacket1 = m_incomingPacketPool.GetObject(); + incomingPacket1.Client = (LLClientView)client; + incomingPacket1.Packet = packet; + } + else + { + incomingPacket1 = new IncomingPacket((LLClientView)client, packet); + } + + packetInbox.Enqueue(incomingPacket1); + } + catch (Exception e) + { + m_log.ErrorFormat( + "[LLUDPSERVER]: CompleteMovementIntoRegion handling from endpoint {0}, client {1} {2} failed. Exception {3}{4}", + endPoint != null ? endPoint.ToString() : "n/a", + client != null ? client.Name : "unknown", + client != null ? client.AgentId.ToString() : "unknown", + e.Message, + e.StackTrace); + } + } + /// /// Send an ack immediately to the given endpoint. /// @@ -1999,6 +2064,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP Packet packet = incomingPacket.Packet; LLClientView client = incomingPacket.Client; + if (packet is CompleteAgentMovementPacket) + m_log.DebugFormat("[ZZZ]: Received CompleteAgentMovementPacket"); + if (client.IsActive) { m_currentIncomingClient = client; -- cgit v1.1 From 14530b260764372df3627206c531b96b9d817dbe Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Mon, 22 Jul 2013 20:20:48 -0700 Subject: Minor adjustment on timings of waits. --- .../Region/ClientStack/Linden/UDP/LLUDPServer.cs | 23 ++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 1b72b26..43167ee 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1631,11 +1631,26 @@ namespace OpenSim.Region.ClientStack.LindenUDP CompleteAgentMovementPacket packet = (CompleteAgentMovementPacket)array[1]; // Determine which agent this packet came from - int count = 10; - while (!m_scene.TryGetClient(endPoint, out client) && count-- > 0) + int count = 20; + bool ready = false; + while (!ready && count-- > 0) { - m_log.Debug("[LLUDPSERVER]: Received a CompleteMovementIntoRegion in " + m_scene.RegionInfo.RegionName + " (not ready yet)"); - Thread.Sleep(200); + if (m_scene.TryGetClient(endPoint, out client) && client.IsActive) + { + LLUDPClient udpClient = ((LLClientView)client).UDPClient; + if (udpClient != null && udpClient.IsConnected) + ready = true; + else + { + m_log.Debug("[LLUDPSERVER]: Received a CompleteMovementIntoRegion in " + m_scene.RegionInfo.RegionName + " (not ready yet)"); + Thread.Sleep(200); + } + } + else + { + m_log.Debug("[LLUDPSERVER]: Received a CompleteMovementIntoRegion in " + m_scene.RegionInfo.RegionName + " (not ready yet)"); + Thread.Sleep(200); + } } if (client == null) -- cgit v1.1 From e6a0f6e428e32356eba14729766aac28cec9989f Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Mon, 22 Jul 2013 20:49:40 -0700 Subject: One more thing to test in order to let CompleteMovement go up the stack. --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 43167ee..f40948f 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1635,9 +1635,10 @@ namespace OpenSim.Region.ClientStack.LindenUDP bool ready = false; while (!ready && count-- > 0) { - if (m_scene.TryGetClient(endPoint, out client) && client.IsActive) + if (m_scene.TryGetClient(endPoint, out client) && client.IsActive && client.SceneAgent != null) { - LLUDPClient udpClient = ((LLClientView)client).UDPClient; + LLClientView llClientView = (LLClientView)client; + LLUDPClient udpClient = llClientView.UDPClient; if (udpClient != null && udpClient.IsConnected) ready = true; else -- cgit v1.1 From cac37e298c267e4fa9c8d4306df85d74bff81fbe Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Wed, 24 Jul 2013 14:24:17 -0700 Subject: Deleted all [ZZZ] debug messages. --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 5 ----- 1 file changed, 5 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index f40948f..25e10be 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1257,7 +1257,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP // UseCircuitCode handling if (packet.Type == PacketType.UseCircuitCode) { - m_log.DebugFormat("[ZZZ]: In the dungeon: UseCircuitCode"); // We need to copy the endpoint so that it doesn't get changed when another thread reuses the // buffer. object[] array = new object[] { new IPEndPoint(endPoint.Address, endPoint.Port), packet }; @@ -1271,7 +1270,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP // Send ack straight away to let the viewer know that we got it. SendAckImmediate(endPoint, packet.Header.Sequence); - m_log.DebugFormat("[ZZZ]: In the dungeon: CompleteAgentMovement"); // We need to copy the endpoint so that it doesn't get changed when another thread reuses the // buffer. object[] array = new object[] { new IPEndPoint(endPoint.Address, endPoint.Port), packet }; @@ -2080,9 +2078,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP Packet packet = incomingPacket.Packet; LLClientView client = incomingPacket.Client; - if (packet is CompleteAgentMovementPacket) - m_log.DebugFormat("[ZZZ]: Received CompleteAgentMovementPacket"); - if (client.IsActive) { m_currentIncomingClient = client; -- cgit v1.1 From 8efe4bfc2ed7086e9fdf4812297e6525f955f6ac Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Mon, 29 Jul 2013 23:18:29 +0100 Subject: Make "abnormal thread terminations" into "ClientLogoutsDueToNoReceives" and add this to the StatsManager This reflects the actual use of this stat - it hasn't recorded general exceptions for some time. Make the sim extra stats collector draw the data from the stats manager rather than maintaing this data itself. --- .../Region/ClientStack/Linden/UDP/LLUDPServer.cs | 26 +++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 25e10be..9e6a401 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -69,9 +69,22 @@ namespace OpenSim.Region.ClientStack.LindenUDP StatsManager.RegisterStat( new Stat( + "ClientLogoutsDueToNoReceives", + "Number of times a client has been logged out because no packets were received before the timeout.", + "", + "", + "clientstack", + scene.Name, + StatType.Pull, + MeasuresOfInterest.None, + stat => stat.Value = m_udpServer.ClientLogoutsDueToNoReceives, + StatVerbosity.Debug)); + + StatsManager.RegisterStat( + new Stat( "IncomingUDPReceivesCount", "Number of UDP receives performed", - "Number of UDP receives performed", + "", "", "clientstack", scene.Name, @@ -84,7 +97,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP new Stat( "IncomingPacketsProcessedCount", "Number of inbound LL protocol packets processed", - "Number of inbound LL protocol packets processed", + "", "", "clientstack", scene.Name, @@ -97,7 +110,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP new Stat( "OutgoingUDPSendsCount", "Number of UDP sends performed", - "Number of UDP sends performed", + "", "", "clientstack", scene.Name, @@ -149,6 +162,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// Maximum transmission unit, or UDP packet size, for the LLUDP protocol public const int MTU = 1400; + /// Number of forced client logouts due to no receipt of packets before timeout. + public int ClientLogoutsDueToNoReceives { get; private set; } + /// /// Default packet debug level given to new clients /// @@ -1037,7 +1053,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP timeoutTicks = m_pausedAckTimeout; if (client.IsActive && - (Environment.TickCount & Int32.MaxValue) - udpClient.TickLastPacketReceived > timeoutTicks) + (Environment.TickCount & Int32.MaxValue) - udpClient.TickLastPacketReceived > -1) { // We must set IsActive synchronously so that we can stop the packet loop reinvoking this method, even // though it's set later on by LLClientView.Close() @@ -1778,7 +1794,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP "[LLUDPSERVER]: Ack timeout, disconnecting {0} agent for {1} in {2}", client.SceneAgent.IsChildAgent ? "child" : "root", client.Name, m_scene.RegionInfo.RegionName); - StatsManager.SimExtraStats.AddAbnormalClientThreadTermination(); + ClientLogoutsDueToNoReceives++; if (!client.SceneAgent.IsChildAgent) client.Kick("Simulator logged you out due to connection timeout"); -- cgit v1.1 From 8004e6f31cb03abc9b6170622099879ccaf5570b Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Mon, 29 Jul 2013 23:38:54 +0100 Subject: Fix issue just introduced in 8efe4bfc2ed7086e9fdf4812297e6525f955f6ac where I accidentally left in a test line to force very quick client unack --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 9e6a401..bf50868 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1053,7 +1053,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP timeoutTicks = m_pausedAckTimeout; if (client.IsActive && - (Environment.TickCount & Int32.MaxValue) - udpClient.TickLastPacketReceived > -1) + (Environment.TickCount & Int32.MaxValue) - udpClient.TickLastPacketReceived > timeoutTicks) { // We must set IsActive synchronously so that we can stop the packet loop reinvoking this method, even // though it's set later on by LLClientView.Close() -- cgit v1.1 From 1416c909326d89566cbe785b6dacac228e31a5a0 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Mon, 29 Jul 2013 23:53:59 +0100 Subject: minor: Add timeout secs to connection timeout message. Change message to reflect it is a timeout due to no data received rather than an ack issue. --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index bf50868..85fe1a4 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1062,7 +1062,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP // Fire this out on a different thread so that we don't hold up outgoing packet processing for // everybody else if this is being called due to an ack timeout. // This is the same as processing as the async process of a logout request. - Util.FireAndForget(o => DeactivateClientDueToTimeout(client)); + Util.FireAndForget(o => DeactivateClientDueToTimeout(client, timeoutTicks)); return; } @@ -1786,18 +1786,19 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// regular client pings. /// /// - private void DeactivateClientDueToTimeout(LLClientView client) + /// + private void DeactivateClientDueToTimeout(LLClientView client, int timeoutTicks) { lock (client.CloseSyncLock) - { - m_log.WarnFormat( - "[LLUDPSERVER]: Ack timeout, disconnecting {0} agent for {1} in {2}", - client.SceneAgent.IsChildAgent ? "child" : "root", client.Name, m_scene.RegionInfo.RegionName); - + { ClientLogoutsDueToNoReceives++; + + m_log.WarnFormat( + "[LLUDPSERVER]: No packets received from {0} agent of {1} for {2}ms in {3}. Disconnecting.", + client.SceneAgent.IsChildAgent ? "child" : "root", client.Name, timeoutTicks, m_scene.Name); if (!client.SceneAgent.IsChildAgent) - client.Kick("Simulator logged you out due to connection timeout"); + client.Kick("Simulator logged you out due to connection timeout."); client.CloseWithoutChecks(); } -- cgit v1.1 From 59b461ac0eaae1cc34bb82431106fdf0476037f3 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Thu, 1 Aug 2013 09:27:44 -0700 Subject: Issue: painfully slow terrain loading. The cause is commit d9d995914c5fba00d4ccaf66b899384c8ea3d5eb (r/23185) -- the WaitOne on the UDPServer. Putting it back to how it was done solves the issue. But this may impact CPU usage, so I'm pushing it to test if it does. --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 85fe1a4..942c044 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -246,7 +246,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// This allows the outbound loop to only operate when there is data to send rather than continuously polling. /// Some data is sent immediately and not queued. That data would not trigger this event. /// - private AutoResetEvent m_dataPresentEvent = new AutoResetEvent(false); + //private AutoResetEvent m_dataPresentEvent = new AutoResetEvent(false); private Pool m_incomingPacketPool; @@ -907,7 +907,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP PacketPool.Instance.ReturnPacket(packet); - m_dataPresentEvent.Set(); + //m_dataPresentEvent.Set(); } /// @@ -1919,12 +1919,12 @@ namespace OpenSim.Region.ClientStack.LindenUDP // If nothing was sent, sleep for the minimum amount of time before a // token bucket could get more tokens - //if (!m_packetSent) - // Thread.Sleep((int)TickCountResolution); + if (!m_packetSent) + Thread.Sleep((int)TickCountResolution); // // Instead, now wait for data present to be explicitly signalled. Evidence so far is that with // modern mono it reduces CPU base load since there is no more continuous polling. - m_dataPresentEvent.WaitOne(100); + //m_dataPresentEvent.WaitOne(100); Watchdog.UpdateThread(); } -- cgit v1.1 From 932c382737d155176bd7e0dcfaf9395d997a88fb Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 1 Aug 2013 18:11:50 +0100 Subject: Revert "Issue: painfully slow terrain loading. The cause is commit d9d995914c5fba00d4ccaf66b899384c8ea3d5eb (r/23185) -- the WaitOne on the UDPServer. Putting it back to how it was done solves the issue. But this may impact CPU usage, so I'm pushing it to test if it does." This reverts commit 59b461ac0eaae1cc34bb82431106fdf0476037f3. --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 942c044..85fe1a4 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -246,7 +246,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// This allows the outbound loop to only operate when there is data to send rather than continuously polling. /// Some data is sent immediately and not queued. That data would not trigger this event. /// - //private AutoResetEvent m_dataPresentEvent = new AutoResetEvent(false); + private AutoResetEvent m_dataPresentEvent = new AutoResetEvent(false); private Pool m_incomingPacketPool; @@ -907,7 +907,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP PacketPool.Instance.ReturnPacket(packet); - //m_dataPresentEvent.Set(); + m_dataPresentEvent.Set(); } /// @@ -1919,12 +1919,12 @@ namespace OpenSim.Region.ClientStack.LindenUDP // If nothing was sent, sleep for the minimum amount of time before a // token bucket could get more tokens - if (!m_packetSent) - Thread.Sleep((int)TickCountResolution); + //if (!m_packetSent) + // Thread.Sleep((int)TickCountResolution); // // Instead, now wait for data present to be explicitly signalled. Evidence so far is that with // modern mono it reduces CPU base load since there is no more continuous polling. - //m_dataPresentEvent.WaitOne(100); + m_dataPresentEvent.WaitOne(100); Watchdog.UpdateThread(); } -- cgit v1.1 From 0c4c084bed5175d8a5b25b8f915363f3b15b6e3a Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 1 Aug 2013 18:12:28 +0100 Subject: Try a different approach to slow terrain update by always cycling the loop immediately if any data was sent, rather than waiting. What I believe is happening is that on initial terrain send, this is done one packet at a time. With WaitOne, the outbound loop has enough time to loop and wait again after the first packet before the second, leading to a slower send. This approach instead does not wait if a packet was just sent but instead loops again, which appears to lead to a quicker send without losing the cpu benefit of not continually looping when there is no outbound data. --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 85fe1a4..558fcb7 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1924,7 +1924,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP // // Instead, now wait for data present to be explicitly signalled. Evidence so far is that with // modern mono it reduces CPU base load since there is no more continuous polling. - m_dataPresentEvent.WaitOne(100); + if (!m_packetSent) + m_dataPresentEvent.WaitOne(100); Watchdog.UpdateThread(); } -- cgit v1.1 From 68b98a8003e0ea7ed082b105ae82a644fb706796 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 1 Aug 2013 23:16:41 +0100 Subject: minor: Add name to debug lludp packet level feedback on console --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 558fcb7..5c38399 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -694,7 +694,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP { DefaultClientPacketDebugLevel = newDebug; MainConsole.Instance.OutputFormat( - "Debug packet debug for new clients set to {0}", DefaultClientPacketDebugLevel); + "Debug packet debug for new clients set to {0} in {1}", DefaultClientPacketDebugLevel, m_scene.Name); } else { -- cgit v1.1 From b1c26a56b3d615f3709363e3a2f91b5423f5891f Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 8 Aug 2013 23:29:30 +0100 Subject: Fix an issue where under teleport v2 protocol, teleporting from regions in an line from A->B->C would not close region A when reaching C The root cause was that v2 was only closing neighbour agents if the root connection also needed a close. However, fixing this requires the neighbour regions also detect when they should not close due to re-teleports re-establishing the child connection. This involves restructuring the code to introduce a scene presence state machine that can serialize the different add and remove client calls that are now possible with the late close of the This commit appears to fix these issues and improve teleport, but still has holes on at least quick reteleporting (and possibly occasionally on ordinary teleports). Also, has not been completely tested yet in scenarios where regions are running on different simulators --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 5c38399..de2f9d4 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1799,9 +1799,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP if (!client.SceneAgent.IsChildAgent) client.Kick("Simulator logged you out due to connection timeout."); - - client.CloseWithoutChecks(); } + + m_scene.IncomingCloseAgent(client.AgentId, true); } private void IncomingPacketHandler() @@ -2142,7 +2142,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP if (!client.IsLoggingOut) { client.IsLoggingOut = true; - client.Close(); + m_scene.IncomingCloseAgent(client.AgentId, false); } } } -- cgit v1.1 From 0d5680e9712a5362c23b0ef6e479654d67e99b8b Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Wed, 14 Aug 2013 22:07:22 +0100 Subject: Count any incoming packet that could not be recognized as an LLUDP packet as a malformed packet. Record this as stat clientstack..IncomingPacketsMalformedCount Used to detect if a simulator is receiving significant junk UDP Decimates the number of packets between which a warning is logged and prints the IP source of the last malformed packet when logging --- .../Region/ClientStack/Linden/UDP/LLUDPServer.cs | 65 +++++++++++++++------- 1 file changed, 44 insertions(+), 21 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index de2f9d4..553250c 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -108,6 +108,19 @@ namespace OpenSim.Region.ClientStack.LindenUDP StatsManager.RegisterStat( new Stat( + "IncomingPacketsMalformedCount", + "Number of inbound UDP packets that could not be recognized as LL protocol packets.", + "", + "", + "clientstack", + scene.Name, + StatType.Pull, + MeasuresOfInterest.AverageChangeOverTime, + stat => stat.Value = m_udpServer.IncomingMalformedPacketCount, + StatVerbosity.Info)); + + StatsManager.RegisterStat( + new Stat( "OutgoingUDPSendsCount", "Number of UDP sends performed", "", @@ -268,7 +281,10 @@ namespace OpenSim.Region.ClientStack.LindenUDP public Socket Server { get { return null; } } - private int m_malformedCount = 0; // Guard against a spamming attack + /// + /// Record how many inbound packets could not be recognized as LLUDP packets. + /// + public int IncomingMalformedPacketCount { get; private set; } /// /// Record current outgoing client for monitoring purposes. @@ -1181,6 +1197,19 @@ namespace OpenSim.Region.ClientStack.LindenUDP outgoingPacket.TickCount = Environment.TickCount & Int32.MaxValue; } + private void RecordMalformedInboundPacket(IPEndPoint endPoint) + { +// if (m_malformedCount < 100) +// m_log.DebugFormat("[LLUDPSERVER]: Dropped malformed packet: " + e.ToString()); + + IncomingMalformedPacketCount++; + + if ((IncomingMalformedPacketCount % 10000) == 0) + m_log.WarnFormat( + "[LLUDPSERVER]: Received {0} malformed packets so far, probable network attack. Last malformed was from {1}", + IncomingMalformedPacketCount, endPoint); + } + public override void PacketReceived(UDPPacketBuffer buffer) { // Debugging/Profiling @@ -1202,6 +1231,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP // "[LLUDPSERVER]: Dropping undersized packet with {0} bytes received from {1} in {2}", // buffer.DataLength, buffer.RemoteEndPoint, m_scene.RegionInfo.RegionName); + RecordMalformedInboundPacket(endPoint); + return; // Drop undersized packet } @@ -1220,6 +1251,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP // "[LLUDPSERVER]: Dropping packet with malformed header received from {0} in {1}", // buffer.RemoteEndPoint, m_scene.RegionInfo.RegionName); + RecordMalformedInboundPacket(endPoint); + return; // Malformed header } @@ -1235,34 +1268,24 @@ namespace OpenSim.Region.ClientStack.LindenUDP // Only allocate a buffer for zerodecoding if the packet is zerocoded ((buffer.Data[0] & Helpers.MSG_ZEROCODED) != 0) ? new byte[4096] : null); } - catch (MalformedDataException) - { - } - catch (IndexOutOfRangeException) - { -// m_log.WarnFormat( -// "[LLUDPSERVER]: Dropping short packet received from {0} in {1}", -// buffer.RemoteEndPoint, m_scene.RegionInfo.RegionName); - - return; // Drop short packet - } catch (Exception e) { - if (m_malformedCount < 100) + if (IncomingMalformedPacketCount < 100) m_log.DebugFormat("[LLUDPSERVER]: Dropped malformed packet: " + e.ToString()); - - m_malformedCount++; - - if ((m_malformedCount % 100000) == 0) - m_log.DebugFormat("[LLUDPSERVER]: Received {0} malformed packets so far, probable network attack.", m_malformedCount); } // Fail-safe check if (packet == null) { - m_log.ErrorFormat("[LLUDPSERVER]: Malformed data, cannot parse {0} byte packet from {1}:", - buffer.DataLength, buffer.RemoteEndPoint); - m_log.Error(Utils.BytesToHexString(buffer.Data, buffer.DataLength, null)); + if (IncomingMalformedPacketCount < 100) + { + m_log.ErrorFormat("[LLUDPSERVER]: Malformed data, cannot parse {0} byte packet from {1}:", + buffer.DataLength, buffer.RemoteEndPoint); + m_log.Error(Utils.BytesToHexString(buffer.Data, buffer.DataLength, null)); + } + + RecordMalformedInboundPacket(endPoint); + return; } -- cgit v1.1 From 93dffe17773ee5af552ac64a7902f66d8acac1b3 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Wed, 14 Aug 2013 22:33:12 +0100 Subject: Add stat clientstack..IncomingPacketsOrphanedCount to record well-formed packets that were not initial connection packets and could not be associated with a connected viewer. --- .../Region/ClientStack/Linden/UDP/LLUDPServer.cs | 33 +++++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 553250c..102e581 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -121,6 +121,19 @@ namespace OpenSim.Region.ClientStack.LindenUDP StatsManager.RegisterStat( new Stat( + "IncomingPacketsOrphanedCount", + "Number of inbound packets that were not initial connections packets and could not be associated with a viewer.", + "", + "", + "clientstack", + scene.Name, + StatType.Pull, + MeasuresOfInterest.AverageChangeOverTime, + stat => stat.Value = m_udpServer.IncomingOrphanedPacketCount, + StatVerbosity.Info)); + + StatsManager.RegisterStat( + new Stat( "OutgoingUDPSendsCount", "Number of UDP sends performed", "", @@ -287,6 +300,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP public int IncomingMalformedPacketCount { get; private set; } /// + /// Record how many inbound packets could not be associated with a simulator circuit. + /// + public int IncomingOrphanedPacketCount { get; private set; } + + /// /// Record current outgoing client for monitoring purposes. /// private IClientAPI m_currentOutgoingClient; @@ -1206,7 +1224,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP if ((IncomingMalformedPacketCount % 10000) == 0) m_log.WarnFormat( - "[LLUDPSERVER]: Received {0} malformed packets so far, probable network attack. Last malformed was from {1}", + "[LLUDPSERVER]: Received {0} malformed packets so far, probable network attack. Last was from {1}", IncomingMalformedPacketCount, endPoint); } @@ -1279,9 +1297,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP { if (IncomingMalformedPacketCount < 100) { - m_log.ErrorFormat("[LLUDPSERVER]: Malformed data, cannot parse {0} byte packet from {1}:", - buffer.DataLength, buffer.RemoteEndPoint); - m_log.Error(Utils.BytesToHexString(buffer.Data, buffer.DataLength, null)); + m_log.WarnFormat("[LLUDPSERVER]: Malformed data, cannot parse {0} byte packet from {1}, data {2}:", + buffer.DataLength, buffer.RemoteEndPoint, Utils.BytesToHexString(buffer.Data, buffer.DataLength, null)); } RecordMalformedInboundPacket(endPoint); @@ -1323,6 +1340,14 @@ namespace OpenSim.Region.ClientStack.LindenUDP if (!m_scene.TryGetClient(endPoint, out client) || !(client is LLClientView)) { //m_log.Debug("[LLUDPSERVER]: Received a " + packet.Type + " packet from an unrecognized source: " + address + " in " + m_scene.RegionInfo.RegionName); + + IncomingOrphanedPacketCount++; + + if ((IncomingOrphanedPacketCount % 10000) == 0) + m_log.WarnFormat( + "[LLUDPSERVER]: Received {0} orphaned packets so far. Last was from {1}", + IncomingOrphanedPacketCount, endPoint); + return; } -- cgit v1.1 From 7dbc93c62a6803fbe201e933b8bc1374b48869ad Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Wed, 18 Sep 2013 21:41:51 +0100 Subject: Change logging to provide more information on LLUDPServer.HandleCompleteMovementIntoRegion() Add more information on which endpoint sent the packet when we have to wait and if we end up dropping the packet Only check if the client is active - other checks are redundant since they can only failed if IsActve = false --- .../Region/ClientStack/Linden/UDP/LLUDPServer.cs | 49 +++++++++++++++++----- 1 file changed, 39 insertions(+), 10 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 102e581..cdc1668 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1693,31 +1693,60 @@ namespace OpenSim.Region.ClientStack.LindenUDP CompleteAgentMovementPacket packet = (CompleteAgentMovementPacket)array[1]; // Determine which agent this packet came from + // We need to wait here because in when using the OpenSimulator V2 teleport protocol to travel to a destination + // simulator with no existing child presence, the viewer (at least LL 3.3.4) will send UseCircuitCode + // and then CompleteAgentMovement immediately without waiting for an ack. As we are now handling these + // packets asynchronously, we need to account for this thread proceeding more quickly than the + // UseCircuitCode thread. int count = 20; - bool ready = false; - while (!ready && count-- > 0) + while (count-- > 0) { - if (m_scene.TryGetClient(endPoint, out client) && client.IsActive && client.SceneAgent != null) + if (m_scene.TryGetClient(endPoint, out client)) { - LLClientView llClientView = (LLClientView)client; - LLUDPClient udpClient = llClientView.UDPClient; - if (udpClient != null && udpClient.IsConnected) - ready = true; + if (client.IsActive) + { + break; + } else { - m_log.Debug("[LLUDPSERVER]: Received a CompleteMovementIntoRegion in " + m_scene.RegionInfo.RegionName + " (not ready yet)"); - Thread.Sleep(200); + // This check exists to catch a condition where the client has been closed by another thread + // but has not yet been removed from the client manager (and possibly a new connection has + // not yet been established). + m_log.DebugFormat( + "[LLUDPSERVER]: Received a CompleteMovementIntoRegion from {0} for {1} in {2} but client is not active. Waiting.", + endPoint, client.Name, m_scene.Name); } } else { - m_log.Debug("[LLUDPSERVER]: Received a CompleteMovementIntoRegion in " + m_scene.RegionInfo.RegionName + " (not ready yet)"); + m_log.DebugFormat( + "[LLUDPSERVER]: Received a CompleteMovementIntoRegion from {0} in {1} but no client exists. Waiting.", + endPoint, m_scene.Name); + Thread.Sleep(200); } } if (client == null) + { + m_log.DebugFormat( + "[LLUDPSERVER]: No client found for CompleteMovementIntoRegion from {0} in {1} after wait. Dropping.", + endPoint, m_scene.Name); + return; + } + else if (!client.IsActive) + { + // This check exists to catch a condition where the client has been closed by another thread + // but has not yet been removed from the client manager. + // The packet could be simply ignored but it is useful to know if this condition occurred for other debugging + // purposes. + m_log.DebugFormat( + "[LLUDPSERVER]: Received a CompleteMovementIntoRegion from {0} for {1} in {2} but client is not active after wait. Dropping.", + endPoint, client.Name, m_scene.Name); + + return; + } IncomingPacket incomingPacket1; -- cgit v1.1 From f4d82a56f4c13bd24305e008418819edcfdc549d Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Wed, 18 Sep 2013 22:09:46 +0100 Subject: Double the time spent waiting for a UseCircuitCode packet in LLUDPServer.HandleCompleteMovementIntoRegion() This is to deal with one aspect of http://opensimulator.org/mantis/view.php?id=6755 With the V2 teleport arrangements, viewers appear to send the single UseCircuitCode and CompleteAgentMovement packets immediately after each other Possibly, on occasion a poor network might drop the initial UseCircuitCode packet and by the time it retries, the CompleteAgementMovement has timed out and the teleport fails. There's no apparant harm in doubling the wait time (most times only one wait will be performed) so trying this. --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index cdc1668..b1752c1 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1698,7 +1698,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP // and then CompleteAgentMovement immediately without waiting for an ack. As we are now handling these // packets asynchronously, we need to account for this thread proceeding more quickly than the // UseCircuitCode thread. - int count = 20; + int count = 40; while (count-- > 0) { if (m_scene.TryGetClient(endPoint, out client)) -- cgit v1.1 From 732554be0451b9e7fa5f96ffb8f9f347b964dd88 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Wed, 25 Sep 2013 18:29:14 +0100 Subject: Reinsert 200ms sleep accidentally removed in commit 7dbc93c (Wed Sep 18 21:41:51 2013 +0100) --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index b1752c1..a130ffe 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1722,9 +1722,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP m_log.DebugFormat( "[LLUDPSERVER]: Received a CompleteMovementIntoRegion from {0} in {1} but no client exists. Waiting.", endPoint, m_scene.Name); - - Thread.Sleep(200); } + + Thread.Sleep(200); } if (client == null) -- cgit v1.1 From 32ddfc274056e68bfd9e7e6d3e7921c8062a59d1 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Wed, 25 Sep 2013 18:45:56 +0100 Subject: Reinsert client.SceneAgent checks into LLUDPServer.HandleCompleteMovementIntoRegion() to fix race condition regression in commit 7dbc93c (Wed Sep 18 21:41:51 2013 +0100) This check is necessary to close a race condition where the CompleteAgentMovement processing could proceed when the UseCircuitCode thread had added the client to the client manager but before the ScenePresence had registered to process the CompleteAgentMovement message. This is most probably why the message appeared to get lost on a proportion of entity transfers. A better long term solution may be to set the IClientAPI.SceneAgent property before the client is added to the manager. --- .../Region/ClientStack/Linden/UDP/LLUDPServer.cs | 37 +++++++++++++++------- 1 file changed, 26 insertions(+), 11 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index a130ffe..9504f15 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1692,6 +1692,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP endPoint = (IPEndPoint)array[0]; CompleteAgentMovementPacket packet = (CompleteAgentMovementPacket)array[1]; + m_log.DebugFormat( + "[LLUDPSERVER]: Handling CompleteAgentMovement request from {0} in {1}", endPoint, m_scene.Name); + // Determine which agent this packet came from // We need to wait here because in when using the OpenSimulator V2 teleport protocol to travel to a destination // simulator with no existing child presence, the viewer (at least LL 3.3.4) will send UseCircuitCode @@ -1703,24 +1706,36 @@ namespace OpenSim.Region.ClientStack.LindenUDP { if (m_scene.TryGetClient(endPoint, out client)) { - if (client.IsActive) - { - break; - } - else + if (!client.IsActive) { // This check exists to catch a condition where the client has been closed by another thread // but has not yet been removed from the client manager (and possibly a new connection has // not yet been established). m_log.DebugFormat( - "[LLUDPSERVER]: Received a CompleteMovementIntoRegion from {0} for {1} in {2} but client is not active. Waiting.", + "[LLUDPSERVER]: Received a CompleteAgentMovement from {0} for {1} in {2} but client is not active yet. Waiting.", + endPoint, client.Name, m_scene.Name); + } + else if (client.SceneAgent == null) + { + // This check exists to catch a condition where the new client has been added to the client + // manager but the SceneAgent has not yet been set in Scene.AddNewClient(). If we are too + // eager, then the new ScenePresence may not have registered a listener for this messsage + // before we try to process it. + // XXX: A better long term fix may be to add the SceneAgent before the client is added to + // the client manager + m_log.DebugFormat( + "[LLUDPSERVER]: Received a CompleteAgentMovement from {0} for {1} in {2} but client SceneAgent not set yet. Waiting.", endPoint, client.Name, m_scene.Name); } + else + { + break; + } } else { m_log.DebugFormat( - "[LLUDPSERVER]: Received a CompleteMovementIntoRegion from {0} in {1} but no client exists. Waiting.", + "[LLUDPSERVER]: Received a CompleteAgentMovement from {0} in {1} but no client exists yet. Waiting.", endPoint, m_scene.Name); } @@ -1730,19 +1745,19 @@ namespace OpenSim.Region.ClientStack.LindenUDP if (client == null) { m_log.DebugFormat( - "[LLUDPSERVER]: No client found for CompleteMovementIntoRegion from {0} in {1} after wait. Dropping.", + "[LLUDPSERVER]: No client found for CompleteAgentMovement from {0} in {1} after wait. Dropping.", endPoint, m_scene.Name); return; } - else if (!client.IsActive) + else if (!client.IsActive || client.SceneAgent == null) { // This check exists to catch a condition where the client has been closed by another thread // but has not yet been removed from the client manager. // The packet could be simply ignored but it is useful to know if this condition occurred for other debugging // purposes. m_log.DebugFormat( - "[LLUDPSERVER]: Received a CompleteMovementIntoRegion from {0} for {1} in {2} but client is not active after wait. Dropping.", + "[LLUDPSERVER]: Received a CompleteAgentMovement from {0} for {1} in {2} but client is not active after wait. Dropping.", endPoint, client.Name, m_scene.Name); return; @@ -1767,7 +1782,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP catch (Exception e) { m_log.ErrorFormat( - "[LLUDPSERVER]: CompleteMovementIntoRegion handling from endpoint {0}, client {1} {2} failed. Exception {3}{4}", + "[LLUDPSERVER]: CompleteAgentMovement handling from endpoint {0}, client {1} {2} failed. Exception {3}{4}", endPoint != null ? endPoint.ToString() : "n/a", client != null ? client.Name : "unknown", client != null ? client.AgentId.ToString() : "unknown", -- cgit v1.1 From b16bc7b01ca0691758e66f85238d657f02271082 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 27 Sep 2013 19:14:21 +0100 Subject: refactor: rename Scene.IncomingCloseAgent() to CloseAgent() in order to make it clear that all non-clientstack callers should be using this rather than RemoveClient() in order to step through the ScenePresence state machine properly. Adds IScene.CloseAgent() to replace RemoveClient() --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 9504f15..5296a6d 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1893,7 +1893,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP client.Kick("Simulator logged you out due to connection timeout."); } - m_scene.IncomingCloseAgent(client.AgentId, true); + m_scene.CloseAgent(client.AgentId, true); } private void IncomingPacketHandler() @@ -2234,7 +2234,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP if (!client.IsLoggingOut) { client.IsLoggingOut = true; - m_scene.IncomingCloseAgent(client.AgentId, false); + m_scene.CloseAgent(client.AgentId, false); } } } -- cgit v1.1 From 2cd95fac736cc99b1a2ad661e4a03810225ffaca Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 27 Sep 2013 22:27:39 +0100 Subject: refactor: Rename Scene.AddNewClient() to AddNewAgent() to make it obvious in the code that this is symmetric with CloseAgent() --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 5296a6d..8d957dc 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1718,7 +1718,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP else if (client.SceneAgent == null) { // This check exists to catch a condition where the new client has been added to the client - // manager but the SceneAgent has not yet been set in Scene.AddNewClient(). If we are too + // manager but the SceneAgent has not yet been set in Scene.AddNewAgent(). If we are too // eager, then the new ScenePresence may not have registered a listener for this messsage // before we try to process it. // XXX: A better long term fix may be to add the SceneAgent before the client is added to -- cgit v1.1 From 5d61c4039dd4f22b20875a0a9dee72179642da9c Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 24 Oct 2013 00:33:14 +0100 Subject: Only set the data present event if we actually queued an outoing packet (not if we sent immediately) --- .../Region/ClientStack/Linden/UDP/LLUDPServer.cs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 8d957dc..981f7ef 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -919,6 +919,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP if (packet.Type == PacketType.CoarseLocationUpdate && allowSplitting) allowSplitting = false; + bool packetQueued = false; + if (allowSplitting && packet.HasVariableBlocks) { byte[][] datas = packet.ToBytesMultiple(); @@ -930,18 +932,21 @@ namespace OpenSim.Region.ClientStack.LindenUDP for (int i = 0; i < packetCount; i++) { byte[] data = datas[i]; - SendPacketData(udpClient, data, packet.Type, category, method); + + if (!SendPacketData(udpClient, data, packet.Type, category, method)) + packetQueued = true; } } else { byte[] data = packet.ToBytes(); - SendPacketData(udpClient, data, packet.Type, category, method); + packetQueued = SendPacketData(udpClient, data, packet.Type, category, method); } PacketPool.Instance.ReturnPacket(packet); - m_dataPresentEvent.Set(); + if (packetQueued) + m_dataPresentEvent.Set(); } /// @@ -955,7 +960,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// The method to call if the packet is not acked by the client. If null, then a standard /// resend of the packet is done. /// - public void SendPacketData( + /// true if the data was sent immediately, false if it was queued for sending + public bool SendPacketData( LLUDPClient udpClient, byte[] data, PacketType type, ThrottleOutPacketType category, UnackedPacketMethod method) { int dataLength = data.Length; @@ -1020,7 +1026,14 @@ namespace OpenSim.Region.ClientStack.LindenUDP // packet so that it isn't sent before a queued update packet. bool requestQueue = type == PacketType.KillObject; if (!outgoingPacket.Client.EnqueueOutgoing(outgoingPacket, requestQueue)) + { SendPacketFinal(outgoingPacket); + return true; + } + else + { + return false; + } #endregion Queue or Send } -- cgit v1.1 From cccdfcb59eea7f8a43fe34ca55ff89b3317904f2 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 24 Oct 2013 00:37:49 +0100 Subject: Comment out LLUDPServer.BroadcastPacket() to reduce code complexity. Appears to be a never used method. --- .../Region/ClientStack/Linden/UDP/LLUDPServer.cs | 76 +++++++++++----------- 1 file changed, 38 insertions(+), 38 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 981f7ef..218c2b2 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -862,44 +862,44 @@ namespace OpenSim.Region.ClientStack.LindenUDP return x == m_location; } - public void BroadcastPacket(Packet packet, ThrottleOutPacketType category, bool sendToPausedAgents, bool allowSplitting) - { - // CoarseLocationUpdate and AvatarGroupsReply packets cannot be split in an automated way - if ((packet.Type == PacketType.CoarseLocationUpdate || packet.Type == PacketType.AvatarGroupsReply) && allowSplitting) - allowSplitting = false; - - if (allowSplitting && packet.HasVariableBlocks) - { - byte[][] datas = packet.ToBytesMultiple(); - int packetCount = datas.Length; - - if (packetCount < 1) - m_log.Error("[LLUDPSERVER]: Failed to split " + packet.Type + " with estimated length " + packet.Length); - - for (int i = 0; i < packetCount; i++) - { - byte[] data = datas[i]; - m_scene.ForEachClient( - delegate(IClientAPI client) - { - if (client is LLClientView) - SendPacketData(((LLClientView)client).UDPClient, data, packet.Type, category, null); - } - ); - } - } - else - { - byte[] data = packet.ToBytes(); - m_scene.ForEachClient( - delegate(IClientAPI client) - { - if (client is LLClientView) - SendPacketData(((LLClientView)client).UDPClient, data, packet.Type, category, null); - } - ); - } - } +// public void BroadcastPacket(Packet packet, ThrottleOutPacketType category, bool sendToPausedAgents, bool allowSplitting) +// { +// // CoarseLocationUpdate and AvatarGroupsReply packets cannot be split in an automated way +// if ((packet.Type == PacketType.CoarseLocationUpdate || packet.Type == PacketType.AvatarGroupsReply) && allowSplitting) +// allowSplitting = false; +// +// if (allowSplitting && packet.HasVariableBlocks) +// { +// byte[][] datas = packet.ToBytesMultiple(); +// int packetCount = datas.Length; +// +// if (packetCount < 1) +// m_log.Error("[LLUDPSERVER]: Failed to split " + packet.Type + " with estimated length " + packet.Length); +// +// for (int i = 0; i < packetCount; i++) +// { +// byte[] data = datas[i]; +// m_scene.ForEachClient( +// delegate(IClientAPI client) +// { +// if (client is LLClientView) +// SendPacketData(((LLClientView)client).UDPClient, data, packet.Type, category, null); +// } +// ); +// } +// } +// else +// { +// byte[] data = packet.ToBytes(); +// m_scene.ForEachClient( +// delegate(IClientAPI client) +// { +// if (client is LLClientView) +// SendPacketData(((LLClientView)client).UDPClient, data, packet.Type, category, null); +// } +// ); +// } +// } /// /// Start the process of sending a packet to the client. -- cgit v1.1 From 3d5a7e9b194d9d6a73091935e316b56d5302dcbb Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 31 Oct 2013 23:45:52 +0000 Subject: Add OutgoingPacketsResentCount clientstack stat. This allows one to monitor the total number of messages resent to clients over time. A constantly increasing stat may indicate a general server network or overloading issue if a fairly high proportion of packets sent A smaller constantly increasing stat may indicate a problem with a particular client-server connection, would need to check "show queues" in this case. --- .../Region/ClientStack/Linden/UDP/LLUDPServer.cs | 27 ++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 218c2b2..ca17771 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -147,6 +147,19 @@ namespace OpenSim.Region.ClientStack.LindenUDP StatsManager.RegisterStat( new Stat( + "OutgoingPacketsResentCount", + "Number of packets resent because a client did not acknowledge receipt", + "", + "", + "clientstack", + scene.Name, + StatType.Pull, + MeasuresOfInterest.AverageChangeOverTime, + stat => stat.Value = m_udpServer.PacketsResentCount, + StatVerbosity.Debug)); + + StatsManager.RegisterStat( + new Stat( "AverageUDPProcessTime", "Average number of milliseconds taken to process each incoming UDP packet in a sample.", "This is for initial receive processing which is separate from the later client LL packet processing stage.", @@ -295,6 +308,16 @@ namespace OpenSim.Region.ClientStack.LindenUDP public Socket Server { get { return null; } } /// + /// Record how many packets have been resent + /// + internal int PacketsResentCount { get; set; } + + /// + /// Record how many packets have been sent + /// + internal int PacketsSentCount { get; set; } + + /// /// Record how many inbound packets could not be recognized as LLUDP packets. /// public int IncomingMalformedPacketCount { get; private set; } @@ -1221,6 +1244,10 @@ namespace OpenSim.Region.ClientStack.LindenUDP // Stats tracking Interlocked.Increment(ref udpClient.PacketsSent); + // We're not going to worry about interlock yet since its not currently critical that this total count + // is 100% correct + PacketsSentCount++; + // Put the UDP payload on the wire AsyncBeginSend(buffer); -- cgit v1.1 From 4c4a1cf71526aa39346f2a8af124a5bcb51e735d Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 31 Oct 2013 23:59:22 +0000 Subject: Start counting resent packets in the places that I missed when the stat was first added a few commits ago --- OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index ca17771..3bd1ef1 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -1237,6 +1237,10 @@ namespace OpenSim.Region.ClientStack.LindenUDP else { Interlocked.Increment(ref udpClient.PacketsResent); + + // We're not going to worry about interlock yet since its not currently critical that this total count + // is 100% correct + PacketsResentCount++; } #endregion Sequence Number Assignment -- cgit v1.1 From 8e1bf55e7b85f8c92237ef2208a4a2c31ba6153c Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Wed, 6 Nov 2013 01:02:20 +0000 Subject: Add IncomingPacketsResentCount clientstack statistics This records how many packets were indicated to be resends by clients Not 100% reliable since clients can lie about resends, but usually would indicate if clients are not receiving UDP acks at all or in a manner they consider timely. --- .../Region/ClientStack/Linden/UDP/LLUDPServer.cs | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 3bd1ef1..c0a4e56 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -134,6 +134,19 @@ namespace OpenSim.Region.ClientStack.LindenUDP StatsManager.RegisterStat( new Stat( + "IncomingPacketsResentCount", + "Number of inbound packets that clients indicate are resends.", + "", + "", + "clientstack", + scene.Name, + StatType.Pull, + MeasuresOfInterest.AverageChangeOverTime, + stat => stat.Value = m_udpServer.IncomingPacketsResentCount, + StatVerbosity.Debug)); + + StatsManager.RegisterStat( + new Stat( "OutgoingUDPSendsCount", "Number of UDP sends performed", "", @@ -318,6 +331,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP internal int PacketsSentCount { get; set; } /// + /// Record how many incoming packets are indicated as resends by clients. + /// + internal int IncomingPacketsResentCount { get; set; } + + /// /// Record how many inbound packets could not be recognized as LLUDP packets. /// public int IncomingMalformedPacketCount { get; private set; } @@ -1467,6 +1485,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP #region Incoming Packet Accounting + // We're not going to worry about interlock yet since its not currently critical that this total count + // is 100% correct + if (packet.Header.Resent) + IncomingPacketsResentCount++; + // Check the archive of received reliable packet IDs to see whether we already received this packet if (packet.Header.Reliable && !udpClient.PacketArchive.TryEnqueue(packet.Header.Sequence)) { -- cgit v1.1