From 2c36106675ad984ba5af674cbb970b17ffa320bb Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 15 Nov 2012 01:14:18 +0000 Subject: Add IncomingPacketsProcessedCount stat for diagnostics. Also puts some packet processing counts in a container named after the scene so that stats can be collected from more than one scene. --- .../Region/ClientStack/Linden/UDP/LLUDPServer.cs | 103 ++++++++++++++++++--- .../ClientStack/Linden/UDP/OpenSimUDPBase.cs | 31 ++----- 2 files changed, 96 insertions(+), 38 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 14cc863..f8391d7 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -70,6 +70,18 @@ namespace OpenSim.Region.ClientStack.LindenUDP public void AddScene(IScene scene) { m_udpServer.AddScene(scene); + + StatsManager.RegisterStat( + new Stat( + "IncomingPacketsProcessedCount", + "Number of inbound UDP packets processed", + "Number of inbound UDP packets processed", + "", + "clientstack", + scene.Name, + StatType.Pull, + stat => stat.Value = m_udpServer.IncomingPacketsProcessed, + StatVerbosity.Debug)); } public bool HandlesRegion(Location x) @@ -170,6 +182,14 @@ namespace OpenSim.Region.ClientStack.LindenUDP private Pool m_incomingPacketPool; + /// + /// Stat for number of packets in the main pool awaiting use. + /// + private Stat m_poolCountStat; + + /// + /// Stat for number of packets in the inbound packet pool awaiting use. + /// private Stat m_incomingPacketPoolStat; private int m_defaultRTO = 0; @@ -342,20 +362,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP m_incomingPacketPool = new Pool(() => new IncomingPacket(), 500); - m_incomingPacketPoolStat - = new Stat( - "IncomingPacketPoolCount", - "Objects within incoming packet pool", - "The number of objects currently stored within the incoming packet pool", - "", - "clientstack", - "packetpool", - StatType.Pull, - stat => stat.Value = m_incomingPacketPool.Count, - StatVerbosity.Debug); - - StatsManager.RegisterStat(m_incomingPacketPoolStat); - return true; } @@ -379,6 +385,53 @@ namespace OpenSim.Region.ClientStack.LindenUDP } /// + /// This is a seperate method so that it can be called once we have an m_scene to distinguish different scene + /// stats. + /// + private void EnablePoolStats() + { + m_poolCountStat + = new Stat( + "UDPPacketBufferPoolCount", + "Objects within the UDPPacketBuffer pool", + "The number of objects currently stored within the UDPPacketBuffer pool", + "", + "clientstack", + m_scene.Name, + StatType.Pull, + stat => stat.Value = Pool.Count, + StatVerbosity.Debug); + + StatsManager.RegisterStat(m_poolCountStat); + + m_incomingPacketPoolStat + = new Stat( + "IncomingPacketPoolCount", + "Objects within incoming packet pool", + "The number of objects currently stored within the incoming packet pool", + "", + "clientstack", + m_scene.Name, + StatType.Pull, + stat => stat.Value = m_incomingPacketPool.Count, + StatVerbosity.Debug); + + StatsManager.RegisterStat(m_incomingPacketPoolStat); + } + + /// + /// Disables pool stats. + /// + private void DisablePoolStats() + { + StatsManager.DeregisterStat(m_poolCountStat); + m_poolCountStat = null; + + StatsManager.DeregisterStat(m_incomingPacketPoolStat); + m_incomingPacketPoolStat = null; + } + + /// /// If the outgoing UDP thread times out, then return client that was being processed to help with debugging. /// /// @@ -416,6 +469,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP m_scene = (Scene)scene; m_location = new Location(m_scene.RegionInfo.RegionHandle); + + // We delay enabling pool stats to AddScene() instead of Initialize() so that we can distinguish pool stats by + // scene name + if (UsePools) + EnablePoolStats(); MainConsole.Instance.Commands.AddCommand( "Debug", @@ -505,12 +563,18 @@ namespace OpenSim.Region.ClientStack.LindenUDP if (enabled == "on") { if (EnablePools()) + { + EnablePoolStats(); MainConsole.Instance.OutputFormat("Packet pools enabled on {0}", m_scene.Name); + } } else if (enabled == "off") { if (DisablePools()) + { + DisablePoolStats(); MainConsole.Instance.OutputFormat("Packet pools disabled on {0}", m_scene.Name); + } } else { @@ -1556,6 +1620,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP private int npacksSent = 0; private int npackNotSent = 0; + /// + /// Number of inbound packets processed since startup. + /// + public long IncomingPacketsProcessed { get; private set; } + private void MonitoredClientOutgoingPacketHandler(IClientAPI client) { nticks++; @@ -1615,7 +1684,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP npacksSent++; } else + { npackNotSent++; + } watch2.Stop(); avgDequeueTicks = (nticks - 1) / (float)nticks * avgDequeueTicks + (watch2.ElapsedTicks / (float)nticks); @@ -1623,7 +1694,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP } else + { m_log.WarnFormat("[LLUDPSERVER]: Client is not connected"); + } } } catch (Exception ex) @@ -1687,6 +1760,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP "[LLUDPSERVER]: Dropped incoming {0} for dead client {1} in {2}", packet.Type, client.Name, m_scene.RegionInfo.RegionName); } + + IncomingPacketsProcessed++; } protected void LogoutHandler(IClientAPI client) diff --git a/OpenSim/Region/ClientStack/Linden/UDP/OpenSimUDPBase.cs b/OpenSim/Region/ClientStack/Linden/UDP/OpenSimUDPBase.cs index 85cbb06..808d177 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/OpenSimUDPBase.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/OpenSimUDPBase.cs @@ -61,14 +61,14 @@ namespace OpenMetaverse private bool m_asyncPacketHandling; /// - /// Pool to use for handling data. May be null if UsePools = false; + /// Are we to use object pool(s) to reduce memory churn when receiving data? /// - protected OpenSim.Framework.Pool m_pool; + public bool UsePools { get; protected set; } /// - /// Are we to use object pool(s) to reduce memory churn when receiving data? + /// Pool to use for handling data. May be null if UsePools = false; /// - public bool UsePools { get; protected set; } + protected OpenSim.Framework.Pool Pool { get; private set; } /// Returns true if the server is currently listening for inbound packets, otherwise false public bool IsRunningInbound { get; private set; } @@ -77,8 +77,6 @@ namespace OpenMetaverse /// If IsRunningOut = false, then any request to send a packet is simply dropped. public bool IsRunningOutbound { get; private set; } - private Stat m_poolCountStat; - /// /// Default constructor /// @@ -182,21 +180,7 @@ namespace OpenMetaverse { if (!UsePools) { - m_pool = new Pool(() => new UDPPacketBuffer(), 500); - - m_poolCountStat - = new Stat( - "UDPPacketBufferPoolCount", - "Objects within the UDPPacketBuffer pool", - "The number of objects currently stored within the UDPPacketBuffer pool", - "", - "clientstack", - "packetpool", - StatType.Pull, - stat => stat.Value = m_pool.Count, - StatVerbosity.Debug); - - StatsManager.RegisterStat(m_poolCountStat); + Pool = new Pool(() => new UDPPacketBuffer(), 500); UsePools = true; @@ -211,7 +195,6 @@ namespace OpenMetaverse if (UsePools) { UsePools = false; - StatsManager.DeregisterStat(m_poolCountStat); // We won't null out the pool to avoid a race condition with code that may be in the middle of using it. @@ -226,7 +209,7 @@ namespace OpenMetaverse UDPPacketBuffer buf; if (UsePools) - buf = m_pool.GetObject(); + buf = Pool.GetObject(); else buf = new UDPPacketBuffer(); @@ -309,7 +292,7 @@ namespace OpenMetaverse finally { if (UsePools) - m_pool.ReturnObject(buffer); + Pool.ReturnObject(buffer); // Synchronous mode waits until the packet callback completes // before starting the receive to fetch another packet -- cgit v1.1 From 038528dc80e0fe03956d61872ba30a887b2890a1 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 15 Nov 2012 02:02:59 +0000 Subject: Make PacketPool class stats pull stats instead of push stats so they can be lifted up into LLUDPServer and be distiguished by scene name --- .../Region/ClientStack/Linden/UDP/LLUDPServer.cs | 54 ++++++++++ .../ClientStack/Linden/UDP/OpenSimUDPBase.cs | 2 +- .../Region/ClientStack/Linden/UDP/PacketPool.cs | 114 ++++++++++----------- 3 files changed, 107 insertions(+), 63 deletions(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index f8391d7..fcc69c0 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -469,6 +469,60 @@ namespace OpenSim.Region.ClientStack.LindenUDP m_scene = (Scene)scene; m_location = new Location(m_scene.RegionInfo.RegionHandle); + + // XXX: These stats are also pool stats but we register them separately since they are currently not + // turned on and off by EnablePools()/DisablePools() + StatsManager.RegisterStat( + new PercentageStat( + "PacketsReused", + "Packets reused", + "Number of packets reused out of all requests to the packet pool", + "clientstack", + m_scene.Name, + StatType.Pull, + stat => + { PercentageStat pstat = (PercentageStat)stat; + pstat.Consequent = PacketPool.Instance.PacketsRequested; + pstat.Antecedent = PacketPool.Instance.PacketsReused; }, + StatVerbosity.Debug)); + + StatsManager.RegisterStat( + new PercentageStat( + "PacketDataBlocksReused", + "Packet data blocks reused", + "Number of data blocks reused out of all requests to the packet pool", + "clientstack", + m_scene.Name, + StatType.Pull, + stat => + { PercentageStat pstat = (PercentageStat)stat; + pstat.Consequent = PacketPool.Instance.BlocksRequested; + pstat.Antecedent = PacketPool.Instance.BlocksReused; }, + StatVerbosity.Debug)); + + StatsManager.RegisterStat( + new Stat( + "PacketsPoolCount", + "Objects within the packet pool", + "The number of objects currently stored within the packet pool", + "", + "clientstack", + m_scene.Name, + StatType.Pull, + stat => stat.Value = PacketPool.Instance.PacketsPooled, + StatVerbosity.Debug)); + + StatsManager.RegisterStat( + new Stat( + "PacketDataBlocksPoolCount", + "Objects within the packet data block pool", + "The number of objects currently stored within the packet data block pool", + "", + "clientstack", + m_scene.Name, + StatType.Pull, + stat => stat.Value = PacketPool.Instance.BlocksPooled, + StatVerbosity.Debug)); // We delay enabling pool stats to AddScene() instead of Initialize() so that we can distinguish pool stats by // scene name diff --git a/OpenSim/Region/ClientStack/Linden/UDP/OpenSimUDPBase.cs b/OpenSim/Region/ClientStack/Linden/UDP/OpenSimUDPBase.cs index 808d177..3f7ca2b 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/OpenSimUDPBase.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/OpenSimUDPBase.cs @@ -334,4 +334,4 @@ namespace OpenMetaverse catch (ObjectDisposedException) { } } } -} +} \ No newline at end of file diff --git a/OpenSim/Region/ClientStack/Linden/UDP/PacketPool.cs b/OpenSim/Region/ClientStack/Linden/UDP/PacketPool.cs index 9f22fb4..1fdc410 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/PacketPool.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/PacketPool.cs @@ -41,29 +41,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP private static readonly PacketPool instance = new PacketPool(); - private bool packetPoolEnabled = true; - private bool dataBlockPoolEnabled = true; - - private PercentageStat m_packetsReusedStat = new PercentageStat( - "PacketsReused", - "Packets reused", - "Number of packets reused out of all requests to the packet pool", - "clientstack", - "packetpool", - StatType.Push, - null, - StatVerbosity.Debug); - - private PercentageStat m_blocksReusedStat = new PercentageStat( - "PacketDataBlocksReused", - "Packet data blocks reused", - "Number of data blocks reused out of all requests to the packet pool", - "clientstack", - "packetpool", - StatType.Push, - null, - StatVerbosity.Debug); - /// /// Pool of packets available for reuse. /// @@ -76,46 +53,59 @@ namespace OpenSim.Region.ClientStack.LindenUDP get { return instance; } } - public bool RecyclePackets + public bool RecyclePackets { get; set; } + + public bool RecycleDataBlocks { get; set; } + + /// + /// The number of packets pooled + /// + public int PacketsPooled { - set { packetPoolEnabled = value; } - get { return packetPoolEnabled; } + get + { + lock (pool) + return pool.Count; + } } - public bool RecycleDataBlocks + /// + /// The number of blocks pooled. + /// + public int BlocksPooled { - set { dataBlockPoolEnabled = value; } - get { return dataBlockPoolEnabled; } + get + { + lock (DataBlocks) + return DataBlocks.Count; + } } + /// + /// Number of packets requested. + /// + public long PacketsRequested { get; private set; } + + /// + /// Number of packets reused. + /// + public long PacketsReused { get; private set; } + + /// + /// Number of packet blocks requested. + /// + public long BlocksRequested { get; private set; } + + /// + /// Number of packet blocks reused. + /// + public long BlocksReused { get; private set; } + private PacketPool() { - StatsManager.RegisterStat(m_packetsReusedStat); - StatsManager.RegisterStat(m_blocksReusedStat); - - StatsManager.RegisterStat( - new Stat( - "PacketsPoolCount", - "Objects within the packet pool", - "The number of objects currently stored within the packet pool", - "", - "clientstack", - "packetpool", - StatType.Pull, - stat => { lock (pool) { stat.Value = pool.Count; } }, - StatVerbosity.Debug)); - - StatsManager.RegisterStat( - new Stat( - "PacketDataBlocksPoolCount", - "Objects within the packet data block pool", - "The number of objects currently stored within the packet data block pool", - "", - "clientstack", - "packetpool", - StatType.Pull, - stat => { lock (DataBlocks) { stat.Value = DataBlocks.Count; } }, - StatVerbosity.Debug)); + // defaults + RecyclePackets = true; + RecycleDataBlocks = true; } /// @@ -125,11 +115,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// Guaranteed to always return a packet, whether from the pool or newly constructed. public Packet GetPacket(PacketType type) { - m_packetsReusedStat.Consequent++; + PacketsRequested++; Packet packet; - if (!packetPoolEnabled) + if (!RecyclePackets) return Packet.BuildPacket(type); lock (pool) @@ -146,7 +136,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP // m_log.DebugFormat("[PACKETPOOL]: Pulling {0} packet", type); // Recycle old packages - m_packetsReusedStat.Antecedent++; + PacketsReused++; packet = pool[type].Pop(); } @@ -215,7 +205,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// public void ReturnPacket(Packet packet) { - if (dataBlockPoolEnabled) + if (RecycleDataBlocks) { switch (packet.Type) { @@ -239,7 +229,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP } } - if (packetPoolEnabled) + if (RecyclePackets) { switch (packet.Type) { @@ -277,7 +267,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP { lock (DataBlocks) { - m_blocksReusedStat.Consequent++; + BlocksRequested++; Stack s; @@ -285,7 +275,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP { if (s.Count > 0) { - m_blocksReusedStat.Antecedent++; + BlocksReused++; return (T)s.Pop(); } } -- cgit v1.1 From df4da51f04ea2900032a6bd2749039bee5338f54 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 15 Nov 2012 03:04:46 +0000 Subject: Following on from 4f982596, launch map name requests on an async thread from LLClientView directly. This releases the inbound packet handling thread marginally quicker and is more consistent with the other async packet handling --- OpenSim/Region/ClientStack/Linden/UDP/LLClientView.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLClientView.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLClientView.cs index 7382e09..c93dbfc 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLClientView.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLClientView.cs @@ -5322,7 +5322,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP AddLocalPacketHandler(PacketType.RezScript, HandleRezScript); AddLocalPacketHandler(PacketType.MapLayerRequest, HandleMapLayerRequest, false); AddLocalPacketHandler(PacketType.MapBlockRequest, HandleMapBlockRequest, false); - AddLocalPacketHandler(PacketType.MapNameRequest, HandleMapNameRequest, false); + AddLocalPacketHandler(PacketType.MapNameRequest, HandleMapNameRequest); AddLocalPacketHandler(PacketType.TeleportLandmarkRequest, HandleTeleportLandmarkRequest); AddLocalPacketHandler(PacketType.TeleportCancel, HandleTeleportCancel); AddLocalPacketHandler(PacketType.TeleportLocationRequest, HandleTeleportLocationRequest); -- cgit v1.1 From 57273ef7b24c5ae466ebb6051a5c614a3c07fe2d Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 15 Nov 2012 03:07:45 +0000 Subject: Do HandleMapLayerRequest on its own thread rather than on the main inbound udp packet handling thread. There's no obvious race condition reason for doing this on the main packet handling thread. --- OpenSim/Region/ClientStack/Linden/UDP/LLClientView.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLClientView.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLClientView.cs index c93dbfc..92a630f 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLClientView.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLClientView.cs @@ -5320,7 +5320,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP AddLocalPacketHandler(PacketType.RemoveTaskInventory, HandleRemoveTaskInventory); AddLocalPacketHandler(PacketType.MoveTaskInventory, HandleMoveTaskInventory); AddLocalPacketHandler(PacketType.RezScript, HandleRezScript); - AddLocalPacketHandler(PacketType.MapLayerRequest, HandleMapLayerRequest, false); + AddLocalPacketHandler(PacketType.MapLayerRequest, HandleMapLayerRequest); AddLocalPacketHandler(PacketType.MapBlockRequest, HandleMapBlockRequest, false); AddLocalPacketHandler(PacketType.MapNameRequest, HandleMapNameRequest); AddLocalPacketHandler(PacketType.TeleportLandmarkRequest, HandleTeleportLandmarkRequest); -- cgit v1.1 From daf03bfb567a7a93a9259945386325fd924f8bd1 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Thu, 15 Nov 2012 03:09:20 +0000 Subject: Handle Map block requests on a separate thread rather than the main packet handling thread. This prevents a slow grid information network call from holding up the main packet handling thread. There's no obvious race condition reason for not doing this asynchronously. --- OpenSim/Region/ClientStack/Linden/UDP/LLClientView.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Region/ClientStack/Linden/UDP') diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLClientView.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLClientView.cs index 92a630f..4fd81fa 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLClientView.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLClientView.cs @@ -5321,7 +5321,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP AddLocalPacketHandler(PacketType.MoveTaskInventory, HandleMoveTaskInventory); AddLocalPacketHandler(PacketType.RezScript, HandleRezScript); AddLocalPacketHandler(PacketType.MapLayerRequest, HandleMapLayerRequest); - AddLocalPacketHandler(PacketType.MapBlockRequest, HandleMapBlockRequest, false); + AddLocalPacketHandler(PacketType.MapBlockRequest, HandleMapBlockRequest); AddLocalPacketHandler(PacketType.MapNameRequest, HandleMapNameRequest); AddLocalPacketHandler(PacketType.TeleportLandmarkRequest, HandleTeleportLandmarkRequest); AddLocalPacketHandler(PacketType.TeleportCancel, HandleTeleportCancel); -- cgit v1.1