1 files changed, 418 insertions, 50 deletions

diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs
index dac3306..fc4e1b2 100644
--- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs
+++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs
@@ -37,6 +37,7 @@ using log4net;
 using Nini.Config;
 using OpenMetaverse.Packets;
 using OpenSim.Framework;
+using OpenSim.Framework.Console;
 using OpenSim.Framework.Monitoring;
 using OpenSim.Region.Framework.Scenes;
 using OpenMetaverse;
@@ -69,6 +70,19 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         public void AddScene(IScene scene)
         {
             m_udpServer.AddScene(scene);
+
+            StatsManager.RegisterStat(
+                new Stat(
+                    "IncomingPacketsProcessedCount",
+                    "Number of inbound UDP packets processed",
+                    "Number of inbound UDP packets processed",
+                    "",
+                    "clientstack",
+                    scene.Name,
+                    StatType.Pull,
+                    MeasuresOfInterest.AverageChangeOverTime,
+                    stat => stat.Value = m_udpServer.IncomingPacketsProcessed,
+                    StatVerbosity.Debug));
         }
 
         public bool HandlesRegion(Location x)
@@ -100,9 +114,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
         /// <summary>The measured resolution of Environment.TickCount</summary>
         public readonly float TickCountResolution;
+
         /// <summary>Number of prim updates to put on the queue each time the 
         /// OnQueueEmpty event is triggered for updates</summary>
         public readonly int PrimUpdatesPerCallback;
+
         /// <summary>Number of texture packets to put on the queue each time the
         /// OnQueueEmpty event is triggered for textures</summary>
         public readonly int TextureSendLimit;
@@ -124,28 +140,37 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         
         /// <summary>Manages authentication for agent circuits</summary>
         private AgentCircuitManager m_circuitManager;
+
         /// <summary>Reference to the scene this UDP server is attached to</summary>
         protected Scene m_scene;
+
         /// <summary>The X/Y coordinates of the scene this UDP server is attached to</summary>
         private Location m_location;
+
         /// <summary>The size of the receive buffer for the UDP socket. This value
         /// is passed up to the operating system and used in the system networking
         /// stack. Use zero to leave this value as the default</summary>
         private int m_recvBufferSize;
+
         /// <summary>Flag to process packets asynchronously or synchronously</summary>
         private bool m_asyncPacketHandling;
+
         /// <summary>Tracks whether or not a packet was sent each round so we know
         /// whether or not to sleep</summary>
         private bool m_packetSent;
 
         /// <summary>Environment.TickCount of the last time that packet stats were reported to the scene</summary>
         private int m_elapsedMSSinceLastStatReport = 0;
+
         /// <summary>Environment.TickCount of the last time the outgoing packet handler executed</summary>
         private int m_tickLastOutgoingPacketHandler;
+
         /// <summary>Keeps track of the number of elapsed milliseconds since the last time the outgoing packet handler looped</summary>
         private int m_elapsedMSOutgoingPacketHandler;
+
         /// <summary>Keeps track of the number of 100 millisecond periods elapsed in the outgoing packet handler executed</summary>
         private int m_elapsed100MSOutgoingPacketHandler;
+
         /// <summary>Keeps track of the number of 500 millisecond periods elapsed in the outgoing packet handler executed</summary>
         private int m_elapsed500MSOutgoingPacketHandler;
 
@@ -159,6 +184,17 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         protected bool m_sendPing;
 
         private ExpiringCache<IPEndPoint, Queue<UDPPacketBuffer>> m_pendingCache = new ExpiringCache<IPEndPoint, Queue<UDPPacketBuffer>>();
+        private Pool<IncomingPacket> m_incomingPacketPool;
+
+        /// <summary>
+        /// Stat for number of packets in the main pool awaiting use.
+        /// </summary>
+        private Stat m_poolCountStat;
+
+        /// <summary>
+        /// Stat for number of packets in the inbound packet pool awaiting use.
+        /// </summary>
+        private Stat m_incomingPacketPoolStat;
 
         private int m_defaultRTO = 0;
         private int m_maxRTO = 0;
@@ -180,7 +216,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         /// </summary>
         private IClientAPI m_currentIncomingClient;
 
-        public LLUDPServer(IPAddress listenIP, ref uint port, int proxyPortOffsetParm, bool allow_alternate_port, IConfigSource configSource, AgentCircuitManager circuitManager)
+        public LLUDPServer(
+            IPAddress listenIP, ref uint port, int proxyPortOffsetParm, bool allow_alternate_port,
+            IConfigSource configSource, AgentCircuitManager circuitManager)
             : base(listenIP, (int)port)
         {
             #region Environment.TickCount Measurement
@@ -202,6 +240,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
             m_circuitManager = circuitManager;
             int sceneThrottleBps = 0;
+            bool usePools = false;
 
             IConfig config = configSource.Configs["ClientStack.LindenUDP"];
             if (config != null)
@@ -227,6 +266,16 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                 m_pausedAckTimeout = 1000 * 300; // 5 minutes
             }
 
+            // FIXME: This actually only needs to be done once since the PacketPool is shared across all servers.
+            // However, there is no harm in temporarily doing it multiple times.
+            IConfig packetConfig = configSource.Configs["PacketPool"];
+            if (packetConfig != null)
+            {
+                PacketPool.Instance.RecyclePackets = packetConfig.GetBoolean("RecyclePackets", true);
+                PacketPool.Instance.RecycleDataBlocks = packetConfig.GetBoolean("RecycleDataBlocks", true);
+                usePools = packetConfig.GetBoolean("RecycleBaseUDPPackets", usePools);
+            }
+
             #region BinaryStats
             config = configSource.Configs["Statistics.Binary"];
             m_shouldCollectStats = false;
@@ -254,20 +303,28 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
             m_throttle = new TokenBucket(null, sceneThrottleBps);
             ThrottleRates = new ThrottleRates(configSource);
+
+            if (usePools)
+                EnablePools();
         }
 
         public void Start()
         {
-            if (m_scene == null)
-                throw new InvalidOperationException("[LLUDPSERVER]: Cannot LLUDPServer.Start() without an IScene reference");
+            StartInbound();
+            StartOutbound();
+
+            m_elapsedMSSinceLastStatReport = Environment.TickCount;
+        }
 
+        private void StartInbound()
+        {
             m_log.InfoFormat(
-                "[LLUDPSERVER]: Starting the LLUDP server in {0} mode",
-                m_asyncPacketHandling ? "asynchronous" : "synchronous");
+                "[LLUDPSERVER]: Starting inbound packet processing for the LLUDP server in {0} mode with UsePools = {1}",
+                m_asyncPacketHandling ? "asynchronous" : "synchronous", UsePools);
 
-            base.Start(m_recvBufferSize, m_asyncPacketHandling);
+            base.StartInbound(m_recvBufferSize, m_asyncPacketHandling);
 
-            // Start the packet processing threads
+            // This thread will process the packets received that are placed on the packetInbox
             Watchdog.StartThread(
                 IncomingPacketHandler,
                 string.Format("Incoming Packets ({0})", m_scene.RegionInfo.RegionName),
@@ -276,6 +333,13 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                 true,
                 GetWatchdogIncomingAlarmData,
                 Watchdog.DEFAULT_WATCHDOG_TIMEOUT_MS);
+        }
+
+        private new void StartOutbound()
+        {
+            m_log.Info("[LLUDPSERVER]: Starting outbound packet processing for the LLUDP server");
+
+            base.StartOutbound();
 
             Watchdog.StartThread(
                 OutgoingPacketHandler,
@@ -285,8 +349,90 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                 true,
                 GetWatchdogOutgoingAlarmData,
                 Watchdog.DEFAULT_WATCHDOG_TIMEOUT_MS);
+        }
 
-            m_elapsedMSSinceLastStatReport = Environment.TickCount;
+        public void Stop()
+        {
+            m_log.Info("[LLUDPSERVER]: Shutting down the LLUDP server for " + m_scene.RegionInfo.RegionName);
+            base.StopOutbound();
+            base.StopInbound();
+        }
+
+        protected override bool EnablePools()
+        {
+            if (!UsePools)
+            {
+                base.EnablePools();
+
+                m_incomingPacketPool = new Pool<IncomingPacket>(() => new IncomingPacket(), 500);
+
+                return true;
+            }
+
+            return false;
+        }
+
+        protected override bool DisablePools()
+        {
+            if (UsePools)
+            {
+                base.DisablePools();
+
+                StatsManager.DeregisterStat(m_incomingPacketPoolStat);
+
+                // We won't null out the pool to avoid a race condition with code that may be in the middle of using it.
+
+                return true;
+            }
+
+            return false;
+        }
+
+        /// <summary>
+        /// This is a seperate method so that it can be called once we have an m_scene to distinguish different scene
+        /// stats.
+        /// </summary>
+        private void EnablePoolStats()
+        {
+            m_poolCountStat
+                = new Stat(
+                    "UDPPacketBufferPoolCount",
+                    "Objects within the UDPPacketBuffer pool",
+                    "The number of objects currently stored within the UDPPacketBuffer pool",
+                    "",
+                    "clientstack",
+                    m_scene.Name,
+                    StatType.Pull,
+                    stat => stat.Value = Pool.Count,
+                    StatVerbosity.Debug);
+
+            StatsManager.RegisterStat(m_poolCountStat);
+
+            m_incomingPacketPoolStat
+                = new Stat(
+                    "IncomingPacketPoolCount",
+                    "Objects within incoming packet pool",
+                    "The number of objects currently stored within the incoming packet pool",
+                    "",
+                    "clientstack",
+                    m_scene.Name,
+                    StatType.Pull,
+                    stat => stat.Value = m_incomingPacketPool.Count,
+                    StatVerbosity.Debug);
+
+            StatsManager.RegisterStat(m_incomingPacketPoolStat);
+        }
+
+        /// <summary>
+        /// Disables pool stats.
+        /// </summary>
+        private void DisablePoolStats()
+        {
+            StatsManager.DeregisterStat(m_poolCountStat);
+            m_poolCountStat = null;
+
+            StatsManager.DeregisterStat(m_incomingPacketPoolStat);
+            m_incomingPacketPoolStat = null;
         }
 
         /// <summary>
@@ -311,12 +457,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                 m_currentOutgoingClient != null ? m_currentOutgoingClient.Name : "none");
         }
 
-        public new void Stop()
-        {
-            m_log.Info("[LLUDPSERVER]: Shutting down the LLUDP server for " + m_scene.RegionInfo.RegionName);
-            base.Stop();
-        }
-
         public void AddScene(IScene scene)
         {
             if (m_scene != null)
@@ -333,6 +473,182 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
             m_scene = (Scene)scene;
             m_location = new Location(m_scene.RegionInfo.RegionHandle);
+
+            // XXX: These stats are also pool stats but we register them separately since they are currently not
+            // turned on and off by EnablePools()/DisablePools()
+            StatsManager.RegisterStat(
+                new PercentageStat(
+                    "PacketsReused",
+                    "Packets reused",
+                    "Number of packets reused out of all requests to the packet pool",
+                    "clientstack",
+                    m_scene.Name,
+                    StatType.Pull,
+                    stat => 
+                        { PercentageStat pstat = (PercentageStat)stat; 
+                          pstat.Consequent = PacketPool.Instance.PacketsRequested; 
+                          pstat.Antecedent = PacketPool.Instance.PacketsReused; },
+                    StatVerbosity.Debug));
+
+            StatsManager.RegisterStat(
+                new PercentageStat(
+                    "PacketDataBlocksReused",
+                    "Packet data blocks reused",
+                    "Number of data blocks reused out of all requests to the packet pool",
+                    "clientstack",
+                    m_scene.Name,
+                    StatType.Pull,
+                    stat =>
+                        { PercentageStat pstat = (PercentageStat)stat; 
+                          pstat.Consequent = PacketPool.Instance.BlocksRequested; 
+                          pstat.Antecedent = PacketPool.Instance.BlocksReused; },
+                    StatVerbosity.Debug));
+
+            StatsManager.RegisterStat(
+                new Stat(
+                    "PacketsPoolCount",
+                    "Objects within the packet pool",
+                    "The number of objects currently stored within the packet pool",
+                    "",
+                    "clientstack",
+                    m_scene.Name,
+                    StatType.Pull,
+                    stat => stat.Value = PacketPool.Instance.PacketsPooled,
+                    StatVerbosity.Debug));
+
+            StatsManager.RegisterStat(
+                new Stat(
+                    "PacketDataBlocksPoolCount",
+                    "Objects within the packet data block pool",
+                    "The number of objects currently stored within the packet data block pool",
+                    "",
+                    "clientstack",
+                    m_scene.Name,
+                    StatType.Pull,
+                    stat => stat.Value = PacketPool.Instance.BlocksPooled,
+                    StatVerbosity.Debug));
+        
+            // We delay enabling pool stats to AddScene() instead of Initialize() so that we can distinguish pool stats by
+            // scene name
+            if (UsePools)
+                EnablePoolStats();
+
+            MainConsole.Instance.Commands.AddCommand(
+                "Debug",
+                false,
+                "debug lludp start",
+                "debug lludp start <in|out|all>",
+                "Control LLUDP packet processing.",
+                "No effect if packet processing has already started.\n"
+                    + "in  - start inbound processing.\n"
+                    + "out - start outbound processing.\n"
+                    + "all - start in and outbound processing.\n",
+                HandleStartCommand);
+
+            MainConsole.Instance.Commands.AddCommand(
+                "Debug",
+                false,
+                "debug lludp stop",
+                "debug lludp stop <in|out|all>",
+                "Stop LLUDP packet processing.",
+                "No effect if packet processing has already stopped.\n"
+                    + "in  - stop inbound processing.\n"
+                    + "out - stop outbound processing.\n"
+                    + "all - stop in and outbound processing.\n",
+                HandleStopCommand);
+
+            MainConsole.Instance.Commands.AddCommand(
+                "Debug",
+                false,
+                "debug lludp pool",
+                "debug lludp pool <on|off>",
+                "Turn object pooling within the lludp component on or off.",
+                HandlePoolCommand);
+
+            MainConsole.Instance.Commands.AddCommand(
+                "Debug",
+                false,
+                "debug lludp status",
+                "debug lludp status",
+                "Return status of LLUDP packet processing.",
+                HandleStatusCommand);
+        }
+
+        private void HandleStartCommand(string module, string[] args)
+        {
+            if (args.Length != 4)
+            {
+                MainConsole.Instance.Output("Usage: debug lludp start <in|out|all>");
+                return;
+            }
+
+            string subCommand = args[3];
+
+            if (subCommand == "in" || subCommand == "all")
+                StartInbound();
+
+            if (subCommand == "out" || subCommand == "all")
+                StartOutbound();
+        }
+
+        private void HandleStopCommand(string module, string[] args)
+        {
+            if (args.Length != 4)
+            {
+                MainConsole.Instance.Output("Usage: debug lludp stop <in|out|all>");
+                return;
+            }
+
+            string subCommand = args[3];
+
+            if (subCommand == "in" || subCommand == "all")
+                StopInbound();
+
+            if (subCommand == "out" || subCommand == "all")
+                StopOutbound();
+        }
+
+        private void HandlePoolCommand(string module, string[] args)
+        {
+            if (args.Length != 4)
+            {
+                MainConsole.Instance.Output("Usage: debug lludp pool <on|off>");
+                return;
+            }
+
+            string enabled = args[3];
+
+            if (enabled == "on")
+            {
+                if (EnablePools())
+                {
+                    EnablePoolStats();
+                    MainConsole.Instance.OutputFormat("Packet pools enabled on {0}", m_scene.Name);
+                }
+            }
+            else if (enabled == "off")
+            {
+                if (DisablePools())
+                {
+                    DisablePoolStats();
+                    MainConsole.Instance.OutputFormat("Packet pools disabled on {0}", m_scene.Name);
+                }
+            }
+            else
+            {
+                MainConsole.Instance.Output("Usage: debug lludp pool <on|off>");
+            }
+        }
+
+        private void HandleStatusCommand(string module, string[] args)
+        {
+            MainConsole.Instance.OutputFormat(
+                "IN  LLUDP packet processing for {0} is {1}", m_scene.Name, IsRunningInbound ? "enabled" : "disabled");
+
+            MainConsole.Instance.OutputFormat(
+                "OUT LLUDP packet processing for {0} is {1}", m_scene.Name, IsRunningOutbound ? "enabled" : "disabled");
+
+            MainConsole.Instance.OutputFormat("LLUDP pools in {0} are {1}", m_scene.Name, UsePools ? "on" : "off");
         }
 
         public bool HandlesRegion(Location x)
@@ -416,6 +732,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                 byte[] data = packet.ToBytes();
                 SendPacketData(udpClient, data, packet.Type, category, method);
             }
+
+            PacketPool.Instance.ReturnPacket(packet);
         }
 
         /// <summary>
@@ -700,7 +1018,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             LLUDPClient udpClient = null;
             Packet packet = null;
             int packetEnd = buffer.DataLength - 1;
-            IPEndPoint address = (IPEndPoint)buffer.RemoteEndPoint;
+            IPEndPoint endPoint = (IPEndPoint)buffer.RemoteEndPoint;
 
             #region Decoding
 
@@ -710,7 +1028,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 //                    "[LLUDPSERVER]: Dropping undersized packet with {0} bytes received from {1} in {2}",
 //                    buffer.DataLength, buffer.RemoteEndPoint, m_scene.RegionInfo.RegionName);
 
-                return; // Drop undersizd packet
+                return; // Drop undersized packet
             }
 
             int headerLen = 7;
@@ -733,7 +1051,13 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
             try
             {
-                packet = Packet.BuildPacket(buffer.Data, ref packetEnd,
+//                packet = Packet.BuildPacket(buffer.Data, ref packetEnd,
+//                    // Only allocate a buffer for zerodecoding if the packet is zerocoded
+//                    ((buffer.Data[0] & Helpers.MSG_ZEROCODED) != 0) ? new byte[4096] : null);
+                // If OpenSimUDPBase.UsePool == true (which is currently separate from the PacketPool) then we
+                // assume that packet construction does not retain a reference to byte[] buffer.Data (instead, all
+                // bytes are copied out).
+                packet = PacketPool.Instance.GetPacket(buffer.Data, ref packetEnd,
                     // Only allocate a buffer for zerodecoding if the packet is zerocoded
                     ((buffer.Data[0] & Helpers.MSG_ZEROCODED) != 0) ? new byte[4096] : null);
             }
@@ -748,11 +1072,13 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
                 return; // Drop short packet
             }
-            catch(Exception e)
+            catch (Exception e)
             {
                 if (m_malformedCount < 100)
                     m_log.DebugFormat("[LLUDPSERVER]: Dropped malformed packet: " + e.ToString());
+
                 m_malformedCount++;
+
                 if ((m_malformedCount % 100000) == 0)
                     m_log.DebugFormat("[LLUDPSERVER]: Received {0} malformed packets so far, probable network attack.", m_malformedCount);
             }
@@ -772,7 +1098,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
             // If there is already a client for this endpoint, don't process UseCircuitCode
             IClientAPI client = null;
-            if (!m_scene.TryGetClient(address, out client))
+            if (!m_scene.TryGetClient(endPoint, out client) || !(client is LLClientView))
             {
                 // UseCircuitCode handling
                 if (packet.Type == PacketType.UseCircuitCode)
@@ -780,13 +1106,15 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     // And if there is a UseCircuitCode pending, also drop it
                     lock (m_pendingCache)
                     {
-                        if (m_pendingCache.Contains(address))
+                        if (m_pendingCache.Contains(endPoint))
                             return;
 
-                        m_pendingCache.AddOrUpdate(address, new Queue<UDPPacketBuffer>(), 60);
+                        m_pendingCache.AddOrUpdate(endPoint, new Queue<UDPPacketBuffer>(), 60);
                     }
 
-                    object[] array = new object[] { buffer, packet };
+                    // We need to copy the endpoint so that it doesn't get changed when another thread reuses the
+                    // buffer.
+                    object[] array = new object[] { new IPEndPoint(endPoint.Address, endPoint.Port), packet };
 
                     Util.FireAndForget(HandleUseCircuitCode, array);
 
@@ -798,7 +1126,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             lock (m_pendingCache)
             {
                 Queue<UDPPacketBuffer> queue;
-                if (m_pendingCache.TryGetValue(address, out queue))
+                if (m_pendingCache.TryGetValue(endPoint, out queue))
                 {
                     //m_log.DebugFormat("[LLUDPSERVER]: Enqueued a {0} packet into the pending queue", packet.Type);
                     queue.Enqueue(buffer);
@@ -834,6 +1162,10 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             // Handle appended ACKs
             if (packet.Header.AppendedAcks && packet.Header.AckList != null)
             {
+//                m_log.DebugFormat(
+//                    "[LLUDPSERVER]: Handling {0} appended acks from {1} in {2}",
+//                    packet.Header.AckList.Length, client.Name, m_scene.Name);
+
                 for (int i = 0; i < packet.Header.AckList.Length; i++)
                     udpClient.NeedAcks.Acknowledge(packet.Header.AckList[i], now, packet.Header.Resent);
             }
@@ -843,6 +1175,10 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             {
                 PacketAckPacket ackPacket = (PacketAckPacket)packet;
 
+//                m_log.DebugFormat(
+//                    "[LLUDPSERVER]: Handling {0} packet acks for {1} in {2}",
+//                    ackPacket.Packets.Length, client.Name, m_scene.Name);
+
                 for (int i = 0; i < ackPacket.Packets.Length; i++)
                     udpClient.NeedAcks.Acknowledge(ackPacket.Packets[i].ID, now, packet.Header.Resent);
 
@@ -856,6 +1192,10 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
             if (packet.Header.Reliable)
             {
+//                m_log.DebugFormat(
+//                    "[LLUDPSERVER]: Adding ack request for {0} {1} from {2} in {3}",
+//                    packet.Type, packet.Header.Sequence, client.Name, m_scene.Name);
+
                 udpClient.PendingAcks.Enqueue(packet.Header.Sequence);
 
                 // This is a somewhat odd sequence of steps to pull the client.BytesSinceLastACK value out,
@@ -902,6 +1242,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
             if (packet.Type == PacketType.StartPingCheck)
             {
+//                m_log.DebugFormat("[LLUDPSERVER]: Handling ping from {0} in {1}", client.Name, m_scene.Name);
+
                 // We don't need to do anything else with ping checks
                 StartPingCheckPacket startPing = (StartPingCheckPacket)packet;
                 CompletePing(udpClient, startPing.PingID.PingID);
@@ -921,13 +1263,25 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
             #endregion Ping Check Handling
 
+            IncomingPacket incomingPacket;
+
             // Inbox insertion
-            if (packet.Type == PacketType.AgentUpdate ||
-                packet.Type == PacketType.ChatFromViewer)
-                packetInbox.EnqueueHigh(new IncomingPacket((LLClientView)client, packet));
+            if (UsePools)
+            {
+                incomingPacket = m_incomingPacketPool.GetObject();
+                incomingPacket.Client = (LLClientView)client;
+                incomingPacket.Packet = packet;
+            }
+            else
+            {
+                incomingPacket = new IncomingPacket((LLClientView)client, packet);
+            }
+
+            if (incomingPacket.Packet.Type == PacketType.AgentUpdate ||
+                incomingPacket.Packet.Type == PacketType.ChatFromViewer)
+                packetInbox.EnqueueHigh(incomingPacket);
             else
-                packetInbox.EnqueueLow(new IncomingPacket((LLClientView)client, packet));
-//            packetInbox.Enqueue(new IncomingPacket((LLClientView)client, packet));
+                packetInbox.EnqueueLow(incomingPacket);
         }
 
         #region BinaryStats
@@ -1013,21 +1367,19 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
         private void HandleUseCircuitCode(object o)
         {
-            IPEndPoint remoteEndPoint = null;
+            IPEndPoint endPoint = null;
             IClientAPI client = null;
 
             try
             {
     //            DateTime startTime = DateTime.Now;
                 object[] array = (object[])o;
-                UDPPacketBuffer buffer = (UDPPacketBuffer)array[0];
+                endPoint = (IPEndPoint)array[0];
                 UseCircuitCodePacket uccp = (UseCircuitCodePacket)array[1];
 
                 m_log.DebugFormat(
                     "[LLUDPSERVER]: Handling UseCircuitCode request for circuit {0} to {1} from IP {2}",
-                    uccp.CircuitCode.Code, m_scene.RegionInfo.RegionName, buffer.RemoteEndPoint);
-    
-                remoteEndPoint = (IPEndPoint)buffer.RemoteEndPoint;
+                    uccp.CircuitCode.Code, m_scene.RegionInfo.RegionName, endPoint);
     
                 AuthenticateResponse sessionInfo;
                 if (IsClientAuthorized(uccp, out sessionInfo))
@@ -1038,13 +1390,13 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                             uccp.CircuitCode.Code,
                             uccp.CircuitCode.ID,
                             uccp.CircuitCode.SessionID,
-                            remoteEndPoint,
+                            endPoint,
                             sessionInfo);
             
                     // Send ack straight away to let the viewer know that the connection is active.
                     // The client will be null if it already exists (e.g. if on a region crossing the client sends a use
                     // circuit code to the existing child agent.  This is not particularly obvious.
-                    SendAckImmediate(remoteEndPoint, uccp.Header.Sequence);
+                    SendAckImmediate(endPoint, uccp.Header.Sequence);
             
                     // We only want to send initial data to new clients, not ones which are being converted from child to root.
                     if (client != null)
@@ -1058,12 +1410,12 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
                     lock (m_pendingCache)
                     {
-                        if (!m_pendingCache.TryGetValue(remoteEndPoint, out queue))
+                        if (!m_pendingCache.TryGetValue(endPoint, out queue))
                         {
                             m_log.DebugFormat("[LLUDPSERVER]: Client created but no pending queue present");
                             return;
                         }
-                        m_pendingCache.Remove(remoteEndPoint);
+                        m_pendingCache.Remove(endPoint);
                     }
 
                     m_log.DebugFormat("[LLUDPSERVER]: Client created, processing pending queue, {0} entries", queue.Count);
@@ -1081,9 +1433,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     // Don't create clients for unauthorized requesters.
                     m_log.WarnFormat(
                         "[LLUDPSERVER]: Ignoring connection request for {0} to {1} with unknown circuit code {2} from IP {3}",
-                        uccp.CircuitCode.ID, m_scene.RegionInfo.RegionName, uccp.CircuitCode.Code, remoteEndPoint);
+                        uccp.CircuitCode.ID, m_scene.RegionInfo.RegionName, uccp.CircuitCode.Code, endPoint);
                     lock (m_pendingCache)
-                        m_pendingCache.Remove(remoteEndPoint);
+                        m_pendingCache.Remove(endPoint);
                 }
     
                 //            m_log.DebugFormat(
@@ -1095,7 +1447,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             {
                 m_log.ErrorFormat(
                     "[LLUDPSERVER]: UseCircuitCode handling from endpoint {0}, client {1} {2} failed.  Exception {3}{4}",
-                    remoteEndPoint != null ? remoteEndPoint.ToString() : "n/a",
+                    endPoint != null ? endPoint.ToString() : "n/a",
                     client != null ? client.Name : "unknown",
                     client != null ? client.AgentId.ToString() : "unknown",
                     e.Message,
@@ -1160,20 +1512,20 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         {
             IClientAPI client = null;
 
-            // In priciple there shouldn't be more than one thread here, ever.
-            // But in case that happens, we need to synchronize this piece of code
-            // because it's too important
-            lock (this) 
+            // We currently synchronize this code across the whole scene to avoid issues such as
+            // http://opensimulator.org/mantis/view.php?id=5365  However, once locking per agent circuit can be done
+            // consistently, this lock could probably be removed.
+            lock (this)
             {
                 if (!m_scene.TryGetClient(agentID, out client))
                 {
                     LLUDPClient udpClient = new LLUDPClient(this, ThrottleRates, m_throttle, circuitCode, agentID, remoteEndPoint, m_defaultRTO, m_maxRTO);
-
+    
                     client = new LLClientView(m_scene, this, udpClient, sessionInfo, agentID, sessionID, circuitCode);
                     client.OnLogout += LogoutHandler;
-
+    
                     ((LLClientView)client).DisableFacelights = m_disableFacelights;
-
+    
                     client.Start();
                 }
             }
@@ -1212,7 +1564,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             // on to en-US to avoid number parsing issues
             Culture.SetCurrentCulture();
 
-            while (base.IsRunning)
+            while (IsRunningInbound)
             {
                 m_scene.ThreadAlive(1);
                 try
@@ -1228,7 +1580,12 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     }
 
                     if (packetInbox.Dequeue(100, ref incomingPacket))
+                    {
                         ProcessInPacket(incomingPacket);//, incomingPacket); Util.FireAndForget(ProcessInPacket, incomingPacket);
+
+                        if (UsePools)
+                            m_incomingPacketPool.ReturnObject(incomingPacket);
+                    }
                 }
                 catch (Exception ex)
                 {
@@ -1255,7 +1612,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             // Action generic every round
             Action<IClientAPI> clientPacketHandler = ClientOutgoingPacketHandler;
 
-            while (base.IsRunning)
+            while (base.IsRunningOutbound)
             {
                 m_scene.ThreadAlive(2);
                 try
@@ -1383,6 +1740,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         private int npacksSent = 0;
         private int npackNotSent = 0;
 
+        /// <summary>
+        /// Number of inbound packets processed since startup.
+        /// </summary>
+        public long IncomingPacketsProcessed { get; private set; }
+
         private void MonitoredClientOutgoingPacketHandler(IClientAPI client)
         {
             nticks++;
@@ -1442,7 +1804,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                             npacksSent++;
                         }
                         else
+                        {
                             npackNotSent++;
+                        }
 
                         watch2.Stop();
                         avgDequeueTicks = (nticks - 1) / (float)nticks * avgDequeueTicks + (watch2.ElapsedTicks / (float)nticks);
@@ -1450,7 +1814,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
                     }
                     else
+                    {
                         m_log.WarnFormat("[LLUDPSERVER]: Client is not connected");
+                    }
                 }
             }
             catch (Exception ex)
@@ -1514,6 +1880,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 //                    "[LLUDPSERVER]: Dropped incoming {0} for dead client {1} in {2}",
 //                    packet.Type, client.Name, m_scene.RegionInfo.RegionName);
 //            }
+
+            IncomingPacketsProcessed++;
         }
 
         protected void LogoutHandler(IClientAPI client)
@@ -1523,7 +1891,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             if (!client.IsLoggingOut)
             {
                 client.IsLoggingOut = true;
-                client.Close(false);
+                client.Close(false, false);
             }
         }
     }