From ac7bc78555c1dd51724790032f0711b24bc8c67d Mon Sep 17 00:00:00 2001
From: Diva Canto
Date: Tue, 8 Feb 2011 12:06:14 -0800
Subject: Added emergency monitoring of UDP Outgoing packets thread. Just type
 "emergency-monitoring on/off"

---
 .../Region/ClientStack/LindenUDP/LLUDPServer.cs    | 100 +++++++++++++++++++++
 1 file changed, 100 insertions(+)

(limited to 'OpenSim/Region/ClientStack/LindenUDP')

diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
index 5ff9aee..72257d2 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
@@ -27,6 +27,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Net;
 using System.Net.Sockets;
@@ -1053,6 +1054,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
                     #endregion Update Timers
 
+                    if (m_scene.EmergencyMonitoring)
+                        clientPacketHandler = MonitoredClientOutgoingPacketHandler;
+
                     // Handle outgoing packets, resends, acknowledgements, and pings for each
                     // client. m_packetSent will be set to true if a packet is sent
                     m_scene.ForEachClient(clientPacketHandler);
@@ -1068,6 +1072,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                 {
                     m_log.Error("[LLUDPSERVER]: OutgoingPacketHandler loop threw an exception: " + ex.Message, ex);
                 }
+
             }
 
             Watchdog.RemoveThread();
@@ -1105,6 +1110,101 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             }
         }
 
+        #region Emergency Monitoring
+        private Stopwatch watch1 = new Stopwatch();
+        private Stopwatch watch2 = new Stopwatch();
+
+        private float avgProcessingTicks = 0;
+        private float avgResendUnackedTicks = 0;
+        private float avgSendAcksTicks = 0;
+        private float avgSendPingTicks = 0;
+        private float avgDequeueTicks = 0;
+        private long nticks = 0;
+        private long nticksUnack = 0;
+        private long nticksAck = 0;
+        private long nticksPing = 0;
+
+        private void MonitoredClientOutgoingPacketHandler(IClientAPI client)
+        {
+            nticks++;
+            watch1.Start();
+            try
+            {
+                if (client is LLClientView)
+                {
+                    LLUDPClient udpClient = ((LLClientView)client).UDPClient;
+
+                    if (udpClient.IsConnected)
+                    {
+                        if (m_resendUnacked)
+                        {
+                            nticksUnack++;
+                            watch2.Start();
+
+                            ResendUnacked(udpClient);
+
+                            watch2.Stop();
+                            avgResendUnackedTicks = (nticksUnack - 1)/(float)nticksUnack * avgResendUnackedTicks + (watch2.ElapsedTicks / (float)nticksUnack);
+                            watch2.Reset();
+                        }
+
+                        if (m_sendAcks)
+                        {
+                            nticksAck++;
+                            watch2.Start();
+                            
+                            SendAcks(udpClient);
+
+                            watch2.Stop();
+                            avgSendAcksTicks = (nticksAck - 1) / (float)nticksAck * avgSendAcksTicks + (watch2.ElapsedTicks / (float)nticksAck);
+                            watch2.Reset();
+                        }
+
+                        if (m_sendPing)
+                        {
+                            nticksPing++;
+                            watch2.Start();
+                            
+                            SendPing(udpClient);
+
+                            watch2.Stop();
+                            avgSendPingTicks = (nticksPing - 1) / (float)nticksPing * avgSendPingTicks + (watch2.ElapsedTicks / (float)nticksPing);
+                            watch2.Reset();
+                        }
+
+                        watch2.Start();
+                        // Dequeue any outgoing packets that are within the throttle limits
+                        if (udpClient.DequeueOutgoing())
+                            m_packetSent = true;
+                        watch2.Stop();
+                        avgDequeueTicks = (nticks - 1) / (float)nticks * avgDequeueTicks + (watch2.ElapsedTicks / (float)nticks);
+                        watch2.Reset();
+
+                    }
+                    else
+                        m_log.WarnFormat("[LLUDPSERVER]: Client is not connected");
+                }
+            }
+            catch (Exception ex)
+            {
+                m_log.Error("[LLUDPSERVER]: OutgoingPacketHandler iteration for " + client.Name +
+                    " threw an exception: " + ex.Message, ex);
+            }
+            watch1.Stop();
+            avgProcessingTicks = (nticks - 1) / (float)nticks * avgProcessingTicks + (watch1.ElapsedTicks / (float)nticks);
+            watch1.Reset();
+
+            // reuse this -- it's every 100ms
+            if (m_scene.EmergencyMonitoring && nticks % 100 == 0)
+            {
+                m_log.InfoFormat("[LLUDPSERVER]: avg processing ticks: {0} avg unacked: {1} avg acks: {2} avg ping: {3} avg dequeue: {4} (pack sent? {5})", 
+                    avgProcessingTicks, avgResendUnackedTicks, avgSendAcksTicks, avgSendPingTicks, avgDequeueTicks, m_packetSent);
+            }
+
+        }
+
+        #endregion 
+
         private void ProcessInPacket(object state)
         {
             IncomingPacket incomingPacket = (IncomingPacket)state;
-- 
cgit v1.1


From 82846afe4b2a141421bff136dec9eef553522aa7 Mon Sep 17 00:00:00 2001
From: Diva Canto
Date: Tue, 8 Feb 2011 12:37:37 -0800
Subject: Minor improvement to previous commit.

---
 OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'OpenSim/Region/ClientStack/LindenUDP')

diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
index 72257d2..703ef6a 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
@@ -1056,6 +1056,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
                     if (m_scene.EmergencyMonitoring)
                         clientPacketHandler = MonitoredClientOutgoingPacketHandler;
+                    else
+                        clientPacketHandler = ClientOutgoingPacketHandler;
 
                     // Handle outgoing packets, resends, acknowledgements, and pings for each
                     // client. m_packetSent will be set to true if a packet is sent
@@ -1197,8 +1199,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             // reuse this -- it's every 100ms
             if (m_scene.EmergencyMonitoring && nticks % 100 == 0)
             {
-                m_log.InfoFormat("[LLUDPSERVER]: avg processing ticks: {0} avg unacked: {1} avg acks: {2} avg ping: {3} avg dequeue: {4} (pack sent? {5})", 
-                    avgProcessingTicks, avgResendUnackedTicks, avgSendAcksTicks, avgSendPingTicks, avgDequeueTicks, m_packetSent);
+                m_log.InfoFormat("[LLUDPSERVER]: avg processing ticks: {0} avg unacked: {1} avg acks: {2} avg ping: {3} avg dequeue: {4} (TickCountRes: {5})", 
+                    avgProcessingTicks, avgResendUnackedTicks, avgSendAcksTicks, avgSendPingTicks, avgDequeueTicks, TickCountResolution);
             }
 
         }
-- 
cgit v1.1


From f431bd20ec18222a69fe1371b6145cf7415fdf9f Mon Sep 17 00:00:00 2001
From: Diva Canto
Date: Tue, 8 Feb 2011 14:49:50 -0800
Subject: Minor addition to the previous commit

---
 OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

(limited to 'OpenSim/Region/ClientStack/LindenUDP')

diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
index 703ef6a..bfbe959 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
@@ -1125,6 +1125,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         private long nticksUnack = 0;
         private long nticksAck = 0;
         private long nticksPing = 0;
+        private int npacksSent = 0;
+        private int npackNotSent = 0;
 
         private void MonitoredClientOutgoingPacketHandler(IClientAPI client)
         {
@@ -1177,7 +1179,13 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                         watch2.Start();
                         // Dequeue any outgoing packets that are within the throttle limits
                         if (udpClient.DequeueOutgoing())
+                        {
                             m_packetSent = true;
+                            npacksSent++;
+                        }
+                        else
+                            npackNotSent++;
+
                         watch2.Stop();
                         avgDequeueTicks = (nticks - 1) / (float)nticks * avgDequeueTicks + (watch2.ElapsedTicks / (float)nticks);
                         watch2.Reset();
@@ -1196,11 +1204,12 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             avgProcessingTicks = (nticks - 1) / (float)nticks * avgProcessingTicks + (watch1.ElapsedTicks / (float)nticks);
             watch1.Reset();
 
-            // reuse this -- it's every 100ms
+            // reuse this -- it's every ~100ms
             if (m_scene.EmergencyMonitoring && nticks % 100 == 0)
             {
-                m_log.InfoFormat("[LLUDPSERVER]: avg processing ticks: {0} avg unacked: {1} avg acks: {2} avg ping: {3} avg dequeue: {4} (TickCountRes: {5})", 
-                    avgProcessingTicks, avgResendUnackedTicks, avgSendAcksTicks, avgSendPingTicks, avgDequeueTicks, TickCountResolution);
+                m_log.InfoFormat("[LLUDPSERVER]: avg processing ticks: {0} avg unacked: {1} avg acks: {2} avg ping: {3} avg dequeue: {4} (TickCountRes: {5} sent: {6} notsent: {7})", 
+                    avgProcessingTicks, avgResendUnackedTicks, avgSendAcksTicks, avgSendPingTicks, avgDequeueTicks, TickCountResolution, npacksSent, npackNotSent);
+                npackNotSent = npacksSent = 0;
             }
 
         }
-- 
cgit v1.1


From 117462cba18fd0f9610d854760b92362b3beea7c Mon Sep 17 00:00:00 2001
From: Diva Canto
Date: Tue, 8 Feb 2011 17:53:01 -0800
Subject: Avoid potential race conditions on UseCircuitCode. I artificially
 made the race condition happen, and got very similar results to those
 described in mantis #5365 -- no prims/avie sent back.

---
 .../Region/ClientStack/LindenUDP/LLUDPServer.cs    | 51 ++++++++++++----------
 1 file changed, 28 insertions(+), 23 deletions(-)

(limited to 'OpenSim/Region/ClientStack/LindenUDP')

diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
index bfbe959..04fec95 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
@@ -640,10 +640,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             {
                 object[] array = new object[] { buffer, packet };
 
-                if (m_asyncPacketHandling)
-                    Util.FireAndForget(HandleUseCircuitCode, array);
-                else
-                    HandleUseCircuitCode(array);
+                Util.FireAndForget(HandleUseCircuitCode, array);
 
                 return;
             }
@@ -857,11 +854,12 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
             IPEndPoint remoteEndPoint = (IPEndPoint)buffer.RemoteEndPoint;
 
+            // Acknowledge the UseCircuitCode packet immediately, even before processing further
+            // This is so that the client doesn't send another one
+            SendAckImmediate(remoteEndPoint, packet.Header.Sequence);
+
             // Begin the process of adding the client to the simulator
             AddNewClient((UseCircuitCodePacket)packet, remoteEndPoint);
-
-            // Acknowledge the UseCircuitCode packet
-            SendAckImmediate(remoteEndPoint, packet.Header.Sequence);
             
 //            m_log.DebugFormat(
 //                "[LLUDPSERVER]: Handling UseCircuitCode request from {0} took {1}ms", 
@@ -927,25 +925,32 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
         protected virtual void AddClient(uint circuitCode, UUID agentID, UUID sessionID, IPEndPoint remoteEndPoint, AuthenticateResponse sessionInfo)
         {
-            // Create the LLUDPClient
-            LLUDPClient udpClient = new LLUDPClient(this, ThrottleRates, m_throttle, circuitCode, agentID, remoteEndPoint, m_defaultRTO, m_maxRTO);
-            IClientAPI existingClient;
-
-            if (!m_scene.TryGetClient(agentID, out existingClient))
+            // In priciple there shouldn't be more than one thread here, ever.
+            // But in case that happens, we need to synchronize this piece of code
+            // because it's too important
+            lock (this) 
             {
-                // Create the LLClientView
-                LLClientView client = new LLClientView(remoteEndPoint, m_scene, this, udpClient, sessionInfo, agentID, sessionID, circuitCode);
-                client.OnLogout += LogoutHandler;
+                IClientAPI existingClient;
 
-                client.DisableFacelights = m_disableFacelights;
+                if (!m_scene.TryGetClient(agentID, out existingClient))
+                {
+                    // Create the LLUDPClient
+                    LLUDPClient udpClient = new LLUDPClient(this, ThrottleRates, m_throttle, circuitCode, agentID, remoteEndPoint, m_defaultRTO, m_maxRTO);
+                    // Create the LLClientView
+                    LLClientView client = new LLClientView(remoteEndPoint, m_scene, this, udpClient, sessionInfo, agentID, sessionID, circuitCode);
+                    client.OnLogout += LogoutHandler;
 
-                // Start the IClientAPI
-                client.Start();
-            }
-            else
-            {
-                m_log.WarnFormat("[LLUDPSERVER]: Ignoring a repeated UseCircuitCode from {0} at {1} for circuit {2}",
-                    udpClient.AgentID, remoteEndPoint, circuitCode);
+                    client.DisableFacelights = m_disableFacelights;
+
+                    // Start the IClientAPI
+                    client.Start();
+
+                }
+                else
+                {
+                    m_log.WarnFormat("[LLUDPSERVER]: Ignoring a repeated UseCircuitCode from {0} at {1} for circuit {2}",
+                        existingClient.AgentId, remoteEndPoint, circuitCode);
+                }
             }
         }
 
-- 
cgit v1.1


From 585473aade100c3ffeef27e0c8e6b6c8c09d0109 Mon Sep 17 00:00:00 2001
From: Diva Canto
Date: Tue, 8 Feb 2011 20:12:33 -0800
Subject: Brute-force debug -- mantis #5365

---
 OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs | 18 +++++++++++++++---
 OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs |  5 +++++
 2 files changed, 20 insertions(+), 3 deletions(-)

(limited to 'OpenSim/Region/ClientStack/LindenUDP')

diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs
index 65a8fe3..2d58b94 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs
@@ -463,12 +463,13 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             bool packetSent = false;
             ThrottleOutPacketTypeFlags emptyCategories = 0;
 
-            //string queueDebugOutput = String.Empty; // Serious debug business
+            string queueDebugOutput = String.Empty; // Serious debug business
 
             for (int i = 0; i < THROTTLE_CATEGORY_COUNT; i++)
             {
                 bucket = m_throttleCategories[i];
-                //queueDebugOutput += m_packetOutboxes[i].Count + " ";  // Serious debug business
+                if (i == 4)
+                    queueDebugOutput += m_packetOutboxes[i].Count + " ";  // Serious debug business
 
                 if (m_nextPackets[i] != null)
                 {
@@ -476,13 +477,17 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     // leaving a dequeued packet still waiting to be sent out. Try to
                     // send it again
                     OutgoingPacket nextPacket = m_nextPackets[i];
+                    if (i == 4) queueDebugOutput += "m_nextPackets[i] != null, " + nextPacket.Buffer.DataLength;
                     if (bucket.RemoveTokens(nextPacket.Buffer.DataLength))
                     {
+                        if (i == 4) queueDebugOutput += " removed tokens ";
                         // Send the packet
                         m_udpServer.SendPacketFinal(nextPacket);
                         m_nextPackets[i] = null;
                         packetSent = true;
                     }
+                    else
+                        if (i == 4) queueDebugOutput += " did not remove tokens ";
                 }
                 else
                 {
@@ -491,6 +496,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     queue = m_packetOutboxes[i];
                     if (queue.Dequeue(out packet))
                     {
+                        if (i == 4) queueDebugOutput += "m_nextPackets[i] == null, dq ok " + packet.Buffer.DataLength;
                         // A packet was pulled off the queue. See if we have
                         // enough tokens in the bucket to send it out
                         if (bucket.RemoveTokens(packet.Buffer.DataLength))
@@ -498,11 +504,13 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                             // Send the packet
                             m_udpServer.SendPacketFinal(packet);
                             packetSent = true;
+                            if (i == 4) queueDebugOutput += " removed tokens ";
                         }
                         else
                         {
                             // Save the dequeued packet for the next iteration
                             m_nextPackets[i] = packet;
+                            if (i == 4) queueDebugOutput += " did not remove tokens ";
                         }
 
                         // If the queue is empty after this dequeue, fire the queue
@@ -513,17 +521,21 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     }
                     else
                     {
+                        if (i == 4) queueDebugOutput += "m_nextPackets[i] == null, dq nok ";
                         // No packets in this queue. Fire the queue empty callback
                         // if it has not been called recently
                         emptyCategories |= CategoryToFlag(i);
                     }
                 }
+
             }
 
             if (emptyCategories != 0)
                 BeginFireQueueEmpty(emptyCategories);
 
-            //m_log.Info("[LLUDPCLIENT]: Queues: " + queueDebugOutput); // Serious debug business
+            if (m_udpServer.EmergencyMonitoring)
+                m_log.Info("[LLUDPCLIENT]: Queues: " + queueDebugOutput); // Serious debug business
+
             return packetSent;
         }
 
diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
index 04fec95..922e2bc 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
@@ -1133,6 +1133,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         private int npacksSent = 0;
         private int npackNotSent = 0;
 
+        public bool EmergencyMonitoring
+        {
+            get { return m_scene.EmergencyMonitoring; }
+        }
+
         private void MonitoredClientOutgoingPacketHandler(IClientAPI client)
         {
             nticks++;
-- 
cgit v1.1


From ba202ea9b08b1205de343c65fd209b6cca4cb6bc Mon Sep 17 00:00:00 2001
From: Diva Canto
Date: Tue, 8 Feb 2011 21:09:10 -0800
Subject: Don't build strings unless we're in emergency debugging.

---
 OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

(limited to 'OpenSim/Region/ClientStack/LindenUDP')

diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs
index 2d58b94..20f0410 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs
@@ -468,7 +468,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             for (int i = 0; i < THROTTLE_CATEGORY_COUNT; i++)
             {
                 bucket = m_throttleCategories[i];
-                if (i == 4)
+                if (m_udpServer.EmergencyMonitoring && i == 4)
                     queueDebugOutput += m_packetOutboxes[i].Count + " ";  // Serious debug business
 
                 if (m_nextPackets[i] != null)
@@ -477,17 +477,17 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     // leaving a dequeued packet still waiting to be sent out. Try to
                     // send it again
                     OutgoingPacket nextPacket = m_nextPackets[i];
-                    if (i == 4) queueDebugOutput += "m_nextPackets[i] != null, " + nextPacket.Buffer.DataLength;
+                    if (m_udpServer.EmergencyMonitoring && i == 4) queueDebugOutput += "m_nextPackets[i] != null, " + nextPacket.Buffer.DataLength;
                     if (bucket.RemoveTokens(nextPacket.Buffer.DataLength))
                     {
-                        if (i == 4) queueDebugOutput += " removed tokens ";
+                        if (m_udpServer.EmergencyMonitoring && i == 4) queueDebugOutput += " removed tokens ";
                         // Send the packet
                         m_udpServer.SendPacketFinal(nextPacket);
                         m_nextPackets[i] = null;
                         packetSent = true;
                     }
                     else
-                        if (i == 4) queueDebugOutput += " did not remove tokens ";
+                        if (m_udpServer.EmergencyMonitoring && i == 4) queueDebugOutput += " did not remove tokens ";
                 }
                 else
                 {
@@ -496,7 +496,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     queue = m_packetOutboxes[i];
                     if (queue.Dequeue(out packet))
                     {
-                        if (i == 4) queueDebugOutput += "m_nextPackets[i] == null, dq ok " + packet.Buffer.DataLength;
+                        if (m_udpServer.EmergencyMonitoring && i == 4) queueDebugOutput += "m_nextPackets[i] == null, dq ok " + packet.Buffer.DataLength;
                         // A packet was pulled off the queue. See if we have
                         // enough tokens in the bucket to send it out
                         if (bucket.RemoveTokens(packet.Buffer.DataLength))
@@ -504,13 +504,13 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                             // Send the packet
                             m_udpServer.SendPacketFinal(packet);
                             packetSent = true;
-                            if (i == 4) queueDebugOutput += " removed tokens ";
+                            if (m_udpServer.EmergencyMonitoring && i == 4) queueDebugOutput += " removed tokens ";
                         }
                         else
                         {
                             // Save the dequeued packet for the next iteration
                             m_nextPackets[i] = packet;
-                            if (i == 4) queueDebugOutput += " did not remove tokens ";
+                            if (m_udpServer.EmergencyMonitoring && i == 4) queueDebugOutput += " did not remove tokens ";
                         }
 
                         // If the queue is empty after this dequeue, fire the queue
@@ -521,7 +521,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     }
                     else
                     {
-                        if (i == 4) queueDebugOutput += "m_nextPackets[i] == null, dq nok ";
+                        if (m_udpServer.EmergencyMonitoring && i == 4) queueDebugOutput += "m_nextPackets[i] == null, dq nok ";
                         // No packets in this queue. Fire the queue empty callback
                         // if it has not been called recently
                         emptyCategories |= CategoryToFlag(i);
-- 
cgit v1.1


From 473fac4dc71858139bd44c1e9ce4fd03d9d1bd91 Mon Sep 17 00:00:00 2001
From: Diva Canto
Date: Wed, 9 Feb 2011 08:06:20 -0800
Subject: Detect negative dripAmounts in TokenBuckets. These negatives result
 from overflown integer operations. Also added Total to the scene throttles 
 in show throttles.

---
 OpenSim/Region/ClientStack/LindenUDP/TokenBucket.cs | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'OpenSim/Region/ClientStack/LindenUDP')

diff --git a/OpenSim/Region/ClientStack/LindenUDP/TokenBucket.cs b/OpenSim/Region/ClientStack/LindenUDP/TokenBucket.cs
index bdbd284..dd52683 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/TokenBucket.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/TokenBucket.cs
@@ -207,6 +207,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
                 int dripAmount = deltaMS * tokensPerMS;
 
+                if (dripAmount < 0)
+                    Console.WriteLine("MAY DAY MAY DAY! Drip amount is " + dripAmount + ". This should not happen");
+
                 content = Math.Min(content + dripAmount, maxBurst);
                 lastDrip = now;
 
-- 
cgit v1.1


From 477a869fb2cc615430d78cbad6a5a0eedf6b9bf8 Mon Sep 17 00:00:00 2001
From: Diva Canto
Date: Wed, 9 Feb 2011 08:08:57 -0800
Subject: More detection of negatives.

---
 OpenSim/Region/ClientStack/LindenUDP/TokenBucket.cs | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'OpenSim/Region/ClientStack/LindenUDP')

diff --git a/OpenSim/Region/ClientStack/LindenUDP/TokenBucket.cs b/OpenSim/Region/ClientStack/LindenUDP/TokenBucket.cs
index dd52683..de8b521 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/TokenBucket.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/TokenBucket.cs
@@ -213,6 +213,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                 content = Math.Min(content + dripAmount, maxBurst);
                 lastDrip = now;
 
+                if (content < 0)
+                    Console.WriteLine("MAY DAY MAY DAY! Content is " + content + ". This should not happen");
+
                 return true;
             }
         }
-- 
cgit v1.1


From 1bba9c6300db10c78f606bf0c21be17004dde704 Mon Sep 17 00:00:00 2001
From: Diva Canto
Date: Wed, 9 Feb 2011 08:35:21 -0800
Subject: Revert "Don't build strings unless we're in emergency debugging."

This reverts commit ba202ea9b08b1205de343c65fd209b6cca4cb6bc.
---
 OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

(limited to 'OpenSim/Region/ClientStack/LindenUDP')

diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs
index 20f0410..2d58b94 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs
@@ -468,7 +468,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             for (int i = 0; i < THROTTLE_CATEGORY_COUNT; i++)
             {
                 bucket = m_throttleCategories[i];
-                if (m_udpServer.EmergencyMonitoring && i == 4)
+                if (i == 4)
                     queueDebugOutput += m_packetOutboxes[i].Count + " ";  // Serious debug business
 
                 if (m_nextPackets[i] != null)
@@ -477,17 +477,17 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     // leaving a dequeued packet still waiting to be sent out. Try to
                     // send it again
                     OutgoingPacket nextPacket = m_nextPackets[i];
-                    if (m_udpServer.EmergencyMonitoring && i == 4) queueDebugOutput += "m_nextPackets[i] != null, " + nextPacket.Buffer.DataLength;
+                    if (i == 4) queueDebugOutput += "m_nextPackets[i] != null, " + nextPacket.Buffer.DataLength;
                     if (bucket.RemoveTokens(nextPacket.Buffer.DataLength))
                     {
-                        if (m_udpServer.EmergencyMonitoring && i == 4) queueDebugOutput += " removed tokens ";
+                        if (i == 4) queueDebugOutput += " removed tokens ";
                         // Send the packet
                         m_udpServer.SendPacketFinal(nextPacket);
                         m_nextPackets[i] = null;
                         packetSent = true;
                     }
                     else
-                        if (m_udpServer.EmergencyMonitoring && i == 4) queueDebugOutput += " did not remove tokens ";
+                        if (i == 4) queueDebugOutput += " did not remove tokens ";
                 }
                 else
                 {
@@ -496,7 +496,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     queue = m_packetOutboxes[i];
                     if (queue.Dequeue(out packet))
                     {
-                        if (m_udpServer.EmergencyMonitoring && i == 4) queueDebugOutput += "m_nextPackets[i] == null, dq ok " + packet.Buffer.DataLength;
+                        if (i == 4) queueDebugOutput += "m_nextPackets[i] == null, dq ok " + packet.Buffer.DataLength;
                         // A packet was pulled off the queue. See if we have
                         // enough tokens in the bucket to send it out
                         if (bucket.RemoveTokens(packet.Buffer.DataLength))
@@ -504,13 +504,13 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                             // Send the packet
                             m_udpServer.SendPacketFinal(packet);
                             packetSent = true;
-                            if (m_udpServer.EmergencyMonitoring && i == 4) queueDebugOutput += " removed tokens ";
+                            if (i == 4) queueDebugOutput += " removed tokens ";
                         }
                         else
                         {
                             // Save the dequeued packet for the next iteration
                             m_nextPackets[i] = packet;
-                            if (m_udpServer.EmergencyMonitoring && i == 4) queueDebugOutput += " did not remove tokens ";
+                            if (i == 4) queueDebugOutput += " did not remove tokens ";
                         }
 
                         // If the queue is empty after this dequeue, fire the queue
@@ -521,7 +521,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     }
                     else
                     {
-                        if (m_udpServer.EmergencyMonitoring && i == 4) queueDebugOutput += "m_nextPackets[i] == null, dq nok ";
+                        if (i == 4) queueDebugOutput += "m_nextPackets[i] == null, dq nok ";
                         // No packets in this queue. Fire the queue empty callback
                         // if it has not been called recently
                         emptyCategories |= CategoryToFlag(i);
-- 
cgit v1.1


From 830fee145d8e8fabc32365ee6f04732d4c85b1e3 Mon Sep 17 00:00:00 2001
From: Diva Canto
Date: Wed, 9 Feb 2011 08:35:36 -0800
Subject: Revert "Brute-force debug -- mantis #5365"

This reverts commit 585473aade100c3ffeef27e0c8e6b6c8c09d0109.
---
 OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs | 18 +++---------------
 OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs |  5 -----
 2 files changed, 3 insertions(+), 20 deletions(-)

(limited to 'OpenSim/Region/ClientStack/LindenUDP')

diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs
index 2d58b94..65a8fe3 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs
@@ -463,13 +463,12 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             bool packetSent = false;
             ThrottleOutPacketTypeFlags emptyCategories = 0;
 
-            string queueDebugOutput = String.Empty; // Serious debug business
+            //string queueDebugOutput = String.Empty; // Serious debug business
 
             for (int i = 0; i < THROTTLE_CATEGORY_COUNT; i++)
             {
                 bucket = m_throttleCategories[i];
-                if (i == 4)
-                    queueDebugOutput += m_packetOutboxes[i].Count + " ";  // Serious debug business
+                //queueDebugOutput += m_packetOutboxes[i].Count + " ";  // Serious debug business
 
                 if (m_nextPackets[i] != null)
                 {
@@ -477,17 +476,13 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     // leaving a dequeued packet still waiting to be sent out. Try to
                     // send it again
                     OutgoingPacket nextPacket = m_nextPackets[i];
-                    if (i == 4) queueDebugOutput += "m_nextPackets[i] != null, " + nextPacket.Buffer.DataLength;
                     if (bucket.RemoveTokens(nextPacket.Buffer.DataLength))
                     {
-                        if (i == 4) queueDebugOutput += " removed tokens ";
                         // Send the packet
                         m_udpServer.SendPacketFinal(nextPacket);
                         m_nextPackets[i] = null;
                         packetSent = true;
                     }
-                    else
-                        if (i == 4) queueDebugOutput += " did not remove tokens ";
                 }
                 else
                 {
@@ -496,7 +491,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     queue = m_packetOutboxes[i];
                     if (queue.Dequeue(out packet))
                     {
-                        if (i == 4) queueDebugOutput += "m_nextPackets[i] == null, dq ok " + packet.Buffer.DataLength;
                         // A packet was pulled off the queue. See if we have
                         // enough tokens in the bucket to send it out
                         if (bucket.RemoveTokens(packet.Buffer.DataLength))
@@ -504,13 +498,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                             // Send the packet
                             m_udpServer.SendPacketFinal(packet);
                             packetSent = true;
-                            if (i == 4) queueDebugOutput += " removed tokens ";
                         }
                         else
                         {
                             // Save the dequeued packet for the next iteration
                             m_nextPackets[i] = packet;
-                            if (i == 4) queueDebugOutput += " did not remove tokens ";
                         }
 
                         // If the queue is empty after this dequeue, fire the queue
@@ -521,21 +513,17 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     }
                     else
                     {
-                        if (i == 4) queueDebugOutput += "m_nextPackets[i] == null, dq nok ";
                         // No packets in this queue. Fire the queue empty callback
                         // if it has not been called recently
                         emptyCategories |= CategoryToFlag(i);
                     }
                 }
-
             }
 
             if (emptyCategories != 0)
                 BeginFireQueueEmpty(emptyCategories);
 
-            if (m_udpServer.EmergencyMonitoring)
-                m_log.Info("[LLUDPCLIENT]: Queues: " + queueDebugOutput); // Serious debug business
-
+            //m_log.Info("[LLUDPCLIENT]: Queues: " + queueDebugOutput); // Serious debug business
             return packetSent;
         }
 
diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
index 922e2bc..04fec95 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
@@ -1133,11 +1133,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         private int npacksSent = 0;
         private int npackNotSent = 0;
 
-        public bool EmergencyMonitoring
-        {
-            get { return m_scene.EmergencyMonitoring; }
-        }
-
         private void MonitoredClientOutgoingPacketHandler(IClientAPI client)
         {
             nticks++;
-- 
cgit v1.1


From 1cd951e5aedad4be809fc2674e57e34d0972b67c Mon Sep 17 00:00:00 2001
From: Diva Canto
Date: Wed, 9 Feb 2011 09:39:53 -0800
Subject: Fix the negative number problem in TokenBucket. mantis #5365

---
 OpenSim/Region/ClientStack/LindenUDP/TokenBucket.cs | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

(limited to 'OpenSim/Region/ClientStack/LindenUDP')

diff --git a/OpenSim/Region/ClientStack/LindenUDP/TokenBucket.cs b/OpenSim/Region/ClientStack/LindenUDP/TokenBucket.cs
index de8b521..0a8331f 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/TokenBucket.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/TokenBucket.cs
@@ -207,14 +207,13 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
                 int dripAmount = deltaMS * tokensPerMS;
 
-                if (dripAmount < 0)
-                    Console.WriteLine("MAY DAY MAY DAY! Drip amount is " + dripAmount + ". This should not happen");
-
                 content = Math.Min(content + dripAmount, maxBurst);
                 lastDrip = now;
 
-                if (content < 0)
-                    Console.WriteLine("MAY DAY MAY DAY! Content is " + content + ". This should not happen");
+                if (dripAmount < 0 || content < 0)
+                    // sim has been idle for too long, integer has overflown
+                    // previous calculation is meaningless, let's put it at correct max
+                    content = maxBurst;
 
                 return true;
             }
-- 
cgit v1.1


From f33e51e2ffd475ccdbc69429b4b4d707e88cc050 Mon Sep 17 00:00:00 2001
From: Diva Canto
Date: Wed, 9 Feb 2011 09:50:26 -0800
Subject: Comment instrumentation out. Not needed anymore. Left in comments, in
 case it is needed again. Mantis #5365

---
 OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

(limited to 'OpenSim/Region/ClientStack/LindenUDP')

diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
index 04fec95..207d213 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
@@ -1059,10 +1059,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
                     #endregion Update Timers
 
-                    if (m_scene.EmergencyMonitoring)
-                        clientPacketHandler = MonitoredClientOutgoingPacketHandler;
-                    else
-                        clientPacketHandler = ClientOutgoingPacketHandler;
+                    // Use this for emergency monitoring -- bug hunting
+                    //if (m_scene.EmergencyMonitoring)
+                    //    clientPacketHandler = MonitoredClientOutgoingPacketHandler;
+                    //else
+                    //    clientPacketHandler = ClientOutgoingPacketHandler;
 
                     // Handle outgoing packets, resends, acknowledgements, and pings for each
                     // client. m_packetSent will be set to true if a packet is sent
@@ -1118,6 +1119,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         }
 
         #region Emergency Monitoring
+        // Alternative packet handler fuull of instrumentation
+        // Handy for hunting bugs
         private Stopwatch watch1 = new Stopwatch();
         private Stopwatch watch2 = new Stopwatch();
 
-- 
cgit v1.1


From 21715396fa0d87bd4936bd37151487346794806d Mon Sep 17 00:00:00 2001
From: Diva Canto
Date: Wed, 9 Feb 2011 17:45:19 -0800
Subject: Put the Ack of UseCircuitCode back to where it used to be. Some ppl
 are reporting login issues.

---
 OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

(limited to 'OpenSim/Region/ClientStack/LindenUDP')

diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
index 207d213..584c577 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
@@ -854,14 +854,13 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
             IPEndPoint remoteEndPoint = (IPEndPoint)buffer.RemoteEndPoint;
 
-            // Acknowledge the UseCircuitCode packet immediately, even before processing further
-            // This is so that the client doesn't send another one
-            SendAckImmediate(remoteEndPoint, packet.Header.Sequence);
-
             // Begin the process of adding the client to the simulator
             AddNewClient((UseCircuitCodePacket)packet, remoteEndPoint);
-            
-//            m_log.DebugFormat(
+
+            // Send ack
+            SendAckImmediate(remoteEndPoint, packet.Header.Sequence);
+
+            //            m_log.DebugFormat(
 //                "[LLUDPSERVER]: Handling UseCircuitCode request from {0} took {1}ms", 
 //                buffer.RemoteEndPoint, (DateTime.Now - startTime).Milliseconds);
         }
-- 
cgit v1.1


From 89bb5c0941625542ce3fe364cf91a2e2fa60cbcf Mon Sep 17 00:00:00 2001
From: Diva Canto
Date: Thu, 10 Feb 2011 06:09:04 -0800
Subject: Revert "Hunting down mantis #5365"

This reverts commit ebeef02fef1a04b5b4cfe13dad588bcce7f9ba21.
---
 .../LindenUDP/UnackedPacketCollection.cs           | 45 ++++++++--------------
 1 file changed, 15 insertions(+), 30 deletions(-)

(limited to 'OpenSim/Region/ClientStack/LindenUDP')

diff --git a/OpenSim/Region/ClientStack/LindenUDP/UnackedPacketCollection.cs b/OpenSim/Region/ClientStack/LindenUDP/UnackedPacketCollection.cs
index d762bef..9d40688 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/UnackedPacketCollection.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/UnackedPacketCollection.cs
@@ -141,46 +141,31 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         private void ProcessQueues()
         {
             // Process all the pending adds
-
             OutgoingPacket pendingAdd;
-            if (m_pendingAdds != null)
-            {
-                while (m_pendingAdds.TryDequeue(out pendingAdd))
-                {
-                    if (pendingAdd != null && m_packets != null)
-                    {
-                        m_packets[pendingAdd.SequenceNumber] = pendingAdd;
-                    }
-                }
-            }
+            while (m_pendingAdds.TryDequeue(out pendingAdd))
+                m_packets[pendingAdd.SequenceNumber] = pendingAdd;
             
             // Process all the pending removes, including updating statistics and round-trip times
             PendingAck pendingRemove;
             OutgoingPacket ackedPacket;
-            if (m_pendingRemoves != null)
+            while (m_pendingRemoves.TryDequeue(out pendingRemove))
             {
-                while (m_pendingRemoves.TryDequeue(out pendingRemove))
+                if (m_packets.TryGetValue(pendingRemove.SequenceNumber, out ackedPacket))
                 {
-                    if (m_pendingRemoves != null && m_packets != null)
+                    m_packets.Remove(pendingRemove.SequenceNumber);
+
+                    // Update stats
+                    Interlocked.Add(ref ackedPacket.Client.UnackedBytes, -ackedPacket.Buffer.DataLength);
+
+                    if (!pendingRemove.FromResend)
                     {
-                        if (m_packets.TryGetValue(pendingRemove.SequenceNumber, out ackedPacket))
-                        {
-                            m_packets.Remove(pendingRemove.SequenceNumber);
-
-                            // Update stats
-                            Interlocked.Add(ref ackedPacket.Client.UnackedBytes, -ackedPacket.Buffer.DataLength);
-
-                            if (!pendingRemove.FromResend)
-                            {
-                                // Calculate the round-trip time for this packet and its ACK
-                                int rtt = pendingRemove.RemoveTime - ackedPacket.TickCount;
-                                if (rtt > 0)
-                                    ackedPacket.Client.UpdateRoundTrip(rtt);
-                            }
-                        }
+                        // Calculate the round-trip time for this packet and its ACK
+                        int rtt = pendingRemove.RemoveTime - ackedPacket.TickCount;
+                        if (rtt > 0)
+                            ackedPacket.Client.UpdateRoundTrip(rtt);
                     }
                 }
             }
         }
     }
-}
+}
\ No newline at end of file
-- 
cgit v1.1


From 45382e6f828395cbbcc1324d366c4f3e0eff7ad7 Mon Sep 17 00:00:00 2001
From: Diva Canto
Date: Thu, 10 Feb 2011 06:26:26 -0800
Subject: Reinstated a couple of null checks related to the previous revert.

---
 .../LindenUDP/UnackedPacketCollection.cs           | 26 +++++++++++++---------
 1 file changed, 15 insertions(+), 11 deletions(-)

(limited to 'OpenSim/Region/ClientStack/LindenUDP')

diff --git a/OpenSim/Region/ClientStack/LindenUDP/UnackedPacketCollection.cs b/OpenSim/Region/ClientStack/LindenUDP/UnackedPacketCollection.cs
index 9d40688..d195110 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/UnackedPacketCollection.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/UnackedPacketCollection.cs
@@ -143,7 +143,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             // Process all the pending adds
             OutgoingPacket pendingAdd;
             while (m_pendingAdds.TryDequeue(out pendingAdd))
-                m_packets[pendingAdd.SequenceNumber] = pendingAdd;
+                if (pendingAdd != null)
+                    m_packets[pendingAdd.SequenceNumber] = pendingAdd;
             
             // Process all the pending removes, including updating statistics and round-trip times
             PendingAck pendingRemove;
@@ -152,17 +153,20 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             {
                 if (m_packets.TryGetValue(pendingRemove.SequenceNumber, out ackedPacket))
                 {
-                    m_packets.Remove(pendingRemove.SequenceNumber);
-
-                    // Update stats
-                    Interlocked.Add(ref ackedPacket.Client.UnackedBytes, -ackedPacket.Buffer.DataLength);
-
-                    if (!pendingRemove.FromResend)
+                    if (ackedPacket != null)
                     {
-                        // Calculate the round-trip time for this packet and its ACK
-                        int rtt = pendingRemove.RemoveTime - ackedPacket.TickCount;
-                        if (rtt > 0)
-                            ackedPacket.Client.UpdateRoundTrip(rtt);
+                        m_packets.Remove(pendingRemove.SequenceNumber);
+
+                        // Update stats
+                        Interlocked.Add(ref ackedPacket.Client.UnackedBytes, -ackedPacket.Buffer.DataLength);
+
+                        if (!pendingRemove.FromResend)
+                        {
+                            // Calculate the round-trip time for this packet and its ACK
+                            int rtt = pendingRemove.RemoveTime - ackedPacket.TickCount;
+                            if (rtt > 0)
+                                ackedPacket.Client.UpdateRoundTrip(rtt);
+                        }
                     }
                 }
             }
-- 
cgit v1.1