Cleared up much confusion in PollServiceRequestManager. Here's the history:

When Melanie added the web fetch inventory throttle to core, she made the long poll requests (EQs) effectively be handled on an active loop. All those requests, if they existed, were being constantly dequeued, checked for events (which most often they didn't have), and requeued again. This was an active loop thread on a 100ms cycle!
This fixes the issue. Now the inventory requests, if they aren't ready to be served, are placed directly back in the queue, but the long poll requests aren't placed there until there are events ready to be sent or timeout has been reached.
This puts the LongPollServiceWatcherThread back to 1sec cycle, as it was before.

4 files changed, 41 insertions, 52 deletions

diff --git a/OpenSim/Framework/BlockingQueue.cs b/OpenSim/Framework/BlockingQueue.cs
index aef1192..e607e64 100644
--- a/OpenSim/Framework/BlockingQueue.cs
+++ b/OpenSim/Framework/BlockingQueue.cs
@@ -76,7 +76,7 @@ namespace OpenSim.Framework
         {
             lock (m_queueSync)
             {
-                if (m_queue.Count < 1 && m_pqueue.Count < 1)
+                while (m_queue.Count < 1 && m_pqueue.Count < 1)
                 {
                     Monitor.Wait(m_queueSync, msTimeout);
                 }
diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceEventArgs.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceEventArgs.cs
index 020bfd5..9477100 100644
--- a/OpenSim/Framework/Servers/HttpServer/PollServiceEventArgs.cs
+++ b/OpenSim/Framework/Servers/HttpServer/PollServiceEventArgs.cs
@@ -50,7 +50,7 @@ namespace OpenSim.Framework.Servers.HttpServer
 
         public enum EventType : int
         {
-            Normal = 0,
+            LongPoll = 0,
             LslHttp = 1,
             Inventory = 2
         }
@@ -80,7 +80,7 @@ namespace OpenSim.Framework.Servers.HttpServer
             NoEvents = pNoEvents;
             Id = pId;
             TimeOutms = pTimeOutms;
-            Type = EventType.Normal;
+            Type = EventType.LongPoll;
         }
     }
 }
diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs
index 1b9010a..4cb551c 100644
--- a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs
+++ b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs
@@ -47,12 +47,11 @@ namespace OpenSim.Framework.Servers.HttpServer
         private readonly BaseHttpServer m_server;
 
         private BlockingQueue<PollServiceHttpRequest> m_requests = new BlockingQueue<PollServiceHttpRequest>();
-        private static Queue<PollServiceHttpRequest> m_slowRequests = new Queue<PollServiceHttpRequest>();
-        private static Queue<PollServiceHttpRequest> m_retryRequests = new Queue<PollServiceHttpRequest>();
+        private static Queue<PollServiceHttpRequest> m_longPollRequests = new Queue<PollServiceHttpRequest>();
 
         private uint m_WorkerThreadCount = 0;
         private Thread[] m_workerThreads;
-        private Thread m_retrysThread;
+        private Thread m_longPollThread;
 
         private bool m_running = true;
         private int slowCount = 0;
@@ -84,9 +83,9 @@ namespace OpenSim.Framework.Servers.HttpServer
                         int.MaxValue);
             }
 
-            m_retrysThread = Watchdog.StartThread(
-                this.CheckRetries,
-                string.Format("PollServiceWatcherThread:{0}", m_server.Port),
+            m_longPollThread = Watchdog.StartThread(
+                this.CheckLongPollThreads,
+                string.Format("LongPollServiceWatcherThread:{0}", m_server.Port),
                 ThreadPriority.Normal,
                 false,
                 true,
@@ -97,48 +96,47 @@ namespace OpenSim.Framework.Servers.HttpServer
         private void ReQueueEvent(PollServiceHttpRequest req)
         {
             if (m_running)
-            {
-                lock (m_retryRequests)
-                    m_retryRequests.Enqueue(req);
-            }
+                m_requests.Enqueue(req);
         }
 
         public void Enqueue(PollServiceHttpRequest req)
         {
             if (m_running)
             {
-                if (req.PollServiceArgs.Type != PollServiceEventArgs.EventType.Normal)
+                if (req.PollServiceArgs.Type == PollServiceEventArgs.EventType.LongPoll)
                 {
-                    m_requests.Enqueue(req);
+                    lock (m_longPollRequests)
+                        m_longPollRequests.Enqueue(req);
                 }
                 else
-                {
-                    lock (m_slowRequests)
-                        m_slowRequests.Enqueue(req);
-                }
+                    m_requests.Enqueue(req);
             }
         }
 
-        private void CheckRetries()
+        private void CheckLongPollThreads()
         {
+            // The only purpose of this thread is to check the EQs for events.
+            // If there are events, that thread will be placed in the "ready-to-serve" queue, m_requests.
+            // If there are no events, that thread will be back to its "waiting" queue, m_longPollRequests.
+            // All other types of tasks (Inventory handlers) don't have the long-poll nature,
+            // so if they aren't ready to be served by a worker thread (no events), they are placed 
+            // directly back in the "ready-to-serve" queue by the worker thread.
             while (m_running)
             {
-                Thread.Sleep(100); // let the world move  .. back to faster rate
+                Thread.Sleep(1000); 
                 Watchdog.UpdateThread();
-                lock (m_retryRequests)
-                {
-                    while (m_retryRequests.Count > 0 && m_running)
-                        m_requests.Enqueue(m_retryRequests.Dequeue());
-                }
-                slowCount++;
-                if (slowCount >= 10)
-                {
-                    slowCount = 0;
 
-                    lock (m_slowRequests)
+                PollServiceHttpRequest req;
+                lock (m_longPollRequests)
+                {
+                    while (m_longPollRequests.Count > 0 && m_running)
                     {
-                        while (m_slowRequests.Count > 0 && m_running)
-                            m_requests.Enqueue(m_slowRequests.Dequeue());
+                        req = m_longPollRequests.Dequeue();
+                        if (req.PollServiceArgs.HasEvents(req.RequestID, req.PollServiceArgs.Id) || // there are events in this EQ
+                            (Environment.TickCount - req.RequestTime) > req.PollServiceArgs.TimeOutms) // no events, but timeout
+                            m_requests.Enqueue(req);
+                        else
+                            m_longPollRequests.Enqueue(req);
                     }
                 }
             }
@@ -153,24 +151,12 @@ namespace OpenSim.Framework.Servers.HttpServer
             foreach (Thread t in m_workerThreads)
                 Watchdog.AbortThread(t.ManagedThreadId);
 
-            try
-            {
-                foreach (PollServiceHttpRequest req in m_retryRequests)
-                {
-                   req.DoHTTPGruntWork(m_server, req.PollServiceArgs.NoEvents(req.RequestID, req.PollServiceArgs.Id));
-                }
-            }
-            catch
-            {
-            }
-
             PollServiceHttpRequest wreq;
-            m_retryRequests.Clear();
 
-            lock (m_slowRequests)
+            lock (m_longPollRequests)
             {
-                while (m_slowRequests.Count > 0 && m_running)
-                    m_requests.Enqueue(m_slowRequests.Dequeue());
+                while (m_longPollRequests.Count > 0 && m_running)
+                    m_requests.Enqueue(m_longPollRequests.Dequeue());
             }
 
             while (m_requests.Count() > 0)
@@ -196,6 +182,7 @@ namespace OpenSim.Framework.Servers.HttpServer
             while (m_running)
             {
                 PollServiceHttpRequest req = m_requests.Dequeue(5000);
+                //m_log.WarnFormat("[YYY]: Dequeued {0}", (req == null ? "null" : req.PollServiceArgs.Type.ToString()));
 
                 Watchdog.UpdateThread();
                 if (req != null)
@@ -209,7 +196,7 @@ namespace OpenSim.Framework.Servers.HttpServer
                             if (responsedata == null)
                                 continue;
 
-                            if (req.PollServiceArgs.Type == PollServiceEventArgs.EventType.Normal) // This is the event queue
+                            if (req.PollServiceArgs.Type == PollServiceEventArgs.EventType.LongPoll) // This is the event queue
                             {
                                 try
                                 {
diff --git a/OpenSim/Region/ClientStack/Linden/Caps/EventQueue/EventQueueGetModule.cs b/OpenSim/Region/ClientStack/Linden/Caps/EventQueue/EventQueueGetModule.cs
index 1835a72..f0445ff 100644
--- a/OpenSim/Region/ClientStack/Linden/Caps/EventQueue/EventQueueGetModule.cs
+++ b/OpenSim/Region/ClientStack/Linden/Caps/EventQueue/EventQueueGetModule.cs
@@ -364,8 +364,7 @@ namespace OpenSim.Region.ClientStack.Linden
 
             caps.RegisterPollHandler(
                 "EventQueueGet", 
-                new PollServiceEventArgs(
-                    null, GenerateEqgCapPath(eventQueueGetUUID), HasEvents, GetEvents, NoEvents, agentID, 40000));
+                new PollServiceEventArgs(null, GenerateEqgCapPath(eventQueueGetUUID), HasEvents, GetEvents, NoEvents, agentID, 40000));
 
             Random rnd = new Random(Environment.TickCount);
             lock (m_ids)
@@ -383,7 +382,10 @@ namespace OpenSim.Region.ClientStack.Linden
             Queue<OSD> queue = GetQueue(agentID);
             if (queue != null)
                 lock (queue)
+                {
+                    //m_log.WarnFormat("POLLED FOR EVENTS BY {0} in {1} -- {2}", agentID, m_scene.RegionInfo.RegionName, queue.Count);
                     return queue.Count > 0;
+                }
 
             return false;
         }
@@ -406,7 +408,7 @@ namespace OpenSim.Region.ClientStack.Linden
         public Hashtable GetEvents(UUID requestID, UUID pAgentId)
         {
             if (DebugLevel >= 2)
-                m_log.DebugFormat("POLLED FOR EQ MESSAGES BY {0} in {1}", pAgentId, m_scene.RegionInfo.RegionName);
+                m_log.WarnFormat("POLLED FOR EQ MESSAGES BY {0} in {1}", pAgentId, m_scene.RegionInfo.RegionName);
 
             Queue<OSD> queue = TryGetQueue(pAgentId);
             OSD element;