From e46459ef21e1ee5ceaeca70365a7c881d33b09ce Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Wed, 17 Jul 2013 11:19:36 -0700 Subject: Cleared up much confusion in PollServiceRequestManager. Here's the history: When Melanie added the web fetch inventory throttle to core, she made the long poll requests (EQs) effectively be handled on an active loop. All those requests, if they existed, were being constantly dequeued, checked for events (which most often they didn't have), and requeued again. This was an active loop thread on a 100ms cycle! This fixes the issue. Now the inventory requests, if they aren't ready to be served, are placed directly back in the queue, but the long poll requests aren't placed there until there are events ready to be sent or timeout has been reached. This puts the LongPollServiceWatcherThread back to 1sec cycle, as it was before. --- .../Servers/HttpServer/PollServiceEventArgs.cs | 4 +- .../HttpServer/PollServiceRequestManager.cs | 79 +++++++++------------- 2 files changed, 35 insertions(+), 48 deletions(-) (limited to 'OpenSim/Framework/Servers') diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceEventArgs.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceEventArgs.cs index 020bfd5..9477100 100644 --- a/OpenSim/Framework/Servers/HttpServer/PollServiceEventArgs.cs +++ b/OpenSim/Framework/Servers/HttpServer/PollServiceEventArgs.cs @@ -50,7 +50,7 @@ namespace OpenSim.Framework.Servers.HttpServer public enum EventType : int { - Normal = 0, + LongPoll = 0, LslHttp = 1, Inventory = 2 } @@ -80,7 +80,7 @@ namespace OpenSim.Framework.Servers.HttpServer NoEvents = pNoEvents; Id = pId; TimeOutms = pTimeOutms; - Type = EventType.Normal; + Type = EventType.LongPoll; } } } diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs index 1b9010a..4cb551c 100644 --- a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs +++ b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs @@ -47,12 +47,11 @@ namespace OpenSim.Framework.Servers.HttpServer private readonly BaseHttpServer m_server; private BlockingQueue m_requests = new BlockingQueue(); - private static Queue m_slowRequests = new Queue(); - private static Queue m_retryRequests = new Queue(); + private static Queue m_longPollRequests = new Queue(); private uint m_WorkerThreadCount = 0; private Thread[] m_workerThreads; - private Thread m_retrysThread; + private Thread m_longPollThread; private bool m_running = true; private int slowCount = 0; @@ -84,9 +83,9 @@ namespace OpenSim.Framework.Servers.HttpServer int.MaxValue); } - m_retrysThread = Watchdog.StartThread( - this.CheckRetries, - string.Format("PollServiceWatcherThread:{0}", m_server.Port), + m_longPollThread = Watchdog.StartThread( + this.CheckLongPollThreads, + string.Format("LongPollServiceWatcherThread:{0}", m_server.Port), ThreadPriority.Normal, false, true, @@ -97,48 +96,47 @@ namespace OpenSim.Framework.Servers.HttpServer private void ReQueueEvent(PollServiceHttpRequest req) { if (m_running) - { - lock (m_retryRequests) - m_retryRequests.Enqueue(req); - } + m_requests.Enqueue(req); } public void Enqueue(PollServiceHttpRequest req) { if (m_running) { - if (req.PollServiceArgs.Type != PollServiceEventArgs.EventType.Normal) + if (req.PollServiceArgs.Type == PollServiceEventArgs.EventType.LongPoll) { - m_requests.Enqueue(req); + lock (m_longPollRequests) + m_longPollRequests.Enqueue(req); } else - { - lock (m_slowRequests) - m_slowRequests.Enqueue(req); - } + m_requests.Enqueue(req); } } - private void CheckRetries() + private void CheckLongPollThreads() { + // The only purpose of this thread is to check the EQs for events. + // If there are events, that thread will be placed in the "ready-to-serve" queue, m_requests. + // If there are no events, that thread will be back to its "waiting" queue, m_longPollRequests. + // All other types of tasks (Inventory handlers) don't have the long-poll nature, + // so if they aren't ready to be served by a worker thread (no events), they are placed + // directly back in the "ready-to-serve" queue by the worker thread. while (m_running) { - Thread.Sleep(100); // let the world move .. back to faster rate + Thread.Sleep(1000); Watchdog.UpdateThread(); - lock (m_retryRequests) - { - while (m_retryRequests.Count > 0 && m_running) - m_requests.Enqueue(m_retryRequests.Dequeue()); - } - slowCount++; - if (slowCount >= 10) - { - slowCount = 0; - lock (m_slowRequests) + PollServiceHttpRequest req; + lock (m_longPollRequests) + { + while (m_longPollRequests.Count > 0 && m_running) { - while (m_slowRequests.Count > 0 && m_running) - m_requests.Enqueue(m_slowRequests.Dequeue()); + req = m_longPollRequests.Dequeue(); + if (req.PollServiceArgs.HasEvents(req.RequestID, req.PollServiceArgs.Id) || // there are events in this EQ + (Environment.TickCount - req.RequestTime) > req.PollServiceArgs.TimeOutms) // no events, but timeout + m_requests.Enqueue(req); + else + m_longPollRequests.Enqueue(req); } } } @@ -153,24 +151,12 @@ namespace OpenSim.Framework.Servers.HttpServer foreach (Thread t in m_workerThreads) Watchdog.AbortThread(t.ManagedThreadId); - try - { - foreach (PollServiceHttpRequest req in m_retryRequests) - { - req.DoHTTPGruntWork(m_server, req.PollServiceArgs.NoEvents(req.RequestID, req.PollServiceArgs.Id)); - } - } - catch - { - } - PollServiceHttpRequest wreq; - m_retryRequests.Clear(); - lock (m_slowRequests) + lock (m_longPollRequests) { - while (m_slowRequests.Count > 0 && m_running) - m_requests.Enqueue(m_slowRequests.Dequeue()); + while (m_longPollRequests.Count > 0 && m_running) + m_requests.Enqueue(m_longPollRequests.Dequeue()); } while (m_requests.Count() > 0) @@ -196,6 +182,7 @@ namespace OpenSim.Framework.Servers.HttpServer while (m_running) { PollServiceHttpRequest req = m_requests.Dequeue(5000); + //m_log.WarnFormat("[YYY]: Dequeued {0}", (req == null ? "null" : req.PollServiceArgs.Type.ToString())); Watchdog.UpdateThread(); if (req != null) @@ -209,7 +196,7 @@ namespace OpenSim.Framework.Servers.HttpServer if (responsedata == null) continue; - if (req.PollServiceArgs.Type == PollServiceEventArgs.EventType.Normal) // This is the event queue + if (req.PollServiceArgs.Type == PollServiceEventArgs.EventType.LongPoll) // This is the event queue { try { -- cgit v1.1 From f4317dc26d670c853d0ea64b401b00f718f09474 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Wed, 17 Jul 2013 12:57:34 -0700 Subject: Putting the requests back in the queue while testing for count >0 is not the smartest move... --- .../Framework/Servers/HttpServer/PollServiceRequestManager.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'OpenSim/Framework/Servers') diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs index 4cb551c..c50df5a 100644 --- a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs +++ b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs @@ -126,18 +126,22 @@ namespace OpenSim.Framework.Servers.HttpServer Thread.Sleep(1000); Watchdog.UpdateThread(); - PollServiceHttpRequest req; + List not_ready = new List(); lock (m_longPollRequests) { while (m_longPollRequests.Count > 0 && m_running) { - req = m_longPollRequests.Dequeue(); + PollServiceHttpRequest req = m_longPollRequests.Dequeue(); if (req.PollServiceArgs.HasEvents(req.RequestID, req.PollServiceArgs.Id) || // there are events in this EQ (Environment.TickCount - req.RequestTime) > req.PollServiceArgs.TimeOutms) // no events, but timeout m_requests.Enqueue(req); else - m_longPollRequests.Enqueue(req); + not_ready.Add(req); } + + foreach (PollServiceHttpRequest req in not_ready) + m_longPollRequests.Enqueue(req); + } } } -- cgit v1.1 From 5f95f4d78e8c7d17b8ba866907156fe6d4444c04 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Wed, 17 Jul 2013 14:09:04 -0700 Subject: Now trying DoubleQueue instead of BlockingQueue for the PollServiceRequestManager. --- OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'OpenSim/Framework/Servers') diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs index c50df5a..bad28e4 100644 --- a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs +++ b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs @@ -46,7 +46,7 @@ namespace OpenSim.Framework.Servers.HttpServer private readonly BaseHttpServer m_server; - private BlockingQueue m_requests = new BlockingQueue(); + private DoubleQueue m_requests = new DoubleQueue(); private static Queue m_longPollRequests = new Queue(); private uint m_WorkerThreadCount = 0; @@ -163,7 +163,7 @@ namespace OpenSim.Framework.Servers.HttpServer m_requests.Enqueue(m_longPollRequests.Dequeue()); } - while (m_requests.Count() > 0) + while (m_requests.Count > 0) { try { -- cgit v1.1 From 5232ab0496eb4fe6903a0fd328974ac69df29ad8 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Wed, 17 Jul 2013 14:36:55 -0700 Subject: This is a completely unreasonable thing to do, effectively defying the purpose of BlockingQueues. Trying this, to see the effect on CPU. --- .../HttpServer/PollServiceRequestManager.cs | 92 ++++++++++++---------- 1 file changed, 52 insertions(+), 40 deletions(-) (limited to 'OpenSim/Framework/Servers') diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs index bad28e4..b8f5743 100644 --- a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs +++ b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs @@ -46,7 +46,7 @@ namespace OpenSim.Framework.Servers.HttpServer private readonly BaseHttpServer m_server; - private DoubleQueue m_requests = new DoubleQueue(); + private BlockingQueue m_requests = new BlockingQueue(); private static Queue m_longPollRequests = new Queue(); private uint m_WorkerThreadCount = 0; @@ -163,7 +163,7 @@ namespace OpenSim.Framework.Servers.HttpServer m_requests.Enqueue(m_longPollRequests.Dequeue()); } - while (m_requests.Count > 0) + while (m_requests.Count() > 0) { try { @@ -185,35 +185,33 @@ namespace OpenSim.Framework.Servers.HttpServer { while (m_running) { - PollServiceHttpRequest req = m_requests.Dequeue(5000); - //m_log.WarnFormat("[YYY]: Dequeued {0}", (req == null ? "null" : req.PollServiceArgs.Type.ToString())); - Watchdog.UpdateThread(); - if (req != null) + + PollServiceHttpRequest req = null; + lock (m_requests) + { + if (m_requests.Count() > 0) + req = m_requests.Dequeue(); + } + if (req == null) + Thread.Sleep(100); + else { - try + //PollServiceHttpRequest req = m_requests.Dequeue(5000); + //m_log.WarnFormat("[YYY]: Dequeued {0}", (req == null ? "null" : req.PollServiceArgs.Type.ToString())); + + if (req != null) { - if (req.PollServiceArgs.HasEvents(req.RequestID, req.PollServiceArgs.Id)) + try { - Hashtable responsedata = req.PollServiceArgs.GetEvents(req.RequestID, req.PollServiceArgs.Id); + if (req.PollServiceArgs.HasEvents(req.RequestID, req.PollServiceArgs.Id)) + { + Hashtable responsedata = req.PollServiceArgs.GetEvents(req.RequestID, req.PollServiceArgs.Id); - if (responsedata == null) - continue; + if (responsedata == null) + continue; - if (req.PollServiceArgs.Type == PollServiceEventArgs.EventType.LongPoll) // This is the event queue - { - try - { - req.DoHTTPGruntWork(m_server, responsedata); - } - catch (ObjectDisposedException) // Browser aborted before we could read body, server closed the stream - { - // Ignore it, no need to reply - } - } - else - { - m_threadPool.QueueWorkItem(x => + if (req.PollServiceArgs.Type == PollServiceEventArgs.EventType.LongPoll) // This is the event queue { try { @@ -223,27 +221,41 @@ namespace OpenSim.Framework.Servers.HttpServer { // Ignore it, no need to reply } + } + else + { + m_threadPool.QueueWorkItem(x => + { + try + { + req.DoHTTPGruntWork(m_server, responsedata); + } + catch (ObjectDisposedException) // Browser aborted before we could read body, server closed the stream + { + // Ignore it, no need to reply + } - return null; - }, null); - } - } - else - { - if ((Environment.TickCount - req.RequestTime) > req.PollServiceArgs.TimeOutms) - { - req.DoHTTPGruntWork( - m_server, req.PollServiceArgs.NoEvents(req.RequestID, req.PollServiceArgs.Id)); + return null; + }, null); + } } else { - ReQueueEvent(req); + if ((Environment.TickCount - req.RequestTime) > req.PollServiceArgs.TimeOutms) + { + req.DoHTTPGruntWork( + m_server, req.PollServiceArgs.NoEvents(req.RequestID, req.PollServiceArgs.Id)); + } + else + { + ReQueueEvent(req); + } } } - } - catch (Exception e) - { - m_log.ErrorFormat("Exception in poll service thread: " + e.ToString()); + catch (Exception e) + { + m_log.ErrorFormat("Exception in poll service thread: " + e.ToString()); + } } } } -- cgit v1.1