From 8fa13e387110233e2a38ceb41906f65992f014e5 Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Wed, 14 Apr 2010 19:48:40 -0700 Subject: Patch from mcortez to add basic caching to the groups module. This prevents database/network explosions when you have a significant number of group-owned prims in a scene --- .../SimianGroupsServicesConnectorModule.cs | 81 +++++++++++-- .../XmlRpcGroupsServicesConnectorModule.cs | 131 ++++++++++++++------- 2 files changed, 160 insertions(+), 52 deletions(-) (limited to 'OpenSim/Region/OptionalModules') diff --git a/OpenSim/Region/OptionalModules/Avatar/XmlRpcGroups/SimianGroupsServicesConnectorModule.cs b/OpenSim/Region/OptionalModules/Avatar/XmlRpcGroups/SimianGroupsServicesConnectorModule.cs index 669373f..9363205 100644 --- a/OpenSim/Region/OptionalModules/Avatar/XmlRpcGroups/SimianGroupsServicesConnectorModule.cs +++ b/OpenSim/Region/OptionalModules/Avatar/XmlRpcGroups/SimianGroupsServicesConnectorModule.cs @@ -167,6 +167,9 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups private bool m_debugEnabled = false; + private ExpiringCache m_memoryCache; + private int m_cacheTimeout = 30; + // private IUserAccountService m_accountService = null; @@ -203,7 +206,7 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups return; } - m_log.InfoFormat("[GROUPS-CONNECTOR]: Initializing {0}", this.Name); + m_log.InfoFormat("[SIMIAN-GROUPS-CONNECTOR]: Initializing {0}", this.Name); m_groupsServerURI = groupsConfig.GetString("GroupsServerURI", string.Empty); if ((m_groupsServerURI == null) || @@ -214,6 +217,22 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups return; } + + m_cacheTimeout = groupsConfig.GetInt("GroupsCacheTimeout", 30); + if (m_cacheTimeout == 0) + { + m_log.WarnFormat("[SIMIAN-GROUPS-CONNECTOR] Groups Cache Disabled."); + } + else + { + m_log.InfoFormat("[SIMIAN-GROUPS-CONNECTOR] Groups Cache Timeout set to {0}.", m_cacheTimeout); + } + + + + m_memoryCache = new ExpiringCache(); + + // If we got all the config options we need, lets start'er'up m_connectorEnabled = true; @@ -224,7 +243,7 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups public void Close() { - m_log.InfoFormat("[GROUPS-CONNECTOR]: Closing {0}", this.Name); + m_log.InfoFormat("[SIMIAN-GROUPS-CONNECTOR]: Closing {0}", this.Name); } public void AddRegion(OpenSim.Region.Framework.Scenes.Scene scene) @@ -657,7 +676,7 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups }; - OSDMap response = WebUtil.PostToService(m_groupsServerURI, requestArgs); + OSDMap response = CachedPostRequest(requestArgs); if (response["Success"].AsBoolean() && response["Entries"] is OSDArray) { OSDArray entryArray = (OSDArray)response["Entries"]; @@ -1086,7 +1105,7 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups }; - OSDMap Response = WebUtil.PostToService(m_groupsServerURI, RequestArgs); + OSDMap Response = CachedPostRequest(RequestArgs); if (Response["Success"].AsBoolean()) { return true; @@ -1113,7 +1132,7 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups }; - OSDMap Response = WebUtil.PostToService(m_groupsServerURI, RequestArgs); + OSDMap Response = CachedPostRequest(RequestArgs); if (Response["Success"].AsBoolean() && Response["Entries"] is OSDArray) { OSDArray entryArray = (OSDArray)Response["Entries"]; @@ -1153,7 +1172,7 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups }; - OSDMap Response = WebUtil.PostToService(m_groupsServerURI, RequestArgs); + OSDMap Response = CachedPostRequest(RequestArgs); if (Response["Success"].AsBoolean() && Response["Entries"] is OSDArray) { OSDArray entryArray = (OSDArray)Response["Entries"]; @@ -1194,7 +1213,7 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups }; - OSDMap Response = WebUtil.PostToService(m_groupsServerURI, RequestArgs); + OSDMap Response = CachedPostRequest(RequestArgs); if (Response["Success"].AsBoolean() && Response["Entries"] is OSDArray) { OSDArray entryArray = (OSDArray)Response["Entries"]; @@ -1234,7 +1253,7 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups - OSDMap response = WebUtil.PostToService(m_groupsServerURI, requestArgs); + OSDMap response = CachedPostRequest(requestArgs); if (response["Success"].AsBoolean() && response["Entries"] is OSDArray) { maps = new Dictionary(); @@ -1272,7 +1291,7 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups - OSDMap response = WebUtil.PostToService(m_groupsServerURI, requestArgs); + OSDMap response = CachedPostRequest(requestArgs); if (response["Success"].AsBoolean() && response["Entries"] is OSDArray) { maps = new Dictionary(); @@ -1310,7 +1329,7 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups }; - OSDMap response = WebUtil.PostToService(m_groupsServerURI, requestArgs); + OSDMap response = CachedPostRequest(requestArgs); if (response["Success"].AsBoolean()) { return true; @@ -1323,6 +1342,48 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups } #endregion + #region CheesyCache + OSDMap CachedPostRequest(NameValueCollection requestArgs) + { + // Immediately forward the request if the cache is disabled. + if (m_cacheTimeout == 0) + { + return WebUtil.PostToService(m_groupsServerURI, requestArgs); + } + + // Check if this is an update or a request + if ( requestArgs["RequestMethod"] == "RemoveGeneric" + || requestArgs["RequestMethod"] == "AddGeneric" + ) + + { + // Any and all updates cause the cache to clear + m_memoryCache.Clear(); + + // Send update to server, return the response without caching it + return WebUtil.PostToService(m_groupsServerURI, requestArgs); + + } + + // If we're not doing an update, we must be requesting data + + // Create the cache key for the request and see if we have it cached + string CacheKey = WebUtil.BuildQueryString(requestArgs); + OSDMap response = null; + if (!m_memoryCache.TryGetValue(CacheKey, out response)) + { + // if it wasn't in the cache, pass the request to the Simian Grid Services + response = WebUtil.PostToService(m_groupsServerURI, requestArgs); + + // and cache the response + m_memoryCache.AddOrUpdate(CacheKey, response, TimeSpan.FromSeconds(m_cacheTimeout)); + } + + // return cached response + return response; + } + #endregion + } } diff --git a/OpenSim/Region/OptionalModules/Avatar/XmlRpcGroups/XmlRpcGroupsServicesConnectorModule.cs b/OpenSim/Region/OptionalModules/Avatar/XmlRpcGroups/XmlRpcGroupsServicesConnectorModule.cs index 523dfbe..e7967d1 100644 --- a/OpenSim/Region/OptionalModules/Avatar/XmlRpcGroups/XmlRpcGroupsServicesConnectorModule.cs +++ b/OpenSim/Region/OptionalModules/Avatar/XmlRpcGroups/XmlRpcGroupsServicesConnectorModule.cs @@ -29,6 +29,7 @@ using System; using System.Collections; using System.Collections.Generic; using System.Reflection; +using System.Text; using Nwc.XmlRpc; @@ -70,6 +71,9 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups private IUserAccountService m_accountService = null; + private ExpiringCache m_memoryCache; + private int m_cacheTimeout = 30; + // Used to track which agents are have dropped from a group chat session // Should be reset per agent, on logon // TODO: move this to Flotsam XmlRpc Service @@ -111,7 +115,7 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups return; } - m_log.InfoFormat("[GROUPS-CONNECTOR]: Initializing {0}", this.Name); + m_log.InfoFormat("[XMLRPC-GROUPS-CONNECTOR]: Initializing {0}", this.Name); m_groupsServerURI = groupsConfig.GetString("GroupsServerURI", string.Empty); if ((m_groupsServerURI == null) || @@ -127,7 +131,16 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups m_groupReadKey = groupsConfig.GetString("XmlRpcServiceReadKey", string.Empty); m_groupWriteKey = groupsConfig.GetString("XmlRpcServiceWriteKey", string.Empty); - + + m_cacheTimeout = groupsConfig.GetInt("GroupsCacheTimeout", 30); + if (m_cacheTimeout == 0) + { + m_log.WarnFormat("[XMLRPC-GROUPS-CONNECTOR]: Groups Cache Disabled."); + } + else + { + m_log.InfoFormat("[XMLRPC-GROUPS-CONNECTOR]: Groups Cache Timeout set to {0}.", m_cacheTimeout); + } // If we got all the config options we need, lets start'er'up @@ -137,7 +150,7 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups public void Close() { - m_log.InfoFormat("[GROUPS-CONNECTOR]: Closing {0}", this.Name); + m_log.InfoFormat("[XMLRPC-GROUPS-CONNECTOR]: Closing {0}", this.Name); } public void AddRegion(OpenSim.Region.Framework.Scenes.Scene scene) @@ -919,50 +932,84 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups /// private Hashtable XmlRpcCall(UUID requestingAgentID, string function, Hashtable param) { - string UserService; - UUID SessionID; - GetClientGroupRequestID(requestingAgentID, out UserService, out SessionID); - param.Add("requestingAgentID", requestingAgentID.ToString()); - param.Add("RequestingAgentUserService", UserService); - param.Add("RequestingSessionID", SessionID.ToString()); - + XmlRpcResponse resp = null; + string CacheKey = null; + + // Only bother with the cache if it isn't disabled. + if (m_cacheTimeout > 0) + { + if (!function.StartsWith("groups.get")) + { + // Any and all updates cause the cache to clear + m_memoryCache.Clear(); + } + else + { + StringBuilder sb = new StringBuilder(requestingAgentID + function); + foreach (object key in param.Keys) + { + if (param[key] != null) + { + sb.AppendFormat(",{0}:{1}", key.ToString(), param[key].ToString()); + } + } + + CacheKey = sb.ToString(); + m_memoryCache.TryGetValue(CacheKey, out resp); + } - param.Add("ReadKey", m_groupReadKey); - param.Add("WriteKey", m_groupWriteKey); + } + + if( resp == null ) + { + string UserService; + UUID SessionID; + GetClientGroupRequestID(requestingAgentID, out UserService, out SessionID); + param.Add("requestingAgentID", requestingAgentID.ToString()); + param.Add("RequestingAgentUserService", UserService); + param.Add("RequestingSessionID", SessionID.ToString()); - IList parameters = new ArrayList(); - parameters.Add(param); + param.Add("ReadKey", m_groupReadKey); + param.Add("WriteKey", m_groupWriteKey); - ConfigurableKeepAliveXmlRpcRequest req; - req = new ConfigurableKeepAliveXmlRpcRequest(function, parameters, m_disableKeepAlive); - XmlRpcResponse resp = null; + IList parameters = new ArrayList(); + parameters.Add(param); - try - { - resp = req.Send(m_groupsServerURI, 10000); - } - catch (Exception e) - { - + ConfigurableKeepAliveXmlRpcRequest req; + req = new ConfigurableKeepAliveXmlRpcRequest(function, parameters, m_disableKeepAlive); - m_log.ErrorFormat("[XMLRPCGROUPDATA]: An error has occured while attempting to access the XmlRpcGroups server method: {0}", function); - m_log.ErrorFormat("[XMLRPCGROUPDATA]: {0} ", e.ToString()); - foreach (string ResponseLine in req.RequestResponse.Split(new string[] { Environment.NewLine },StringSplitOptions.None)) + try { - m_log.ErrorFormat("[XMLRPCGROUPDATA]: {0} ", ResponseLine); - } + resp = req.Send(m_groupsServerURI, 10000); + + if ((m_cacheTimeout > 0) && (CacheKey != null)) + { + m_memoryCache.AddOrUpdate(CacheKey, resp, TimeSpan.FromSeconds(m_cacheTimeout)); + } - foreach (string key in param.Keys) + } + catch (Exception e) { - m_log.WarnFormat("[XMLRPCGROUPDATA]: {0} :: {1}", key, param[key].ToString()); + m_log.ErrorFormat("[XMLRPC-GROUPS-CONNECTOR]: An error has occured while attempting to access the XmlRpcGroups server method: {0}", function); + m_log.ErrorFormat("[XMLRPC-GROUPS-CONNECTOR]: {0} ", e.ToString()); + + foreach (string ResponseLine in req.RequestResponse.Split(new string[] { Environment.NewLine }, StringSplitOptions.None)) + { + m_log.ErrorFormat("[XMLRPC-GROUPS-CONNECTOR]: {0} ", ResponseLine); + } + + foreach (string key in param.Keys) + { + m_log.WarnFormat("[XMLRPC-GROUPS-CONNECTOR]: {0} :: {1}", key, param[key].ToString()); + } + + Hashtable respData = new Hashtable(); + respData.Add("error", e.ToString()); + return respData; } - - Hashtable respData = new Hashtable(); - respData.Add("error", e.ToString()); - return respData; } if (resp.Value is Hashtable) @@ -976,21 +1023,21 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups return respData; } - m_log.ErrorFormat("[XMLRPCGROUPDATA]: The XmlRpc server returned a {1} instead of a hashtable for {0}", function, resp.Value.GetType().ToString()); + m_log.ErrorFormat("[XMLRPC-GROUPS-CONNECTOR]: The XmlRpc server returned a {1} instead of a hashtable for {0}", function, resp.Value.GetType().ToString()); if (resp.Value is ArrayList) { ArrayList al = (ArrayList)resp.Value; - m_log.ErrorFormat("[XMLRPCGROUPDATA]: Contains {0} elements", al.Count); + m_log.ErrorFormat("[XMLRPC-GROUPS-CONNECTOR]: Contains {0} elements", al.Count); foreach (object o in al) { - m_log.ErrorFormat("[XMLRPCGROUPDATA]: {0} :: {1}", o.GetType().ToString(), o.ToString()); + m_log.ErrorFormat("[XMLRPC-GROUPS-CONNECTOR]: {0} :: {1}", o.GetType().ToString(), o.ToString()); } } else { - m_log.ErrorFormat("[XMLRPCGROUPDATA]: Function returned: {0}", resp.Value.ToString()); + m_log.ErrorFormat("[XMLRPC-GROUPS-CONNECTOR]: Function returned: {0}", resp.Value.ToString()); } Hashtable error = new Hashtable(); @@ -1000,16 +1047,16 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups private void LogRespDataToConsoleError(Hashtable respData) { - m_log.Error("[XMLRPCGROUPDATA]: Error:"); + m_log.Error("[XMLRPC-GROUPS-CONNECTOR]: Error:"); foreach (string key in respData.Keys) { - m_log.ErrorFormat("[XMLRPCGROUPDATA]: Key: {0}", key); + m_log.ErrorFormat("[XMLRPC-GROUPS-CONNECTOR]: Key: {0}", key); string[] lines = respData[key].ToString().Split(new char[] { '\n' }); foreach (string line in lines) { - m_log.ErrorFormat("[XMLRPCGROUPDATA]: {0}", line); + m_log.ErrorFormat("[XMLRPC-GROUPS-CONNECTOR]: {0}", line); } } -- cgit v1.1