Moved DataSnapshot feature to OptionalModules too.

1 files changed, 487 insertions, 0 deletions

diff --git a/OpenSim/Region/OptionalModules/DataSnapshot/DataSnapshotManager.cs b/OpenSim/Region/OptionalModules/DataSnapshot/DataSnapshotManager.cs
new file mode 100644
index 0000000..0c3446d
--- /dev/null
+++ b/OpenSim/Region/OptionalModules/DataSnapshot/DataSnapshotManager.cs
@@ -0,0 +1,487 @@
+/*
+* Copyright (c) Contributors, http://opensimulator.org/
+* See CONTRIBUTORS.TXT for a full list of copyright holders.
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following conditions are met:
+*     * Redistributions of source code must retain the above copyright
+*       notice, this list of conditions and the following disclaimer.
+*     * Redistributions in binary form must reproduce the above copyright
+*       notice, this list of conditions and the following disclaimer in the
+*       documentation and/or other materials provided with the distribution.
+*     * Neither the name of the OpenSimulator Project nor the
+*       names of its contributors may be used to endorse or promote products
+*       derived from this software without specific prior written permission.
+*
+* THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
+* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+* DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
+* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*
+*/
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Reflection;
+using System.Text;
+using System.Xml;
+using log4net;
+using Nini.Config;
+using OpenMetaverse;
+using Mono.Addins;
+using OpenSim.Framework;
+using OpenSim.Region.DataSnapshot.Interfaces;
+using OpenSim.Region.Framework.Interfaces;
+using OpenSim.Region.Framework.Scenes;
+
+namespace OpenSim.Region.DataSnapshot
+{
+    [Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "DataSnapshotManager")]
+    public class DataSnapshotManager : ISharedRegionModule, IDataSnapshot
+    {
+        #region Class members
+        //Information from config
+        private bool m_enabled = false;
+        private bool m_configLoaded = false;
+        private List<string> m_disabledModules = new List<string>();
+        private Dictionary<string, string> m_gridinfo = new Dictionary<string, string>();
+        private string m_snapsDir = "DataSnapshot";
+        private string m_exposure_level = "minimum";
+
+        //Lists of stuff we need
+        private List<Scene> m_scenes = new List<Scene>();
+        private List<IDataSnapshotProvider> m_dataproviders = new List<IDataSnapshotProvider>();
+
+        //Various internal objects
+        private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
+        internal object m_syncInit = new object();
+
+        //DataServices and networking
+        private string m_dataServices = "noservices";
+        public string m_listener_port = ConfigSettings.DefaultRegionHttpPort.ToString();
+        public string m_hostname = "127.0.0.1";
+        private UUID m_Secret = UUID.Random();
+        private bool m_servicesNotified = false;
+
+        //Update timers
+        private int m_period = 20; // in seconds
+        private int m_maxStales = 500;
+        private int m_stales = 0;
+        private int m_lastUpdate = 0;
+
+        //Program objects
+        private SnapshotStore m_snapStore = null;
+
+        #endregion
+
+        #region Properties
+
+        public string ExposureLevel
+        {
+            get { return m_exposure_level; }
+        }
+
+        public UUID Secret
+        {
+            get { return m_Secret; }
+        }
+
+        #endregion
+
+        #region Region Module interface
+
+        public void Initialise(IConfigSource config)
+        {
+            if (!m_configLoaded) 
+            {
+                m_configLoaded = true;
+                //m_log.Debug("[DATASNAPSHOT]: Loading configuration");
+                //Read from the config for options
+                lock (m_syncInit)
+                {
+                    try
+                    {
+                        m_enabled = config.Configs["DataSnapshot"].GetBoolean("index_sims", m_enabled);
+                        string gatekeeper = Util.GetConfigVarFromSections<string>(config, "GatekeeperURI",
+                            new string[] { "Startup", "Hypergrid", "GridService" }, String.Empty);
+                        // Legacy. Remove soon!
+                        if (string.IsNullOrEmpty(gatekeeper))
+                        {
+                            IConfig conf = config.Configs["GridService"];
+                            if (conf != null)
+                                gatekeeper = conf.GetString("Gatekeeper", gatekeeper);
+                        }
+                        if (!string.IsNullOrEmpty(gatekeeper))
+                            m_gridinfo.Add("gatekeeperURL", gatekeeper);
+
+                        m_gridinfo.Add(
+                            "name", config.Configs["DataSnapshot"].GetString("gridname", "the lost continent of hippo"));
+                        m_exposure_level = config.Configs["DataSnapshot"].GetString("data_exposure", m_exposure_level);
+                        m_period = config.Configs["DataSnapshot"].GetInt("default_snapshot_period", m_period);
+                        m_maxStales = config.Configs["DataSnapshot"].GetInt("max_changes_before_update", m_maxStales);
+                        m_snapsDir = config.Configs["DataSnapshot"].GetString("snapshot_cache_directory", m_snapsDir);
+                        m_listener_port = config.Configs["Network"].GetString("http_listener_port", m_listener_port);
+
+                        m_dataServices = config.Configs["DataSnapshot"].GetString("data_services", m_dataServices);
+                        // New way of spec'ing data services, one per line
+                        AddDataServicesVars(config.Configs["DataSnapshot"]);
+
+                        String[] annoying_string_array = config.Configs["DataSnapshot"].GetString("disable_modules", "").Split(".".ToCharArray());
+                        foreach (String bloody_wanker in annoying_string_array)
+                        {
+                            m_disabledModules.Add(bloody_wanker);
+                        }
+                        m_lastUpdate = Environment.TickCount;
+                    }
+                    catch (Exception)
+                    {
+                        m_log.Warn("[DATASNAPSHOT]: Could not load configuration. DataSnapshot will be disabled.");
+                        m_enabled = false;
+                        return;
+                    }
+
+                }
+
+            }
+
+        }
+
+        public void AddRegion(Scene scene)
+        {
+            if (!m_enabled)
+                return;
+
+            m_log.DebugFormat("[DATASNAPSHOT]: Module added to Scene {0}.", scene.RegionInfo.RegionName);
+
+            if (!m_servicesNotified)
+            {
+                m_hostname = scene.RegionInfo.ExternalHostName;
+                m_snapStore = new SnapshotStore(m_snapsDir, m_gridinfo, m_listener_port, m_hostname);
+
+                //Hand it the first scene, assuming that all scenes have the same BaseHTTPServer
+                new DataRequestHandler(scene, this);
+
+                if (m_dataServices != "" && m_dataServices != "noservices")
+                    NotifyDataServices(m_dataServices, "online");
+
+                m_servicesNotified = true;
+            }
+
+            m_scenes.Add(scene);
+            m_snapStore.AddScene(scene);
+
+            Assembly currentasm = Assembly.GetExecutingAssembly();
+
+            foreach (Type pluginType in currentasm.GetTypes())
+            {
+                if (pluginType.IsPublic)
+                {
+                    if (!pluginType.IsAbstract)
+                    {
+                        if (pluginType.GetInterface("IDataSnapshotProvider") != null)
+                        {
+                            IDataSnapshotProvider module = (IDataSnapshotProvider)Activator.CreateInstance(pluginType);
+                            module.Initialize(scene, this);
+                            module.OnStale += MarkDataStale;
+
+                            m_dataproviders.Add(module);
+                            m_snapStore.AddProvider(module);
+
+                            m_log.Debug("[DATASNAPSHOT]: Added new data provider type: " + pluginType.Name);
+                        }
+                    }
+                }
+            }
+
+        }
+
+        public void RemoveRegion(Scene scene)
+        {
+            if (!m_enabled)
+                return;
+
+            m_log.Info("[DATASNAPSHOT]: Region " + scene.RegionInfo.RegionName + " is being removed, removing from indexing");
+            Scene restartedScene = SceneForUUID(scene.RegionInfo.RegionID);
+
+            m_scenes.Remove(restartedScene);
+            m_snapStore.RemoveScene(restartedScene);
+
+            //Getting around the fact that we can't remove objects from a collection we are enumerating over
+            List<IDataSnapshotProvider> providersToRemove = new List<IDataSnapshotProvider>();
+
+            foreach (IDataSnapshotProvider provider in m_dataproviders)
+            {
+                if (provider.GetParentScene == restartedScene)
+                {
+                    providersToRemove.Add(provider);
+                }
+            }
+
+            foreach (IDataSnapshotProvider provider in providersToRemove)
+            {
+                m_dataproviders.Remove(provider);
+                m_snapStore.RemoveProvider(provider);
+            }
+
+            m_snapStore.RemoveScene(restartedScene);
+        }
+
+        public void PostInitialise()
+        {
+        }
+
+        public void RegionLoaded(Scene scene)
+        {
+            if (!m_enabled)
+                return;
+
+            m_log.DebugFormat("[DATASNAPSHOT]: Marking scene {0} as stale.", scene.RegionInfo.RegionName);
+            m_snapStore.ForceSceneStale(scene);
+        }
+
+        public void Close() 
+        {
+            if (!m_enabled)
+                return;
+
+            if (m_enabled && m_dataServices != "" && m_dataServices != "noservices")
+                NotifyDataServices(m_dataServices, "offline");
+        }
+
+
+        public string Name
+        {
+            get { return "External Data Generator"; }
+        }
+
+        public Type ReplaceableInterface
+        {
+            get { return null; }
+        }
+
+        #endregion
+
+        #region Associated helper functions
+
+        public Scene SceneForName(string name)
+        {
+            foreach (Scene scene in m_scenes)
+                if (scene.RegionInfo.RegionName == name)
+                    return scene;
+
+            return null;
+        }
+
+        public Scene SceneForUUID(UUID id)
+        {
+            foreach (Scene scene in m_scenes)
+                if (scene.RegionInfo.RegionID == id)
+                    return scene;
+
+            return null;
+        }
+
+        private void AddDataServicesVars(IConfig config)
+        {
+            // Make sure the services given this way aren't in m_dataServices already
+            List<string> servs = new List<string>(m_dataServices.Split(new char[] { ';' }));
+
+            StringBuilder sb = new StringBuilder();
+            string[] keys = config.GetKeys();
+
+            if (keys.Length > 0)
+            {
+                IEnumerable<string> serviceKeys = keys.Where(value => value.StartsWith("DATA_SRV_"));
+                foreach (string serviceKey in serviceKeys)
+                {
+                    string keyValue = config.GetString(serviceKey, string.Empty).Trim();
+                    if (!servs.Contains(keyValue))
+                        sb.Append(keyValue).Append(";");
+                }
+            }
+
+            m_dataServices = (m_dataServices == "noservices") ? sb.ToString() : sb.Append(m_dataServices).ToString();
+        }
+
+        #endregion
+
+        #region [Public] Snapshot storage functions
+
+        /**
+         * Reply to the http request
+         */
+        public XmlDocument GetSnapshot(string regionName)
+        {
+            CheckStale();
+
+            XmlDocument requestedSnap = new XmlDocument();
+            requestedSnap.AppendChild(requestedSnap.CreateXmlDeclaration("1.0", null, null));
+            requestedSnap.AppendChild(requestedSnap.CreateWhitespace("\r\n"));
+
+            XmlNode regiondata = requestedSnap.CreateNode(XmlNodeType.Element, "regiondata", "");
+            try
+            {
+                if (string.IsNullOrEmpty(regionName))
+                {
+                    XmlNode timerblock = requestedSnap.CreateNode(XmlNodeType.Element, "expire", "");
+                    timerblock.InnerText = m_period.ToString();
+                    regiondata.AppendChild(timerblock);
+
+                    regiondata.AppendChild(requestedSnap.CreateWhitespace("\r\n"));
+                    foreach (Scene scene in m_scenes)
+                    {
+                        regiondata.AppendChild(m_snapStore.GetScene(scene, requestedSnap));
+                    }
+                }
+                else
+                {
+                    Scene scene = SceneForName(regionName);
+                    regiondata.AppendChild(m_snapStore.GetScene(scene, requestedSnap));
+                }
+                requestedSnap.AppendChild(regiondata);
+                regiondata.AppendChild(requestedSnap.CreateWhitespace("\r\n"));
+            }
+            catch (XmlException e)
+            {
+                m_log.Warn("[DATASNAPSHOT]: XmlException while trying to load snapshot: " + e.ToString());
+                requestedSnap = GetErrorMessage(regionName, e);
+            }
+            catch (Exception e)
+            {
+                m_log.Warn("[DATASNAPSHOT]: Caught unknown exception while trying to load snapshot: " + e.StackTrace);
+                requestedSnap = GetErrorMessage(regionName, e);
+            }
+
+
+            return requestedSnap;
+        }
+
+        private XmlDocument GetErrorMessage(string regionName, Exception e)
+        {
+            XmlDocument errorMessage = new XmlDocument();
+            XmlNode error = errorMessage.CreateNode(XmlNodeType.Element, "error", "");
+            XmlNode region = errorMessage.CreateNode(XmlNodeType.Element, "region", "");
+            region.InnerText = regionName;
+
+            XmlNode exception = errorMessage.CreateNode(XmlNodeType.Element, "exception", "");
+            exception.InnerText = e.ToString();
+
+            error.AppendChild(region);
+            error.AppendChild(exception);
+            errorMessage.AppendChild(error);
+
+            return errorMessage;
+        }
+
+        #endregion
+
+        #region External data services
+        private void NotifyDataServices(string servicesStr, string serviceName)
+        {
+            Stream reply = null;
+            string delimStr = ";";
+            char [] delimiter = delimStr.ToCharArray();
+
+            string[] services = servicesStr.Split(delimiter, StringSplitOptions.RemoveEmptyEntries);
+
+            for (int i = 0; i < services.Length; i++)
+            {
+                string url = services[i].Trim();
+                using (RestClient cli = new RestClient(url))
+                {
+                    cli.AddQueryParameter("service", serviceName);
+                    cli.AddQueryParameter("host", m_hostname);
+                    cli.AddQueryParameter("port", m_listener_port);
+                    cli.AddQueryParameter("secret", m_Secret.ToString());
+                    cli.RequestMethod = "GET";
+                    try
+                    {
+                        reply = cli.Request(null);
+                    }
+                    catch (WebException)
+                    {
+                        m_log.Warn("[DATASNAPSHOT]: Unable to notify " + url);
+                    }
+                    catch (Exception e)
+                    {
+                        m_log.Warn("[DATASNAPSHOT]: Ignoring unknown exception " + e.ToString());
+                    }
+
+                    byte[] response = new byte[1024];
+                    // int n = 0;
+                    try
+                    {
+                        // n = reply.Read(response, 0, 1024);
+                        reply.Read(response, 0, 1024);
+                    }
+                    catch (Exception e)
+                    {
+                        m_log.WarnFormat("[DATASNAPSHOT]: Unable to decode reply from data service. Ignoring. {0}", e.StackTrace);
+                    }
+                    // This is not quite working, so...
+                    // string responseStr = Util.UTF8.GetString(response);
+                    m_log.Info("[DATASNAPSHOT]: data service " + url + " notified. Secret: " + m_Secret);
+                }
+            }
+
+        }
+        #endregion
+
+        #region Latency-based update functions
+
+        public void MarkDataStale(IDataSnapshotProvider provider)
+        {
+            //Behavior here: Wait m_period seconds, then update if there has not been a request in m_period seconds
+            //or m_maxStales has been exceeded
+            m_stales++;
+        }
+
+        private void CheckStale()
+        {
+            // Wrap check
+            if (Environment.TickCount < m_lastUpdate)
+            {
+                m_lastUpdate = Environment.TickCount;
+            }
+
+            if (m_stales >= m_maxStales)
+            {
+                if (Environment.TickCount - m_lastUpdate >= 20000)
+                {
+                    m_stales = 0;
+                    m_lastUpdate = Environment.TickCount;
+                    MakeEverythingStale();
+                }
+            }
+            else
+            {
+                if (m_lastUpdate + 1000 * m_period < Environment.TickCount)
+                {
+                    m_stales = 0;
+                    m_lastUpdate = Environment.TickCount;
+                    MakeEverythingStale();
+                }
+            }
+        }
+
+        public void MakeEverythingStale()
+        {
+            m_log.Debug("[DATASNAPSHOT]: Marking all scenes as stale.");
+            foreach (Scene scene in m_scenes)
+            {
+                m_snapStore.ForceSceneStale(scene);
+            }
+        }
+        #endregion
+
+    }
+}