2 files changed, 227 insertions, 44 deletions

diff --git a/OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs b/OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs
index 84fbcd3..c753c6a 100644
--- a/OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs
+++ b/OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs
@@ -30,11 +30,11 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Reflection;
+using System.Timers;
 using Nini.Config;
 using OpenSim.Framework;
 using OpenSim.Framework.Console;
 using OpenSim.Framework.Communications;
-using OpenSim.Framework.Servers.HttpServer;
 using OpenSim.Services.Interfaces;
 using OpenMetaverse;
 
@@ -48,6 +48,15 @@ namespace OpenSim.Services.Connectors
 
         private string m_ServerURI = String.Empty;
         private IImprovedAssetCache m_Cache = null;
+        private int m_retryCounter;
+        private Dictionary<int, List<AssetBase>> m_retryQueue = new Dictionary<int, List<AssetBase>>();
+        private Timer m_retryTimer;
+        private delegate void AssetRetrievedEx(AssetBase asset);
+
+        // Keeps track of concurrent requests for the same asset, so that it's only loaded once.
+        // Maps: Asset ID -> Handlers which will be called when the asset has been loaded
+        private Dictionary<string, AssetRetrievedEx> m_AssetHandlers = new Dictionary<string, AssetRetrievedEx>();
+        private Dictionary<string, string> m_UriMap = new Dictionary<string, string>();
 
         public AssetServicesConnector()
         {
@@ -68,7 +77,7 @@ namespace OpenSim.Services.Connectors
             IConfig assetConfig = source.Configs["AssetService"];
             if (assetConfig == null)
             {
-                m_log.Error("[ASSET CONNECTOR]: AssetService missing from OpanSim.ini");
+                m_log.Error("[ASSET CONNECTOR]: AssetService missing from OpenSim.ini");
                 throw new Exception("Asset connector init error");
             }
 
@@ -85,6 +94,83 @@ namespace OpenSim.Services.Connectors
             MainConsole.Instance.Commands.AddCommand("asset", false, "dump asset",
                                           "dump asset <id> <file>",
                                           "dump one cached asset", HandleDumpAsset);
+
+            m_retryTimer = new Timer();
+            m_retryTimer.Elapsed += new ElapsedEventHandler(retryCheck);
+            m_retryTimer.Interval = 60000;
+
+            Uri serverUri = new Uri(m_ServerURI);
+
+            string groupHost = serverUri.Host;
+
+            for (int i = 0 ; i < 256 ; i++)
+            {
+                string prefix = i.ToString("x2");
+                groupHost = assetConfig.GetString("AssetServerHost_"+prefix, groupHost);
+
+                m_UriMap[prefix] = groupHost;
+                //m_log.DebugFormat("[ASSET]: Using {0} for prefix {1}", groupHost, prefix);
+            }
+        }
+
+        private string MapServer(string id)
+        {
+            UriBuilder serverUri = new UriBuilder(m_ServerURI);
+
+            string prefix = id.Substring(0, 2).ToLower();
+
+            string host = m_UriMap[prefix];
+
+            serverUri.Host = host;
+
+            // m_log.DebugFormat("[ASSET]: Using {0} for host name for prefix {1}", host, prefix);
+
+            return serverUri.Uri.AbsoluteUri;
+        }
+
+        protected void retryCheck(object source, ElapsedEventArgs e)
+        {
+            m_retryCounter++;
+            if (m_retryCounter > 60) m_retryCounter -= 60;
+            List<int> keys = new List<int>();
+            foreach (int a in m_retryQueue.Keys)
+            {
+                keys.Add(a);
+            }
+            foreach (int a in keys)
+            {
+                //We exponentially fall back on frequency until we reach one attempt per hour
+                //The net result is that we end up in the queue for roughly 24 hours..
+                //24 hours worth of assets could be a lot, so the hope is that the region admin
+                //will have gotten the asset connector back online quickly!
+
+                int timefactor = a ^ 2; 
+                if (timefactor > 60)
+                {
+                    timefactor = 60;
+                }
+
+                //First, find out if we care about this timefactor
+                if (timefactor % a == 0)
+                {
+                    //Yes, we do!   
+                    List<AssetBase> retrylist = m_retryQueue[a];
+                    m_retryQueue.Remove(a);
+
+                    foreach(AssetBase ass in retrylist)
+                    {
+                        Store(ass); //Store my ass. This function will put it back in the dictionary if it fails
+                    }
+                }
+            }
+
+            if (m_retryQueue.Count == 0)
+            {
+                //It might only be one tick per minute, but I have
+                //repented and abandoned my wasteful ways
+                m_retryCounter = 0;
+                m_retryTimer.Stop();
+            }
         }
 
         protected void SetCache(IImprovedAssetCache cache)
@@ -94,13 +180,13 @@ namespace OpenSim.Services.Connectors
 
         public AssetBase Get(string id)
         {
-            string uri = m_ServerURI + "/assets/" + id;
+            string uri = MapServer(id) + "/assets/" + id;
 
             AssetBase asset = null;
             if (m_Cache != null)
                 asset = m_Cache.Get(id);
-
-            if (asset == null)
+            
+            if (asset == null || asset.Data == null || asset.Data.Length == 0)
             {
                 asset = SynchronousRestObjectRequester.
                         MakeRequest<int, AssetBase>("GET", uri, 0);
@@ -129,7 +215,7 @@ namespace OpenSim.Services.Connectors
                     return fullAsset.Metadata;
             }
 
-            string uri = m_ServerURI + "/assets/" + id + "/metadata";
+            string uri = MapServer(id) + "/assets/" + id + "/metadata";
 
             AssetMetadata asset = SynchronousRestObjectRequester.
                     MakeRequest<int, AssetMetadata>("GET", uri, 0);
@@ -146,7 +232,7 @@ namespace OpenSim.Services.Connectors
                     return fullAsset.Data;
             }
 
-            RestClient rc = new RestClient(m_ServerURI);
+            RestClient rc = new RestClient(MapServer(id));
             rc.AddResourcePath("assets");
             rc.AddResourcePath(id);
             rc.AddResourcePath("data");
@@ -171,31 +257,64 @@ namespace OpenSim.Services.Connectors
 
         public bool Get(string id, Object sender, AssetRetrieved handler)
         {
-            string uri = m_ServerURI + "/assets/" + id;
+            string uri = MapServer(id) + "/assets/" + id;
 
             AssetBase asset = null;
             if (m_Cache != null)
                 asset = m_Cache.Get(id);
 
-            if (asset == null)
+            if (asset == null || asset.Data == null || asset.Data.Length == 0)
             {
-                bool result = false;
-
-                AsynchronousRestObjectRequester.
-                        MakeRequest<int, AssetBase>("GET", uri, 0,
+                lock (m_AssetHandlers)
+                {
+                    AssetRetrievedEx handlerEx = new AssetRetrievedEx(delegate(AssetBase _asset) { handler(id, sender, _asset); });
+
+                    AssetRetrievedEx handlers;
+                    if (m_AssetHandlers.TryGetValue(id, out handlers))
+                    {
+                        // Someone else is already loading this asset. It will notify our handler when done.
+                        handlers += handlerEx;
+                        return true;
+                    }
+
+                    // Load the asset ourselves
+                    handlers += handlerEx;
+                    m_AssetHandlers.Add(id, handlers);
+                }
+
+                bool success = false;
+                try
+                {
+                    AsynchronousRestObjectRequester.MakeRequest<int, AssetBase>("GET", uri, 0,
                         delegate(AssetBase a)
                         {
                             if (m_Cache != null)
                                 m_Cache.Cache(a);
-                            handler(id, sender, a);
-                            result = true;
-                        });
 
-                return result;
+                            AssetRetrievedEx handlers;
+                            lock (m_AssetHandlers)
+                            {
+                                handlers = m_AssetHandlers[id];
+                                m_AssetHandlers.Remove(id);
+                            }
+                            handlers.Invoke(a);
+                        });
+                    
+                    success = true;
+                }
+                finally
+                {
+                    if (!success)
+                    {
+                        lock (m_AssetHandlers)
+                        {
+                            m_AssetHandlers.Remove(id);
+                        }
+                    }
+                }
             }
             else
             {
-                //Util.FireAndForget(delegate { handler(id, sender, asset); });
                 handler(id, sender, asset);
             }
 
@@ -204,38 +323,95 @@ namespace OpenSim.Services.Connectors
 
         public string Store(AssetBase asset)
         {
-            if (asset.Temporary || asset.Local)
+            // Have to assign the asset ID here. This isn't likely to
+            // trigger since current callers don't pass emtpy IDs
+            // We need the asset ID to route the request to the proper
+            // cluster member, so we can't have the server assign one.
+            if (asset.ID == string.Empty)
             {
-                if (m_Cache != null)
-                    m_Cache.Cache(asset);
+                if (asset.FullID == UUID.Zero)
+                {
+                    asset.FullID = UUID.Random();
+                }
+                asset.ID = asset.FullID.ToString();
+            }
+            else if (asset.FullID == UUID.Zero)
+            {
+                UUID uuid = UUID.Zero;
+                if (UUID.TryParse(asset.ID, out uuid))
+                {
+                    asset.FullID = uuid;
+                }
+                else
+                {
+                    asset.FullID = UUID.Random();
+                }
+            }
 
+            if (m_Cache != null)
+                m_Cache.Cache(asset);
+            if (asset.Temporary || asset.Local)
+            {
                 return asset.ID;
             }
 
-            string uri = m_ServerURI + "/assets/";
+            string uri = MapServer(asset.FullID.ToString()) + "/assets/";
 
             string newID = string.Empty;
             try
             {
                 newID = SynchronousRestObjectRequester.
-                        MakeRequest<AssetBase, string>("POST", uri, asset);
+                        MakeRequest<AssetBase, string>("POST", uri, asset, 25);
+                if (newID == null || newID == "")
+                {
+                    newID = UUID.Zero.ToString();
+                }
             }
             catch (Exception e)
             {
-                m_log.WarnFormat("[ASSET CONNECTOR]: Unable to send asset {0} to asset server. Reason: {1}", asset.ID, e.Message);
+                newID = UUID.Zero.ToString();
             }
 
-            if (newID != String.Empty)
+            if (newID == UUID.Zero.ToString())
             {
-                // Placing this here, so that this work with old asset servers that don't send any reply back
-                // SynchronousRestObjectRequester returns somethins that is not an empty string
-                if (newID != null)
-                    asset.ID = newID;
-
-                if (m_Cache != null)
-                    m_Cache.Cache(asset);
+                //The asset upload failed, put it in a queue for later
+                asset.UploadAttempts++;
+                if (asset.UploadAttempts > 30)
+                {
+                    //By this stage we've been in the queue for a good few hours;
+                    //We're going to drop the asset.
+                    m_log.ErrorFormat("[Assets] Dropping asset {0} - Upload has been in the queue for too long.", asset.ID.ToString());
+                }
+                else
+                {
+                    if (!m_retryQueue.ContainsKey(asset.UploadAttempts))
+                    {
+                        m_retryQueue.Add(asset.UploadAttempts, new List<AssetBase>());
+                    }
+                    List<AssetBase> m_queue = m_retryQueue[asset.UploadAttempts];
+                    m_queue.Add(asset);
+                    m_log.WarnFormat("[Assets] Upload failed: {0} - Requeuing asset for another run.", asset.ID.ToString());
+                    m_retryTimer.Start();
+                }
+            }
+            else
+            {
+                if (asset.UploadAttempts > 0)
+                {
+                    m_log.InfoFormat("[Assets] Upload of {0} succeeded after {1} failed attempts", asset.ID.ToString(), asset.UploadAttempts.ToString());
+                }
+                if (newID != String.Empty)
+                {
+                    // Placing this here, so that this work with old asset servers that don't send any reply back
+                    // SynchronousRestObjectRequester returns somethins that is not an empty string
+                    if (newID != null)
+                        asset.ID = newID;
+
+                    if (m_Cache != null)
+                        m_Cache.Cache(asset);
+                }
             }
-            return newID;
+            return asset.ID;
         }
 
         public bool UpdateContent(string id, byte[] data)
@@ -251,12 +427,12 @@ namespace OpenSim.Services.Connectors
                 if (metadata == null)
                     return false;
 
-                asset = new AssetBase(metadata.FullID, metadata.Name, metadata.Type);
+                asset = new AssetBase(metadata.FullID, metadata.Name, metadata.Type, UUID.Zero.ToString());
                 asset.Metadata = metadata;
             }
             asset.Data = data;
 
-            string uri = m_ServerURI + "/assets/" + id;
+            string uri = MapServer(id) + "/assets/" + id;
 
             if (SynchronousRestObjectRequester.
                     MakeRequest<AssetBase, bool>("POST", uri, asset))
@@ -271,7 +447,7 @@ namespace OpenSim.Services.Connectors
 
         public bool Delete(string id)
         {
-            string uri = m_ServerURI + "/assets/" + id;
+            string uri = MapServer(id) + "/assets/" + id;
 
             if (SynchronousRestObjectRequester.
                     MakeRequest<int, bool>("DELETE", uri, 0))
diff --git a/OpenSim/Services/Connectors/Asset/HGAssetServiceConnector.cs b/OpenSim/Services/Connectors/Asset/HGAssetServiceConnector.cs
index 34df54a..5c31639 100644
--- a/OpenSim/Services/Connectors/Asset/HGAssetServiceConnector.cs
+++ b/OpenSim/Services/Connectors/Asset/HGAssetServiceConnector.cs
@@ -32,6 +32,8 @@ using System.Collections.Generic;
 using System.Reflection;
 using OpenSim.Framework;
 using OpenSim.Services.Interfaces;
+using OpenSim.Services.Connectors.Hypergrid;
+using OpenSim.Services.Connectors.SimianGrid;
 
 namespace OpenSim.Services.Connectors
 {
@@ -41,7 +43,7 @@ namespace OpenSim.Services.Connectors
                 LogManager.GetLogger(
                 MethodBase.GetCurrentMethod().DeclaringType);
 
-        private Dictionary<string, AssetServicesConnector> m_connectors = new Dictionary<string, AssetServicesConnector>();
+        private Dictionary<string, IAssetService> m_connectors = new Dictionary<string, IAssetService>();
 
         public HGAssetServiceConnector(IConfigSource source)
         {
@@ -81,7 +83,7 @@ namespace OpenSim.Services.Connectors
 
         private IAssetService GetConnector(string url)
         {
-            AssetServicesConnector connector = null;
+            IAssetService connector = null;
             lock (m_connectors)
             {
                 if (m_connectors.ContainsKey(url))
@@ -90,12 +92,17 @@ namespace OpenSim.Services.Connectors
                 }
                 else
                 {
-                    // We're instantiating this class explicitly, but this won't
-                    // work in general, because the remote grid may be running
-                    // an asset server that has a different protocol.
-                    // Eventually we will want a piece of protocol asking
-                    // the remote server about its kind. Definitely cool thing to do!
-                    connector = new AssetServicesConnector(url);
+                    // Still not as flexible as I would like this to be,
+                    // but good enough for now
+                    string connectorType = new HeloServicesConnector(url).Helo();
+                    m_log.DebugFormat("[HG ASSET SERVICE]: HELO returned {0}", connectorType);
+                    if (connectorType == "opensim-simian")
+                    {
+                        connector = new SimianAssetServiceConnector(url);
+                    }
+                    else
+                        connector = new AssetServicesConnector(url);
+
                     m_connectors.Add(url, connector);
                 }
             }