10 files changed, 116 insertions, 19 deletions

diff --git a/OpenSim/Data/AssetDataBase.cs b/OpenSim/Data/AssetDataBase.cs
index e1a810c..b4ae913 100644
--- a/OpenSim/Data/AssetDataBase.cs
+++ b/OpenSim/Data/AssetDataBase.cs
@@ -38,7 +38,7 @@ namespace OpenSim.Data
     {
         public abstract AssetBase GetAsset(UUID uuid);
         
-        public abstract void StoreAsset(AssetBase asset);
+        public abstract bool StoreAsset(AssetBase asset);
         public abstract bool ExistsAsset(UUID uuid);
 
         public abstract List<AssetMetadata> FetchAssetMetadataSet(int start, int count);
diff --git a/OpenSim/Data/IAssetData.cs b/OpenSim/Data/IAssetData.cs
index 90d5eeb..065d3a5 100644
--- a/OpenSim/Data/IAssetData.cs
+++ b/OpenSim/Data/IAssetData.cs
@@ -34,7 +34,7 @@ namespace OpenSim.Data
     public interface IAssetDataPlugin : IPlugin
     {
         AssetBase GetAsset(UUID uuid);
-        void StoreAsset(AssetBase asset);
+        bool StoreAsset(AssetBase asset);
         bool ExistsAsset(UUID uuid);
         List<AssetMetadata> FetchAssetMetadataSet(int start, int count);
         void Initialise(string connect);
diff --git a/OpenSim/Data/MSSQL/MSSQLAssetData.cs b/OpenSim/Data/MSSQL/MSSQLAssetData.cs
index c7488d8..c882555 100644
--- a/OpenSim/Data/MSSQL/MSSQLAssetData.cs
+++ b/OpenSim/Data/MSSQL/MSSQLAssetData.cs
@@ -143,7 +143,7 @@ namespace OpenSim.Data.MSSQL
         /// Create asset in m_database
         /// </summary>
         /// <param name="asset">the asset</param>
-        override public void StoreAsset(AssetBase asset)
+        override public bool StoreAsset(AssetBase asset)
         {
            
             string sql =
@@ -192,10 +192,12 @@ namespace OpenSim.Data.MSSQL
                 try
                 {
                     command.ExecuteNonQuery();
+                    return true;
                 }
                 catch(Exception e)
                 {
                     m_log.Error("[ASSET DB]: Error storing item :" + e.Message);
+                    return false;
                 }
             }
         }
diff --git a/OpenSim/Data/MySQL/MySQLAssetData.cs b/OpenSim/Data/MySQL/MySQLAssetData.cs
index fe5152a..f9ce3d9 100644
--- a/OpenSim/Data/MySQL/MySQLAssetData.cs
+++ b/OpenSim/Data/MySQL/MySQLAssetData.cs
@@ -153,7 +153,7 @@ namespace OpenSim.Data.MySQL
         /// </summary>
         /// <param name="asset">Asset UUID to create</param>
         /// <remarks>On failure : Throw an exception and attempt to reconnect to database</remarks>
-        override public void StoreAsset(AssetBase asset)
+        override public bool StoreAsset(AssetBase asset)
         {
             lock (m_dbLock)
             {
@@ -201,12 +201,14 @@ namespace OpenSim.Data.MySQL
                             cmd.Parameters.AddWithValue("?data", asset.Data);
                             cmd.ExecuteNonQuery();
                             cmd.Dispose();
+                            return true;
                         }
                     }
                     catch (Exception e)
                     {
                         m_log.ErrorFormat("[ASSET DB]: MySQL failure creating asset {0} with name \"{1}\". Error: {2}",
                             asset.FullID, asset.Name, e.Message);
+                        return false;
                     }
                 }
             }
diff --git a/OpenSim/Data/SQLite/SQLiteAssetData.cs b/OpenSim/Data/SQLite/SQLiteAssetData.cs
index 16e560c..75e51a3 100644
--- a/OpenSim/Data/SQLite/SQLiteAssetData.cs
+++ b/OpenSim/Data/SQLite/SQLiteAssetData.cs
@@ -119,7 +119,7 @@ namespace OpenSim.Data.SQLite
         /// Create an asset
         /// </summary>
         /// <param name="asset">Asset Base</param>
-        override public void StoreAsset(AssetBase asset)
+        override public bool StoreAsset(AssetBase asset)
         {
             //m_log.Info("[ASSET DB]: Creating Asset " + asset.FullID.ToString());
             if (ExistsAsset(asset.FullID))
@@ -141,6 +141,7 @@ namespace OpenSim.Data.SQLite
                         cmd.Parameters.Add(new SqliteParameter(":Data", asset.Data));
 
                         cmd.ExecuteNonQuery();
+                        return true;
                     }
                 }
             }
@@ -161,6 +162,7 @@ namespace OpenSim.Data.SQLite
                         cmd.Parameters.Add(new SqliteParameter(":Data", asset.Data));
 
                         cmd.ExecuteNonQuery();
+                        return true;
                     }
                 }
             }
diff --git a/OpenSim/Data/SQLiteLegacy/SQLiteAssetData.cs b/OpenSim/Data/SQLiteLegacy/SQLiteAssetData.cs
index df50902..3da298b 100644
--- a/OpenSim/Data/SQLiteLegacy/SQLiteAssetData.cs
+++ b/OpenSim/Data/SQLiteLegacy/SQLiteAssetData.cs
@@ -119,7 +119,7 @@ namespace OpenSim.Data.SQLiteLegacy
         /// Create an asset
         /// </summary>
         /// <param name="asset">Asset Base</param>
-        override public void StoreAsset(AssetBase asset)
+        override public bool StoreAsset(AssetBase asset)
         {
             //m_log.Info("[ASSET DB]: Creating Asset " + asset.FullID.ToString());
             if (ExistsAsset(asset.FullID))
@@ -139,6 +139,7 @@ namespace OpenSim.Data.SQLiteLegacy
                         cmd.Parameters.Add(new SqliteParameter(":Data", asset.Data));
  
                         cmd.ExecuteNonQuery();
+                        return true;
                     }
                 }
             }
@@ -157,6 +158,7 @@ namespace OpenSim.Data.SQLiteLegacy
                         cmd.Parameters.Add(new SqliteParameter(":Data", asset.Data));
 
                         cmd.ExecuteNonQuery();
+                        return true;
                     }
                 }
             }
diff --git a/OpenSim/Framework/AssetBase.cs b/OpenSim/Framework/AssetBase.cs
index 53d28be..98fa846 100644
--- a/OpenSim/Framework/AssetBase.cs
+++ b/OpenSim/Framework/AssetBase.cs
@@ -60,6 +60,8 @@ namespace OpenSim.Framework
         /// </summary>
         private AssetMetadata m_metadata;
 
+        private int m_uploadAttempts;
+
         // This is needed for .NET serialization!!!
         // Do NOT "Optimize" away!
         public AssetBase()
@@ -197,6 +199,12 @@ namespace OpenSim.Framework
             set { m_metadata.Type = value; }
         }
 
+        public int UploadAttempts
+        {
+            get { return m_uploadAttempts; }
+            set { m_uploadAttempts = value; }
+        }
+
         /// <summary>
         /// Is this a region only asset, or does this exist on the asset server also
         /// </summary>
diff --git a/OpenSim/Services/AssetService/AssetService.cs b/OpenSim/Services/AssetService/AssetService.cs
index 470a4dd..3122382 100644
--- a/OpenSim/Services/AssetService/AssetService.cs
+++ b/OpenSim/Services/AssetService/AssetService.cs
@@ -144,7 +144,10 @@ namespace OpenSim.Services.AssetService
         public string Store(AssetBase asset)
         {
             //m_log.DebugFormat("[ASSET SERVICE]: Store asset {0} {1}", asset.Name, asset.ID);
-            m_Database.StoreAsset(asset);
+            if (!m_Database.StoreAsset(asset))
+            {
+                return UUID.Zero.ToString();
+            }
 
             return asset.ID;
         }
diff --git a/OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs b/OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs
index 65b3537..ae600bb 100644
--- a/OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs
+++ b/OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs
@@ -30,6 +30,7 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Reflection;
+using System.Timers;
 using Nini.Config;
 using OpenSim.Framework;
 using OpenSim.Framework.Console;
@@ -48,7 +49,9 @@ namespace OpenSim.Services.Connectors
 
         private string m_ServerURI = String.Empty;
         private IImprovedAssetCache m_Cache = null;
-
+        private int m_retryCounter;
+        private Dictionary<int, List<AssetBase>> m_retryQueue = new Dictionary<int, List<AssetBase>>();
+        private Timer m_retryTimer;
         public AssetServicesConnector()
         {
         }
@@ -85,6 +88,55 @@ namespace OpenSim.Services.Connectors
             MainConsole.Instance.Commands.AddCommand("asset", false, "dump asset",
                                           "dump asset <id> <file>",
                                           "dump one cached asset", HandleDumpAsset);
+
+            m_retryTimer = new Timer();
+            m_retryTimer.Elapsed += new ElapsedEventHandler(retryCheck);
+            m_retryTimer.Interval = 60000;
+        }
+
+        protected void retryCheck(object source, ElapsedEventArgs e)
+        {
+            m_retryCounter++;
+            if (m_retryCounter > 60) m_retryCounter -= 60;
+            List<int> keys = new List<int>();
+            foreach (int a in m_retryQueue.Keys)
+            {
+                keys.Add(a);
+            }
+            foreach (int a in keys)
+            {
+                //We exponentially fall back on frequency until we reach one attempt per hour
+                //The net result is that we end up in the queue for roughly 24 hours..
+                //24 hours worth of assets could be a lot, so the hope is that the region admin
+                //will have gotten the asset connector back online quickly!
+
+                int timefactor = a ^ 2; 
+                if (timefactor > 60)
+                {
+                    timefactor = 60;
+                }
+
+                //First, find out if we care about this timefactor
+                if (timefactor % a == 0)
+                {
+                    //Yes, we do!   
+                    List<AssetBase> retrylist = m_retryQueue[a];
+                    m_retryQueue.Remove(a);
+
+                    foreach(AssetBase ass in retrylist)
+                    {
+                        Store(ass); //Store my ass. This function will put it back in the dictionary if it fails
+                    }
+                }
+            }
+
+            if (m_retryQueue.Count == 0)
+            {
+                //It might only be one tick per minute, but I have
+                //repented and abandoned my wasteful ways
+                m_retryCounter = 0;
+                m_retryTimer.Stop();
+            }
         }
 
         protected void SetCache(IImprovedAssetCache cache)
@@ -222,20 +274,45 @@ namespace OpenSim.Services.Connectors
             }
             catch (Exception e)
             {
-                m_log.WarnFormat("[ASSET CONNECTOR]: Unable to send asset {0} to asset server. Reason: {1}", asset.ID, e.Message);
+                newID = UUID.Zero.ToString();
             }
 
-            if (newID != String.Empty)
+            if (newID == UUID.Zero.ToString())
             {
-                // Placing this here, so that this work with old asset servers that don't send any reply back
-                // SynchronousRestObjectRequester returns somethins that is not an empty string
-                if (newID != null)
-                    asset.ID = newID;
-
-                if (m_Cache != null)
-                    m_Cache.Cache(asset);
+                //The asset upload failed, put it in a queue for later
+                asset.UploadAttempts++;
+                if (asset.UploadAttempts > 30)
+                {
+                    //By this stage we've been in the queue for a good few hours;
+                    //We're going to drop the asset.
+                    m_log.ErrorFormat("[Assets] Dropping asset {0} - Upload has been in the queue for too long.", asset.ID.ToString());
+                }
+                else
+                {
+                    if (!m_retryQueue.ContainsKey(asset.UploadAttempts))
+                    {
+                        m_retryQueue.Add(asset.UploadAttempts, new List<AssetBase>());
+                    }
+                    List<AssetBase> m_queue = m_retryQueue[asset.UploadAttempts];
+                    m_queue.Add(asset);
+                    m_log.WarnFormat("[Assets] Upload failed: {0} - Requeuing asset for another run.", asset.ID.ToString());
+                    m_retryTimer.Start();
+                }
+            }
+            else
+            {
+                if (newID != String.Empty)
+                {
+                    // Placing this here, so that this work with old asset servers that don't send any reply back
+                    // SynchronousRestObjectRequester returns somethins that is not an empty string
+                    if (newID != null)
+                        asset.ID = newID;
+
+                    if (m_Cache != null)
+                        m_Cache.Cache(asset);
+                }
             }
-            return newID;
+            return asset.ID;
         }
 
         public bool UpdateContent(string id, byte[] data)
diff --git a/OpenSim/Tests/Common/Mock/MockAssetDataPlugin.cs b/OpenSim/Tests/Common/Mock/MockAssetDataPlugin.cs
index 4a15cf2..5bab62c 100644
--- a/OpenSim/Tests/Common/Mock/MockAssetDataPlugin.cs
+++ b/OpenSim/Tests/Common/Mock/MockAssetDataPlugin.cs
@@ -54,9 +54,10 @@ namespace OpenSim.Tests.Common.Mock
             return assets.Find(x=>x.FullID == uuid);
         }
 
-        public void StoreAsset(AssetBase asset)
+        public bool StoreAsset(AssetBase asset)
         {
             assets.Add(asset);
+            return true;
         }
 
         public List<AssetMetadata> FetchAssetMetadataSet(int start, int count) { return new List<AssetMetadata>(count); }