From 8dc7c0849eecaea09ce3571bab28b44c17bb3743 Mon Sep 17 00:00:00 2001
From: Tom Grimshaw
Date: Tue, 29 Jun 2010 23:18:48 -0700
Subject: The other half of the asset fix. Implement an exponentially
incrementing retry timer for asset upload failures. Total queue time in the
ballpark of 24 hours, which should be a reasonable time for any grid admin to
get their asset service back online. This should stop lost assets.
---
OpenSim/Framework/AssetBase.cs | 8 ++
.../Connectors/Asset/AssetServiceConnector.cs | 99 +++++++++++++++++++---
2 files changed, 96 insertions(+), 11 deletions(-)
(limited to 'OpenSim')
diff --git a/OpenSim/Framework/AssetBase.cs b/OpenSim/Framework/AssetBase.cs
index 53d28be..98fa846 100644
--- a/OpenSim/Framework/AssetBase.cs
+++ b/OpenSim/Framework/AssetBase.cs
@@ -60,6 +60,8 @@ namespace OpenSim.Framework
///
private AssetMetadata m_metadata;
+ private int m_uploadAttempts;
+
// This is needed for .NET serialization!!!
// Do NOT "Optimize" away!
public AssetBase()
@@ -197,6 +199,12 @@ namespace OpenSim.Framework
set { m_metadata.Type = value; }
}
+ public int UploadAttempts
+ {
+ get { return m_uploadAttempts; }
+ set { m_uploadAttempts = value; }
+ }
+
///
/// Is this a region only asset, or does this exist on the asset server also
///
diff --git a/OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs b/OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs
index 65b3537..ae600bb 100644
--- a/OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs
+++ b/OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs
@@ -30,6 +30,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Reflection;
+using System.Timers;
using Nini.Config;
using OpenSim.Framework;
using OpenSim.Framework.Console;
@@ -48,7 +49,9 @@ namespace OpenSim.Services.Connectors
private string m_ServerURI = String.Empty;
private IImprovedAssetCache m_Cache = null;
-
+ private int m_retryCounter;
+ private Dictionary> m_retryQueue = new Dictionary>();
+ private Timer m_retryTimer;
public AssetServicesConnector()
{
}
@@ -85,6 +88,55 @@ namespace OpenSim.Services.Connectors
MainConsole.Instance.Commands.AddCommand("asset", false, "dump asset",
"dump asset ",
"dump one cached asset", HandleDumpAsset);
+
+ m_retryTimer = new Timer();
+ m_retryTimer.Elapsed += new ElapsedEventHandler(retryCheck);
+ m_retryTimer.Interval = 60000;
+ }
+
+ protected void retryCheck(object source, ElapsedEventArgs e)
+ {
+ m_retryCounter++;
+ if (m_retryCounter > 60) m_retryCounter -= 60;
+ List keys = new List();
+ foreach (int a in m_retryQueue.Keys)
+ {
+ keys.Add(a);
+ }
+ foreach (int a in keys)
+ {
+ //We exponentially fall back on frequency until we reach one attempt per hour
+ //The net result is that we end up in the queue for roughly 24 hours..
+ //24 hours worth of assets could be a lot, so the hope is that the region admin
+ //will have gotten the asset connector back online quickly!
+
+ int timefactor = a ^ 2;
+ if (timefactor > 60)
+ {
+ timefactor = 60;
+ }
+
+ //First, find out if we care about this timefactor
+ if (timefactor % a == 0)
+ {
+ //Yes, we do!
+ List retrylist = m_retryQueue[a];
+ m_retryQueue.Remove(a);
+
+ foreach(AssetBase ass in retrylist)
+ {
+ Store(ass); //Store my ass. This function will put it back in the dictionary if it fails
+ }
+ }
+ }
+
+ if (m_retryQueue.Count == 0)
+ {
+ //It might only be one tick per minute, but I have
+ //repented and abandoned my wasteful ways
+ m_retryCounter = 0;
+ m_retryTimer.Stop();
+ }
}
protected void SetCache(IImprovedAssetCache cache)
@@ -222,20 +274,45 @@ namespace OpenSim.Services.Connectors
}
catch (Exception e)
{
- m_log.WarnFormat("[ASSET CONNECTOR]: Unable to send asset {0} to asset server. Reason: {1}", asset.ID, e.Message);
+ newID = UUID.Zero.ToString();
}
- if (newID != String.Empty)
+ if (newID == UUID.Zero.ToString())
{
- // Placing this here, so that this work with old asset servers that don't send any reply back
- // SynchronousRestObjectRequester returns somethins that is not an empty string
- if (newID != null)
- asset.ID = newID;
-
- if (m_Cache != null)
- m_Cache.Cache(asset);
+ //The asset upload failed, put it in a queue for later
+ asset.UploadAttempts++;
+ if (asset.UploadAttempts > 30)
+ {
+ //By this stage we've been in the queue for a good few hours;
+ //We're going to drop the asset.
+ m_log.ErrorFormat("[Assets] Dropping asset {0} - Upload has been in the queue for too long.", asset.ID.ToString());
+ }
+ else
+ {
+ if (!m_retryQueue.ContainsKey(asset.UploadAttempts))
+ {
+ m_retryQueue.Add(asset.UploadAttempts, new List());
+ }
+ List m_queue = m_retryQueue[asset.UploadAttempts];
+ m_queue.Add(asset);
+ m_log.WarnFormat("[Assets] Upload failed: {0} - Requeuing asset for another run.", asset.ID.ToString());
+ m_retryTimer.Start();
+ }
+ }
+ else
+ {
+ if (newID != String.Empty)
+ {
+ // Placing this here, so that this work with old asset servers that don't send any reply back
+ // SynchronousRestObjectRequester returns somethins that is not an empty string
+ if (newID != null)
+ asset.ID = newID;
+
+ if (m_Cache != null)
+ m_Cache.Cache(asset);
+ }
}
- return newID;
+ return asset.ID;
}
public bool UpdateContent(string id, byte[] data)
--
cgit v1.1