aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/OpenSim
diff options
context:
space:
mode:
authorTom Grimshaw2010-06-29 23:18:48 -0700
committerTom Grimshaw2010-06-29 23:18:48 -0700
commit8dc7c0849eecaea09ce3571bab28b44c17bb3743 (patch)
treed8bc683e18ec75368517ceec56d4770beb624dff /OpenSim
parentReturn a UUID.Zero if the asset fails to upload (diff)
downloadopensim-SC-8dc7c0849eecaea09ce3571bab28b44c17bb3743.zip
opensim-SC-8dc7c0849eecaea09ce3571bab28b44c17bb3743.tar.gz
opensim-SC-8dc7c0849eecaea09ce3571bab28b44c17bb3743.tar.bz2
opensim-SC-8dc7c0849eecaea09ce3571bab28b44c17bb3743.tar.xz
The other half of the asset fix. Implement an exponentially incrementing retry timer for asset upload failures. Total queue time in the ballpark of 24 hours, which should be a reasonable time for any grid admin to get their asset service back online. This should stop lost assets.
Diffstat (limited to '')
-rw-r--r--OpenSim/Framework/AssetBase.cs8
-rw-r--r--OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs99
2 files changed, 96 insertions, 11 deletions
diff --git a/OpenSim/Framework/AssetBase.cs b/OpenSim/Framework/AssetBase.cs
index 53d28be..98fa846 100644
--- a/OpenSim/Framework/AssetBase.cs
+++ b/OpenSim/Framework/AssetBase.cs
@@ -60,6 +60,8 @@ namespace OpenSim.Framework
60 /// </summary> 60 /// </summary>
61 private AssetMetadata m_metadata; 61 private AssetMetadata m_metadata;
62 62
63 private int m_uploadAttempts;
64
63 // This is needed for .NET serialization!!! 65 // This is needed for .NET serialization!!!
64 // Do NOT "Optimize" away! 66 // Do NOT "Optimize" away!
65 public AssetBase() 67 public AssetBase()
@@ -197,6 +199,12 @@ namespace OpenSim.Framework
197 set { m_metadata.Type = value; } 199 set { m_metadata.Type = value; }
198 } 200 }
199 201
202 public int UploadAttempts
203 {
204 get { return m_uploadAttempts; }
205 set { m_uploadAttempts = value; }
206 }
207
200 /// <summary> 208 /// <summary>
201 /// Is this a region only asset, or does this exist on the asset server also 209 /// Is this a region only asset, or does this exist on the asset server also
202 /// </summary> 210 /// </summary>
diff --git a/OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs b/OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs
index 65b3537..ae600bb 100644
--- a/OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs
+++ b/OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs
@@ -30,6 +30,7 @@ using System;
30using System.Collections.Generic; 30using System.Collections.Generic;
31using System.IO; 31using System.IO;
32using System.Reflection; 32using System.Reflection;
33using System.Timers;
33using Nini.Config; 34using Nini.Config;
34using OpenSim.Framework; 35using OpenSim.Framework;
35using OpenSim.Framework.Console; 36using OpenSim.Framework.Console;
@@ -48,7 +49,9 @@ namespace OpenSim.Services.Connectors
48 49
49 private string m_ServerURI = String.Empty; 50 private string m_ServerURI = String.Empty;
50 private IImprovedAssetCache m_Cache = null; 51 private IImprovedAssetCache m_Cache = null;
51 52 private int m_retryCounter;
53 private Dictionary<int, List<AssetBase>> m_retryQueue = new Dictionary<int, List<AssetBase>>();
54 private Timer m_retryTimer;
52 public AssetServicesConnector() 55 public AssetServicesConnector()
53 { 56 {
54 } 57 }
@@ -85,6 +88,55 @@ namespace OpenSim.Services.Connectors
85 MainConsole.Instance.Commands.AddCommand("asset", false, "dump asset", 88 MainConsole.Instance.Commands.AddCommand("asset", false, "dump asset",
86 "dump asset <id> <file>", 89 "dump asset <id> <file>",
87 "dump one cached asset", HandleDumpAsset); 90 "dump one cached asset", HandleDumpAsset);
91
92 m_retryTimer = new Timer();
93 m_retryTimer.Elapsed += new ElapsedEventHandler(retryCheck);
94 m_retryTimer.Interval = 60000;
95 }
96
97 protected void retryCheck(object source, ElapsedEventArgs e)
98 {
99 m_retryCounter++;
100 if (m_retryCounter > 60) m_retryCounter -= 60;
101 List<int> keys = new List<int>();
102 foreach (int a in m_retryQueue.Keys)
103 {
104 keys.Add(a);
105 }
106 foreach (int a in keys)
107 {
108 //We exponentially fall back on frequency until we reach one attempt per hour
109 //The net result is that we end up in the queue for roughly 24 hours..
110 //24 hours worth of assets could be a lot, so the hope is that the region admin
111 //will have gotten the asset connector back online quickly!
112
113 int timefactor = a ^ 2;
114 if (timefactor > 60)
115 {
116 timefactor = 60;
117 }
118
119 //First, find out if we care about this timefactor
120 if (timefactor % a == 0)
121 {
122 //Yes, we do!
123 List<AssetBase> retrylist = m_retryQueue[a];
124 m_retryQueue.Remove(a);
125
126 foreach(AssetBase ass in retrylist)
127 {
128 Store(ass); //Store my ass. This function will put it back in the dictionary if it fails
129 }
130 }
131 }
132
133 if (m_retryQueue.Count == 0)
134 {
135 //It might only be one tick per minute, but I have
136 //repented and abandoned my wasteful ways
137 m_retryCounter = 0;
138 m_retryTimer.Stop();
139 }
88 } 140 }
89 141
90 protected void SetCache(IImprovedAssetCache cache) 142 protected void SetCache(IImprovedAssetCache cache)
@@ -222,20 +274,45 @@ namespace OpenSim.Services.Connectors
222 } 274 }
223 catch (Exception e) 275 catch (Exception e)
224 { 276 {
225 m_log.WarnFormat("[ASSET CONNECTOR]: Unable to send asset {0} to asset server. Reason: {1}", asset.ID, e.Message); 277 newID = UUID.Zero.ToString();
226 } 278 }
227 279
228 if (newID != String.Empty) 280 if (newID == UUID.Zero.ToString())
229 { 281 {
230 // Placing this here, so that this work with old asset servers that don't send any reply back 282 //The asset upload failed, put it in a queue for later
231 // SynchronousRestObjectRequester returns somethins that is not an empty string 283 asset.UploadAttempts++;
232 if (newID != null) 284 if (asset.UploadAttempts > 30)
233 asset.ID = newID; 285 {
234 286 //By this stage we've been in the queue for a good few hours;
235 if (m_Cache != null) 287 //We're going to drop the asset.
236 m_Cache.Cache(asset); 288 m_log.ErrorFormat("[Assets] Dropping asset {0} - Upload has been in the queue for too long.", asset.ID.ToString());
289 }
290 else
291 {
292 if (!m_retryQueue.ContainsKey(asset.UploadAttempts))
293 {
294 m_retryQueue.Add(asset.UploadAttempts, new List<AssetBase>());
295 }
296 List<AssetBase> m_queue = m_retryQueue[asset.UploadAttempts];
297 m_queue.Add(asset);
298 m_log.WarnFormat("[Assets] Upload failed: {0} - Requeuing asset for another run.", asset.ID.ToString());
299 m_retryTimer.Start();
300 }
301 }
302 else
303 {
304 if (newID != String.Empty)
305 {
306 // Placing this here, so that this work with old asset servers that don't send any reply back
307 // SynchronousRestObjectRequester returns somethins that is not an empty string
308 if (newID != null)
309 asset.ID = newID;
310
311 if (m_Cache != null)
312 m_Cache.Cache(asset);
313 }
237 } 314 }
238 return newID; 315 return asset.ID;
239 } 316 }
240 317
241 public bool UpdateContent(string id, byte[] data) 318 public bool UpdateContent(string id, byte[] data)