diff options
author | Tom Grimshaw | 2010-06-29 23:18:48 -0700 |
---|---|---|
committer | Tom Grimshaw | 2010-06-29 23:18:48 -0700 |
commit | 8dc7c0849eecaea09ce3571bab28b44c17bb3743 (patch) | |
tree | d8bc683e18ec75368517ceec56d4770beb624dff /OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs | |
parent | Return a UUID.Zero if the asset fails to upload (diff) | |
download | opensim-SC_OLD-8dc7c0849eecaea09ce3571bab28b44c17bb3743.zip opensim-SC_OLD-8dc7c0849eecaea09ce3571bab28b44c17bb3743.tar.gz opensim-SC_OLD-8dc7c0849eecaea09ce3571bab28b44c17bb3743.tar.bz2 opensim-SC_OLD-8dc7c0849eecaea09ce3571bab28b44c17bb3743.tar.xz |
The other half of the asset fix. Implement an exponentially incrementing retry timer for asset upload failures. Total queue time in the ballpark of 24 hours, which should be a reasonable time for any grid admin to get their asset service back online. This should stop lost assets.
Diffstat (limited to '')
-rw-r--r-- | OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs | 99 |
1 files changed, 88 insertions, 11 deletions
diff --git a/OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs b/OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs index 65b3537..ae600bb 100644 --- a/OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs +++ b/OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs | |||
@@ -30,6 +30,7 @@ using System; | |||
30 | using System.Collections.Generic; | 30 | using System.Collections.Generic; |
31 | using System.IO; | 31 | using System.IO; |
32 | using System.Reflection; | 32 | using System.Reflection; |
33 | using System.Timers; | ||
33 | using Nini.Config; | 34 | using Nini.Config; |
34 | using OpenSim.Framework; | 35 | using OpenSim.Framework; |
35 | using OpenSim.Framework.Console; | 36 | using OpenSim.Framework.Console; |
@@ -48,7 +49,9 @@ namespace OpenSim.Services.Connectors | |||
48 | 49 | ||
49 | private string m_ServerURI = String.Empty; | 50 | private string m_ServerURI = String.Empty; |
50 | private IImprovedAssetCache m_Cache = null; | 51 | private IImprovedAssetCache m_Cache = null; |
51 | 52 | private int m_retryCounter; | |
53 | private Dictionary<int, List<AssetBase>> m_retryQueue = new Dictionary<int, List<AssetBase>>(); | ||
54 | private Timer m_retryTimer; | ||
52 | public AssetServicesConnector() | 55 | public AssetServicesConnector() |
53 | { | 56 | { |
54 | } | 57 | } |
@@ -85,6 +88,55 @@ namespace OpenSim.Services.Connectors | |||
85 | MainConsole.Instance.Commands.AddCommand("asset", false, "dump asset", | 88 | MainConsole.Instance.Commands.AddCommand("asset", false, "dump asset", |
86 | "dump asset <id> <file>", | 89 | "dump asset <id> <file>", |
87 | "dump one cached asset", HandleDumpAsset); | 90 | "dump one cached asset", HandleDumpAsset); |
91 | |||
92 | m_retryTimer = new Timer(); | ||
93 | m_retryTimer.Elapsed += new ElapsedEventHandler(retryCheck); | ||
94 | m_retryTimer.Interval = 60000; | ||
95 | } | ||
96 | |||
97 | protected void retryCheck(object source, ElapsedEventArgs e) | ||
98 | { | ||
99 | m_retryCounter++; | ||
100 | if (m_retryCounter > 60) m_retryCounter -= 60; | ||
101 | List<int> keys = new List<int>(); | ||
102 | foreach (int a in m_retryQueue.Keys) | ||
103 | { | ||
104 | keys.Add(a); | ||
105 | } | ||
106 | foreach (int a in keys) | ||
107 | { | ||
108 | //We exponentially fall back on frequency until we reach one attempt per hour | ||
109 | //The net result is that we end up in the queue for roughly 24 hours.. | ||
110 | //24 hours worth of assets could be a lot, so the hope is that the region admin | ||
111 | //will have gotten the asset connector back online quickly! | ||
112 | |||
113 | int timefactor = a ^ 2; | ||
114 | if (timefactor > 60) | ||
115 | { | ||
116 | timefactor = 60; | ||
117 | } | ||
118 | |||
119 | //First, find out if we care about this timefactor | ||
120 | if (timefactor % a == 0) | ||
121 | { | ||
122 | //Yes, we do! | ||
123 | List<AssetBase> retrylist = m_retryQueue[a]; | ||
124 | m_retryQueue.Remove(a); | ||
125 | |||
126 | foreach(AssetBase ass in retrylist) | ||
127 | { | ||
128 | Store(ass); //Store my ass. This function will put it back in the dictionary if it fails | ||
129 | } | ||
130 | } | ||
131 | } | ||
132 | |||
133 | if (m_retryQueue.Count == 0) | ||
134 | { | ||
135 | //It might only be one tick per minute, but I have | ||
136 | //repented and abandoned my wasteful ways | ||
137 | m_retryCounter = 0; | ||
138 | m_retryTimer.Stop(); | ||
139 | } | ||
88 | } | 140 | } |
89 | 141 | ||
90 | protected void SetCache(IImprovedAssetCache cache) | 142 | protected void SetCache(IImprovedAssetCache cache) |
@@ -222,20 +274,45 @@ namespace OpenSim.Services.Connectors | |||
222 | } | 274 | } |
223 | catch (Exception e) | 275 | catch (Exception e) |
224 | { | 276 | { |
225 | m_log.WarnFormat("[ASSET CONNECTOR]: Unable to send asset {0} to asset server. Reason: {1}", asset.ID, e.Message); | 277 | newID = UUID.Zero.ToString(); |
226 | } | 278 | } |
227 | 279 | ||
228 | if (newID != String.Empty) | 280 | if (newID == UUID.Zero.ToString()) |
229 | { | 281 | { |
230 | // Placing this here, so that this work with old asset servers that don't send any reply back | 282 | //The asset upload failed, put it in a queue for later |
231 | // SynchronousRestObjectRequester returns somethins that is not an empty string | 283 | asset.UploadAttempts++; |
232 | if (newID != null) | 284 | if (asset.UploadAttempts > 30) |
233 | asset.ID = newID; | 285 | { |
234 | 286 | //By this stage we've been in the queue for a good few hours; | |
235 | if (m_Cache != null) | 287 | //We're going to drop the asset. |
236 | m_Cache.Cache(asset); | 288 | m_log.ErrorFormat("[Assets] Dropping asset {0} - Upload has been in the queue for too long.", asset.ID.ToString()); |
289 | } | ||
290 | else | ||
291 | { | ||
292 | if (!m_retryQueue.ContainsKey(asset.UploadAttempts)) | ||
293 | { | ||
294 | m_retryQueue.Add(asset.UploadAttempts, new List<AssetBase>()); | ||
295 | } | ||
296 | List<AssetBase> m_queue = m_retryQueue[asset.UploadAttempts]; | ||
297 | m_queue.Add(asset); | ||
298 | m_log.WarnFormat("[Assets] Upload failed: {0} - Requeuing asset for another run.", asset.ID.ToString()); | ||
299 | m_retryTimer.Start(); | ||
300 | } | ||
301 | } | ||
302 | else | ||
303 | { | ||
304 | if (newID != String.Empty) | ||
305 | { | ||
306 | // Placing this here, so that this work with old asset servers that don't send any reply back | ||
307 | // SynchronousRestObjectRequester returns somethins that is not an empty string | ||
308 | if (newID != null) | ||
309 | asset.ID = newID; | ||
310 | |||
311 | if (m_Cache != null) | ||
312 | m_Cache.Cache(asset); | ||
313 | } | ||
237 | } | 314 | } |
238 | return newID; | 315 | return asset.ID; |
239 | } | 316 | } |
240 | 317 | ||
241 | public bool UpdateContent(string id, byte[] data) | 318 | public bool UpdateContent(string id, byte[] data) |