aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
authorJustin Clarke Casey2009-06-05 13:48:43 +0000
committerJustin Clarke Casey2009-06-05 13:48:43 +0000
commitbfea07750835ab7b14f56e94bff869505a88ebb4 (patch)
treeda8dd21507a1e322d100ec05dd1803df9c6da0d3
parentThank you, thomax, for a patch to provide finer-grained access control to (diff)
downloadopensim-SC_OLD-bfea07750835ab7b14f56e94bff869505a88ebb4.zip
opensim-SC_OLD-bfea07750835ab7b14f56e94bff869505a88ebb4.tar.gz
opensim-SC_OLD-bfea07750835ab7b14f56e94bff869505a88ebb4.tar.bz2
opensim-SC_OLD-bfea07750835ab7b14f56e94bff869505a88ebb4.tar.xz
* Add oar saving timeout
* If an oar save fails to get responses to all asset requests to the asset service then timeout after 60 seconds * Timeout executes abort, since missing assets in an OAR seems bad * This means that oar saves won't permanently hang and instead can be retried if something goes wrong with the asset service * This is not a solution to mantis 3714. Hopefully a fix will be along shortly since I can now consistently reproduce that problem
-rw-r--r--OpenSim/Framework/Servers/BaseGetAssetStreamHandler.cs2
-rw-r--r--OpenSim/Framework/Servers/HttpServer/AsynchronousRestObjectRequester.cs10
-rw-r--r--OpenSim/Region/Application/OpenSimBase.cs1
-rw-r--r--OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs10
-rw-r--r--OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs119
5 files changed, 138 insertions, 4 deletions
diff --git a/OpenSim/Framework/Servers/BaseGetAssetStreamHandler.cs b/OpenSim/Framework/Servers/BaseGetAssetStreamHandler.cs
index 9eacf24..83a5676 100644
--- a/OpenSim/Framework/Servers/BaseGetAssetStreamHandler.cs
+++ b/OpenSim/Framework/Servers/BaseGetAssetStreamHandler.cs
@@ -61,7 +61,7 @@ namespace OpenSim.Framework.Servers
61 if (p.Length > 0) 61 if (p.Length > 0)
62 { 62 {
63 UUID assetID; 63 UUID assetID;
64 64
65 if (!UUID.TryParse(p[0], out assetID)) 65 if (!UUID.TryParse(p[0], out assetID))
66 { 66 {
67 m_log.InfoFormat( 67 m_log.InfoFormat(
diff --git a/OpenSim/Framework/Servers/HttpServer/AsynchronousRestObjectRequester.cs b/OpenSim/Framework/Servers/HttpServer/AsynchronousRestObjectRequester.cs
index 9b44cc1..76d0b6f 100644
--- a/OpenSim/Framework/Servers/HttpServer/AsynchronousRestObjectRequester.cs
+++ b/OpenSim/Framework/Servers/HttpServer/AsynchronousRestObjectRequester.cs
@@ -28,18 +28,22 @@
28using System; 28using System;
29using System.IO; 29using System.IO;
30using System.Net; 30using System.Net;
31using System.Reflection;
31using System.Text; 32using System.Text;
32using System.Xml; 33using System.Xml;
33using System.Xml.Serialization; 34using System.Xml.Serialization;
35using log4net;
34 36
35namespace OpenSim.Framework.Servers.HttpServer 37namespace OpenSim.Framework.Servers.HttpServer
36{ 38{
37 public class AsynchronousRestObjectRequester 39 public class AsynchronousRestObjectRequester
38 { 40 {
41 //private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
42
39 /// <summary> 43 /// <summary>
40 /// Perform an asynchronous REST request. 44 /// Perform an asynchronous REST request.
41 /// </summary> 45 /// </summary>
42 /// <param name="verb"></param> 46 /// <param name="verb">GET or POST</param>
43 /// <param name="requestUrl"></param> 47 /// <param name="requestUrl"></param>
44 /// <param name="obj"></param> 48 /// <param name="obj"></param>
45 /// <param name="action"></param> 49 /// <param name="action"></param>
@@ -52,6 +56,8 @@ namespace OpenSim.Framework.Servers.HttpServer
52 public static void MakeRequest<TRequest, TResponse>(string verb, 56 public static void MakeRequest<TRequest, TResponse>(string verb,
53 string requestUrl, TRequest obj, Action<TResponse> action) 57 string requestUrl, TRequest obj, Action<TResponse> action)
54 { 58 {
59 //m_log.DebugFormat("[ASYNC REQUEST]: Starting {0} on {1}", verb, requestUrl);
60
55 Type type = typeof (TRequest); 61 Type type = typeof (TRequest);
56 62
57 WebRequest request = WebRequest.Create(requestUrl); 63 WebRequest request = WebRequest.Create(requestUrl);
@@ -119,6 +125,8 @@ namespace OpenSim.Framework.Servers.HttpServer
119 { 125 {
120 } 126 }
121 127
128 // m_log.DebugFormat("[ASYNC REQUEST]: Received {0}", deserial.ToString());
129
122 action(deserial); 130 action(deserial);
123 }, null); 131 }, null);
124 } 132 }
diff --git a/OpenSim/Region/Application/OpenSimBase.cs b/OpenSim/Region/Application/OpenSimBase.cs
index 8f3f3e8..a1d4b31 100644
--- a/OpenSim/Region/Application/OpenSimBase.cs
+++ b/OpenSim/Region/Application/OpenSimBase.cs
@@ -193,7 +193,6 @@ namespace OpenSim
193 CreatePIDFile(pidFile); 193 CreatePIDFile(pidFile);
194 194
195 userStatsURI = startupConfig.GetString("Stats_URI", String.Empty); 195 userStatsURI = startupConfig.GetString("Stats_URI", String.Empty);
196
197 } 196 }
198 197
199 base.StartupSpecific(); 198 base.StartupSpecific();
diff --git a/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs b/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs
index 6972188..330fa3f 100644
--- a/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs
+++ b/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs
@@ -147,7 +147,15 @@ namespace OpenSim.Region.CoreModules.World.Archiver
147 //m_log.DebugFormat("[ARCHIVER]: Added asset {0}", m_assetsWritten); 147 //m_log.DebugFormat("[ARCHIVER]: Added asset {0}", m_assetsWritten);
148 148
149 if (m_assetsWritten % LOG_ASSET_LOAD_NOTIFICATION_INTERVAL == 0) 149 if (m_assetsWritten % LOG_ASSET_LOAD_NOTIFICATION_INTERVAL == 0)
150 m_log.InfoFormat("[ARCHIVER]: Added {0} assets to archive", m_assetsWritten); 150 m_log.InfoFormat("[ARCHIVER]: Added {0} assets to archive", m_assetsWritten);
151 }
152
153 /// <summary>
154 /// Only call this if you need to force a close on the underlying writer.
155 /// </summary>
156 public void ForceClose()
157 {
158 m_archiveWriter.Close();
151 } 159 }
152 } 160 }
153} 161}
diff --git a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
index c459a66..14804a4 100644
--- a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
+++ b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
@@ -29,6 +29,7 @@ using System;
29using System.Collections.Generic; 29using System.Collections.Generic;
30using System.Reflection; 30using System.Reflection;
31using System.Threading; 31using System.Threading;
32using System.Timers;
32using log4net; 33using log4net;
33using OpenMetaverse; 34using OpenMetaverse;
34using OpenSim.Framework; 35using OpenSim.Framework;
@@ -44,6 +45,37 @@ namespace OpenSim.Region.CoreModules.World.Archiver
44 { 45 {
45 private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); 46 private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
46 47
48 enum RequestState
49 {
50 Initial,
51 Running,
52 Completed,
53 Aborted
54 };
55
56 /// <value>
57 /// Timeout threshold if we still need assets or missing asset notifications but have stopped receiving them
58 /// from the asset service
59 /// </value>
60 protected const int TIMEOUT = 60 * 1000;
61
62 /// <value>
63 /// If a timeout does occur, limit the amount of UUID information put to the console.
64 /// </value>
65 protected const int MAX_UUID_DISPLAY_ON_TIMEOUT = 3;
66
67 protected System.Timers.Timer m_requestCallbackTimer;
68
69 /// <value>
70 /// State of this request
71 /// </value>
72 private RequestState m_requestState = RequestState.Initial;
73
74 /// <value>
75 /// Record whether the request has completed.
76 /// </value>
77 private bool m_requestCompleted;
78
47 /// <value> 79 /// <value>
48 /// uuids to request 80 /// uuids to request
49 /// </value> 81 /// </value>
@@ -85,20 +117,92 @@ namespace OpenSim.Region.CoreModules.World.Archiver
85 m_assetsRequestCallback = assetsRequestCallback; 117 m_assetsRequestCallback = assetsRequestCallback;
86 m_assetService = assetService; 118 m_assetService = assetService;
87 m_repliesRequired = uuids.Count; 119 m_repliesRequired = uuids.Count;
120
121 m_requestCallbackTimer = new System.Timers.Timer(TIMEOUT);
122 m_requestCallbackTimer.AutoReset = false;
123 m_requestCallbackTimer.Elapsed += new ElapsedEventHandler(OnRequestCallbackTimeout);
88 } 124 }
89 125
90 protected internal void Execute() 126 protected internal void Execute()
91 { 127 {
128 m_requestState = RequestState.Running;
129
92 m_log.DebugFormat("[ARCHIVER]: AssetsRequest executed looking for {0} assets", m_repliesRequired); 130 m_log.DebugFormat("[ARCHIVER]: AssetsRequest executed looking for {0} assets", m_repliesRequired);
93 131
94 // We can stop here if there are no assets to fetch 132 // We can stop here if there are no assets to fetch
95 if (m_repliesRequired == 0) 133 if (m_repliesRequired == 0)
134 {
135 m_requestState = RequestState.Completed;
96 PerformAssetsRequestCallback(); 136 PerformAssetsRequestCallback();
137 return;
138 }
97 139
98 foreach (UUID uuid in m_uuids) 140 foreach (UUID uuid in m_uuids)
99 { 141 {
100 m_assetService.Get(uuid.ToString(), this, AssetRequestCallback); 142 m_assetService.Get(uuid.ToString(), this, AssetRequestCallback);
101 } 143 }
144
145 m_requestCallbackTimer.Enabled = true;
146 }
147
148 protected void OnRequestCallbackTimeout(object source, ElapsedEventArgs args)
149 {
150 try
151 {
152 lock (this)
153 {
154 // Take care of the possibilty that this thread started but was paused just outside the lock before
155 // the final request came in (assuming that such a thing is possible)
156 if (m_requestState == RequestState.Completed)
157 return;
158
159 m_requestState = RequestState.Aborted;
160 }
161
162 // Calculate which uuids were not found. This is an expensive way of doing it, but this is a failure
163 // case anyway.
164 List<UUID> uuids = new List<UUID>();
165 foreach (UUID uuid in m_uuids)
166 {
167 uuids.Add(uuid);
168 }
169
170 foreach (UUID uuid in m_foundAssetUuids)
171 {
172 uuids.Remove(uuid);
173 }
174
175 foreach (UUID uuid in m_notFoundAssetUuids)
176 {
177 uuids.Remove(uuid);
178 }
179
180 m_log.ErrorFormat(
181 "[ARCHIVER]: Asset service failed to return information about {0} requested assets", uuids.Count);
182
183 int i = 0;
184 foreach (UUID uuid in uuids)
185 {
186 m_log.ErrorFormat("[ARCHIVER]: No information about asset {0} received", uuid);
187
188 if (++i >= MAX_UUID_DISPLAY_ON_TIMEOUT)
189 break;
190 }
191
192 if (uuids.Count > MAX_UUID_DISPLAY_ON_TIMEOUT)
193 m_log.ErrorFormat(
194 "[ARCHIVER]: (... {0} more not shown)", uuids.Count - MAX_UUID_DISPLAY_ON_TIMEOUT);
195
196 m_log.Error("[ARCHIVER]: OAR save aborted.");
197 }
198 catch (Exception e)
199 {
200 m_log.ErrorFormat("[ARCHIVER]: Timeout handler exception {0}", e);
201 }
202 finally
203 {
204 m_assetsArchiver.ForceClose();
205 }
102 } 206 }
103 207
104 /// <summary> 208 /// <summary>
@@ -114,6 +218,15 @@ namespace OpenSim.Region.CoreModules.World.Archiver
114 { 218 {
115 //m_log.DebugFormat("[ARCHIVER]: Received callback for asset {0}", id); 219 //m_log.DebugFormat("[ARCHIVER]: Received callback for asset {0}", id);
116 220
221 m_requestCallbackTimer.Stop();
222
223 if (m_requestState == RequestState.Aborted)
224 {
225 m_log.WarnFormat(
226 "[ARCHIVER]: Received information about asset {0} after archive save abortion. Ignoring.",
227 id);
228 }
229
117 if (asset != null) 230 if (asset != null)
118 { 231 {
119 m_foundAssetUuids.Add(asset.FullID); 232 m_foundAssetUuids.Add(asset.FullID);
@@ -126,6 +239,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver
126 239
127 if (m_foundAssetUuids.Count + m_notFoundAssetUuids.Count == m_repliesRequired) 240 if (m_foundAssetUuids.Count + m_notFoundAssetUuids.Count == m_repliesRequired)
128 { 241 {
242 m_requestState = RequestState.Completed;
243
129 m_log.DebugFormat( 244 m_log.DebugFormat(
130 "[ARCHIVER]: Successfully added {0} assets ({1} assets notified missing)", 245 "[ARCHIVER]: Successfully added {0} assets ({1} assets notified missing)",
131 m_foundAssetUuids.Count, m_notFoundAssetUuids.Count); 246 m_foundAssetUuids.Count, m_notFoundAssetUuids.Count);
@@ -136,6 +251,10 @@ namespace OpenSim.Region.CoreModules.World.Archiver
136 newThread.Name = "OpenSimulator archiving thread post assets receipt"; 251 newThread.Name = "OpenSimulator archiving thread post assets receipt";
137 newThread.Start(); 252 newThread.Start();
138 } 253 }
254 else
255 {
256 m_requestCallbackTimer.Start();
257 }
139 } 258 }
140 } 259 }
141 catch (Exception e) 260 catch (Exception e)