diff options
Diffstat (limited to 'OpenSim/Services/FSAssetService')
-rw-r--r-- | OpenSim/Services/FSAssetService/FSAssetService.cs | 211 |
1 files changed, 158 insertions, 53 deletions
diff --git a/OpenSim/Services/FSAssetService/FSAssetService.cs b/OpenSim/Services/FSAssetService/FSAssetService.cs index aba8b33..27b04f7 100644 --- a/OpenSim/Services/FSAssetService/FSAssetService.cs +++ b/OpenSim/Services/FSAssetService/FSAssetService.cs | |||
@@ -76,6 +76,8 @@ namespace OpenSim.Services.FSAssetService | |||
76 | protected int m_missingAssets = 0; | 76 | protected int m_missingAssets = 0; |
77 | protected int m_missingAssetsFS = 0; | 77 | protected int m_missingAssetsFS = 0; |
78 | protected string m_FSBase; | 78 | protected string m_FSBase; |
79 | protected bool m_useOsgridFormat = false; | ||
80 | protected bool m_showStats = true; | ||
79 | 81 | ||
80 | private static bool m_Initialized; | 82 | private static bool m_Initialized; |
81 | private bool m_MainInstance; | 83 | private bool m_MainInstance; |
@@ -113,34 +115,34 @@ namespace OpenSim.Services.FSAssetService | |||
113 | } | 115 | } |
114 | 116 | ||
115 | IConfig assetConfig = config.Configs[configName]; | 117 | IConfig assetConfig = config.Configs[configName]; |
116 | 118 | ||
117 | if (assetConfig == null) | 119 | if (assetConfig == null) |
118 | throw new Exception("No AssetService configuration"); | 120 | throw new Exception("No AssetService configuration"); |
119 | 121 | ||
120 | // Get Database Connector from Asset Config (If present) | 122 | // Get Database Connector from Asset Config (If present) |
121 | string dllName = assetConfig.GetString("StorageProvider", string.Empty); | 123 | string dllName = assetConfig.GetString("StorageProvider", string.Empty); |
122 | string m_ConnectionString = assetConfig.GetString("ConnectionString", string.Empty); | 124 | string connectionString = assetConfig.GetString("ConnectionString", string.Empty); |
123 | string m_Realm = assetConfig.GetString("Realm", "fsassets"); | 125 | string realm = assetConfig.GetString("Realm", "fsassets"); |
124 | 126 | ||
125 | int SkipAccessTimeDays = assetConfig.GetInt("DaysBetweenAccessTimeUpdates", 0); | 127 | int SkipAccessTimeDays = assetConfig.GetInt("DaysBetweenAccessTimeUpdates", 0); |
126 | 128 | ||
127 | // If not found above, fallback to Database defaults | 129 | // If not found above, fallback to Database defaults |
128 | IConfig dbConfig = config.Configs["DatabaseService"]; | 130 | IConfig dbConfig = config.Configs["DatabaseService"]; |
129 | 131 | ||
130 | if (dbConfig != null) | 132 | if (dbConfig != null) |
131 | { | 133 | { |
132 | if (dllName == String.Empty) | 134 | if (dllName == String.Empty) |
133 | dllName = dbConfig.GetString("StorageProvider", String.Empty); | 135 | dllName = dbConfig.GetString("StorageProvider", String.Empty); |
134 | 136 | ||
135 | if (m_ConnectionString == String.Empty) | 137 | if (connectionString == String.Empty) |
136 | m_ConnectionString = dbConfig.GetString("ConnectionString", String.Empty); | 138 | connectionString = dbConfig.GetString("ConnectionString", String.Empty); |
137 | } | 139 | } |
138 | 140 | ||
139 | // No databse connection found in either config | 141 | // No databse connection found in either config |
140 | if (dllName.Equals(String.Empty)) | 142 | if (dllName.Equals(String.Empty)) |
141 | throw new Exception("No StorageProvider configured"); | 143 | throw new Exception("No StorageProvider configured"); |
142 | 144 | ||
143 | if (m_ConnectionString.Equals(String.Empty)) | 145 | if (connectionString.Equals(String.Empty)) |
144 | throw new Exception("Missing database connection string"); | 146 | throw new Exception("Missing database connection string"); |
145 | 147 | ||
146 | // Create Storage Provider | 148 | // Create Storage Provider |
@@ -150,11 +152,11 @@ namespace OpenSim.Services.FSAssetService | |||
150 | throw new Exception(string.Format("Could not find a storage interface in the module {0}", dllName)); | 152 | throw new Exception(string.Format("Could not find a storage interface in the module {0}", dllName)); |
151 | 153 | ||
152 | // Initialize DB And perform any migrations required | 154 | // Initialize DB And perform any migrations required |
153 | m_DataConnector.Initialise(m_ConnectionString, m_Realm, SkipAccessTimeDays); | 155 | m_DataConnector.Initialise(connectionString, realm, SkipAccessTimeDays); |
154 | 156 | ||
155 | // Setup Fallback Service | 157 | // Setup Fallback Service |
156 | string str = assetConfig.GetString("FallbackService", string.Empty); | 158 | string str = assetConfig.GetString("FallbackService", string.Empty); |
157 | 159 | ||
158 | if (str != string.Empty) | 160 | if (str != string.Empty) |
159 | { | 161 | { |
160 | object[] args = new object[] { config }; | 162 | object[] args = new object[] { config }; |
@@ -177,12 +179,29 @@ namespace OpenSim.Services.FSAssetService | |||
177 | Directory.CreateDirectory(spoolTmp); | 179 | Directory.CreateDirectory(spoolTmp); |
178 | 180 | ||
179 | m_FSBase = assetConfig.GetString("BaseDirectory", String.Empty); | 181 | m_FSBase = assetConfig.GetString("BaseDirectory", String.Empty); |
182 | m_log.InfoFormat("[FSASSETS]: Assets base and spool directory are {0} and {1}", m_FSBase, m_SpoolDirectory); | ||
180 | if (m_FSBase == String.Empty) | 183 | if (m_FSBase == String.Empty) |
181 | { | 184 | { |
182 | m_log.ErrorFormat("[FSASSETS]: BaseDirectory not specified"); | 185 | m_log.ErrorFormat("[FSASSETS]: BaseDirectory not specified"); |
183 | throw new Exception("Configuration error"); | 186 | throw new Exception("Configuration error"); |
184 | } | 187 | } |
185 | 188 | ||
189 | // OSGrid format is /3/3/, OpenSims default format is /2/2/2/4/. Try to figure out which we have. | ||
190 | List<string> dirs = new List<string>(Directory.EnumerateDirectories(m_FSBase)); | ||
191 | foreach (var dir in dirs) | ||
192 | { | ||
193 | if (3 == dir.Substring(dir.LastIndexOf(Path.DirectorySeparatorChar) + 1).Length) | ||
194 | { | ||
195 | m_useOsgridFormat = true; | ||
196 | break; | ||
197 | } | ||
198 | } | ||
199 | m_useOsgridFormat = assetConfig.GetBoolean("UseOsgridFormat", m_useOsgridFormat); | ||
200 | m_log.InfoFormat("[FSASSETS]: UseOsgridFormat is {0}", m_useOsgridFormat); | ||
201 | |||
202 | // Default is to show stats to retain original behaviour | ||
203 | m_showStats = assetConfig.GetBoolean("ShowConsoleStats", m_showStats); | ||
204 | |||
186 | if (m_MainInstance) | 205 | if (m_MainInstance) |
187 | { | 206 | { |
188 | string loader = assetConfig.GetString("DefaultAssetLoader", string.Empty); | 207 | string loader = assetConfig.GetString("DefaultAssetLoader", string.Empty); |
@@ -197,13 +216,17 @@ namespace OpenSim.Services.FSAssetService | |||
197 | Store(a, false); | 216 | Store(a, false); |
198 | }); | 217 | }); |
199 | } | 218 | } |
200 | 219 | ||
201 | m_WriterThread = new Thread(Writer); | 220 | m_WriterThread = new Thread(Writer); |
202 | m_WriterThread.Start(); | 221 | m_WriterThread.Start(); |
203 | m_StatsThread = new Thread(Stats); | 222 | |
204 | m_StatsThread.Start(); | 223 | if (m_showStats) |
224 | { | ||
225 | m_StatsThread = new Thread(Stats); | ||
226 | m_StatsThread.Start(); | ||
227 | } | ||
205 | } | 228 | } |
206 | 229 | ||
207 | m_log.Info("[FSASSETS]: FS asset service enabled"); | 230 | m_log.Info("[FSASSETS]: FS asset service enabled"); |
208 | } | 231 | } |
209 | 232 | ||
@@ -212,7 +235,7 @@ namespace OpenSim.Services.FSAssetService | |||
212 | while (true) | 235 | while (true) |
213 | { | 236 | { |
214 | Thread.Sleep(60000); | 237 | Thread.Sleep(60000); |
215 | 238 | ||
216 | lock (m_statsLock) | 239 | lock (m_statsLock) |
217 | { | 240 | { |
218 | if (m_readCount > 0) | 241 | if (m_readCount > 0) |
@@ -220,6 +243,7 @@ namespace OpenSim.Services.FSAssetService | |||
220 | double avg = (double)m_readTicks / (double)m_readCount; | 243 | double avg = (double)m_readTicks / (double)m_readCount; |
221 | // if (avg > 10000) | 244 | // if (avg > 10000) |
222 | // Environment.Exit(0); | 245 | // Environment.Exit(0); |
246 | //// m_log.InfoFormat("[FSASSETS]: Read stats: {0} files, {1} ticks, avg {2:F2}, missing {3}, FS {4}", m_readCount, m_readTicks, (double)m_readTicks / (double)m_readCount, m_missingAssets, m_missingAssetsFS); | ||
223 | } | 247 | } |
224 | m_readCount = 0; | 248 | m_readCount = 0; |
225 | m_readTicks = 0; | 249 | m_readTicks = 0; |
@@ -231,7 +255,7 @@ namespace OpenSim.Services.FSAssetService | |||
231 | 255 | ||
232 | private void Writer() | 256 | private void Writer() |
233 | { | 257 | { |
234 | m_log.Info("[FSASSETS]: Writer started"); | 258 | m_log.Info("[ASSET]: Writer started"); |
235 | 259 | ||
236 | while (true) | 260 | while (true) |
237 | { | 261 | { |
@@ -245,33 +269,98 @@ namespace OpenSim.Services.FSAssetService | |||
245 | string hash = Path.GetFileNameWithoutExtension(files[i]); | 269 | string hash = Path.GetFileNameWithoutExtension(files[i]); |
246 | string s = HashToFile(hash); | 270 | string s = HashToFile(hash); |
247 | string diskFile = Path.Combine(m_FSBase, s); | 271 | string diskFile = Path.Combine(m_FSBase, s); |
272 | bool pathOk = false; | ||
248 | 273 | ||
249 | Directory.CreateDirectory(Path.GetDirectoryName(diskFile)); | 274 | // The cure for chicken bones! |
250 | try | 275 | while(true) |
251 | { | 276 | { |
252 | byte[] data = File.ReadAllBytes(files[i]); | 277 | try |
253 | |||
254 | using (GZipStream gz = new GZipStream(new FileStream(diskFile + ".gz", FileMode.Create), CompressionMode.Compress)) | ||
255 | { | 278 | { |
256 | gz.Write(data, 0, data.Length); | 279 | // Try to make the directory we need for this file |
257 | gz.Close(); | 280 | Directory.CreateDirectory(Path.GetDirectoryName(diskFile)); |
281 | pathOk = true; | ||
282 | break; | ||
258 | } | 283 | } |
259 | File.Delete(files[i]); | 284 | catch (System.IO.IOException) |
260 | 285 | { | |
261 | //File.Move(files[i], diskFile); | 286 | // Creating the directory failed. This can't happen unless |
287 | // a part of the path already exists as a file. Sadly the | ||
288 | // SRAS data contains such files. | ||
289 | string d = Path.GetDirectoryName(diskFile); | ||
290 | |||
291 | // Test each path component in turn. If we can successfully | ||
292 | // make a directory, the level below must be the chicken bone. | ||
293 | while (d.Length > 0) | ||
294 | { | ||
295 | Console.WriteLine(d); | ||
296 | try | ||
297 | { | ||
298 | Directory.CreateDirectory(Path.GetDirectoryName(d)); | ||
299 | } | ||
300 | catch (System.IO.IOException) | ||
301 | { | ||
302 | d = Path.GetDirectoryName(d); | ||
303 | |||
304 | // We failed making the directory and need to | ||
305 | // go up a bit more | ||
306 | continue; | ||
307 | } | ||
308 | |||
309 | // We succeeded in making the directory and (d) is | ||
310 | // the chicken bone | ||
311 | break; | ||
312 | } | ||
313 | |||
314 | // Is the chicken alive? | ||
315 | if (d.Length > 0) | ||
316 | { | ||
317 | Console.WriteLine(d); | ||
318 | |||
319 | FileAttributes attr = File.GetAttributes(d); | ||
320 | |||
321 | if ((attr & FileAttributes.Directory) == 0) | ||
322 | { | ||
323 | // The chicken bone should be resolved. | ||
324 | // Return to writing the file. | ||
325 | File.Delete(d); | ||
326 | continue; | ||
327 | } | ||
328 | } | ||
329 | } | ||
330 | // Could not resolve, skipping | ||
331 | m_log.ErrorFormat("[ASSET]: Could not resolve path creation error for {0}", diskFile); | ||
332 | break; | ||
262 | } | 333 | } |
263 | catch(System.IO.IOException e) | 334 | |
335 | if (pathOk) | ||
264 | { | 336 | { |
265 | if (e.Message.StartsWith("Win32 IO returned ERROR_ALREADY_EXISTS")) | 337 | try |
338 | { | ||
339 | byte[] data = File.ReadAllBytes(files[i]); | ||
340 | |||
341 | using (GZipStream gz = new GZipStream(new FileStream(diskFile + ".gz", FileMode.Create), CompressionMode.Compress)) | ||
342 | { | ||
343 | gz.Write(data, 0, data.Length); | ||
344 | gz.Close(); | ||
345 | } | ||
266 | File.Delete(files[i]); | 346 | File.Delete(files[i]); |
267 | else | 347 | |
268 | throw; | 348 | //File.Move(files[i], diskFile); |
349 | } | ||
350 | catch(System.IO.IOException e) | ||
351 | { | ||
352 | if (e.Message.StartsWith("Win32 IO returned ERROR_ALREADY_EXISTS")) | ||
353 | File.Delete(files[i]); | ||
354 | else | ||
355 | throw; | ||
356 | } | ||
269 | } | 357 | } |
270 | } | 358 | } |
359 | |||
271 | int totalTicks = System.Environment.TickCount - tickCount; | 360 | int totalTicks = System.Environment.TickCount - tickCount; |
272 | if (totalTicks > 0) // Wrap? | 361 | if (totalTicks > 0) // Wrap? |
273 | { | 362 | { |
274 | m_log.InfoFormat("[FSASSETS]: Write cycle complete, {0} files, {1} ticks, avg {2:F2}", files.Length, totalTicks, (double)totalTicks / (double)files.Length); | 363 | m_log.InfoFormat("[ASSET]: Write cycle complete, {0} files, {1} ticks, avg {2:F2}", files.Length, totalTicks, (double)totalTicks / (double)files.Length); |
275 | } | 364 | } |
276 | } | 365 | } |
277 | 366 | ||
@@ -291,20 +380,27 @@ namespace OpenSim.Services.FSAssetService | |||
291 | if (hash == null || hash.Length < 10) | 380 | if (hash == null || hash.Length < 10) |
292 | return "junkyard"; | 381 | return "junkyard"; |
293 | 382 | ||
294 | return Path.Combine(hash.Substring(0, 3), | 383 | if (m_useOsgridFormat) |
295 | Path.Combine(hash.Substring(3, 3))); | 384 | { |
296 | /* | 385 | /* |
297 | * The below is what core would normally use. | 386 | * The code below is the OSGrid code. |
298 | * This is modified to work in OSGrid, as seen | 387 | */ |
299 | * above, because the SRAS data is structured | 388 | return Path.Combine(hash.Substring(0, 3), |
300 | * that way. | 389 | Path.Combine(hash.Substring(3, 3))); |
301 | */ | 390 | } |
302 | /* | 391 | else |
303 | return Path.Combine(hash.Substring(0, 2), | 392 | { |
304 | Path.Combine(hash.Substring(2, 2), | 393 | /* |
305 | Path.Combine(hash.Substring(4, 2), | 394 | * The below is what core would normally use. |
306 | hash.Substring(6, 4)))); | 395 | * This is modified to work in OSGrid, as seen |
307 | */ | 396 | * above, because the SRAS data is structured |
397 | * that way. | ||
398 | */ | ||
399 | return Path.Combine(hash.Substring(0, 2), | ||
400 | Path.Combine(hash.Substring(2, 2), | ||
401 | Path.Combine(hash.Substring(4, 2), | ||
402 | hash.Substring(6, 4)))); | ||
403 | } | ||
308 | } | 404 | } |
309 | 405 | ||
310 | private bool AssetExists(string hash) | 406 | private bool AssetExists(string hash) |
@@ -365,7 +461,7 @@ namespace OpenSim.Services.FSAssetService | |||
365 | Store(asset); | 461 | Store(asset); |
366 | } | 462 | } |
367 | } | 463 | } |
368 | if (asset == null) | 464 | if (asset == null && m_showStats) |
369 | { | 465 | { |
370 | // m_log.InfoFormat("[FSASSETS]: Asset {0} not found", id); | 466 | // m_log.InfoFormat("[FSASSETS]: Asset {0} not found", id); |
371 | m_missingAssets++; | 467 | m_missingAssets++; |
@@ -393,8 +489,11 @@ namespace OpenSim.Services.FSAssetService | |||
393 | } | 489 | } |
394 | } | 490 | } |
395 | if (asset == null) | 491 | if (asset == null) |
396 | m_missingAssetsFS++; | 492 | { |
397 | // m_log.InfoFormat("[FSASSETS]: Asset {0}, hash {1} not found in FS", id, hash); | 493 | if (m_showStats) |
494 | m_missingAssetsFS++; | ||
495 | // m_log.InfoFormat("[FSASSETS]: Asset {0}, hash {1} not found in FS", id, hash); | ||
496 | } | ||
398 | else | 497 | else |
399 | { | 498 | { |
400 | // Deal with bug introduced in Oct. 20 (1eb3e6cc43e2a7b4053bc1185c7c88e22356c5e8) | 499 | // Deal with bug introduced in Oct. 20 (1eb3e6cc43e2a7b4053bc1185c7c88e22356c5e8) |
@@ -408,10 +507,13 @@ namespace OpenSim.Services.FSAssetService | |||
408 | } | 507 | } |
409 | } | 508 | } |
410 | 509 | ||
411 | lock (m_statsLock) | 510 | if (m_showStats) |
412 | { | 511 | { |
413 | m_readTicks += Environment.TickCount - startTime; | 512 | lock (m_statsLock) |
414 | m_readCount++; | 513 | { |
514 | m_readTicks += Environment.TickCount - startTime; | ||
515 | m_readCount++; | ||
516 | } | ||
415 | } | 517 | } |
416 | 518 | ||
417 | // Deal with bug introduced in Oct. 20 (1eb3e6cc43e2a7b4053bc1185c7c88e22356c5e8) | 519 | // Deal with bug introduced in Oct. 20 (1eb3e6cc43e2a7b4053bc1185c7c88e22356c5e8) |
@@ -578,6 +680,9 @@ namespace OpenSim.Services.FSAssetService | |||
578 | 680 | ||
579 | if (!m_DataConnector.Store(asset.Metadata, hash)) | 681 | if (!m_DataConnector.Store(asset.Metadata, hash)) |
580 | { | 682 | { |
683 | if (asset.Metadata.Type == -2) | ||
684 | return asset.ID; | ||
685 | |||
581 | return UUID.Zero.ToString(); | 686 | return UUID.Zero.ToString(); |
582 | } | 687 | } |
583 | else | 688 | else |
@@ -630,7 +735,7 @@ namespace OpenSim.Services.FSAssetService | |||
630 | AssetBase asset = Get(args[2], out hash); | 735 | AssetBase asset = Get(args[2], out hash); |
631 | 736 | ||
632 | if (asset == null || asset.Data.Length == 0) | 737 | if (asset == null || asset.Data.Length == 0) |
633 | { | 738 | { |
634 | MainConsole.Instance.Output("Asset not found"); | 739 | MainConsole.Instance.Output("Asset not found"); |
635 | return; | 740 | return; |
636 | } | 741 | } |
@@ -672,7 +777,7 @@ namespace OpenSim.Services.FSAssetService | |||
672 | AssetBase asset = Get(args[2]); | 777 | AssetBase asset = Get(args[2]); |
673 | 778 | ||
674 | if (asset == null || asset.Data.Length == 0) | 779 | if (asset == null || asset.Data.Length == 0) |
675 | { | 780 | { |
676 | MainConsole.Instance.Output("Asset not found"); | 781 | MainConsole.Instance.Output("Asset not found"); |
677 | return; | 782 | return; |
678 | } | 783 | } |