aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/OpenSim/Services/FSAssetService/FSAssetService.cs
diff options
context:
space:
mode:
Diffstat (limited to 'OpenSim/Services/FSAssetService/FSAssetService.cs')
-rw-r--r--OpenSim/Services/FSAssetService/FSAssetService.cs723
1 files changed, 723 insertions, 0 deletions
diff --git a/OpenSim/Services/FSAssetService/FSAssetService.cs b/OpenSim/Services/FSAssetService/FSAssetService.cs
new file mode 100644
index 0000000..1bab687
--- /dev/null
+++ b/OpenSim/Services/FSAssetService/FSAssetService.cs
@@ -0,0 +1,723 @@
1/*
2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the OpenSimulator Project nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28using System;
29using System.Diagnostics;
30using System.Collections.Generic;
31using System.IO;
32using System.IO.Compression;
33using System.Text;
34using System.Threading;
35using System.Reflection;
36using OpenSim.Data;
37using OpenSim.Framework;
38using OpenSim.Framework.Serialization.External;
39using OpenSim.Framework.Console;
40using OpenSim.Server.Base;
41using OpenSim.Services.Base;
42using OpenSim.Services.Interfaces;
43using Nini.Config;
44using log4net;
45using OpenMetaverse;
46using System.Security.Cryptography;
47
48namespace OpenSim.Services.FSAssetService
49{
50 public class FSAssetConnector : ServiceBase, IAssetService
51 {
52 private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
53
54 static System.Text.ASCIIEncoding enc = new System.Text.ASCIIEncoding();
55 static SHA256CryptoServiceProvider SHA256 = new SHA256CryptoServiceProvider();
56
57 static byte[] ToCString(string s)
58 {
59 byte[] ret = enc.GetBytes(s);
60 Array.Resize(ref ret, ret.Length + 1);
61 ret[ret.Length - 1] = 0;
62
63 return ret;
64 }
65
66 protected IAssetLoader m_AssetLoader = null;
67 protected IFSAssetDataPlugin m_DataConnector = null;
68 protected IAssetService m_FallbackService;
69 protected Thread m_WriterThread;
70 protected Thread m_StatsThread;
71 protected string m_SpoolDirectory;
72 protected object m_readLock = new object();
73 protected object m_statsLock = new object();
74 protected int m_readCount = 0;
75 protected int m_readTicks = 0;
76 protected int m_missingAssets = 0;
77 protected int m_missingAssetsFS = 0;
78 protected string m_FSBase;
79
80 private static bool m_Initialized;
81 private bool m_MainInstance;
82
83 public FSAssetConnector(IConfigSource config)
84 : this(config, "AssetService")
85 {
86 }
87
88 public FSAssetConnector(IConfigSource config, string configName) : base(config)
89 {
90 if (!m_Initialized)
91 {
92 m_Initialized = true;
93 m_MainInstance = true;
94
95 MainConsole.Instance.Commands.AddCommand("fs", false,
96 "show assets", "show assets", "Show asset stats",
97 HandleShowAssets);
98 MainConsole.Instance.Commands.AddCommand("fs", false,
99 "show digest", "show digest <ID>", "Show asset digest",
100 HandleShowDigest);
101 MainConsole.Instance.Commands.AddCommand("fs", false,
102 "delete asset", "delete asset <ID>",
103 "Delete asset from database",
104 HandleDeleteAsset);
105 MainConsole.Instance.Commands.AddCommand("fs", false,
106 "import", "import <conn> <table> [<start> <count>]",
107 "Import legacy assets",
108 HandleImportAssets);
109 MainConsole.Instance.Commands.AddCommand("fs", false,
110 "force import", "force import <conn> <table> [<start> <count>]",
111 "Import legacy assets, overwriting current content",
112 HandleImportAssets);
113 }
114
115 IConfig assetConfig = config.Configs[configName];
116
117 if (assetConfig == null)
118 throw new Exception("No AssetService configuration");
119
120 // Get Database Connector from Asset Config (If present)
121 string dllName = assetConfig.GetString("StorageProvider", string.Empty);
122 string m_ConnectionString = assetConfig.GetString("ConnectionString", string.Empty);
123 string m_Realm = assetConfig.GetString("Realm", "fsassets");
124
125 int SkipAccessTimeDays = assetConfig.GetInt("DaysBetweenAccessTimeUpdates", 0);
126
127 // If not found above, fallback to Database defaults
128 IConfig dbConfig = config.Configs["DatabaseService"];
129
130 if (dbConfig != null)
131 {
132 if (dllName == String.Empty)
133 dllName = dbConfig.GetString("StorageProvider", String.Empty);
134
135 if (m_ConnectionString == String.Empty)
136 m_ConnectionString = dbConfig.GetString("ConnectionString", String.Empty);
137 }
138
139 // No databse connection found in either config
140 if (dllName.Equals(String.Empty))
141 throw new Exception("No StorageProvider configured");
142
143 if (m_ConnectionString.Equals(String.Empty))
144 throw new Exception("Missing database connection string");
145
146 // Create Storage Provider
147 m_DataConnector = LoadPlugin<IFSAssetDataPlugin>(dllName);
148
149 if (m_DataConnector == null)
150 throw new Exception(string.Format("Could not find a storage interface in the module {0}", dllName));
151
152 // Initialize DB And perform any migrations required
153 m_DataConnector.Initialise(m_ConnectionString, m_Realm, SkipAccessTimeDays);
154
155 // Setup Fallback Service
156 string str = assetConfig.GetString("FallbackService", string.Empty);
157
158 if (str != string.Empty)
159 {
160 object[] args = new object[] { config };
161 m_FallbackService = LoadPlugin<IAssetService>(str, args);
162 if (m_FallbackService != null)
163 {
164 m_log.Info("[FSASSETS]: Fallback service loaded");
165 }
166 else
167 {
168 m_log.Error("[FSASSETS]: Failed to load fallback service");
169 }
170 }
171
172 // Setup directory structure including temp directory
173 m_SpoolDirectory = assetConfig.GetString("SpoolDirectory", "/tmp");
174
175 string spoolTmp = Path.Combine(m_SpoolDirectory, "spool");
176
177 Directory.CreateDirectory(spoolTmp);
178
179 m_FSBase = assetConfig.GetString("BaseDirectory", String.Empty);
180 if (m_FSBase == String.Empty)
181 {
182 m_log.ErrorFormat("[FSASSETS]: BaseDirectory not specified");
183 throw new Exception("Configuration error");
184 }
185
186 if (m_MainInstance)
187 {
188 string loader = assetConfig.GetString("DefaultAssetLoader", string.Empty);
189 if (loader != string.Empty)
190 {
191 m_AssetLoader = LoadPlugin<IAssetLoader>(loader);
192 string loaderArgs = assetConfig.GetString("AssetLoaderArgs", string.Empty);
193 m_log.InfoFormat("[FSASSETS]: Loading default asset set from {0}", loaderArgs);
194 m_AssetLoader.ForEachDefaultXmlAsset(loaderArgs,
195 delegate(AssetBase a)
196 {
197 Store(a, false);
198 });
199 }
200
201 m_WriterThread = new Thread(Writer);
202 m_WriterThread.Start();
203 m_StatsThread = new Thread(Stats);
204 m_StatsThread.Start();
205 }
206
207 m_log.Info("[FSASSETS]: FS asset service enabled");
208 }
209
210 private void Stats()
211 {
212 while (true)
213 {
214 Thread.Sleep(60000);
215
216 lock (m_statsLock)
217 {
218 if (m_readCount > 0)
219 {
220 double avg = (double)m_readTicks / (double)m_readCount;
221// if (avg > 10000)
222// Environment.Exit(0);
223 m_log.InfoFormat("[FSASSETS]: Read stats: {0} files, {1} ticks, avg {2:F2}, missing {3}, FS {4}", m_readCount, m_readTicks, (double)m_readTicks / (double)m_readCount, m_missingAssets, m_missingAssetsFS);
224 }
225 m_readCount = 0;
226 m_readTicks = 0;
227 m_missingAssets = 0;
228 m_missingAssetsFS = 0;
229 }
230 }
231 }
232
233 private void Writer()
234 {
235 m_log.Info("[FSASSETS]: Writer started");
236
237 while (true)
238 {
239 string[] files = Directory.GetFiles(m_SpoolDirectory);
240
241 if (files.Length > 0)
242 {
243 int tickCount = Environment.TickCount;
244 for (int i = 0 ; i < files.Length ; i++)
245 {
246 string hash = Path.GetFileNameWithoutExtension(files[i]);
247 string s = HashToFile(hash);
248 string diskFile = Path.Combine(m_FSBase, s);
249
250 Directory.CreateDirectory(Path.GetDirectoryName(diskFile));
251 try
252 {
253 byte[] data = File.ReadAllBytes(files[i]);
254
255 using (GZipStream gz = new GZipStream(new FileStream(diskFile + ".gz", FileMode.Create), CompressionMode.Compress))
256 {
257 gz.Write(data, 0, data.Length);
258 gz.Close();
259 }
260 File.Delete(files[i]);
261
262 //File.Move(files[i], diskFile);
263 }
264 catch(System.IO.IOException e)
265 {
266 if (e.Message.StartsWith("Win32 IO returned ERROR_ALREADY_EXISTS"))
267 File.Delete(files[i]);
268 else
269 throw;
270 }
271 }
272 int totalTicks = System.Environment.TickCount - tickCount;
273 if (totalTicks > 0) // Wrap?
274 {
275 m_log.InfoFormat("[FSASSETS]: Write cycle complete, {0} files, {1} ticks, avg {2:F2}", files.Length, totalTicks, (double)totalTicks / (double)files.Length);
276 }
277 }
278
279 Thread.Sleep(1000);
280 }
281 }
282
283 string GetSHA256Hash(byte[] data)
284 {
285 byte[] hash = SHA256.ComputeHash(data);
286
287 return BitConverter.ToString(hash).Replace("-", String.Empty);
288 }
289
290 public string HashToPath(string hash)
291 {
292 if (hash == null || hash.Length < 10)
293 return "junkyard";
294
295 return Path.Combine(hash.Substring(0, 3),
296 Path.Combine(hash.Substring(3, 3)));
297 /*
298 * The below is what core would normally use.
299 * This is modified to work in OSGrid, as seen
300 * above, because the SRAS data is structured
301 * that way.
302 */
303 /*
304 return Path.Combine(hash.Substring(0, 2),
305 Path.Combine(hash.Substring(2, 2),
306 Path.Combine(hash.Substring(4, 2),
307 hash.Substring(6, 4))));
308 */
309 }
310
311 private bool AssetExists(string hash)
312 {
313 string s = HashToFile(hash);
314 string diskFile = Path.Combine(m_FSBase, s);
315
316 if (File.Exists(diskFile + ".gz") || File.Exists(diskFile))
317 return true;
318
319 return false;
320 }
321
322 public virtual bool[] AssetsExist(string[] ids)
323 {
324 UUID[] uuid = Array.ConvertAll(ids, id => UUID.Parse(id));
325 return m_DataConnector.AssetsExist(uuid);
326 }
327
328 public string HashToFile(string hash)
329 {
330 return Path.Combine(HashToPath(hash), hash);
331 }
332
333 public virtual AssetBase Get(string id)
334 {
335 string hash;
336
337 return Get(id, out hash);
338 }
339
340 private AssetBase Get(string id, out string sha)
341 {
342 string hash = string.Empty;
343
344 int startTime = System.Environment.TickCount;
345 AssetMetadata metadata;
346
347 lock (m_readLock)
348 {
349 metadata = m_DataConnector.Get(id, out hash);
350 }
351
352 sha = hash;
353
354 if (metadata == null)
355 {
356 AssetBase asset = null;
357 if (m_FallbackService != null)
358 {
359 asset = m_FallbackService.Get(id);
360 if (asset != null)
361 {
362 asset.Metadata.ContentType =
363 SLUtil.SLAssetTypeToContentType((int)asset.Type);
364 sha = GetSHA256Hash(asset.Data);
365 m_log.InfoFormat("[FSASSETS]: Added asset {0} from fallback to local store", id);
366 Store(asset);
367 }
368 }
369 if (asset == null)
370 {
371 // m_log.InfoFormat("[FSASSETS]: Asset {0} not found", id);
372 m_missingAssets++;
373 }
374 return asset;
375 }
376 AssetBase newAsset = new AssetBase();
377 newAsset.Metadata = metadata;
378 try
379 {
380 newAsset.Data = GetFsData(hash);
381 if (newAsset.Data.Length == 0)
382 {
383 AssetBase asset = null;
384 if (m_FallbackService != null)
385 {
386 asset = m_FallbackService.Get(id);
387 if (asset != null)
388 {
389 asset.Metadata.ContentType =
390 SLUtil.SLAssetTypeToContentType((int)asset.Type);
391 sha = GetSHA256Hash(asset.Data);
392 m_log.InfoFormat("[FSASSETS]: Added asset {0} from fallback to local store", id);
393 Store(asset);
394 }
395 }
396 if (asset == null)
397 m_missingAssetsFS++;
398 // m_log.InfoFormat("[FSASSETS]: Asset {0}, hash {1} not found in FS", id, hash);
399 else
400 {
401 // Deal with bug introduced in Oct. 20 (1eb3e6cc43e2a7b4053bc1185c7c88e22356c5e8)
402 // Fix bad assets before sending them elsewhere
403 if (asset.Type == (int)AssetType.Object && asset.Data != null)
404 {
405 string xml = ExternalRepresentationUtils.SanitizeXml(Utils.BytesToString(asset.Data));
406 asset.Data = Utils.StringToBytes(xml);
407 }
408 return asset;
409 }
410 }
411
412 lock (m_statsLock)
413 {
414 m_readTicks += Environment.TickCount - startTime;
415 m_readCount++;
416 }
417
418 // Deal with bug introduced in Oct. 20 (1eb3e6cc43e2a7b4053bc1185c7c88e22356c5e8)
419 // Fix bad assets before sending them elsewhere
420 if (newAsset.Type == (int)AssetType.Object && newAsset.Data != null)
421 {
422 string xml = ExternalRepresentationUtils.SanitizeXml(Utils.BytesToString(newAsset.Data));
423 newAsset.Data = Utils.StringToBytes(xml);
424 }
425
426 return newAsset;
427 }
428 catch (Exception exception)
429 {
430 m_log.Error(exception.ToString());
431 Thread.Sleep(5000);
432 Environment.Exit(1);
433 return null;
434 }
435 }
436
437 public virtual AssetMetadata GetMetadata(string id)
438 {
439 string hash;
440 return m_DataConnector.Get(id, out hash);
441 }
442
443 public virtual byte[] GetData(string id)
444 {
445 string hash;
446 if (m_DataConnector.Get(id, out hash) == null)
447 return null;
448
449 return GetFsData(hash);
450 }
451
452 public bool Get(string id, Object sender, AssetRetrieved handler)
453 {
454 AssetBase asset = Get(id);
455
456 handler(id, sender, asset);
457
458 return true;
459 }
460
461 public byte[] GetFsData(string hash)
462 {
463 string spoolFile = Path.Combine(m_SpoolDirectory, hash + ".asset");
464
465 if (File.Exists(spoolFile))
466 {
467 try
468 {
469 byte[] content = File.ReadAllBytes(spoolFile);
470
471 return content;
472 }
473 catch
474 {
475 }
476 }
477
478 string file = HashToFile(hash);
479 string diskFile = Path.Combine(m_FSBase, file);
480
481 if (File.Exists(diskFile + ".gz"))
482 {
483 try
484 {
485 using (GZipStream gz = new GZipStream(new FileStream(diskFile + ".gz", FileMode.Open, FileAccess.Read), CompressionMode.Decompress))
486 {
487 using (MemoryStream ms = new MemoryStream())
488 {
489 byte[] data = new byte[32768];
490 int bytesRead;
491
492 do
493 {
494 bytesRead = gz.Read(data, 0, 32768);
495 if (bytesRead > 0)
496 ms.Write(data, 0, bytesRead);
497 } while (bytesRead > 0);
498
499 return ms.ToArray();
500 }
501 }
502 }
503 catch (Exception)
504 {
505 return new Byte[0];
506 }
507 }
508 else if (File.Exists(diskFile))
509 {
510 try
511 {
512 byte[] content = File.ReadAllBytes(diskFile);
513
514 return content;
515 }
516 catch
517 {
518 }
519 }
520 return new Byte[0];
521
522 }
523
524 public virtual string Store(AssetBase asset)
525 {
526 return Store(asset, false);
527 }
528
529 private string Store(AssetBase asset, bool force)
530 {
531 int tickCount = Environment.TickCount;
532 string hash = GetSHA256Hash(asset.Data);
533
534 if (!AssetExists(hash))
535 {
536 string tempFile = Path.Combine(Path.Combine(m_SpoolDirectory, "spool"), hash + ".asset");
537 string finalFile = Path.Combine(m_SpoolDirectory, hash + ".asset");
538
539 if (!File.Exists(finalFile))
540 {
541 // Deal with bug introduced in Oct. 20 (1eb3e6cc43e2a7b4053bc1185c7c88e22356c5e8)
542 // Fix bad assets before storing on this server
543 if (asset.Type == (int)AssetType.Object && asset.Data != null)
544 {
545 string xml = ExternalRepresentationUtils.SanitizeXml(Utils.BytesToString(asset.Data));
546 asset.Data = Utils.StringToBytes(xml);
547 }
548
549 FileStream fs = File.Create(tempFile);
550
551 fs.Write(asset.Data, 0, asset.Data.Length);
552
553 fs.Close();
554
555 File.Move(tempFile, finalFile);
556 }
557 }
558
559 if (asset.ID == string.Empty)
560 {
561 if (asset.FullID == UUID.Zero)
562 {
563 asset.FullID = UUID.Random();
564 }
565 asset.ID = asset.FullID.ToString();
566 }
567 else if (asset.FullID == UUID.Zero)
568 {
569 UUID uuid = UUID.Zero;
570 if (UUID.TryParse(asset.ID, out uuid))
571 {
572 asset.FullID = uuid;
573 }
574 else
575 {
576 asset.FullID = UUID.Random();
577 }
578 }
579
580 if (!m_DataConnector.Store(asset.Metadata, hash))
581 {
582 return UUID.Zero.ToString();
583 }
584 else
585 {
586 return asset.ID;
587 }
588 }
589
590 public bool UpdateContent(string id, byte[] data)
591 {
592 return false;
593
594// string oldhash;
595// AssetMetadata meta = m_DataConnector.Get(id, out oldhash);
596//
597// if (meta == null)
598// return false;
599//
600// AssetBase asset = new AssetBase();
601// asset.Metadata = meta;
602// asset.Data = data;
603//
604// Store(asset);
605//
606// return true;
607 }
608
609 public virtual bool Delete(string id)
610 {
611 m_DataConnector.Delete(id);
612
613 return true;
614 }
615
616 private void HandleShowAssets(string module, string[] args)
617 {
618 int num = m_DataConnector.Count();
619 MainConsole.Instance.Output(string.Format("Total asset count: {0}", num));
620 }
621
622 private void HandleShowDigest(string module, string[] args)
623 {
624 if (args.Length < 3)
625 {
626 MainConsole.Instance.Output("Syntax: show digest <ID>");
627 return;
628 }
629
630 string hash;
631 AssetBase asset = Get(args[2], out hash);
632
633 if (asset == null || asset.Data.Length == 0)
634 {
635 MainConsole.Instance.Output("Asset not found");
636 return;
637 }
638
639 int i;
640
641 MainConsole.Instance.Output(String.Format("Name: {0}", asset.Name));
642 MainConsole.Instance.Output(String.Format("Description: {0}", asset.Description));
643 MainConsole.Instance.Output(String.Format("Type: {0}", asset.Type));
644 MainConsole.Instance.Output(String.Format("Content-type: {0}", asset.Metadata.ContentType));
645 MainConsole.Instance.Output(String.Format("Flags: {0}", asset.Metadata.Flags.ToString()));
646 MainConsole.Instance.Output(String.Format("FS file: {0}", HashToFile(hash)));
647
648 for (i = 0 ; i < 5 ; i++)
649 {
650 int off = i * 16;
651 if (asset.Data.Length <= off)
652 break;
653 int len = 16;
654 if (asset.Data.Length < off + len)
655 len = asset.Data.Length - off;
656
657 byte[] line = new byte[len];
658 Array.Copy(asset.Data, off, line, 0, len);
659
660 string text = BitConverter.ToString(line);
661 MainConsole.Instance.Output(String.Format("{0:x4}: {1}", off, text));
662 }
663 }
664
665 private void HandleDeleteAsset(string module, string[] args)
666 {
667 if (args.Length < 3)
668 {
669 MainConsole.Instance.Output("Syntax: delete asset <ID>");
670 return;
671 }
672
673 AssetBase asset = Get(args[2]);
674
675 if (asset == null || asset.Data.Length == 0)
676 {
677 MainConsole.Instance.Output("Asset not found");
678 return;
679 }
680
681 m_DataConnector.Delete(args[2]);
682
683 MainConsole.Instance.Output("Asset deleted");
684 }
685
686 private void HandleImportAssets(string module, string[] args)
687 {
688 bool force = false;
689 if (args[0] == "force")
690 {
691 force = true;
692 List<string> list = new List<string>(args);
693 list.RemoveAt(0);
694 args = list.ToArray();
695 }
696 if (args.Length < 3)
697 {
698 MainConsole.Instance.Output("Syntax: import <conn> <table> [<start> <count>]");
699 }
700 else
701 {
702 string conn = args[1];
703 string table = args[2];
704 int start = 0;
705 int count = -1;
706 if (args.Length > 3)
707 {
708 start = Convert.ToInt32(args[3]);
709 }
710 if (args.Length > 4)
711 {
712 count = Convert.ToInt32(args[4]);
713 }
714 m_DataConnector.Import(conn, table, start, count, force, new FSStoreDelegate(Store));
715 }
716 }
717
718 public AssetBase GetCached(string id)
719 {
720 return Get(id);
721 }
722 }
723}