aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/OpenSim/Services/FSAssetService
diff options
context:
space:
mode:
Diffstat (limited to 'OpenSim/Services/FSAssetService')
-rw-r--r--OpenSim/Services/FSAssetService/FSAssetService.cs664
1 files changed, 664 insertions, 0 deletions
diff --git a/OpenSim/Services/FSAssetService/FSAssetService.cs b/OpenSim/Services/FSAssetService/FSAssetService.cs
new file mode 100644
index 0000000..0e578d2
--- /dev/null
+++ b/OpenSim/Services/FSAssetService/FSAssetService.cs
@@ -0,0 +1,664 @@
1/*
2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the OpenSimulator Project nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28using System;
29using System.Diagnostics;
30using System.Collections.Generic;
31using System.IO;
32using System.IO.Compression;
33using System.Text;
34using System.Threading;
35using System.Reflection;
36using OpenSim.Framework;
37using OpenSim.Framework.Console;
38using OpenSim.Server.Base;
39using OpenSim.Services.Base;
40using OpenSim.Services.Interfaces;
41using Nini.Config;
42using log4net;
43using MySql.Data.MySqlClient;
44using System.Data;
45using Careminster;
46using OpenMetaverse;
47using System.Security.Cryptography;
48
49namespace OpenSim.Services.FSAssetService
50{
51 public class FSAssetConnector : ServiceBase, IAssetService
52 {
53 private static readonly ILog m_log =
54 LogManager.GetLogger(
55 MethodBase.GetCurrentMethod().DeclaringType);
56
57 static System.Text.ASCIIEncoding enc = new System.Text.ASCIIEncoding();
58 static SHA256CryptoServiceProvider SHA256 = new SHA256CryptoServiceProvider();
59
60 static byte[] ToCString(string s)
61 {
62 byte[] ret = enc.GetBytes(s);
63 Array.Resize(ref ret, ret.Length + 1);
64 ret[ret.Length - 1] = 0;
65
66 return ret;
67 }
68
69 protected IAssetLoader m_AssetLoader = null;
70 protected string m_ConnectionString;
71 protected FSAssetConnectorData m_DataConnector = null;
72 protected string m_FsckProgram;
73 protected IAssetService m_FallbackService;
74 protected Thread m_WriterThread;
75 protected Thread m_StatsThread;
76 protected string m_SpoolDirectory;
77 protected object m_readLock = new object();
78 protected object m_statsLock = new object();
79 protected int m_readCount = 0;
80 protected int m_readTicks = 0;
81 protected int m_missingAssets = 0;
82 protected int m_missingAssetsFS = 0;
83 protected string m_FSBase;
84 protected string m_Realm;
85
86 public FSAssetConnector(IConfigSource config)
87 : this(config, "AssetService")
88 {
89 }
90
91 public FSAssetConnector(IConfigSource config, string configName) : base(config)
92 {
93 m_FsckProgram = string.Empty;
94
95 MainConsole.Instance.Commands.AddCommand("fs", false,
96 "show assets", "show assets", "Show asset stats",
97 HandleShowAssets);
98 MainConsole.Instance.Commands.AddCommand("fs", false,
99 "show digest", "show digest <ID>", "Show asset digest",
100 HandleShowDigest);
101 MainConsole.Instance.Commands.AddCommand("fs", false,
102 "delete asset", "delete asset <ID>",
103 "Delete asset from database",
104 HandleDeleteAsset);
105 MainConsole.Instance.Commands.AddCommand("fs", false,
106 "import", "import <conn> <table> [<start> <count>]",
107 "Import legacy assets",
108 HandleImportAssets);
109 MainConsole.Instance.Commands.AddCommand("fs", false,
110 "force import", "force import <conn> <table> [<start> <count>]",
111 "Import legacy assets, overwriting current content",
112 HandleImportAssets);
113
114 IConfig assetConfig = config.Configs[configName];
115 if (assetConfig == null)
116 {
117 throw new Exception("No AssetService configuration");
118 }
119
120 m_ConnectionString = assetConfig.GetString("ConnectionString", string.Empty);
121 if (m_ConnectionString == string.Empty)
122 {
123 throw new Exception("Missing database connection string");
124 }
125
126 m_Realm = assetConfig.GetString("Realm", "fsassets");
127
128 m_DataConnector = new FSAssetConnectorData(m_ConnectionString, m_Realm);
129 string str = assetConfig.GetString("FallbackService", string.Empty);
130 if (str != string.Empty)
131 {
132 object[] args = new object[] { config };
133 m_FallbackService = LoadPlugin<IAssetService>(str, args);
134 if (m_FallbackService != null)
135 {
136 m_log.Info("[FALLBACK]: Fallback service loaded");
137 }
138 else
139 {
140 m_log.Error("[FALLBACK]: Failed to load fallback service");
141 }
142 }
143
144 m_SpoolDirectory = assetConfig.GetString("SpoolDirectory", "/tmp");
145
146 string spoolTmp = Path.Combine(m_SpoolDirectory, "spool");
147
148 Directory.CreateDirectory(spoolTmp);
149
150 m_FSBase = assetConfig.GetString("BaseDirectory", String.Empty);
151 if (m_FSBase == String.Empty)
152 {
153 m_log.ErrorFormat("[ASSET]: BaseDirectory not specified");
154 throw new Exception("Configuration error");
155 }
156
157 string loader = assetConfig.GetString("DefaultAssetLoader", string.Empty);
158 if (loader != string.Empty)
159 {
160 m_AssetLoader = LoadPlugin<IAssetLoader>(loader);
161 string loaderArgs = assetConfig.GetString("AssetLoaderArgs", string.Empty);
162 m_log.InfoFormat("[ASSET]: Loading default asset set from {0}", loaderArgs);
163 m_AssetLoader.ForEachDefaultXmlAsset(loaderArgs,
164 delegate(AssetBase a)
165 {
166 Store(a, false);
167 });
168 }
169 m_log.Info("[ASSET]: FS asset service enabled");
170
171 m_WriterThread = new Thread(Writer);
172 m_WriterThread.Start();
173 m_StatsThread = new Thread(Stats);
174 m_StatsThread.Start();
175 }
176
177 private void Stats()
178 {
179 while (true)
180 {
181 Thread.Sleep(60000);
182
183 lock (m_statsLock)
184 {
185 if (m_readCount > 0)
186 {
187 double avg = (double)m_readTicks / (double)m_readCount;
188// if (avg > 10000)
189// Environment.Exit(0);
190 m_log.InfoFormat("[ASSET]: Read stats: {0} files, {1} ticks, avg {2:F2}, missing {3}, FS {4}", m_readCount, m_readTicks, (double)m_readTicks / (double)m_readCount, m_missingAssets, m_missingAssetsFS);
191 }
192 m_readCount = 0;
193 m_readTicks = 0;
194 m_missingAssets = 0;
195 m_missingAssetsFS = 0;
196 }
197 }
198 }
199
200 private void Writer()
201 {
202 m_log.Info("[ASSET]: Writer started");
203
204 while (true)
205 {
206 string[] files = Directory.GetFiles(m_SpoolDirectory);
207
208 if (files.Length > 0)
209 {
210 int tickCount = Environment.TickCount;
211 for (int i = 0 ; i < files.Length ; i++)
212 {
213 string hash = Path.GetFileNameWithoutExtension(files[i]);
214 string s = HashToFile(hash);
215 string diskFile = Path.Combine(m_FSBase, s);
216
217 Directory.CreateDirectory(Path.GetDirectoryName(diskFile));
218 try
219 {
220 byte[] data = File.ReadAllBytes(files[i]);
221
222 using (GZipStream gz = new GZipStream(new FileStream(diskFile + ".gz", FileMode.Create), CompressionMode.Compress))
223 {
224 gz.Write(data, 0, data.Length);
225 gz.Close();
226 }
227 File.Delete(files[i]);
228
229 //File.Move(files[i], diskFile);
230 }
231 catch(System.IO.IOException e)
232 {
233 if (e.Message.StartsWith("Win32 IO returned ERROR_ALREADY_EXISTS"))
234 File.Delete(files[i]);
235 else
236 throw;
237 }
238 }
239 int totalTicks = System.Environment.TickCount - tickCount;
240 if (totalTicks > 0) // Wrap?
241 {
242 m_log.InfoFormat("[ASSET]: Write cycle complete, {0} files, {1} ticks, avg {2:F2}", files.Length, totalTicks, (double)totalTicks / (double)files.Length);
243 }
244 }
245
246 Thread.Sleep(1000);
247 }
248 }
249
250 string GetSHA256Hash(byte[] data)
251 {
252 byte[] hash = SHA256.ComputeHash(data);
253
254 return BitConverter.ToString(hash).Replace("-", String.Empty);
255 }
256
257 public string HashToPath(string hash)
258 {
259 if (hash == null || hash.Length < 10)
260 return "junkyard";
261
262 return Path.Combine(hash.Substring(0, 3),
263 Path.Combine(hash.Substring(3, 3)));
264 /*
265 * The below is what core would normally use.
266 * This is modified to work in OSGrid, as seen
267 * above, because the SRAS data is structured
268 * that way.
269 */
270 /*
271 return Path.Combine(hash.Substring(0, 2),
272 Path.Combine(hash.Substring(2, 2),
273 Path.Combine(hash.Substring(4, 2),
274 hash.Substring(6, 4))));
275 */
276 }
277
278 private bool AssetExists(string hash)
279 {
280 string s = HashToFile(hash);
281 string diskFile = Path.Combine(m_FSBase, s);
282
283 if (File.Exists(diskFile + ".gz") || File.Exists(diskFile))
284 return true;
285
286 return false;
287 }
288
289 public virtual bool[] AssetsExist(string[] ids)
290 {
291 UUID[] uuid = Array.ConvertAll(ids, id => UUID.Parse(id));
292 return m_DataConnector.AssetsExist(uuid);
293 }
294
295 public string HashToFile(string hash)
296 {
297 return Path.Combine(HashToPath(hash), hash);
298 }
299
300 public AssetBase Get(string id)
301 {
302 string hash;
303
304 return Get(id, out hash);
305 }
306
307 private AssetBase Get(string id, out string sha)
308 {
309 string hash = string.Empty;
310
311 int startTime = System.Environment.TickCount;
312 AssetMetadata metadata;
313
314 lock (m_readLock)
315 {
316 metadata = m_DataConnector.Get(id, out hash);
317 }
318
319 sha = hash;
320
321 if (metadata == null)
322 {
323 AssetBase asset = null;
324 if (m_FallbackService != null)
325 {
326 asset = m_FallbackService.Get(id);
327 if (asset != null)
328 {
329 asset.Metadata.ContentType =
330 SLUtil.SLAssetTypeToContentType((int)asset.Type);
331 sha = GetSHA256Hash(asset.Data);
332 m_log.InfoFormat("[FALLBACK]: Added asset {0} from fallback to local store", id);
333 Store(asset);
334 }
335 }
336 if (asset == null)
337 {
338// m_log.InfoFormat("[ASSET]: Asset {0} not found", id);
339 m_missingAssets++;
340 }
341 return asset;
342 }
343 AssetBase newAsset = new AssetBase();
344 newAsset.Metadata = metadata;
345 try
346 {
347 newAsset.Data = GetFsData(hash);
348 if (newAsset.Data.Length == 0)
349 {
350 AssetBase asset = null;
351 if (m_FallbackService != null)
352 {
353 asset = m_FallbackService.Get(id);
354 if (asset != null)
355 {
356 asset.Metadata.ContentType =
357 SLUtil.SLAssetTypeToContentType((int)asset.Type);
358 sha = GetSHA256Hash(asset.Data);
359 m_log.InfoFormat("[FALLBACK]: Added asset {0} from fallback to local store", id);
360 Store(asset);
361 }
362 }
363 if (asset == null)
364 m_missingAssetsFS++;
365// m_log.InfoFormat("[ASSET]: Asset {0}, hash {1} not found in FS", id, hash);
366 else
367 return asset;
368 }
369
370 lock (m_statsLock)
371 {
372 m_readTicks += Environment.TickCount - startTime;
373 m_readCount++;
374 }
375 return newAsset;
376 }
377 catch (Exception exception)
378 {
379 m_log.Error(exception.ToString());
380 Thread.Sleep(5000);
381 Environment.Exit(1);
382 return null;
383 }
384 }
385
386 public AssetMetadata GetMetadata(string id)
387 {
388 string hash;
389 return m_DataConnector.Get(id, out hash);
390 }
391
392 public byte[] GetData(string id)
393 {
394 string hash;
395 if (m_DataConnector.Get(id, out hash) == null)
396 return null;
397
398 return GetFsData(hash);
399 }
400
401 public bool Get(string id, Object sender, AssetRetrieved handler)
402 {
403 AssetBase asset = Get(id);
404
405 handler(id, sender, asset);
406
407 return true;
408 }
409
410 public byte[] GetFsData(string hash)
411 {
412 string spoolFile = Path.Combine(m_SpoolDirectory, hash + ".asset");
413
414 if (File.Exists(spoolFile))
415 {
416 try
417 {
418 byte[] content = File.ReadAllBytes(spoolFile);
419
420 return content;
421 }
422 catch
423 {
424 }
425 }
426
427 string file = HashToFile(hash);
428 string diskFile = Path.Combine(m_FSBase, file);
429
430 if (File.Exists(diskFile + ".gz"))
431 {
432 try
433 {
434 using (GZipStream gz = new GZipStream(new FileStream(diskFile + ".gz", FileMode.Open, FileAccess.Read), CompressionMode.Decompress))
435 {
436 using (MemoryStream ms = new MemoryStream())
437 {
438 byte[] data = new byte[32768];
439 int bytesRead;
440
441 do
442 {
443 bytesRead = gz.Read(data, 0, 32768);
444 if (bytesRead > 0)
445 ms.Write(data, 0, bytesRead);
446 } while (bytesRead > 0);
447
448 return ms.ToArray();
449 }
450 }
451 }
452 catch (Exception)
453 {
454 return new Byte[0];
455 }
456 }
457 else if (File.Exists(diskFile))
458 {
459 try
460 {
461 byte[] content = File.ReadAllBytes(diskFile);
462
463 return content;
464 }
465 catch
466 {
467 }
468 }
469 return new Byte[0];
470
471 }
472
473 public string Store(AssetBase asset)
474 {
475 return Store(asset, false);
476 }
477
478 private string Store(AssetBase asset, bool force)
479 {
480 int tickCount = Environment.TickCount;
481 string hash = GetSHA256Hash(asset.Data);
482
483 if (!AssetExists(hash))
484 {
485 string tempFile = Path.Combine(Path.Combine(m_SpoolDirectory, "spool"), hash + ".asset");
486 string finalFile = Path.Combine(m_SpoolDirectory, hash + ".asset");
487
488 if (!File.Exists(finalFile))
489 {
490 FileStream fs = File.Create(tempFile);
491
492 fs.Write(asset.Data, 0, asset.Data.Length);
493
494 fs.Close();
495
496 File.Move(tempFile, finalFile);
497 }
498 }
499
500 if (asset.ID == string.Empty)
501 {
502 if (asset.FullID == UUID.Zero)
503 {
504 asset.FullID = UUID.Random();
505 }
506 asset.ID = asset.FullID.ToString();
507 }
508 else if (asset.FullID == UUID.Zero)
509 {
510 UUID uuid = UUID.Zero;
511 if (UUID.TryParse(asset.ID, out uuid))
512 {
513 asset.FullID = uuid;
514 }
515 else
516 {
517 asset.FullID = UUID.Random();
518 }
519 }
520
521 if (!m_DataConnector.Store(asset.Metadata, hash))
522 {
523 return UUID.Zero.ToString();
524 }
525 else
526 {
527 return asset.ID;
528 }
529 }
530
531 public bool UpdateContent(string id, byte[] data)
532 {
533 return false;
534
535// string oldhash;
536// AssetMetadata meta = m_DataConnector.Get(id, out oldhash);
537//
538// if (meta == null)
539// return false;
540//
541// AssetBase asset = new AssetBase();
542// asset.Metadata = meta;
543// asset.Data = data;
544//
545// Store(asset);
546//
547// return true;
548 }
549
550 public bool Delete(string id)
551 {
552 m_DataConnector.Delete(id);
553
554 return true;
555 }
556
557 private void HandleShowAssets(string module, string[] args)
558 {
559 int num = m_DataConnector.Count();
560 MainConsole.Instance.Output(string.Format("Total asset count: {0}", num));
561 }
562
563 private void HandleShowDigest(string module, string[] args)
564 {
565 if (args.Length < 3)
566 {
567 MainConsole.Instance.Output("Syntax: show digest <ID>");
568 return;
569 }
570
571 string hash;
572 AssetBase asset = Get(args[2], out hash);
573
574 if (asset == null || asset.Data.Length == 0)
575 {
576 MainConsole.Instance.Output("Asset not found");
577 return;
578 }
579
580 int i;
581
582 MainConsole.Instance.Output(String.Format("Name: {0}", asset.Name));
583 MainConsole.Instance.Output(String.Format("Description: {0}", asset.Description));
584 MainConsole.Instance.Output(String.Format("Type: {0}", asset.Type));
585 MainConsole.Instance.Output(String.Format("Content-type: {0}", asset.Metadata.ContentType));
586 MainConsole.Instance.Output(String.Format("Flags: {0}", asset.Metadata.Flags.ToString()));
587 MainConsole.Instance.Output(String.Format("FS file: {0}", HashToFile(hash)));
588
589 for (i = 0 ; i < 5 ; i++)
590 {
591 int off = i * 16;
592 if (asset.Data.Length <= off)
593 break;
594 int len = 16;
595 if (asset.Data.Length < off + len)
596 len = asset.Data.Length - off;
597
598 byte[] line = new byte[len];
599 Array.Copy(asset.Data, off, line, 0, len);
600
601 string text = BitConverter.ToString(line);
602 MainConsole.Instance.Output(String.Format("{0:x4}: {1}", off, text));
603 }
604 }
605
606 private void HandleDeleteAsset(string module, string[] args)
607 {
608 if (args.Length < 3)
609 {
610 MainConsole.Instance.Output("Syntax: delete asset <ID>");
611 return;
612 }
613
614 AssetBase asset = Get(args[2]);
615
616 if (asset == null || asset.Data.Length == 0)
617 {
618 MainConsole.Instance.Output("Asset not found");
619 return;
620 }
621
622 m_DataConnector.Delete(args[2]);
623
624 MainConsole.Instance.Output("Asset deleted");
625 }
626
627 private void HandleImportAssets(string module, string[] args)
628 {
629 bool force = false;
630 if (args[0] == "force")
631 {
632 force = true;
633 List<string> list = new List<string>(args);
634 list.RemoveAt(0);
635 args = list.ToArray();
636 }
637 if (args.Length < 3)
638 {
639 MainConsole.Instance.Output("Syntax: import <conn> <table> [<start> <count>]");
640 }
641 else
642 {
643 string conn = args[1];
644 string table = args[2];
645 int start = 0;
646 int count = -1;
647 if (args.Length > 3)
648 {
649 start = Convert.ToInt32(args[3]);
650 }
651 if (args.Length > 4)
652 {
653 count = Convert.ToInt32(args[4]);
654 }
655 m_DataConnector.Import(conn, table, start, count, force, new StoreDelegate(Store));
656 }
657 }
658
659 public AssetBase GetCached(string id)
660 {
661 return Get(id);
662 }
663 }
664}
generated by cgit v1.1 at 2024-12-14 08:26:02 +0000