aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/OpenSim/Framework/Communications/Cache/AssetCache.cs
diff options
context:
space:
mode:
Diffstat (limited to 'OpenSim/Framework/Communications/Cache/AssetCache.cs')
-rw-r--r--OpenSim/Framework/Communications/Cache/AssetCache.cs695
1 files changed, 0 insertions, 695 deletions
diff --git a/OpenSim/Framework/Communications/Cache/AssetCache.cs b/OpenSim/Framework/Communications/Cache/AssetCache.cs
deleted file mode 100644
index 7d25ca9..0000000
--- a/OpenSim/Framework/Communications/Cache/AssetCache.cs
+++ /dev/null
@@ -1,695 +0,0 @@
1/*
2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the OpenSim Project nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28using System;
29using System.Collections.Generic;
30using System.Reflection;
31using System.Text.RegularExpressions;
32using System.Threading;
33using GlynnTucker.Cache;
34using log4net;
35using OpenMetaverse;
36using OpenMetaverse.Packets;
37using OpenSim.Framework.Statistics;
38using System.Text;
39
40namespace OpenSim.Framework.Communications.Cache
41{
42 /// <summary>
43 /// Manages local cache of assets and their sending to viewers.
44 /// </summary>
45 ///
46 /// This class actually encapsulates two largely separate mechanisms. One mechanism fetches assets either
47 /// synchronously or async and passes the data back to the requester. The second mechanism fetches assets and
48 /// sends packetised data directly back to the client. The only point where they meet is AssetReceived() and
49 /// AssetNotFound(), which means they do share the same asset and texture caches.
50 public class AssetCache : IAssetCache
51 {
52 private static readonly ILog m_log
53 = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
54
55 protected ICache m_memcache = new SimpleMemoryCache();
56
57 /// <value>
58 /// Assets requests which are waiting for asset server data. This includes texture requests
59 /// </value>
60 private Dictionary<UUID, AssetRequest> RequestedAssets;
61
62 /// <value>
63 /// Asset requests with data which are ready to be sent back to requesters. This includes textures.
64 /// </value>
65 private List<AssetRequest> AssetRequests;
66
67 /// <value>
68 /// Until the asset request is fulfilled, each asset request is associated with a list of requesters
69 /// </value>
70 private Dictionary<UUID, AssetRequestsList> RequestLists;
71
72 #region IPlugin
73
74 /// <summary>
75 /// The methods and properties in this section are needed to
76 /// support the IPlugin interface. They cann all be overridden
77 /// as needed by a derived class.
78 /// </summary>
79
80 public virtual string Name
81 {
82 get { return "OpenSim.Framework.Communications.Cache.AssetCache"; }
83 }
84
85 public virtual string Version
86 {
87 get { return "1.0"; }
88 }
89
90 public virtual void Initialise()
91 {
92 m_log.Debug("[ASSET CACHE]: Asset cache null initialisation");
93 }
94
95 public virtual void Initialise(IAssetServer assetServer)
96 {
97 m_log.Debug("[ASSET CACHE]: Asset cache server-specified initialisation");
98 m_log.InfoFormat("[ASSET CACHE]: Asset cache initialisation [{0}/{1}]", Name, Version);
99
100 Reset();
101
102 m_assetServer = assetServer;
103 m_assetServer.SetReceiver(this);
104
105 Thread assetCacheThread = new Thread(RunAssetManager);
106 assetCacheThread.Name = "AssetCacheThread";
107 assetCacheThread.IsBackground = true;
108 assetCacheThread.Start();
109 ThreadTracker.Add(assetCacheThread);
110 }
111
112 public virtual void Initialise(ConfigSettings settings, IAssetServer assetServer)
113 {
114 m_log.Debug("[ASSET CACHE]: Asset cache configured initialisation");
115 Initialise(assetServer);
116 }
117
118 public void Dispose()
119 {
120 }
121
122 #endregion
123
124 public IAssetServer AssetServer
125 {
126 get { return m_assetServer; }
127 }
128 private IAssetServer m_assetServer;
129
130 public AssetCache()
131 {
132 m_log.Debug("[ASSET CACHE]: Asset cache (plugin constructor)");
133 }
134
135 /// <summary>
136 /// Constructor.
137 /// </summary>
138 /// <param name="assetServer"></param>
139 public AssetCache(IAssetServer assetServer)
140 {
141 Initialise(assetServer);
142 }
143
144 public void ShowState()
145 {
146 m_log.InfoFormat("Memcache:{0} RequestLists:{1}",
147 m_memcache.Count,
148 // AssetRequests.Count,
149 // RequestedAssets.Count,
150 RequestLists.Count);
151 }
152
153 public void Clear()
154 {
155 m_log.Info("[ASSET CACHE]: Clearing Asset cache");
156
157 if (StatsManager.SimExtraStats != null)
158 StatsManager.SimExtraStats.ClearAssetCacheStatistics();
159
160 Reset();
161 }
162
163 /// <summary>
164 /// Reset the cache.
165 /// </summary>
166 private void Reset()
167 {
168 AssetRequests = new List<AssetRequest>();
169 RequestedAssets = new Dictionary<UUID, AssetRequest>();
170 RequestLists = new Dictionary<UUID, AssetRequestsList>();
171 }
172
173 /// <summary>
174 /// Process the asset queue which holds data which is packeted up and sent
175 /// directly back to the client.
176 /// </summary>
177 private void RunAssetManager()
178 {
179 while (true)
180 {
181 try
182 {
183 ProcessAssetQueue();
184 Thread.Sleep(500);
185 }
186 catch (Exception e)
187 {
188 if (e != null)
189 {
190 m_log.ErrorFormat("[ASSET CACHE]: {0}", e);
191 }
192 else
193 {
194 // this looks weird, but we've seen this show up as an issue in unit tests, so leave it here until we know why
195 m_log.Error("[ASSET CACHE]: an exception was thrown in RunAssetManager, but is now null. Something is very wrong.");
196 }
197 }
198 }
199 }
200
201 public bool TryGetCachedAsset(UUID assetId, out AssetBase asset)
202 {
203 Object tmp;
204 if (m_memcache.TryGet(assetId, out tmp))
205 {
206 asset = (AssetBase)tmp;
207 //m_log.Info("Retrieved from cache " + assetId);
208 return true;
209 }
210
211 asset = null;
212 return false;
213 }
214
215 public void GetAsset(UUID assetId, AssetRequestCallback callback, bool isTexture)
216 {
217 //m_log.DebugFormat("[ASSET CACHE]: Requesting {0} {1}", isTexture ? "texture" : "asset", assetId);
218
219 // Xantor 20080526:
220 // if a request is made for an asset which is not in the cache yet, but has already been requested by
221 // something else, queue up the callbacks on that requestor instead of swamping the assetserver
222 // with multiple requests for the same asset.
223
224 AssetBase asset;
225
226 if (TryGetCachedAsset(assetId, out asset))
227 {
228 callback(assetId, asset);
229 }
230 else
231 {
232 // m_log.DebugFormat("[ASSET CACHE]: Adding request for {0} {1}", isTexture ? "texture" : "asset", assetId);
233
234 NewAssetRequest req = new NewAssetRequest(callback);
235 AssetRequestsList requestList;
236
237 lock (RequestLists)
238 {
239 if (RequestLists.TryGetValue(assetId, out requestList)) // do we already have a request pending?
240 {
241 // m_log.DebugFormat("[ASSET CACHE]: Intercepted Duplicate request for {0} {1}", isTexture ? "texture" : "asset", assetId);
242 // add to callbacks for this assetId
243 RequestLists[assetId].Requests.Add(req);
244 }
245 else
246 {
247 // m_log.DebugFormat("[ASSET CACHE]: Adding request for {0} {1}", isTexture ? "texture" : "asset", assetId);
248 requestList = new AssetRequestsList();
249 requestList.TimeRequested = DateTime.Now;
250 requestList.Requests.Add(req);
251
252 RequestLists.Add(assetId, requestList);
253
254 m_assetServer.RequestAsset(assetId, isTexture);
255 }
256 }
257 }
258 }
259
260 public AssetBase GetAsset(UUID assetID, bool isTexture)
261 {
262 // I'm not going over 3 seconds since this will be blocking processing of all the other inbound
263 // packets from the client.
264 const int pollPeriod = 200;
265 int maxPolls = 15;
266
267 AssetBase asset;
268
269 if (TryGetCachedAsset(assetID, out asset))
270 {
271 return asset;
272 }
273
274 m_assetServer.RequestAsset(assetID, isTexture);
275
276 do
277 {
278 Thread.Sleep(pollPeriod);
279
280 if (TryGetCachedAsset(assetID, out asset))
281 {
282 return asset;
283 }
284 }
285 while (--maxPolls > 0);
286
287 m_log.WarnFormat("[ASSET CACHE]: {0} {1} was not received before the retrieval timeout was reached",
288 isTexture ? "texture" : "asset", assetID.ToString());
289
290 return null;
291 }
292
293 public void AddAsset(AssetBase asset)
294 {
295 if (!m_memcache.Contains(asset.FullID))
296 {
297 m_log.Info("[CACHE] Caching " + asset.FullID + " for 24 hours from last access");
298 // Use 24 hour rolling asset cache.
299 m_memcache.AddOrUpdate(asset.FullID, asset, TimeSpan.FromHours(24));
300
301 // According to http://wiki.secondlife.com/wiki/AssetUploadRequest, Local signifies that the
302 // information is stored locally. It could disappear, in which case we could send the
303 // ImageNotInDatabase packet to tell the client this.
304 //
305 // However, this doesn't quite appear to work with local textures that are part of an avatar's
306 // appearance texture set. Whilst sending an ImageNotInDatabase does trigger an automatic rebake
307 // and reupload by the client, if those assets aren't pushed to the asset server anyway, then
308 // on crossing onto another region server, other avatars can no longer get the required textures.
309 // There doesn't appear to be any signal from the sim to the newly region border crossed client
310 // asking it to reupload its local texture assets to that region server.
311 //
312 // One can think of other cunning ways around this. For instance, on a region crossing or teleport,
313 // the original sim could squirt local assets to the new sim. Or the new sim could have pointers
314 // to the original sim to fetch the 'local' assets (this is getting more complicated).
315 //
316 // But for now, we're going to take the easy way out and store local assets globally.
317 //
318 // TODO: Also, Temporary is now deprecated. We should start ignoring it and not passing it out from LLClientView.
319 if (!asset.Temporary || asset.Local)
320 {
321 m_assetServer.StoreAsset(asset);
322 }
323
324 if (StatsManager.SimExtraStats != null)
325 StatsManager.SimExtraStats.AddAsset(asset);
326 }
327 }
328
329 public void ExpireAsset(UUID uuid)
330 {
331 if (m_memcache.Contains(uuid))
332 {
333 m_memcache.Remove(uuid);
334
335 if (StatsManager.SimExtraStats != null)
336 StatsManager.SimExtraStats.RemoveAsset(uuid);
337 }
338 }
339
340 // See IAssetReceiver
341 public virtual void AssetReceived(AssetBase asset, bool IsTexture)
342 {
343 AssetInfo assetInf = new AssetInfo(asset);
344
345 ProcessReceivedAsset(IsTexture, assetInf, null);
346
347 if (!m_memcache.Contains(assetInf.FullID))
348 {
349 m_memcache.AddOrUpdate(assetInf.FullID, assetInf, TimeSpan.FromHours(24));
350
351 if (StatsManager.SimExtraStats != null)
352 StatsManager.SimExtraStats.AddAsset(assetInf);
353
354 if (RequestedAssets.ContainsKey(assetInf.FullID))
355 {
356 AssetRequest req = RequestedAssets[assetInf.FullID];
357 req.AssetInf = assetInf;
358 req.NumPackets = CalculateNumPackets(assetInf.Data);
359
360 RequestedAssets.Remove(assetInf.FullID);
361
362 if (req.AssetRequestSource == 2 && assetInf.Type == 10)
363 {
364 // If it's a direct request for a script, drop it
365 // because it's a hacked client
366 }
367 else
368 {
369 lock (AssetRequests)
370 {
371 AssetRequests.Add(req);
372 }
373 }
374 }
375 }
376
377 // Notify requesters for this asset
378 AssetRequestsList reqList;
379
380 lock (RequestLists)
381 {
382 if (RequestLists.TryGetValue(asset.FullID, out reqList))
383 RequestLists.Remove(asset.FullID);
384 }
385
386 if (reqList != null)
387 {
388 if (StatsManager.SimExtraStats != null)
389 StatsManager.SimExtraStats.AddAssetRequestTimeAfterCacheMiss(DateTime.Now - reqList.TimeRequested);
390
391 foreach (NewAssetRequest req in reqList.Requests)
392 {
393 // Xantor 20080526 are we really calling all the callbacks if multiple queued for 1 request? -- Yes, checked
394 // m_log.DebugFormat("[ASSET CACHE]: Callback for asset {0}", asset.FullID);
395 req.Callback(asset.FullID, asset);
396 }
397 }
398 }
399
400 protected void ProcessReceivedAsset(bool IsTexture, AssetInfo assetInf, IUserService userService)
401 {
402 // if (!IsTexture && assetInf.ContainsReferences && false)
403 // {
404 // assetInf.Data = ProcessAssetData(assetInf.Data, userService);
405 // }
406 }
407
408 // See IAssetReceiver
409 public virtual void AssetNotFound(UUID assetId, bool isTexture)
410 {
411// m_log.WarnFormat("[ASSET CACHE]: AssetNotFound for {0}", assetId);
412
413 // Remember the fact that this asset could not be found to prevent delays from repeated requests
414 m_memcache.Add(assetId, null, TimeSpan.FromHours(24));
415
416 // Notify requesters for this asset
417 AssetRequestsList reqList;
418 lock (RequestLists)
419 {
420 if (RequestLists.TryGetValue(assetId, out reqList))
421 RequestLists.Remove(assetId);
422 }
423
424 if (reqList != null)
425 {
426 if (StatsManager.SimExtraStats != null)
427 StatsManager.SimExtraStats.AddAssetRequestTimeAfterCacheMiss(DateTime.Now - reqList.TimeRequested);
428
429 foreach (NewAssetRequest req in reqList.Requests)
430 {
431 req.Callback(assetId, null);
432 }
433 }
434 }
435
436 /// <summary>
437 /// Calculate the number of packets required to send the asset to the client.
438 /// </summary>
439 /// <param name="data"></param>
440 /// <returns></returns>
441 private static int CalculateNumPackets(byte[] data)
442 {
443 const uint m_maxPacketSize = 600;
444 int numPackets = 1;
445
446 if (data.LongLength > m_maxPacketSize)
447 {
448 // over max number of bytes so split up file
449 long restData = data.LongLength - m_maxPacketSize;
450 int restPackets = (int)((restData + m_maxPacketSize - 1) / m_maxPacketSize);
451 numPackets += restPackets;
452 }
453
454 return numPackets;
455 }
456
457 public void AddAssetRequest(IClientAPI userInfo, TransferRequestPacket transferRequest)
458 {
459 UUID requestID = UUID.Zero;
460 byte source = 2;
461 if (transferRequest.TransferInfo.SourceType == 2)
462 {
463 //direct asset request
464 requestID = new UUID(transferRequest.TransferInfo.Params, 0);
465 }
466 else if (transferRequest.TransferInfo.SourceType == 3)
467 {
468 //inventory asset request
469 requestID = new UUID(transferRequest.TransferInfo.Params, 80);
470 source = 3;
471 //m_log.Debug("asset request " + requestID);
472 }
473
474 //check to see if asset is in local cache, if not we need to request it from asset server.
475 //m_log.Debug("asset request " + requestID);
476 if (!m_memcache.Contains(requestID))
477 {
478 //not found asset
479 // so request from asset server
480 if (!RequestedAssets.ContainsKey(requestID))
481 {
482 AssetRequest request = new AssetRequest();
483 request.RequestUser = userInfo;
484 request.RequestAssetID = requestID;
485 request.TransferRequestID = transferRequest.TransferInfo.TransferID;
486 request.AssetRequestSource = source;
487 request.Params = transferRequest.TransferInfo.Params;
488 RequestedAssets.Add(requestID, request);
489 m_assetServer.RequestAsset(requestID, false);
490 }
491
492 return;
493 }
494
495 // It has an entry in our cache
496 AssetBase asset = (AssetBase)m_memcache[requestID];
497
498 // FIXME: We never tell the client about assets which do not exist when requested by this transfer mechanism, which can't be right.
499 if (null == asset)
500 {
501 //m_log.DebugFormat("[ASSET CACHE]: Asset transfer request for asset which is {0} already known to be missing. Dropping", requestID);
502 return;
503 }
504
505 // Scripts cannot be retrieved by direct request
506 if (transferRequest.TransferInfo.SourceType == 2 && asset.Type == 10)
507 return;
508
509 // The asset is knosn to exist and is in our cache, so add it to the AssetRequests list
510 AssetRequest req = new AssetRequest();
511 req.RequestUser = userInfo;
512 req.RequestAssetID = requestID;
513 req.TransferRequestID = transferRequest.TransferInfo.TransferID;
514 req.AssetRequestSource = source;
515 req.Params = transferRequest.TransferInfo.Params;
516 req.AssetInf = new AssetInfo(asset);
517 req.NumPackets = CalculateNumPackets(asset.Data);
518 lock (AssetRequests)
519 {
520 AssetRequests.Add(req);
521 }
522 }
523
524 /// <summary>
525 /// Process the asset queue which sends packets directly back to the client.
526 /// </summary>
527 private void ProcessAssetQueue()
528 {
529 //should move the asset downloading to a module, like has been done with texture downloading
530 if (AssetRequests.Count == 0)
531 {
532 //no requests waiting
533 return;
534 }
535
536 // if less than 5, do all of them
537 int num = Math.Min(5, AssetRequests.Count);
538
539 AssetRequest req;
540 AssetRequestToClient req2 = new AssetRequestToClient();
541
542 lock (AssetRequests)
543 {
544 for (int i = 0; i < num; i++)
545 {
546 req = AssetRequests[0];
547 AssetRequests.RemoveAt(0);
548 req2.AssetInf = req.AssetInf;
549 req2.AssetRequestSource = req.AssetRequestSource;
550 req2.DataPointer = req.DataPointer;
551 req2.DiscardLevel = req.DiscardLevel;
552 req2.ImageInfo = req.ImageInfo;
553 req2.IsTextureRequest = req.IsTextureRequest;
554 req2.NumPackets = req.NumPackets;
555 req2.PacketCounter = req.PacketCounter;
556 req2.Params = req.Params;
557 req2.RequestAssetID = req.RequestAssetID;
558 req2.TransferRequestID = req.TransferRequestID;
559 req.RequestUser.SendAsset(req2);
560 }
561 }
562 }
563
564 public byte[] ProcessAssetData(byte[] assetData, IUserService userService)
565 {
566 string data = Encoding.ASCII.GetString(assetData);
567
568 data = ProcessAssetDataString(data, userService);
569
570 return Encoding.ASCII.GetBytes(data);
571 }
572
573 public string ProcessAssetDataString(string data, IUserService userService)
574 {
575 Regex regex = new Regex("(creator_url|owner_url)\\s+(\\S+)");
576
577 data = regex.Replace(data, delegate(Match m)
578 {
579 string result = String.Empty;
580
581 string key = m.Groups[1].Captures[0].Value;
582
583 string value = m.Groups[2].Captures[0].Value;
584
585 Uri userUri;
586
587 switch (key)
588 {
589 case "creator_url":
590 userUri = new Uri(value);
591 result = "creator_id " + ResolveUserUri(userService, userUri);
592 break;
593
594 case "owner_url":
595 userUri = new Uri(value);
596 result = "owner_id " + ResolveUserUri(userService, userUri);
597 break;
598 }
599
600 return result;
601 });
602
603 return data;
604 }
605
606 private Guid ResolveUserUri(IUserService userService, Uri userUri)
607 {
608 Guid id;
609 UserProfileData userProfile = userService.GetUserProfile(userUri);
610 if (userProfile == null)
611 {
612 id = Guid.Empty;
613 }
614 else
615 {
616 id = userProfile.ID.Guid;
617 }
618 return id;
619 }
620
621
622 public class AssetRequest
623 {
624 public IClientAPI RequestUser;
625 public UUID RequestAssetID;
626 public AssetInfo AssetInf;
627 public TextureImage ImageInfo;
628 public UUID TransferRequestID;
629 public long DataPointer = 0;
630 public int NumPackets = 0;
631 public int PacketCounter = 0;
632 public bool IsTextureRequest;
633 public byte AssetRequestSource = 2;
634 public byte[] Params = null;
635 //public bool AssetInCache;
636 //public int TimeRequested;
637 public int DiscardLevel = -1;
638 }
639
640 public class AssetInfo : AssetBase
641 {
642 public AssetInfo(AssetBase aBase)
643 {
644 Data = aBase.Data;
645 FullID = aBase.FullID;
646 Type = aBase.Type;
647 Name = aBase.Name;
648 Description = aBase.Description;
649 }
650
651 public const string Secret = "secret";
652 }
653
654 public class TextureImage : AssetBase
655 {
656 public TextureImage(AssetBase aBase)
657 {
658 Data = aBase.Data;
659 FullID = aBase.FullID;
660 Type = aBase.Type;
661 Name = aBase.Name;
662 Description = aBase.Description;
663 }
664 }
665
666 /// <summary>
667 /// A list of requests for a particular asset.
668 /// </summary>
669 public class AssetRequestsList
670 {
671 /// <summary>
672 /// A list of requests for assets
673 /// </summary>
674 public List<NewAssetRequest> Requests = new List<NewAssetRequest>();
675
676 /// <summary>
677 /// Record the time that this request was first made.
678 /// </summary>
679 public DateTime TimeRequested;
680 }
681
682 /// <summary>
683 /// Represent a request for an asset that has yet to be fulfilled.
684 /// </summary>
685 public class NewAssetRequest
686 {
687 public AssetRequestCallback Callback;
688
689 public NewAssetRequest(AssetRequestCallback callback)
690 {
691 Callback = callback;
692 }
693 }
694 }
695}
generated by cgit v1.1 at 2024-07-03 05:16:54 +0000