aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/OpenSim/Capabilities
diff options
context:
space:
mode:
authorDiva Canto2015-05-07 19:24:08 -0700
committerDiva Canto2015-05-07 19:24:08 -0700
commitc74cef0f4261191962959e42c7e349adafd42a04 (patch)
tree95ad098d2606b7d37a6b287816d9f6a02311860d /OpenSim/Capabilities
parentMerge branch 'master' of ssh://opensimulator.org/var/git/opensim (diff)
downloadopensim-SC_OLD-c74cef0f4261191962959e42c7e349adafd42a04.zip
opensim-SC_OLD-c74cef0f4261191962959e42c7e349adafd42a04.tar.gz
opensim-SC_OLD-c74cef0f4261191962959e42c7e349adafd42a04.tar.bz2
opensim-SC_OLD-c74cef0f4261191962959e42c7e349adafd42a04.tar.xz
Major change in the way inventory is downloaded: added a method throughout IIventoryService that fetches sets of folders at once. Also added folder id in the InventoryCollection data structure, so that we don't need to go to inventory server again just for that. This reduces the chatter between sims and inventory server by... a lot. On my tests, this reduces initial inventory download down to 30% of what it currently is.
Diffstat (limited to 'OpenSim/Capabilities')
-rw-r--r--OpenSim/Capabilities/Handlers/FetchInventoryDescendents/FetchInvDescHandler.cs449
1 files changed, 365 insertions, 84 deletions
diff --git a/OpenSim/Capabilities/Handlers/FetchInventoryDescendents/FetchInvDescHandler.cs b/OpenSim/Capabilities/Handlers/FetchInventoryDescendents/FetchInvDescHandler.cs
index 451575f..a2f6740 100644
--- a/OpenSim/Capabilities/Handlers/FetchInventoryDescendents/FetchInvDescHandler.cs
+++ b/OpenSim/Capabilities/Handlers/FetchInventoryDescendents/FetchInvDescHandler.cs
@@ -57,104 +57,113 @@ namespace OpenSim.Capabilities.Handlers
57 m_LibraryService = libService; 57 m_LibraryService = libService;
58 } 58 }
59 59
60
60 public string FetchInventoryDescendentsRequest(string request, string path, string param, IOSHttpRequest httpRequest, IOSHttpResponse httpResponse) 61 public string FetchInventoryDescendentsRequest(string request, string path, string param, IOSHttpRequest httpRequest, IOSHttpResponse httpResponse)
61 { 62 {
62// lock (m_fetchLock)
63// {
64// m_log.DebugFormat("[WEB FETCH INV DESC HANDLER]: Received request {0}", request);
65 63
66 // nasty temporary hack here, the linden client falsely 64 // nasty temporary hack here, the linden client falsely
67 // identifies the uuid 00000000-0000-0000-0000-000000000000 65 // identifies the uuid 00000000-0000-0000-0000-000000000000
68 // as a string which breaks us 66 // as a string which breaks us
69 // 67 //
70 // correctly mark it as a uuid 68 // correctly mark it as a uuid
71 // 69 //
72 request = request.Replace("<string>00000000-0000-0000-0000-000000000000</string>", "<uuid>00000000-0000-0000-0000-000000000000</uuid>"); 70 request = request.Replace("<string>00000000-0000-0000-0000-000000000000</string>", "<uuid>00000000-0000-0000-0000-000000000000</uuid>");
73 71
74 // another hack <integer>1</integer> results in a 72 // another hack <integer>1</integer> results in a
75 // System.ArgumentException: Object type System.Int32 cannot 73 // System.ArgumentException: Object type System.Int32 cannot
76 // be converted to target type: System.Boolean 74 // be converted to target type: System.Boolean
77 // 75 //
78 request = request.Replace("<key>fetch_folders</key><integer>0</integer>", "<key>fetch_folders</key><boolean>0</boolean>"); 76 request = request.Replace("<key>fetch_folders</key><integer>0</integer>", "<key>fetch_folders</key><boolean>0</boolean>");
79 request = request.Replace("<key>fetch_folders</key><integer>1</integer>", "<key>fetch_folders</key><boolean>1</boolean>"); 77 request = request.Replace("<key>fetch_folders</key><integer>1</integer>", "<key>fetch_folders</key><boolean>1</boolean>");
80 78
81 Hashtable hash = new Hashtable(); 79 Hashtable hash = new Hashtable();
80 try
81 {
82 hash = (Hashtable)LLSD.LLSDDeserialize(Utils.StringToBytes(request));
83 }
84 catch (LLSD.LLSDParseException e)
85 {
86 m_log.ErrorFormat("[WEB FETCH INV DESC HANDLER]: Fetch error: {0}{1}" + e.Message, e.StackTrace);
87 m_log.Error("Request: " + request);
88 }
89
90 ArrayList foldersrequested = (ArrayList)hash["folders"];
91
92 string response = "";
93 string bad_folders_response = "";
94
95 List<LLSDFetchInventoryDescendents> folders = new List<LLSDFetchInventoryDescendents>();
96 for (int i = 0; i < foldersrequested.Count; i++)
97 {
98 Hashtable inventoryhash = (Hashtable)foldersrequested[i];
99
100 LLSDFetchInventoryDescendents llsdRequest = new LLSDFetchInventoryDescendents();
101
82 try 102 try
83 { 103 {
84 hash = (Hashtable)LLSD.LLSDDeserialize(Utils.StringToBytes(request)); 104 LLSDHelpers.DeserialiseOSDMap(inventoryhash, llsdRequest);
85 } 105 }
86 catch (LLSD.LLSDParseException e) 106 catch (Exception e)
87 { 107 {
88 m_log.ErrorFormat("[WEB FETCH INV DESC HANDLER]: Fetch error: {0}{1}" + e.Message, e.StackTrace); 108 m_log.Debug("[WEB FETCH INV DESC HANDLER]: caught exception doing OSD deserialize" + e);
89 m_log.Error("Request: " + request); 109 continue;
90 } 110 }
91
92 ArrayList foldersrequested = (ArrayList)hash["folders"];
93
94 string response = "";
95 string bad_folders_response = "";
96 111
97 for (int i = 0; i < foldersrequested.Count; i++) 112 folders.Add(llsdRequest);
98 { 113 }
99 string inventoryitemstr = "";
100 Hashtable inventoryhash = (Hashtable)foldersrequested[i];
101 114
102 LLSDFetchInventoryDescendents llsdRequest = new LLSDFetchInventoryDescendents(); 115 if (folders.Count > 0)
116 {
117 List<InventoryCollectionWithDescendents> invcollSet = Fetch(folders);
118 //m_log.DebugFormat("[XXX]: Got {0} folders from a request of {1}", invcollSet.Count, folders.Count);
119 if (invcollSet == null)
120 {
121 m_log.DebugFormat("[WEB FETCH INV DESC HANDLER]: Multiple folder fetch failed. Trying old protocol.");
122 return FetchInventoryDescendentsRequest(foldersrequested, httpRequest, httpResponse);
123 }
103 124
104 try 125 string inventoryitemstr = string.Empty;
105 { 126 foreach (InventoryCollectionWithDescendents icoll in invcollSet)
106 LLSDHelpers.DeserialiseOSDMap(inventoryhash, llsdRequest); 127 {
107 } 128 LLSDInventoryDescendents reply = ToLLSD(icoll.Collection, icoll.Descendents);
108 catch (Exception e)
109 {
110 m_log.Debug("[WEB FETCH INV DESC HANDLER]: caught exception doing OSD deserialize" + e);
111 }
112 LLSDInventoryDescendents reply = FetchInventoryReply(llsdRequest);
113 129
114 if (null == reply) 130 inventoryitemstr = LLSDHelpers.SerialiseLLSDReply(reply);
115 { 131 inventoryitemstr = inventoryitemstr.Replace("<llsd><map><key>folders</key><array>", "");
116 bad_folders_response += "<uuid>" + llsdRequest.folder_id.ToString() + "</uuid>"; 132 inventoryitemstr = inventoryitemstr.Replace("</array></map></llsd>", "");
117 }
118 else
119 {
120 inventoryitemstr = LLSDHelpers.SerialiseLLSDReply(reply);
121 inventoryitemstr = inventoryitemstr.Replace("<llsd><map><key>folders</key><array>", "");
122 inventoryitemstr = inventoryitemstr.Replace("</array></map></llsd>", "");
123 }
124 133
125 response += inventoryitemstr; 134 response += inventoryitemstr;
126 } 135 }
136 }
127 137
128 if (response.Length == 0) 138 if (response.Length == 0)
139 {
140 /* Viewers expect a bad_folders array when not available */
141 if (bad_folders_response.Length != 0)
129 { 142 {
130 /* Viewers expect a bad_folders array when not available */ 143 response = "<llsd><map><key>bad_folders</key><array>" + bad_folders_response + "</array></map></llsd>";
131 if (bad_folders_response.Length != 0)
132 {
133 response = "<llsd><map><key>bad_folders</key><array>" + bad_folders_response + "</array></map></llsd>";
134 }
135 else
136 {
137 response = "<llsd><map><key>folders</key><array /></map></llsd>";
138 }
139 } 144 }
140 else 145 else
141 { 146 {
142 if (bad_folders_response.Length != 0) 147 response = "<llsd><map><key>folders</key><array /></map></llsd>";
143 {
144 response = "<llsd><map><key>folders</key><array>" + response + "</array><key>bad_folders</key><array>" + bad_folders_response + "</array></map></llsd>";
145 }
146 else
147 {
148 response = "<llsd><map><key>folders</key><array>" + response + "</array></map></llsd>";
149 }
150 } 148 }
149 }
150 else
151 {
152 if (bad_folders_response.Length != 0)
153 {
154 response = "<llsd><map><key>folders</key><array>" + response + "</array><key>bad_folders</key><array>" + bad_folders_response + "</array></map></llsd>";
155 }
156 else
157 {
158 response = "<llsd><map><key>folders</key><array>" + response + "</array></map></llsd>";
159 }
160 }
151 161
152// m_log.DebugFormat("[WEB FETCH INV DESC HANDLER]: Replying to CAPS fetch inventory request"); 162// m_log.DebugFormat("[WEB FETCH INV DESC HANDLER]: Replying to CAPS fetch inventory request");
153 //m_log.Debug("[WEB FETCH INV DESC HANDLER] "+response); 163// m_log.Debug("[WEB FETCH INV DESC HANDLER] "+response);
154 164
155 return response; 165 return response;
156 166
157// }
158 } 167 }
159 168
160 /// <summary> 169 /// <summary>
@@ -203,18 +212,130 @@ namespace OpenSim.Capabilities.Handlers
203 contents.descendents = descendents; 212 contents.descendents = descendents;
204 contents.version = version; 213 contents.version = version;
205 214
206// m_log.DebugFormat( 215 //m_log.DebugFormat(
207// "[WEB FETCH INV DESC HANDLER]: Replying to request for folder {0} (fetch items {1}, fetch folders {2}) with {3} items and {4} folders for agent {5}", 216 // "[WEB FETCH INV DESC HANDLER]: Replying to request for folder {0} (fetch items {1}, fetch folders {2}) with {3} items and {4} folders for agent {5}",
208// invFetch.folder_id, 217 // invFetch.folder_id,
209// invFetch.fetch_items, 218 // invFetch.fetch_items,
210// invFetch.fetch_folders, 219 // invFetch.fetch_folders,
211// contents.items.Array.Count, 220 // contents.items.Array.Count,
212// contents.categories.Array.Count, 221 // contents.categories.Array.Count,
213// invFetch.owner_id); 222 // invFetch.owner_id);
214 223
215 return reply; 224 return reply;
216 } 225 }
217 226
227 private LLSDInventoryDescendents ToLLSD(InventoryCollection inv, int descendents)
228 {
229 LLSDInventoryDescendents reply = new LLSDInventoryDescendents();
230 LLSDInventoryFolderContents contents = new LLSDInventoryFolderContents();
231 contents.agent_id = inv.OwnerID;
232 contents.owner_id = inv.OwnerID;
233 contents.folder_id = inv.FolderID;
234
235 reply.folders.Array.Add(contents);
236
237 if (inv.Folders != null)
238 {
239 foreach (InventoryFolderBase invFolder in inv.Folders)
240 {
241 contents.categories.Array.Add(ConvertInventoryFolder(invFolder));
242 }
243
244 descendents += inv.Folders.Count;
245 }
246
247 if (inv.Items != null)
248 {
249 foreach (InventoryItemBase invItem in inv.Items)
250 {
251 contents.items.Array.Add(ConvertInventoryItem(invItem));
252 }
253 }
254
255 contents.descendents = descendents;
256 contents.version = inv.Version;
257
258 return reply;
259 }
260 /// <summary>
261 /// Old style. Soon to be deprecated.
262 /// </summary>
263 /// <param name="request"></param>
264 /// <param name="httpRequest"></param>
265 /// <param name="httpResponse"></param>
266 /// <returns></returns>
267 [Obsolete]
268 private string FetchInventoryDescendentsRequest(ArrayList foldersrequested, IOSHttpRequest httpRequest, IOSHttpResponse httpResponse)
269 {
270 //m_log.DebugFormat("[WEB FETCH INV DESC HANDLER]: Received request for {0} folders", foldersrequested.Count);
271
272 string response = "";
273 string bad_folders_response = "";
274
275 for (int i = 0; i < foldersrequested.Count; i++)
276 {
277 string inventoryitemstr = "";
278 Hashtable inventoryhash = (Hashtable)foldersrequested[i];
279
280 LLSDFetchInventoryDescendents llsdRequest = new LLSDFetchInventoryDescendents();
281
282 try
283 {
284 LLSDHelpers.DeserialiseOSDMap(inventoryhash, llsdRequest);
285 }
286 catch (Exception e)
287 {
288 m_log.Debug("[WEB FETCH INV DESC HANDLER]: caught exception doing OSD deserialize" + e);
289 }
290
291 LLSDInventoryDescendents reply = FetchInventoryReply(llsdRequest);
292
293 if (null == reply)
294 {
295 bad_folders_response += "<uuid>" + llsdRequest.folder_id.ToString() + "</uuid>";
296 }
297 else
298 {
299 inventoryitemstr = LLSDHelpers.SerialiseLLSDReply(reply);
300 inventoryitemstr = inventoryitemstr.Replace("<llsd><map><key>folders</key><array>", "");
301 inventoryitemstr = inventoryitemstr.Replace("</array></map></llsd>", "");
302 }
303
304 response += inventoryitemstr;
305 }
306
307 if (response.Length == 0)
308 {
309 /* Viewers expect a bad_folders array when not available */
310 if (bad_folders_response.Length != 0)
311 {
312 response = "<llsd><map><key>bad_folders</key><array>" + bad_folders_response + "</array></map></llsd>";
313 }
314 else
315 {
316 response = "<llsd><map><key>folders</key><array /></map></llsd>";
317 }
318 }
319 else
320 {
321 if (bad_folders_response.Length != 0)
322 {
323 response = "<llsd><map><key>folders</key><array>" + response + "</array><key>bad_folders</key><array>" + bad_folders_response + "</array></map></llsd>";
324 }
325 else
326 {
327 response = "<llsd><map><key>folders</key><array>" + response + "</array></map></llsd>";
328 }
329 }
330
331 // m_log.DebugFormat("[WEB FETCH INV DESC HANDLER]: Replying to CAPS fetch inventory request");
332 //m_log.Debug("[WEB FETCH INV DESC HANDLER] "+response);
333
334 return response;
335
336 // }
337 }
338
218 /// <summary> 339 /// <summary>
219 /// Handle the caps inventory descendents fetch. 340 /// Handle the caps inventory descendents fetch.
220 /// </summary> 341 /// </summary>
@@ -226,6 +347,7 @@ namespace OpenSim.Capabilities.Handlers
226 /// <param name="sortOrder"></param> 347 /// <param name="sortOrder"></param>
227 /// <param name="version"></param> 348 /// <param name="version"></param>
228 /// <returns>An empty InventoryCollection if the inventory look up failed</returns> 349 /// <returns>An empty InventoryCollection if the inventory look up failed</returns>
350 [Obsolete]
229 private InventoryCollection Fetch( 351 private InventoryCollection Fetch(
230 UUID agentID, UUID folderID, UUID ownerID, 352 UUID agentID, UUID folderID, UUID ownerID,
231 bool fetchFolders, bool fetchItems, int sortOrder, out int version, out int descendents) 353 bool fetchFolders, bool fetchItems, int sortOrder, out int version, out int descendents)
@@ -264,7 +386,6 @@ namespace OpenSim.Capabilities.Handlers
264 m_log.WarnFormat("[WEB FETCH INV DESC HANDLER]: Could not get contents of folder {0} for user {1}", folderID, agentID); 386 m_log.WarnFormat("[WEB FETCH INV DESC HANDLER]: Could not get contents of folder {0} for user {1}", folderID, agentID);
265 return contents; 387 return contents;
266 } 388 }
267
268 contents = fetchedContents; 389 contents = fetchedContents;
269 InventoryFolderBase containingFolder = new InventoryFolderBase(); 390 InventoryFolderBase containingFolder = new InventoryFolderBase();
270 containingFolder.ID = folderID; 391 containingFolder.ID = folderID;
@@ -273,9 +394,9 @@ namespace OpenSim.Capabilities.Handlers
273 394
274 if (containingFolder != null) 395 if (containingFolder != null)
275 { 396 {
276// m_log.DebugFormat( 397 //m_log.DebugFormat(
277// "[WEB FETCH INV DESC HANDLER]: Retrieved folder {0} {1} for agent id {2}", 398 // "[WEB FETCH INV DESC HANDLER]: Retrieved folder {0} {1} for agent id {2}",
278// containingFolder.Name, containingFolder.ID, agentID); 399 // containingFolder.Name, containingFolder.ID, agentID);
279 400
280 version = containingFolder.Version; 401 version = containingFolder.Version;
281 402
@@ -410,6 +531,160 @@ namespace OpenSim.Capabilities.Handlers
410 return contents; 531 return contents;
411 532
412 } 533 }
534
535 private void AddLibraryFolders(List<LLSDFetchInventoryDescendents> fetchFolders, List<InventoryCollectionWithDescendents> result)
536 {
537 InventoryFolderImpl fold;
538 if (m_LibraryService != null && m_LibraryService.LibraryRootFolder != null)
539 {
540 List<LLSDFetchInventoryDescendents> libfolders = fetchFolders.FindAll(f => f.owner_id == m_LibraryService.LibraryRootFolder.Owner);
541 fetchFolders.RemoveAll(f => libfolders.Contains(f));
542
543 foreach (LLSDFetchInventoryDescendents f in libfolders)
544 {
545 if ((fold = m_LibraryService.LibraryRootFolder.FindFolder(f.folder_id)) != null)
546 {
547 InventoryCollectionWithDescendents ret = new InventoryCollectionWithDescendents();
548 ret.Collection = new InventoryCollection();
549 ret.Collection.Folders = new List<InventoryFolderBase>();
550 ret.Collection.Items = fold.RequestListOfItems();
551 ret.Collection.OwnerID = m_LibraryService.LibraryRootFolder.Owner;
552 ret.Collection.FolderID = f.folder_id;
553 ret.Collection.Version = fold.Version;
554
555 ret.Descendents = ret.Collection.Items.Count;
556
557 result.Add(ret);
558 }
559 }
560 }
561 }
562
563 private List<InventoryCollectionWithDescendents> Fetch(List<LLSDFetchInventoryDescendents> fetchFolders)
564 {
565 //m_log.DebugFormat(
566 // "[WEB FETCH INV DESC HANDLER]: Fetching {0} folders for owner {1}", fetchFolders.Count, fetchFolders[0].owner_id);
567
568 // FIXME MAYBE: We're not handling sortOrder!
569
570 List<InventoryCollectionWithDescendents> result = new List<InventoryCollectionWithDescendents>();
571
572 AddLibraryFolders(fetchFolders, result);
573
574 if (fetchFolders.Count > 0)
575 {
576 UUID[] fids = new UUID[fetchFolders.Count];
577 int i = 0;
578 foreach (LLSDFetchInventoryDescendents f in fetchFolders)
579 fids[i++] = f.folder_id;
580
581 InventoryCollection[] fetchedContents = m_InventoryService.GetMultipleFoldersContent(fetchFolders[0].owner_id, fids);
582
583 if (fetchedContents == null || (fetchedContents != null && fetchedContents.Length == 0))
584 {
585 //m_log.WarnFormat("[WEB FETCH INV DESC HANDLER]: Could not get contents of multiple folders for user {0}", fetchFolders[0].owner_id);
586 return null;
587 }
588
589 i = 0;
590 // Do some post-processing. May need to fetch more from inv server for links
591 foreach (InventoryCollection contents in fetchedContents)
592 {
593 InventoryCollectionWithDescendents coll = new InventoryCollectionWithDescendents();
594 coll.Collection = contents;
595
596 // Find the original request
597 LLSDFetchInventoryDescendents freq = fetchFolders[i++];
598
599 // The inventory server isn't sending FolderID in the collection...
600 // Must fetch it individually
601 if (contents.FolderID == UUID.Zero)
602 {
603 InventoryFolderBase containingFolder = new InventoryFolderBase();
604 containingFolder.ID = freq.folder_id;
605 containingFolder.Owner = freq.owner_id;
606 containingFolder = m_InventoryService.GetFolder(containingFolder);
607
608 if (containingFolder != null)
609 {
610 contents.FolderID = containingFolder.ID;
611 contents.OwnerID = containingFolder.Owner;
612 contents.Version = containingFolder.Version;
613 }
614 else
615 {
616 m_log.WarnFormat("[WEB FETCH INV DESC HANDLER]: Unable to fetch folder {0}", freq.folder_id);
617 continue;
618 }
619 }
620
621 if (freq.fetch_items && contents.Items != null)
622 {
623 List<InventoryItemBase> itemsToReturn = contents.Items;
624 List<InventoryItemBase> originalItems = new List<InventoryItemBase>(itemsToReturn);
625
626 // descendents must only include the links, not the linked items we add
627 coll.Descendents = originalItems.Count;
628
629 // Add target items for links in this folder before the links themselves.
630 foreach (InventoryItemBase item in originalItems)
631 {
632 if (item.AssetType == (int)AssetType.Link)
633 {
634 InventoryItemBase linkedItem = m_InventoryService.GetItem(new InventoryItemBase(item.AssetID));
635
636 // Take care of genuinely broken links where the target doesn't exist
637 // HACK: Also, don't follow up links that just point to other links. In theory this is legitimate,
638 // but no viewer has been observed to set these up and this is the lazy way of avoiding cycles
639 // rather than having to keep track of every folder requested in the recursion.
640 if (linkedItem != null && linkedItem.AssetType != (int)AssetType.Link)
641 {
642 itemsToReturn.Insert(0, linkedItem);
643 }
644 }
645 }
646
647 // Now scan for folder links and insert the items they target and those links at the head of the return data
648 foreach (InventoryItemBase item in originalItems)
649 {
650 if (item.AssetType == (int)AssetType.LinkFolder)
651 {
652 InventoryCollection linkedFolderContents = m_InventoryService.GetFolderContent(coll.Collection.OwnerID, item.AssetID);
653 List<InventoryItemBase> links = linkedFolderContents.Items;
654
655 itemsToReturn.InsertRange(0, links);
656
657 foreach (InventoryItemBase link in linkedFolderContents.Items)
658 {
659 // Take care of genuinely broken links where the target doesn't exist
660 // HACK: Also, don't follow up links that just point to other links. In theory this is legitimate,
661 // but no viewer has been observed to set these up and this is the lazy way of avoiding cycles
662 // rather than having to keep track of every folder requested in the recursion.
663 if (link != null)
664 {
665 //m_log.DebugFormat(
666 // "[WEB FETCH INV DESC HANDLER]: Adding item {0} {1} from folder {2} linked from {3}",
667 // link.Name, (AssetType)link.AssetType, item.AssetID, contents.FolderID);
668
669 InventoryItemBase linkedItem
670 = m_InventoryService.GetItem(new InventoryItemBase(link.AssetID));
671
672 if (linkedItem != null)
673 itemsToReturn.Insert(0, linkedItem);
674 }
675 }
676 }
677 }
678 }
679
680 result.Add(coll);
681 }
682 }
683
684 return result;
685 }
686
687
413 /// <summary> 688 /// <summary>
414 /// Convert an internal inventory folder object into an LLSD object. 689 /// Convert an internal inventory folder object into an LLSD object.
415 /// </summary> 690 /// </summary>
@@ -462,4 +737,10 @@ namespace OpenSim.Capabilities.Handlers
462 return llsdItem; 737 return llsdItem;
463 } 738 }
464 } 739 }
740
741 struct InventoryCollectionWithDescendents
742 {
743 public InventoryCollection Collection;
744 public int Descendents;
745 }
465} \ No newline at end of file 746} \ No newline at end of file