diff options
Diffstat (limited to 'linden/indra/llmessage/llhttpassetstorage.cpp')
-rw-r--r-- | linden/indra/llmessage/llhttpassetstorage.cpp | 428 |
1 files changed, 350 insertions, 78 deletions
diff --git a/linden/indra/llmessage/llhttpassetstorage.cpp b/linden/indra/llmessage/llhttpassetstorage.cpp index 2b64385..6c57499 100644 --- a/linden/indra/llmessage/llhttpassetstorage.cpp +++ b/linden/indra/llmessage/llhttpassetstorage.cpp | |||
@@ -30,12 +30,15 @@ | |||
30 | 30 | ||
31 | #include "llhttpassetstorage.h" | 31 | #include "llhttpassetstorage.h" |
32 | 32 | ||
33 | #include <sys/stat.h> | ||
34 | |||
33 | #include "indra_constants.h" | 35 | #include "indra_constants.h" |
34 | #include "llvfile.h" | 36 | #include "llvfile.h" |
35 | #include "llvfs.h" | 37 | #include "llvfs.h" |
36 | 38 | ||
37 | #include "zlib/zlib.h" | 39 | #include "zlib/zlib.h" |
38 | 40 | ||
41 | const U32 MAX_RUNNING_REQUESTS = 4; | ||
39 | const F32 MAX_PROCESSING_TIME = 0.005f; | 42 | const F32 MAX_PROCESSING_TIME = 0.005f; |
40 | const S32 CURL_XFER_BUFFER_SIZE = 65536; | 43 | const S32 CURL_XFER_BUFFER_SIZE = 65536; |
41 | // Try for 30 minutes for now. | 44 | // Try for 30 minutes for now. |
@@ -68,7 +71,9 @@ struct LLTempAssetData | |||
68 | class LLHTTPAssetRequest : public LLAssetRequest | 71 | class LLHTTPAssetRequest : public LLAssetRequest |
69 | { | 72 | { |
70 | public: | 73 | public: |
71 | LLHTTPAssetRequest(LLHTTPAssetStorage *asp, const LLUUID &uuid, LLAssetType::EType type, const char *url, CURLM *curl_multi); | 74 | LLHTTPAssetRequest(LLHTTPAssetStorage *asp, const LLUUID &uuid, |
75 | LLAssetType::EType type, LLAssetStorage::ERequestType rt, | ||
76 | const char *url, CURLM *curl_multi); | ||
72 | virtual ~LLHTTPAssetRequest(); | 77 | virtual ~LLHTTPAssetRequest(); |
73 | 78 | ||
74 | void setupCurlHandle(); | 79 | void setupCurlHandle(); |
@@ -80,6 +85,9 @@ public: | |||
80 | static size_t curlCompressedUploadCallback( | 85 | static size_t curlCompressedUploadCallback( |
81 | void *data, size_t size, size_t nmemb, void *user_data); | 86 | void *data, size_t size, size_t nmemb, void *user_data); |
82 | 87 | ||
88 | virtual LLSD getTerseDetails() const; | ||
89 | virtual LLSD getFullDetails() const; | ||
90 | |||
83 | public: | 91 | public: |
84 | LLHTTPAssetStorage *mAssetStoragep; | 92 | LLHTTPAssetStorage *mAssetStoragep; |
85 | 93 | ||
@@ -89,9 +97,7 @@ public: | |||
89 | struct curl_slist *mHTTPHeaders; | 97 | struct curl_slist *mHTTPHeaders; |
90 | LLVFile *mVFile; | 98 | LLVFile *mVFile; |
91 | LLUUID mTmpUUID; | 99 | LLUUID mTmpUUID; |
92 | BOOL mIsUpload; | 100 | LLAssetStorage::ERequestType mRequestType; |
93 | BOOL mIsLocalUpload; | ||
94 | BOOL mIsDownload; | ||
95 | 101 | ||
96 | bool mZInitialized; | 102 | bool mZInitialized; |
97 | z_stream mZStream; | 103 | z_stream mZStream; |
@@ -102,7 +108,12 @@ public: | |||
102 | }; | 108 | }; |
103 | 109 | ||
104 | 110 | ||
105 | LLHTTPAssetRequest::LLHTTPAssetRequest(LLHTTPAssetStorage *asp, const LLUUID &uuid, LLAssetType::EType type, const char *url, CURLM *curl_multi) | 111 | LLHTTPAssetRequest::LLHTTPAssetRequest(LLHTTPAssetStorage *asp, |
112 | const LLUUID &uuid, | ||
113 | LLAssetType::EType type, | ||
114 | LLAssetStorage::ERequestType rt, | ||
115 | const char *url, | ||
116 | CURLM *curl_multi) | ||
106 | : LLAssetRequest(uuid, type), | 117 | : LLAssetRequest(uuid, type), |
107 | mZInitialized(false) | 118 | mZInitialized(false) |
108 | { | 119 | { |
@@ -110,15 +121,16 @@ LLHTTPAssetRequest::LLHTTPAssetRequest(LLHTTPAssetStorage *asp, const LLUUID &uu | |||
110 | mCurlHandle = NULL; | 121 | mCurlHandle = NULL; |
111 | mCurlMultiHandle = curl_multi; | 122 | mCurlMultiHandle = curl_multi; |
112 | mVFile = NULL; | 123 | mVFile = NULL; |
113 | mIsUpload = FALSE; | 124 | mRequestType = rt; |
114 | mIsLocalUpload = FALSE; | ||
115 | mIsDownload = FALSE; | ||
116 | mHTTPHeaders = NULL; | 125 | mHTTPHeaders = NULL; |
126 | mFP = NULL; | ||
127 | mZInputBuffer = NULL; | ||
128 | mZInputExhausted = false; | ||
117 | 129 | ||
118 | mURLBuffer = new char[strlen(url) + 1]; /*Flawfinder: ignore*/ | 130 | mURLBuffer = new char[strlen(url) + 1]; /*Flawfinder: ignore*/ |
119 | if (mURLBuffer) | 131 | if (mURLBuffer) |
120 | { | 132 | { |
121 | strcpy(mURLBuffer, url); | 133 | strcpy(mURLBuffer, url); /*Flawfinder: ignore*/ |
122 | } | 134 | } |
123 | } | 135 | } |
124 | 136 | ||
@@ -132,22 +144,7 @@ LLHTTPAssetRequest::~LLHTTPAssetRequest() | |||
132 | if (mAssetStoragep) | 144 | if (mAssetStoragep) |
133 | { | 145 | { |
134 | // Terminating a request. Thus upload or download is no longer pending. | 146 | // Terminating a request. Thus upload or download is no longer pending. |
135 | if (mIsUpload) | 147 | mAssetStoragep->removeRunningRequest(mRequestType, this); |
136 | { | ||
137 | mAssetStoragep->clearPendingUpload(); | ||
138 | } | ||
139 | else if (mIsLocalUpload) | ||
140 | { | ||
141 | mAssetStoragep->clearPendingLocalUpload(); | ||
142 | } | ||
143 | else if (mIsDownload) | ||
144 | { | ||
145 | mAssetStoragep->clearPendingDownload(); | ||
146 | } | ||
147 | else | ||
148 | { | ||
149 | llerrs << "LLHTTPAssetRequest::~LLHTTPAssetRequest - Destroyed request is not upload OR download, this is bad!" << llendl; | ||
150 | } | ||
151 | } | 148 | } |
152 | else | 149 | else |
153 | { | 150 | { |
@@ -163,6 +160,82 @@ LLHTTPAssetRequest::~LLHTTPAssetRequest() | |||
163 | finishCompressedUpload(); | 160 | finishCompressedUpload(); |
164 | } | 161 | } |
165 | 162 | ||
163 | // virtual | ||
164 | LLSD LLHTTPAssetRequest::getTerseDetails() const | ||
165 | { | ||
166 | LLSD sd = LLAssetRequest::getTerseDetails(); | ||
167 | |||
168 | sd["url"] = mURLBuffer; | ||
169 | |||
170 | return sd; | ||
171 | } | ||
172 | |||
173 | // virtual | ||
174 | LLSD LLHTTPAssetRequest::getFullDetails() const | ||
175 | { | ||
176 | LLSD sd = LLAssetRequest::getFullDetails(); | ||
177 | |||
178 | if (mCurlHandle) | ||
179 | { | ||
180 | long curl_response = -1; | ||
181 | long curl_connect = -1; | ||
182 | double curl_total_time = -1.0f; | ||
183 | double curl_size_upload = -1.0f; | ||
184 | double curl_size_download = -1.0f; | ||
185 | long curl_content_length_upload = -1; | ||
186 | long curl_content_length_download = -1; | ||
187 | long curl_request_size = -1; | ||
188 | const char* curl_content_type = NULL; | ||
189 | |||
190 | curl_easy_getinfo(mCurlHandle, CURLINFO_HTTP_CODE, &curl_response); | ||
191 | curl_easy_getinfo(mCurlHandle, CURLINFO_HTTP_CONNECTCODE, &curl_connect); | ||
192 | curl_easy_getinfo(mCurlHandle, CURLINFO_TOTAL_TIME, &curl_total_time); | ||
193 | curl_easy_getinfo(mCurlHandle, CURLINFO_SIZE_UPLOAD, &curl_size_upload); | ||
194 | curl_easy_getinfo(mCurlHandle, CURLINFO_SIZE_DOWNLOAD, &curl_size_download); | ||
195 | curl_easy_getinfo(mCurlHandle, CURLINFO_CONTENT_LENGTH_UPLOAD, &curl_content_length_upload); | ||
196 | curl_easy_getinfo(mCurlHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &curl_content_length_download); | ||
197 | curl_easy_getinfo(mCurlHandle, CURLINFO_REQUEST_SIZE, &curl_request_size); | ||
198 | curl_easy_getinfo(mCurlHandle, CURLINFO_CONTENT_TYPE, &curl_content_type); | ||
199 | |||
200 | sd["curl_response_code"] = (int) curl_response; | ||
201 | sd["curl_http_connect_code"] = (int) curl_connect; | ||
202 | sd["curl_total_time"] = curl_total_time; | ||
203 | sd["curl_size_upload"] = curl_size_upload; | ||
204 | sd["curl_size_download"] = curl_size_download; | ||
205 | sd["curl_content_length_upload"] = (int) curl_content_length_upload; | ||
206 | sd["curl_content_length_download"] = (int) curl_content_length_download; | ||
207 | sd["curl_request_size"] = (int) curl_request_size; | ||
208 | if (curl_content_type) | ||
209 | { | ||
210 | sd["curl_content_type"] = curl_content_type; | ||
211 | } | ||
212 | else | ||
213 | { | ||
214 | sd["curl_content_type"] = ""; | ||
215 | } | ||
216 | } | ||
217 | |||
218 | sd["temp_id"] = mTmpUUID; | ||
219 | sd["request_type"] = LLAssetStorage::getRequestName(mRequestType); | ||
220 | sd["z_initialized"] = mZInitialized; | ||
221 | sd["z_input_exhausted"] = mZInputExhausted; | ||
222 | |||
223 | S32 file_size = -1; | ||
224 | if (mFP) | ||
225 | { | ||
226 | struct stat file_stat; | ||
227 | int file_desc = fileno(mFP); | ||
228 | if ( fstat(file_desc, &file_stat) == 0) | ||
229 | { | ||
230 | file_size = file_stat.st_size; | ||
231 | } | ||
232 | } | ||
233 | sd["file_size"] = file_size; | ||
234 | |||
235 | return sd; | ||
236 | } | ||
237 | |||
238 | |||
166 | void LLHTTPAssetRequest::setupCurlHandle() | 239 | void LLHTTPAssetRequest::setupCurlHandle() |
167 | { | 240 | { |
168 | mCurlHandle = curl_easy_init(); | 241 | mCurlHandle = curl_easy_init(); |
@@ -170,7 +243,7 @@ void LLHTTPAssetRequest::setupCurlHandle() | |||
170 | curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1); | 243 | curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1); |
171 | curl_easy_setopt(mCurlHandle, CURLOPT_URL, mURLBuffer); | 244 | curl_easy_setopt(mCurlHandle, CURLOPT_URL, mURLBuffer); |
172 | curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this); | 245 | curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this); |
173 | if (mIsDownload) | 246 | if (LLAssetStorage::RT_DOWNLOAD == mRequestType) |
174 | { | 247 | { |
175 | curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); | 248 | curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); |
176 | // only do this on downloads, as uploads | 249 | // only do this on downloads, as uploads |
@@ -193,22 +266,7 @@ void LLHTTPAssetRequest::setupCurlHandle() | |||
193 | if (mAssetStoragep) | 266 | if (mAssetStoragep) |
194 | { | 267 | { |
195 | // Set the appropriate pending upload or download flag | 268 | // Set the appropriate pending upload or download flag |
196 | if (mIsUpload) | 269 | mAssetStoragep->addRunningRequest(mRequestType, this); |
197 | { | ||
198 | mAssetStoragep->setPendingUpload(); | ||
199 | } | ||
200 | else if (mIsLocalUpload) | ||
201 | { | ||
202 | mAssetStoragep->setPendingLocalUpload(); | ||
203 | } | ||
204 | else if (mIsDownload) | ||
205 | { | ||
206 | mAssetStoragep->setPendingDownload(); | ||
207 | } | ||
208 | else | ||
209 | { | ||
210 | llerrs << "LLHTTPAssetRequest::setupCurlHandle - Request is not upload OR download, this is bad!" << llendl; | ||
211 | } | ||
212 | } | 270 | } |
213 | else | 271 | else |
214 | { | 272 | { |
@@ -342,10 +400,6 @@ void LLHTTPAssetStorage::_init(const char *web_host, const char *local_web_host, | |||
342 | curl_global_init(CURL_GLOBAL_ALL); | 400 | curl_global_init(CURL_GLOBAL_ALL); |
343 | 401 | ||
344 | mCurlMultiHandle = curl_multi_init(); | 402 | mCurlMultiHandle = curl_multi_init(); |
345 | |||
346 | mPendingDownload = FALSE; | ||
347 | mPendingUpload = FALSE; | ||
348 | mPendingLocalUpload = FALSE; | ||
349 | } | 403 | } |
350 | 404 | ||
351 | LLHTTPAssetStorage::~LLHTTPAssetStorage() | 405 | LLHTTPAssetStorage::~LLHTTPAssetStorage() |
@@ -457,6 +511,113 @@ void LLHTTPAssetStorage::storeAssetData( | |||
457 | } | 511 | } |
458 | } | 512 | } |
459 | 513 | ||
514 | // virtual | ||
515 | LLSD LLHTTPAssetStorage::getPendingDetails(LLAssetStorage::ERequestType rt, | ||
516 | LLAssetType::EType asset_type, | ||
517 | const std::string& detail_prefix) const | ||
518 | { | ||
519 | LLSD sd = LLAssetStorage::getPendingDetails(rt, asset_type, detail_prefix); | ||
520 | const request_list_t* running = getRunningList(rt); | ||
521 | if (running) | ||
522 | { | ||
523 | // Loop through the pending requests sd, and add extra info about its running status. | ||
524 | S32 num_pending = sd["requests"].size(); | ||
525 | S32 i; | ||
526 | for (i = 0; i < num_pending; ++i) | ||
527 | { | ||
528 | LLSD& pending = sd["requests"][i]; | ||
529 | // See if this pending request is running. | ||
530 | const LLAssetRequest* req = findRequest(running, | ||
531 | LLAssetType::lookup(pending["type"].asString().c_str()), | ||
532 | pending["asset_id"]); | ||
533 | if (req) | ||
534 | { | ||
535 | // Keep the detail_url so we don't have to rebuild it. | ||
536 | LLURI detail_url = pending["detail"]; | ||
537 | pending = req->getTerseDetails(); | ||
538 | pending["detail"] = detail_url; | ||
539 | pending["is_running"] = true; | ||
540 | } | ||
541 | else | ||
542 | { | ||
543 | pending["is_running"] = false; | ||
544 | } | ||
545 | } | ||
546 | } | ||
547 | return sd; | ||
548 | } | ||
549 | |||
550 | // virtual | ||
551 | LLSD LLHTTPAssetStorage::getPendingRequest(LLAssetStorage::ERequestType rt, | ||
552 | LLAssetType::EType asset_type, | ||
553 | const LLUUID& asset_id) const | ||
554 | { | ||
555 | // Look for this asset in the running list first. | ||
556 | const request_list_t* running = getRunningList(rt); | ||
557 | if (running) | ||
558 | { | ||
559 | LLSD sd = LLAssetStorage::getPendingRequest(running, asset_type, asset_id); | ||
560 | if (sd) | ||
561 | { | ||
562 | sd["is_running"] = true; | ||
563 | return sd; | ||
564 | } | ||
565 | } | ||
566 | LLSD sd = LLAssetStorage::getPendingRequest(rt, asset_type, asset_id); | ||
567 | if (sd) | ||
568 | { | ||
569 | sd["is_running"] = false; | ||
570 | } | ||
571 | return sd; | ||
572 | } | ||
573 | |||
574 | // virtual | ||
575 | bool LLHTTPAssetStorage::deletePendingRequest(LLAssetStorage::ERequestType rt, | ||
576 | LLAssetType::EType asset_type, | ||
577 | const LLUUID& asset_id) | ||
578 | { | ||
579 | // Try removing this from the running list first. | ||
580 | request_list_t* running = getRunningList(rt); | ||
581 | if (running) | ||
582 | { | ||
583 | LLAssetRequest* req = findRequest(running, asset_type, asset_id); | ||
584 | if (req) | ||
585 | { | ||
586 | // Remove this request from the running list to get it out of curl. | ||
587 | running->remove(req); | ||
588 | |||
589 | // Find this request in the pending list, so we can move it to the end of the line. | ||
590 | request_list_t* pending = getRequestList(rt); | ||
591 | if (pending) | ||
592 | { | ||
593 | request_list_t::iterator result = std::find_if(pending->begin(), pending->end(), | ||
594 | std::bind2nd(ll_asset_request_equal<LLAssetRequest*>(), req)); | ||
595 | if (pending->end() != result) | ||
596 | { | ||
597 | // This request was found in the pending list. Move it to the end! | ||
598 | LLAssetRequest* pending_req = *result; | ||
599 | pending->remove(pending_req); | ||
600 | pending->push_back(pending_req); | ||
601 | |||
602 | llinfos << "Asset " << getRequestName(rt) << " request for " | ||
603 | << asset_id << "." << LLAssetType::lookup(asset_type) | ||
604 | << " removed from curl and placed at the end of the pending queue." | ||
605 | << llendl; | ||
606 | } | ||
607 | else | ||
608 | { | ||
609 | llwarns << "Unable to find pending " << getRequestName(rt) << " request for " | ||
610 | << asset_id << "." << LLAssetType::lookup(asset_type) << llendl; | ||
611 | } | ||
612 | } | ||
613 | delete req; | ||
614 | |||
615 | return true; | ||
616 | } | ||
617 | } | ||
618 | return LLAssetStorage::deletePendingRequest(rt, asset_type, asset_id); | ||
619 | } | ||
620 | |||
460 | // internal requester, used by getAssetData in superclass | 621 | // internal requester, used by getAssetData in superclass |
461 | void LLHTTPAssetStorage::_queueDataRequest(const LLUUID& uuid, LLAssetType::EType type, | 622 | void LLHTTPAssetStorage::_queueDataRequest(const LLUUID& uuid, LLAssetType::EType type, |
462 | void (*callback)(LLVFS *vfs, const LLUUID&, LLAssetType::EType, void *, S32), | 623 | void (*callback)(LLVFS *vfs, const LLUUID&, LLAssetType::EType, void *, S32), |
@@ -488,13 +649,41 @@ void LLHTTPAssetStorage::_queueDataRequest(const LLUUID& uuid, LLAssetType::ETyp | |||
488 | } | 649 | } |
489 | } | 650 | } |
490 | 651 | ||
652 | LLAssetRequest* LLHTTPAssetStorage::findNextRequest(LLAssetStorage::request_list_t& pending, | ||
653 | LLAssetStorage::request_list_t& running) | ||
654 | { | ||
655 | // Early exit if the running list is full, or we don't have more pending than running. | ||
656 | if (running.size() >= MAX_RUNNING_REQUESTS | ||
657 | || pending.size() <= running.size()) return NULL; | ||
658 | |||
659 | // Look for the first pending request that is not already running. | ||
660 | request_list_t::iterator running_begin = running.begin(); | ||
661 | request_list_t::iterator running_end = running.end(); | ||
662 | |||
663 | request_list_t::iterator pending_iter = pending.begin(); | ||
664 | request_list_t::iterator pending_end = pending.end(); | ||
665 | // Loop over all pending requests until we miss finding it in the running list. | ||
666 | for (; pending_iter != pending.end(); ++pending_iter) | ||
667 | { | ||
668 | LLAssetRequest* req = *pending_iter; | ||
669 | // Look for this pending request in the running list. | ||
670 | if (running_end == std::find_if(running_begin, running_end, | ||
671 | std::bind2nd(ll_asset_request_equal<LLAssetRequest*>(), req))) | ||
672 | { | ||
673 | // It isn't running! Return it. | ||
674 | return req; | ||
675 | } | ||
676 | } | ||
677 | return NULL; | ||
678 | } | ||
679 | |||
491 | // overloaded to additionally move data to/from the webserver | 680 | // overloaded to additionally move data to/from the webserver |
492 | void LLHTTPAssetStorage::checkForTimeouts() | 681 | void LLHTTPAssetStorage::checkForTimeouts() |
493 | { | 682 | { |
494 | LLAssetRequest *req = NULL; | 683 | CURLMcode mcode; |
495 | if (mPendingDownloads.size() > 0 && !mPendingDownload) | 684 | LLAssetRequest *req; |
496 | { | 685 | while (req = findNextRequest(mPendingDownloads, mRunningDownloads)) |
497 | req = mPendingDownloads.front(); | 686 | { |
498 | // Setup this curl download request | 687 | // Setup this curl download request |
499 | // We need to generate a new request here | 688 | // We need to generate a new request here |
500 | // since the one in the list could go away | 689 | // since the one in the list could go away |
@@ -504,9 +693,9 @@ void LLHTTPAssetStorage::checkForTimeouts() | |||
504 | std::string base_url = getBaseURL(req->getUUID(), req->getType()); | 693 | std::string base_url = getBaseURL(req->getUUID(), req->getType()); |
505 | snprintf(tmp_url, sizeof(tmp_url), "%s/%36s.%s", base_url.c_str() , uuid_str, LLAssetType::lookup(req->getType())); /*Flawfinder: ignore*/ | 694 | snprintf(tmp_url, sizeof(tmp_url), "%s/%36s.%s", base_url.c_str() , uuid_str, LLAssetType::lookup(req->getType())); /*Flawfinder: ignore*/ |
506 | 695 | ||
507 | LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(), req->getType(), tmp_url, mCurlMultiHandle); | 696 | LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(), |
697 | req->getType(), RT_DOWNLOAD, tmp_url, mCurlMultiHandle); | ||
508 | new_req->mTmpUUID.generate(); | 698 | new_req->mTmpUUID.generate(); |
509 | new_req->mIsDownload = TRUE; | ||
510 | 699 | ||
511 | // Sets pending download flag internally | 700 | // Sets pending download flag internally |
512 | new_req->setupCurlHandle(); | 701 | new_req->setupCurlHandle(); |
@@ -514,15 +703,22 @@ void LLHTTPAssetStorage::checkForTimeouts() | |||
514 | curl_easy_setopt(new_req->mCurlHandle, CURLOPT_WRITEFUNCTION, &curlDownCallback); | 703 | curl_easy_setopt(new_req->mCurlHandle, CURLOPT_WRITEFUNCTION, &curlDownCallback); |
515 | curl_easy_setopt(new_req->mCurlHandle, CURLOPT_WRITEDATA, new_req->mCurlHandle); | 704 | curl_easy_setopt(new_req->mCurlHandle, CURLOPT_WRITEDATA, new_req->mCurlHandle); |
516 | 705 | ||
517 | curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle); | 706 | mcode = curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle); |
518 | llinfos << "Requesting " << new_req->mURLBuffer << llendl; | 707 | if (mcode > CURLM_OK) |
519 | 708 | { | |
709 | // Failure. Deleting the pending request will remove it from the running | ||
710 | // queue, and push it to the end of the pending queue. | ||
711 | deletePendingRequest(RT_DOWNLOAD, req->getType(), req->getUUID()); | ||
712 | break; | ||
713 | } | ||
714 | else | ||
715 | { | ||
716 | llinfos << "Requesting " << new_req->mURLBuffer << llendl; | ||
717 | } | ||
520 | } | 718 | } |
521 | 719 | ||
522 | 720 | while (req = findNextRequest(mPendingUploads, mRunningUploads)) | |
523 | if (mPendingUploads.size() > 0 && !mPendingUpload) | ||
524 | { | 721 | { |
525 | req = mPendingUploads.front(); | ||
526 | // setup this curl upload request | 722 | // setup this curl upload request |
527 | 723 | ||
528 | bool do_compress = req->getType() == LLAssetType::AT_OBJECT; | 724 | bool do_compress = req->getType() == LLAssetType::AT_OBJECT; |
@@ -534,8 +730,8 @@ void LLHTTPAssetStorage::checkForTimeouts() | |||
534 | do_compress ? "%s/%s.%s.gz" : "%s/%s.%s", | 730 | do_compress ? "%s/%s.%s.gz" : "%s/%s.%s", |
535 | mBaseURL.c_str(), uuid_str, LLAssetType::lookup(req->getType())); | 731 | mBaseURL.c_str(), uuid_str, LLAssetType::lookup(req->getType())); |
536 | 732 | ||
537 | LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(), req->getType(), tmp_url, mCurlMultiHandle); | 733 | LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(), |
538 | new_req->mIsUpload = TRUE; | 734 | req->getType(), RT_UPLOAD, tmp_url, mCurlMultiHandle); |
539 | if (do_compress) | 735 | if (do_compress) |
540 | { | 736 | { |
541 | new_req->prepareCompressedUpload(); | 737 | new_req->prepareCompressedUpload(); |
@@ -560,15 +756,23 @@ void LLHTTPAssetStorage::checkForTimeouts() | |||
560 | } | 756 | } |
561 | curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READDATA, new_req->mCurlHandle); | 757 | curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READDATA, new_req->mCurlHandle); |
562 | 758 | ||
563 | curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle); | 759 | mcode = curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle); |
564 | llinfos << "Requesting PUT " << new_req->mURLBuffer << llendl; | 760 | if (mcode > CURLM_OK) |
761 | { | ||
762 | // Failure. Deleting the pending request will remove it from the running | ||
763 | // queue, and push it to the end of the pending queue. | ||
764 | deletePendingRequest(RT_UPLOAD, req->getType(), req->getUUID()); | ||
765 | break; | ||
766 | } | ||
767 | else | ||
768 | { | ||
769 | llinfos << "Requesting PUT " << new_req->mURLBuffer << llendl; | ||
770 | } | ||
565 | // Pending upload will have been flagged by the request | 771 | // Pending upload will have been flagged by the request |
566 | } | 772 | } |
567 | 773 | ||
568 | 774 | while (req = findNextRequest(mPendingLocalUploads, mRunningLocalUploads)) | |
569 | if (mPendingLocalUploads.size() > 0 && !mPendingLocalUpload) | ||
570 | { | 775 | { |
571 | req = mPendingLocalUploads.front(); | ||
572 | // setup this curl upload request | 776 | // setup this curl upload request |
573 | LLVFile file(mVFS, req->getUUID(), req->getType()); | 777 | LLVFile file(mVFS, req->getUUID(), req->getType()); |
574 | 778 | ||
@@ -579,8 +783,8 @@ void LLHTTPAssetStorage::checkForTimeouts() | |||
579 | // KLW - All temporary uploads are saved locally "http://localhost:12041/asset" | 783 | // KLW - All temporary uploads are saved locally "http://localhost:12041/asset" |
580 | snprintf(tmp_url, sizeof(tmp_url), "%s/%36s.%s", mLocalBaseURL.c_str(), uuid_str, LLAssetType::lookup(req->getType())); /*Flawfinder: ignore*/ | 784 | snprintf(tmp_url, sizeof(tmp_url), "%s/%36s.%s", mLocalBaseURL.c_str(), uuid_str, LLAssetType::lookup(req->getType())); /*Flawfinder: ignore*/ |
581 | 785 | ||
582 | LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(), req->getType(), tmp_url, mCurlMultiHandle); | 786 | LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(), |
583 | new_req->mIsLocalUpload = TRUE; | 787 | req->getType(), RT_LOCALUPLOAD, tmp_url, mCurlMultiHandle); |
584 | new_req->mRequestingAgentID = req->mRequestingAgentID; | 788 | new_req->mRequestingAgentID = req->mRequestingAgentID; |
585 | 789 | ||
586 | // Sets pending upload flag internally | 790 | // Sets pending upload flag internally |
@@ -591,13 +795,22 @@ void LLHTTPAssetStorage::checkForTimeouts() | |||
591 | curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READFUNCTION, &curlUpCallback); | 795 | curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READFUNCTION, &curlUpCallback); |
592 | curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READDATA, new_req->mCurlHandle); | 796 | curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READDATA, new_req->mCurlHandle); |
593 | 797 | ||
594 | curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle); | 798 | mcode = curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle); |
595 | llinfos << "TAT: LLHTTPAssetStorage::checkForTimeouts() : pending local!" | 799 | if (mcode > CURLM_OK) |
596 | << " Requesting PUT " << new_req->mURLBuffer << llendl; | 800 | { |
801 | // Failure. Deleting the pending request will remove it from the running | ||
802 | // queue, and push it to the end of the pending queue. | ||
803 | deletePendingRequest(RT_LOCALUPLOAD, req->getType(), req->getUUID()); | ||
804 | break; | ||
805 | } | ||
806 | else | ||
807 | { | ||
808 | llinfos << "TAT: LLHTTPAssetStorage::checkForTimeouts() : pending local!" | ||
809 | << " Requesting PUT " << new_req->mURLBuffer << llendl; | ||
810 | } | ||
597 | // Pending upload will have been flagged by the request | 811 | // Pending upload will have been flagged by the request |
598 | } | 812 | } |
599 | S32 count = 0; | 813 | S32 count = 0; |
600 | CURLMcode mcode; | ||
601 | int queue_length; | 814 | int queue_length; |
602 | do | 815 | do |
603 | { | 816 | { |
@@ -618,12 +831,15 @@ void LLHTTPAssetStorage::checkForTimeouts() | |||
618 | curl_easy_getinfo(curl_msg->easy_handle, CURLINFO_PRIVATE, &req); | 831 | curl_easy_getinfo(curl_msg->easy_handle, CURLINFO_PRIVATE, &req); |
619 | 832 | ||
620 | curl_easy_getinfo(curl_msg->easy_handle, CURLINFO_HTTP_CODE, &curl_result); | 833 | curl_easy_getinfo(curl_msg->easy_handle, CURLINFO_HTTP_CODE, &curl_result); |
621 | if (req->mIsUpload || req->mIsLocalUpload) | 834 | if (RT_UPLOAD == req->mRequestType || RT_LOCALUPLOAD == req->mRequestType) |
622 | { | 835 | { |
623 | if (curl_msg->data.result == CURLE_OK && (curl_result == HTTP_OK || curl_result == HTTP_PUT_OK || curl_result == HTTP_NO_CONTENT)) | 836 | if (curl_msg->data.result == CURLE_OK && |
837 | ( curl_result == HTTP_OK | ||
838 | || curl_result == HTTP_PUT_OK | ||
839 | || curl_result == HTTP_NO_CONTENT)) | ||
624 | { | 840 | { |
625 | llinfos << "Success uploading " << req->getUUID() << " to " << req->mURLBuffer << llendl; | 841 | llinfos << "Success uploading " << req->getUUID() << " to " << req->mURLBuffer << llendl; |
626 | if (req->mIsLocalUpload) | 842 | if (RT_LOCALUPLOAD == req->mRequestType) |
627 | { | 843 | { |
628 | addTempAssetData(req->getUUID(), req->mRequestingAgentID, mHostName); | 844 | addTempAssetData(req->getUUID(), req->mRequestingAgentID, mHostName); |
629 | } | 845 | } |
@@ -655,7 +871,7 @@ void LLHTTPAssetStorage::checkForTimeouts() | |||
655 | // Pending upload flag will get cleared when the request is deleted | 871 | // Pending upload flag will get cleared when the request is deleted |
656 | } | 872 | } |
657 | } | 873 | } |
658 | else if (req->mIsDownload) | 874 | else if (RT_DOWNLOAD == req->mRequestType) |
659 | { | 875 | { |
660 | if (curl_result == HTTP_OK && curl_msg->data.result == CURLE_OK) | 876 | if (curl_result == HTTP_OK && curl_msg->data.result == CURLE_OK) |
661 | { | 877 | { |
@@ -667,7 +883,7 @@ void LLHTTPAssetStorage::checkForTimeouts() | |||
667 | } | 883 | } |
668 | else | 884 | else |
669 | { | 885 | { |
670 | // TODO: if this actually indicates a bad asset on the server | 886 | // *TODO: if this actually indicates a bad asset on the server |
671 | // (not certain at this point), then delete it | 887 | // (not certain at this point), then delete it |
672 | llwarns << "Found " << req->mURLBuffer << " to be zero size" << llendl; | 888 | llwarns << "Found " << req->mURLBuffer << " to be zero size" << llendl; |
673 | xfer_result = LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE; | 889 | xfer_result = LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE; |
@@ -793,9 +1009,8 @@ S32 LLHTTPAssetStorage::getURLToFile(const LLUUID& uuid, LLAssetType::EType asse | |||
793 | } | 1009 | } |
794 | 1010 | ||
795 | // make sure we use the normal curl setup, even though we don't really need a request object | 1011 | // make sure we use the normal curl setup, even though we don't really need a request object |
796 | LLHTTPAssetRequest req(this, uuid, asset_type, url.c_str(), mCurlMultiHandle); | 1012 | LLHTTPAssetRequest req(this, uuid, asset_type, RT_DOWNLOAD, url.c_str(), mCurlMultiHandle); |
797 | req.mFP = fp; | 1013 | req.mFP = fp; |
798 | req.mIsDownload = TRUE; | ||
799 | 1014 | ||
800 | req.setupCurlHandle(); | 1015 | req.setupCurlHandle(); |
801 | curl_easy_setopt(req.mCurlHandle, CURLOPT_FOLLOWLOCATION, TRUE); | 1016 | curl_easy_setopt(req.mCurlHandle, CURLOPT_FOLLOWLOCATION, TRUE); |
@@ -887,6 +1102,63 @@ size_t LLHTTPAssetStorage::curlFileDownCallback(void *data, size_t size, size_t | |||
887 | return fwrite(data, size, nmemb, req->mFP); | 1102 | return fwrite(data, size, nmemb, req->mFP); |
888 | } | 1103 | } |
889 | 1104 | ||
1105 | LLAssetStorage::request_list_t* LLHTTPAssetStorage::getRunningList(LLAssetStorage::ERequestType rt) | ||
1106 | { | ||
1107 | switch (rt) | ||
1108 | { | ||
1109 | case RT_DOWNLOAD: | ||
1110 | return &mRunningDownloads; | ||
1111 | case RT_UPLOAD: | ||
1112 | return &mRunningUploads; | ||
1113 | case RT_LOCALUPLOAD: | ||
1114 | return &mRunningLocalUploads; | ||
1115 | default: | ||
1116 | return NULL; | ||
1117 | } | ||
1118 | } | ||
1119 | |||
1120 | const LLAssetStorage::request_list_t* LLHTTPAssetStorage::getRunningList(LLAssetStorage::ERequestType rt) const | ||
1121 | { | ||
1122 | switch (rt) | ||
1123 | { | ||
1124 | case RT_DOWNLOAD: | ||
1125 | return &mRunningDownloads; | ||
1126 | case RT_UPLOAD: | ||
1127 | return &mRunningUploads; | ||
1128 | case RT_LOCALUPLOAD: | ||
1129 | return &mRunningLocalUploads; | ||
1130 | default: | ||
1131 | return NULL; | ||
1132 | } | ||
1133 | } | ||
1134 | |||
1135 | |||
1136 | void LLHTTPAssetStorage::addRunningRequest(ERequestType rt, LLHTTPAssetRequest* request) | ||
1137 | { | ||
1138 | request_list_t* requests = getRunningList(rt); | ||
1139 | if (requests) | ||
1140 | { | ||
1141 | requests->push_back(request); | ||
1142 | } | ||
1143 | else | ||
1144 | { | ||
1145 | llerrs << "LLHTTPAssetStorage::addRunningRequest - Request is not an upload OR download, this is bad!" << llendl; | ||
1146 | } | ||
1147 | } | ||
1148 | |||
1149 | void LLHTTPAssetStorage::removeRunningRequest(ERequestType rt, LLHTTPAssetRequest* request) | ||
1150 | { | ||
1151 | request_list_t* requests = getRunningList(rt); | ||
1152 | if (requests) | ||
1153 | { | ||
1154 | requests->remove(request); | ||
1155 | } | ||
1156 | else | ||
1157 | { | ||
1158 | llerrs << "LLHTTPAssetStorage::removeRunningRequest - Destroyed request is not an upload OR download, this is bad!" << llendl; | ||
1159 | } | ||
1160 | } | ||
1161 | |||
890 | // virtual | 1162 | // virtual |
891 | void LLHTTPAssetStorage::addTempAssetData(const LLUUID& asset_id, const LLUUID& agent_id, const std::string& host_name) | 1163 | void LLHTTPAssetStorage::addTempAssetData(const LLUUID& asset_id, const LLUUID& agent_id, const std::string& host_name) |
892 | { | 1164 | { |