From 4ca4594ec264f390be981568ef5ff3ff0f2f77e6 Mon Sep 17 00:00:00 2001 From: McCabe Maxsted Date: Mon, 30 Aug 2010 13:11:36 -0700 Subject: Updated lltexturecache to the latest in viewer-development (see #425) --- linden/indra/llcommon/llworkerthread.cpp | 39 +- linden/indra/llcommon/llworkerthread.h | 8 +- linden/indra/newview/lltexturecache.cpp | 679 +++++++++++++++++++++---------- linden/indra/newview/lltexturecache.h | 37 +- linden/indra/newview/lltexturefetch.cpp | 2 + 5 files changed, 533 insertions(+), 232 deletions(-) diff --git a/linden/indra/llcommon/llworkerthread.cpp b/linden/indra/llcommon/llworkerthread.cpp index 8195e1c..67664c7 100644 --- a/linden/indra/llcommon/llworkerthread.cpp +++ b/linden/indra/llcommon/llworkerthread.cpp @@ -60,6 +60,27 @@ LLWorkerThread::~LLWorkerThread() // ~LLQueuedThread() will be called here } +//called only in destructor. +void LLWorkerThread::clearDeleteList() +{ + // Delete any workers in the delete queue (should be safe - had better be!) + if (!mDeleteList.empty()) + { + llwarns << "Worker Thread: " << mName << " destroyed with " << mDeleteList.size() + << " entries in delete list." << llendl; + + mDeleteMutex->lock(); + for (delete_list_t::iterator iter = mDeleteList.begin(); iter != mDeleteList.end(); ++iter) + { + (*iter)->mRequestHandle = LLWorkerThread::nullHandle(); + (*iter)->clearFlags(LLWorkerClass::WCF_HAVE_WORK); + delete *iter ; + } + mDeleteList.clear() ; + mDeleteMutex->unlock() ; + } +} + // virtual S32 LLWorkerThread::update(U32 max_time_ms) { @@ -183,6 +204,7 @@ LLWorkerClass::LLWorkerClass(LLWorkerThread* workerthread, const std::string& na : mWorkerThread(workerthread), mWorkerClassName(name), mRequestHandle(LLWorkerThread::nullHandle()), + mRequestPriority(LLWorkerThread::PRIORITY_NORMAL), mMutex(NULL), mWorkFlags(0) { @@ -314,7 +336,20 @@ bool LLWorkerClass::checkWork(bool aborting) if (mRequestHandle != LLWorkerThread::nullHandle()) { LLWorkerThread::WorkRequest* workreq = (LLWorkerThread::WorkRequest*)mWorkerThread->getRequest(mRequestHandle); - llassert_always(workreq); + if(!workreq) + { + if(mWorkerThread->isQuitting() || mWorkerThread->isStopped()) //the mWorkerThread is not running + { + mRequestHandle = LLWorkerThread::nullHandle(); + clearFlags(WCF_HAVE_WORK); + } + else + { + llassert_always(workreq); + } + return true ; + } + LLQueuedThread::status_t status = workreq->getStatus(); if (status == LLWorkerThread::STATUS_ABORTED) { @@ -364,7 +399,7 @@ void LLWorkerClass::scheduleDelete() void LLWorkerClass::setPriority(U32 priority) { mMutex.lock(); - if (mRequestHandle != LLWorkerThread::nullHandle()) + if (mRequestHandle != LLWorkerThread::nullHandle() && mRequestPriority != priority) { mRequestPriority = priority; mWorkerThread->setPriority(mRequestHandle, priority); diff --git a/linden/indra/llcommon/llworkerthread.h b/linden/indra/llcommon/llworkerthread.h index 708d812..d1868bc 100644 --- a/linden/indra/llcommon/llworkerthread.h +++ b/linden/indra/llcommon/llworkerthread.h @@ -80,6 +80,9 @@ public: S32 mParam; }; +protected: + void clearDeleteList() ; + private: typedef std::list delete_list_t; delete_list_t mDeleteList; @@ -93,8 +96,11 @@ public: handle_t addWorkRequest(LLWorkerClass* workerclass, S32 param, U32 priority = PRIORITY_NORMAL); - void deleteWorker(LLWorkerClass* workerclass); // schedule for deletion S32 getNumDeletes() { return (S32)mDeleteList.size(); } // debug + +private: + void deleteWorker(LLWorkerClass* workerclass); // schedule for deletion + }; //============================================================================ diff --git a/linden/indra/newview/lltexturecache.cpp b/linden/indra/newview/lltexturecache.cpp index 46c125f..0a76742 100644 --- a/linden/indra/newview/lltexturecache.cpp +++ b/linden/indra/newview/lltexturecache.cpp @@ -48,11 +48,11 @@ // Unordered array of Entry structs // cache/texture.cache // First TEXTURE_CACHE_ENTRY_SIZE bytes of each texture in texture.entries in same order -// Entry size same as header packet, so we're not 0-padding unless whole image is contained in header. // cache/textures/[0-F]/UUID.texture // Actual texture body files -const S32 TEXTURE_CACHE_ENTRY_SIZE = FIRST_PACKET_SIZE; +//note: there is no good to define 1024 for TEXTURE_CACHE_ENTRY_SIZE while FIRST_PACKET_SIZE is 600 on sim side. +const S32 TEXTURE_CACHE_ENTRY_SIZE = FIRST_PACKET_SIZE;//1024; const F32 TEXTURE_CACHE_PURGE_AMOUNT = .20f; // % amount to reduce the cache by when it exceeds its limit const F32 TEXTURE_CACHE_LRU_SIZE = .10f; // % amount for LRU list (low overhead to regenerate) @@ -390,6 +390,7 @@ bool LLTextureCacheRemoteWorker::doRead() } else { + //llinfos << "texture " << mID.asString() << " found in local_assets" << llendl; mImageSize = local_size; mImageLocal = TRUE; } @@ -400,7 +401,8 @@ bool LLTextureCacheRemoteWorker::doRead() // Second state / stage : identify the cache or not... if (!done && (mState == CACHE)) { - idx = mCache->getHeaderCacheEntry(mID, mImageSize); + LLTextureCache::Entry entry ; + idx = mCache->getHeaderCacheEntry(mID, entry); if (idx < 0) { // The texture is *not* cached. We're done here... @@ -409,6 +411,7 @@ bool LLTextureCacheRemoteWorker::doRead() } else { + mImageSize = entry.mImageSize ; // If the read offset is bigger than the header cache, we read directly from the body // Note that currently, we *never* read with offset from the cache, so the result is *always* HEADER mState = mOffset < TEXTURE_CACHE_ENTRY_SIZE ? HEADER : BODY; @@ -536,6 +539,7 @@ bool LLTextureCacheRemoteWorker::doWrite() { llassert_always(mOffset == 0); // We currently do not support write offsets llassert_always(mDataSize > 0); // Things will go badly wrong if mDataSize is nul or negative... + llassert_always(mImageSize >= mDataSize); mState = CACHE; } @@ -545,14 +549,19 @@ bool LLTextureCacheRemoteWorker::doWrite() if (!done && (mState == CACHE)) { bool alreadyCached = false; - S32 cur_imagesize = 0; + LLTextureCache::Entry entry ; + // Checks if this image is already in the entry list - idx = mCache->getHeaderCacheEntry(mID, cur_imagesize); - if (idx >= 0 && (cur_imagesize >= 0)) + idx = mCache->getHeaderCacheEntry(mID, entry); + if(idx < 0) + { + idx = mCache->setHeaderCacheEntry(mID, entry, mImageSize, mDataSize); // create the new entry. + } + else { - alreadyCached = true; // already there and non empty + alreadyCached = mCache->updateEntry(idx, entry, mImageSize, mDataSize); // update the existing entry. } - idx = mCache->setHeaderCacheEntry(mID, mImageSize); // create or touch the entry + if (idx < 0) { llwarns << "LLTextureCacheWorker: " << mID @@ -562,10 +571,6 @@ bool LLTextureCacheRemoteWorker::doWrite() } else { - if (cur_imagesize > 0 && (mImageSize != cur_imagesize)) - { - alreadyCached = false; // re-write the header if the size changed in all cases - } if (alreadyCached && (mDataSize <= TEXTURE_CACHE_ENTRY_SIZE)) { // Small texture already cached case: we're done with writing @@ -628,7 +633,7 @@ bool LLTextureCacheRemoteWorker::doWrite() { llassert(mDataSize > TEXTURE_CACHE_ENTRY_SIZE); // wouldn't make sense to be here otherwise... S32 file_size = mDataSize - TEXTURE_CACHE_ENTRY_SIZE; - if ((file_size > 0) && mCache->updateTextureEntryList(mID, file_size)) + { // build the cache file name from the UUID std::string filename = mCache->getTextureFileName(mID); @@ -645,10 +650,7 @@ bool LLTextureCacheRemoteWorker::doWrite() done = true; } } - else - { - mDataSize = 0; // no data written - } + // Nothing else to do at that point... done = true; } @@ -739,7 +741,7 @@ LLTextureCache::LLTextureCache(bool threaded) mHeaderMutex(NULL), mListMutex(NULL), mHeaderAPRFile(NULL), - mReadOnly(FALSE), + mReadOnly(TRUE), //do not allow to change the texture cache until setReadOnly() is called. mTexturesSizeTotal(0), mDoPurge(FALSE) { @@ -747,6 +749,8 @@ LLTextureCache::LLTextureCache(bool threaded) LLTextureCache::~LLTextureCache() { + clearDeleteList() ; + writeUpdatedEntries() ; } ////////////////////////////////////////////////////////////////////////////// @@ -754,6 +758,9 @@ LLTextureCache::~LLTextureCache() //virtual S32 LLTextureCache::update(U32 max_time_ms) { + static LLFrameTimer timer ; + static const F32 MAX_TIME_INTERVAL = 300.f ; //seconds. + S32 res; res = LLWorkerThread::update(max_time_ms); @@ -789,6 +796,12 @@ S32 LLTextureCache::update(U32 max_time_ms) responder->completed(success); } + if(!res && timer.getElapsedTimeF32() > MAX_TIME_INTERVAL) + { + timer.reset() ; + writeUpdatedEntries() ; + } + return res; } @@ -811,58 +824,54 @@ std::string LLTextureCache::getTextureFileName(const LLUUID& id) return filename; } -bool LLTextureCache::updateTextureEntryList(const LLUUID& id, S32 bodysize) +//debug +BOOL LLTextureCache::isInCache(const LLUUID& id) { - bool res = false; - bool purge = false; + LLMutexLock lock(&mHeaderMutex); + id_map_t::const_iterator iter = mHeaderIDMap.find(id); + + return (iter != mHeaderIDMap.end()) ; +} + +//debug +BOOL LLTextureCache::isInLocal(const LLUUID& id) +{ + S32 local_size = 0; + std::string local_filename; + + std::string filename = getLocalFileName(id); + // Is it a JPEG2000 file? { - mHeaderMutex.lock(); - size_map_t::iterator iter1 = mTexturesSizeMap.find(id); - if (iter1 == mTexturesSizeMap.end() || iter1->second < bodysize) + local_filename = filename + ".j2c"; + local_size = LLAPRFile::size(local_filename); + if (local_size > 0) { - llassert_always(bodysize > 0); - - S32 oldbodysize = 0; - if (iter1 != mTexturesSizeMap.end()) - { - oldbodysize = iter1->second; - } - - Entry entry; - S32 idx = openAndReadEntry(id, entry, false); - if (idx < 0) - { - llwarns << "Failed to open entry: " << id << llendl; - mHeaderMutex.unlock(); - removeFromCache(id); - return false; - } - else if (oldbodysize != entry.mBodySize) - { - llwarns << "Entry mismatch in mTextureSizeMap / mHeaderIDMap" - << " idx=" << idx << " oldsize=" << oldbodysize << " entrysize=" << entry.mBodySize << llendl; - } - entry.mBodySize = bodysize; - writeEntryAndClose(idx, entry); - - mTexturesSizeTotal -= oldbodysize; - mTexturesSizeTotal += bodysize; - - if (mTexturesSizeTotal > sCacheMaxTexturesSize) - { - purge = true; - } - res = true; + return TRUE ; } } - if (purge) + + // If not, is it a jpeg file? { - mDoPurge = TRUE; + local_filename = filename + ".jpg"; + local_size = LLAPRFile::size(local_filename); + if (local_size > 0) + { + return TRUE ; + } } - mHeaderMutex.unlock(); - return res; + + // Hmm... What about a targa file? (used for UI texture mostly) + { + local_filename = filename + ".tga"; + local_size = LLAPRFile::size(local_filename); + if (local_size > 0) + { + return TRUE ; + } + } + + return FALSE ; } - ////////////////////////////////////////////////////////////////////////////// //static @@ -893,13 +902,21 @@ void LLTextureCache::purgeCache(ELLPath location) LLAPRFile::remove(mHeaderEntriesFileName); LLAPRFile::remove(mHeaderDataFileName); } + //remove the current texture cache. purgeAllTextures(true); } -S64 LLTextureCache::initCache(ELLPath location, S64 max_size, BOOL read_only) +//is called in the main thread before initCache(...) is called. +void LLTextureCache::setReadOnly(BOOL read_only) { mReadOnly = read_only; - +} + +//called in the main thread. +S64 LLTextureCache::initCache(ELLPath location, S64 max_size, BOOL disable_texture_cache) +{ + llassert_always(getPending() == 0) ; //should not start accessing the texture cache before initialized. + S64 header_size = (max_size * 2) / 10; S64 max_entries = header_size / TEXTURE_CACHE_ENTRY_SIZE; sCacheMaxEntries = (S32)(llmin((S64)sCacheMaxEntries, max_entries)); @@ -911,6 +928,15 @@ S64 LLTextureCache::initCache(ELLPath location, S64 max_size, BOOL read_only) sCacheMaxTexturesSize = max_size; max_size -= sCacheMaxTexturesSize; + if(disable_texture_cache) //the texture cache is disabled + { + llinfos << "The texture cache is disabled!" << llendl ; + setReadOnly(TRUE) ; + purgeAllTextures(true); + + return max_size ; + } + LL_INFOS("TextureCache") << "Headers: " << sCacheMaxEntries << " Textures size: " << sCacheMaxTexturesSize/(1024*1024) << " MB" << LL_ENDL; @@ -919,6 +945,7 @@ S64 LLTextureCache::initCache(ELLPath location, S64 max_size, BOOL read_only) if (!mReadOnly) { LLFile::mkdir(mTexturesDirName); + const char* subdirs = "0123456789abcdef"; for (S32 i=0; i<16; i++) { @@ -929,6 +956,8 @@ S64 LLTextureCache::initCache(ELLPath location, S64 max_size, BOOL read_only) readHeaderCache(); purgeTextures(true); // calc mTexturesSize and make some room in the texture cache if we need it + llassert_always(getPending() == 0) ; //should not start accessing the texture cache before initialized. + return max_size; // unused cache space } @@ -940,13 +969,20 @@ LLAPRFile* LLTextureCache::openHeaderEntriesFile(bool readonly, S32 offset) llassert_always(mHeaderAPRFile == NULL); apr_int32_t flags = readonly ? APR_READ|APR_BINARY : APR_READ|APR_WRITE|APR_BINARY; mHeaderAPRFile = new LLAPRFile(mHeaderEntriesFileName, flags, LLAPRFile::local); - mHeaderAPRFile->seek(APR_SET, offset); + if(offset > 0) + { + mHeaderAPRFile->seek(APR_SET, offset); + } return mHeaderAPRFile; } void LLTextureCache::closeHeaderEntriesFile() { - llassert_always(mHeaderAPRFile != NULL); + if(!mHeaderAPRFile) + { + return ; + } + delete mHeaderAPRFile; mHeaderAPRFile = NULL; } @@ -959,6 +995,12 @@ void LLTextureCache::readEntriesHeader() { LLAPRFile::readEx(mHeaderEntriesFileName, (U8*)&mHeaderEntriesInfo, 0, sizeof(EntriesInfo)); } + else //create an empty entries header. + { + mHeaderEntriesInfo.mVersion = sHeaderCacheVersion ; + mHeaderEntriesInfo.mEntries = 0 ; + writeEntriesHeader() ; + } } void LLTextureCache::writeEntriesHeader() @@ -970,8 +1012,7 @@ void LLTextureCache::writeEntriesHeader() } } -static S32 mHeaderEntriesMaxWriteIdx = 0; - +//mHeaderMutex is locked before calling this. S32 LLTextureCache::openAndReadEntry(const LLUUID& id, Entry& entry, bool create) { S32 idx = -1; @@ -1011,8 +1052,7 @@ S32 LLTextureCache::openAndReadEntry(const LLUUID& id, Entry& entry, bool create if (iter3 != mHeaderIDMap.end() && iter3->second >= 0) { idx = iter3->second; - mHeaderIDMap.erase(oldid); - mTexturesSizeMap.erase(oldid); + removeCachedTexture(oldid) ;//remove the existing cached texture to release the entry index. break; } } @@ -1022,20 +1062,9 @@ S32 LLTextureCache::openAndReadEntry(const LLUUID& id, Entry& entry, bool create } if (idx >= 0) { - // Set the header index - mHeaderIDMap[id] = idx; - llassert_always(mTexturesSizeMap.erase(id) == 0); - // Initialize the entry (will get written later) - entry.init(id, time(NULL)); - // Update Header - writeEntriesHeader(); - // Write Entry - S32 offset = sizeof(EntriesInfo) + idx * sizeof(Entry); - LLAPRFile* aprfile = openHeaderEntriesFile(false, offset); - S32 bytes_written = aprfile->write((void*)&entry, (S32)sizeof(Entry)); - llassert_always(bytes_written == sizeof(Entry)); - mHeaderEntriesMaxWriteIdx = llmax(mHeaderEntriesMaxWriteIdx, idx); - closeHeaderEntriesFile(); + entry.mID = id ; + entry.mImageSize = -1 ; //mark it is a brand-new entry. + entry.mBodySize = 0 ; } } } @@ -1044,44 +1073,153 @@ S32 LLTextureCache::openAndReadEntry(const LLUUID& id, Entry& entry, bool create // Remove this entry from the LRU if it exists mLRU.erase(id); // Read the entry - S32 offset = sizeof(EntriesInfo) + idx * sizeof(Entry); - LLAPRFile* aprfile = openHeaderEntriesFile(true, offset); - S32 bytes_read = aprfile->read((void*)&entry, (S32)sizeof(Entry)); - llassert_always(bytes_read == sizeof(Entry)); - llassert_always(entry.mImageSize == 0 || entry.mImageSize == -1 || entry.mImageSize > entry.mBodySize); - closeHeaderEntriesFile(); + idx_entry_map_t::iterator iter = mUpdatedEntryMap.find(idx) ; + if(iter != mUpdatedEntryMap.end()) + { + entry = iter->second ; + } + else + { + readEntryFromHeaderImmediately(idx, entry) ; + } + if(entry.mImageSize <= entry.mBodySize)//it happens on 64-bit systems, do not know why + { + llwarns << "corrupted entry: " << id << " entry image size: " << entry.mImageSize << " entry body size: " << entry.mBodySize << llendl ; + + //erase this entry and the cached texture from the cache. + std::string tex_filename = getTextureFileName(id); + removeEntry(idx, entry, tex_filename) ; + mUpdatedEntryMap.erase(idx) ; + idx = -1 ; + } } return idx; } -void LLTextureCache::writeEntryAndClose(S32 idx, Entry& entry) +//mHeaderMutex is locked before calling this. +void LLTextureCache::writeEntryToHeaderImmediately(S32& idx, Entry& entry, bool write_header) +{ + LLAPRFile* aprfile ; + S32 bytes_written ; + S32 offset = sizeof(EntriesInfo) + idx * sizeof(Entry); + if(write_header) + { + aprfile = openHeaderEntriesFile(false, 0); + bytes_written = aprfile->write((U8*)&mHeaderEntriesInfo, sizeof(EntriesInfo)) ; + if(bytes_written != sizeof(EntriesInfo)) + { + clearCorruptedCache() ; //clear the cache. + idx = -1 ;//mark the idx invalid. + return ; + } + + mHeaderAPRFile->seek(APR_SET, offset); + } + else + { + aprfile = openHeaderEntriesFile(false, offset); + } + bytes_written = aprfile->write((void*)&entry, (S32)sizeof(Entry)); + if(bytes_written != sizeof(Entry)) + { + clearCorruptedCache() ; //clear the cache. + idx = -1 ;//mark the idx invalid. + + return ; + } + + closeHeaderEntriesFile(); + mUpdatedEntryMap.erase(idx) ; +} + +//mHeaderMutex is locked before calling this. +void LLTextureCache::readEntryFromHeaderImmediately(S32& idx, Entry& entry) +{ + S32 offset = sizeof(EntriesInfo) + idx * sizeof(Entry); + LLAPRFile* aprfile = openHeaderEntriesFile(true, offset); + S32 bytes_read = aprfile->read((void*)&entry, (S32)sizeof(Entry)); + closeHeaderEntriesFile(); + + if(bytes_read != sizeof(Entry)) + { + clearCorruptedCache() ; //clear the cache. + idx = -1 ;//mark the idx invalid. + } +} + +//mHeaderMutex is locked before calling this. +//update an existing entry time stamp, delay writing. +void LLTextureCache::updateEntryTimeStamp(S32 idx, Entry& entry) { + static const U32 MAX_ENTRIES_WITHOUT_TIME_STAMP = (U32)(LLTextureCache::sCacheMaxEntries * 0.75f) ; + + if(mHeaderEntriesInfo.mEntries < MAX_ENTRIES_WITHOUT_TIME_STAMP) + { + return ; //there are enough empty entry index space, no need to stamp time. + } + if (idx >= 0) { if (!mReadOnly) { - entry.mTime = time(NULL); - if(entry.mImageSize < entry.mBodySize) - { - // Just say no, due to my messing around to cache discards other than 0 we can end up here - // after recalling an image from cache at a lower discard than cached. RC - return; - } + entry.mTime = time(NULL); + mUpdatedEntryMap[idx] = entry ; + } + } +} - llassert_always(entry.mImageSize == 0 || entry.mImageSize == -1 || entry.mImageSize > entry.mBodySize); - if (entry.mBodySize > 0) - { - mTexturesSizeMap[entry.mID] = entry.mBodySize; - } -// llinfos << "Updating TE: " << idx << ": " << id << " Size: " << entry.mBodySize << " Time: " << entry.mTime << llendl; - S32 offset = sizeof(EntriesInfo) + idx * sizeof(Entry); - LLAPRFile* aprfile = openHeaderEntriesFile(false, offset); - S32 bytes_written = aprfile->write((void*)&entry, (S32)sizeof(Entry)); - llassert_always(bytes_written == sizeof(Entry)); - mHeaderEntriesMaxWriteIdx = llmax(mHeaderEntriesMaxWriteIdx, idx); - closeHeaderEntriesFile(); +//update an existing entry, write to header file immediately. +bool LLTextureCache::updateEntry(S32& idx, Entry& entry, S32 new_image_size, S32 new_data_size) +{ + S32 new_body_size = llmax(0, new_data_size - TEXTURE_CACHE_ENTRY_SIZE) ; + + if(new_image_size == entry.mImageSize && new_body_size == entry.mBodySize) + { + return true ; //nothing changed. + } + else + { + bool purge = false ; + + lockHeaders() ; + + bool update_header = false ; + if(entry.mImageSize < 0) //is a brand-new entry + { + mHeaderIDMap[entry.mID] = idx; + mTexturesSizeMap[entry.mID] = new_body_size ; + mTexturesSizeTotal += new_body_size ; + + // Update Header + update_header = true ; + } + else if (entry.mBodySize != new_body_size) + { + //already in mHeaderIDMap. + mTexturesSizeMap[entry.mID] = new_body_size ; + mTexturesSizeTotal -= entry.mBodySize ; + mTexturesSizeTotal += new_body_size ; + } + entry.mTime = time(NULL); + entry.mImageSize = new_image_size ; + entry.mBodySize = new_body_size ; + + writeEntryToHeaderImmediately(idx, entry, update_header) ; + + if (mTexturesSizeTotal > sCacheMaxTexturesSize) + { + purge = true; + } + + unlockHeaders() ; + + if (purge) + { + mDoPurge = TRUE; } } + + return false ; } U32 LLTextureCache::openAndReadEntries(std::vector& entries) @@ -1093,7 +1231,21 @@ U32 LLTextureCache::openAndReadEntries(std::vector& entries) mFreeList.clear(); mTexturesSizeTotal = 0; - LLAPRFile* aprfile = openHeaderEntriesFile(false, (S32)sizeof(EntriesInfo)); + LLAPRFile* aprfile = NULL; + if(mUpdatedEntryMap.empty()) + { + aprfile = openHeaderEntriesFile(true, (S32)sizeof(EntriesInfo)); + } + else //update the header file first. + { + aprfile = openHeaderEntriesFile(false, 0); + updatedHeaderEntriesFile() ; + if(!aprfile) + { + return 0; + } + aprfile->seek(APR_SET, (S32)sizeof(EntriesInfo)); + } for (U32 idx=0; idx& entries) } entries.push_back(entry); // llinfos << "ENTRY: " << entry.mTime << " TEX: " << entry.mID << " IDX: " << idx << " Size: " << entry.mImageSize << llendl; - if (entry.mImageSize < 0) + if(entry.mImageSize > entry.mBodySize) { - mFreeList.insert(idx); + mHeaderIDMap[entry.mID] = idx; + mTexturesSizeMap[entry.mID] = entry.mBodySize; + mTexturesSizeTotal += entry.mBodySize; } else { - mHeaderIDMap[entry.mID] = idx; - if (entry.mBodySize > 0) - { - mTexturesSizeMap[entry.mID] = entry.mBodySize; - mTexturesSizeTotal += entry.mBodySize; - } - llassert_always(entry.mImageSize == 0 || entry.mImageSize > entry.mBodySize); + mFreeList.insert(idx); } } closeHeaderEntriesFile(); @@ -1137,13 +1285,65 @@ void LLTextureCache::writeEntriesAndClose(const std::vector& entries) for (S32 idx=0; idxwrite((void*)(&entries[idx]), (S32)sizeof(Entry)); - llassert_always(bytes_written == sizeof(Entry)); + if(bytes_written != sizeof(Entry)) + { + clearCorruptedCache() ; //clear the cache. + return ; + } } - mHeaderEntriesMaxWriteIdx = llmax(mHeaderEntriesMaxWriteIdx, num_entries-1); closeHeaderEntriesFile(); } } +void LLTextureCache::writeUpdatedEntries() +{ + lockHeaders() ; + if (!mReadOnly && !mUpdatedEntryMap.empty()) + { + openHeaderEntriesFile(false, 0); + updatedHeaderEntriesFile() ; + closeHeaderEntriesFile(); + } + unlockHeaders() ; +} + +//mHeaderMutex is locked and mHeaderAPRFile is created before calling this. +void LLTextureCache::updatedHeaderEntriesFile() +{ + if (!mReadOnly && !mUpdatedEntryMap.empty() && mHeaderAPRFile) + { + //entriesInfo + mHeaderAPRFile->seek(APR_SET, 0); + S32 bytes_written = mHeaderAPRFile->write((U8*)&mHeaderEntriesInfo, sizeof(EntriesInfo)) ; + if(bytes_written != sizeof(EntriesInfo)) + { + clearCorruptedCache() ; //clear the cache. + return ; + } + + //write each updated entry + S32 entry_size = (S32)sizeof(Entry) ; + S32 prev_idx = -1 ; + S32 delta_idx ; + for (idx_entry_map_t::iterator iter = mUpdatedEntryMap.begin(); iter != mUpdatedEntryMap.end(); ++iter) + { + delta_idx = iter->first - prev_idx - 1; + prev_idx = iter->first ; + if(delta_idx) + { + mHeaderAPRFile->seek(APR_CUR, delta_idx * entry_size); + } + + bytes_written = mHeaderAPRFile->write((void*)(&iter->second), entry_size); + if(bytes_written != entry_size) + { + clearCorruptedCache() ; //clear the cache. + return ; + } + } + mUpdatedEntryMap.clear() ; + } +} //---------------------------------------------------------------------------- // Called from either the main thread or the worker thread @@ -1169,28 +1369,27 @@ void LLTextureCache::readHeaderCache() if (num_entries) { U32 empty_entries = 0; - typedef std::pair lru_data_t; + typedef std::pair lru_data_t; std::set lru; - std::vector purge_list; + std::set purge_list; for (U32 i=0; i 0) { if (entry.mBodySize > entry.mImageSize) { // Shouldn't happen, failsafe only - llwarns << "Bad entry: " << i << ": " << id << ": BodySize: " << entry.mBodySize << llendl; - purge_list.push_back(id); + llwarns << "Bad entry: " << i << ": " << entry.mID << ": BodySize: " << entry.mBodySize << llendl; + purge_list.insert(i); } } } @@ -1200,22 +1399,24 @@ void LLTextureCache::readHeaderCache() // Special case: cache size was reduced, need to remove entries // Note: After we prune entries, we will call this again and create the LRU U32 entries_to_purge = (num_entries-empty_entries) - sCacheMaxEntries; + llinfos << "Texture Cache Entries: " << num_entries << " Max: " << sCacheMaxEntries << " Empty: " << empty_entries << " Purging: " << entries_to_purge << llendl; if (entries_to_purge > 0) { for (std::set::iterator iter = lru.begin(); iter != lru.end(); ++iter) { - purge_list.push_back(iter->second); - if (--entries_to_purge <= 0) + purge_list.insert(iter->second); + if (purge_list.size() >= entries_to_purge) break; } } + llassert_always(purge_list.size() >= entries_to_purge); } else { S32 lru_entries = (S32)((F32)sCacheMaxEntries * TEXTURE_CACHE_LRU_SIZE); for (std::set::iterator iter = lru.begin(); iter != lru.end(); ++iter) { - mLRU.insert(iter->second); + mLRU.insert(entries[iter->second].mID); // llinfos << "LRU: " << iter->first << " : " << iter->second << llendl; if (--lru_entries <= 0) break; @@ -1224,11 +1425,10 @@ void LLTextureCache::readHeaderCache() if (purge_list.size() > 0) { - for (std::vector::iterator iter = purge_list.begin(); iter != purge_list.end(); ++iter) + for (std::set::iterator iter = purge_list.begin(); iter != purge_list.end(); ++iter) { - mHeaderMutex.unlock(); - removeFromCache(*iter); - mHeaderMutex.lock(); + std::string tex_filename = getTextureFileName(entries[*iter].mID); + removeEntry((S32)*iter, entries[*iter], tex_filename); } // If we removed any entries, we need to rebuild the entries list, // write the header, and call this again @@ -1236,13 +1436,14 @@ void LLTextureCache::readHeaderCache() for (U32 i=0; i=0) + if (entry.mImageSize > 0) { new_entries.push_back(entry); } } llassert_always(new_entries.size() <= sCacheMaxEntries); mHeaderEntriesInfo.mEntries = new_entries.size(); + writeEntriesHeader(); writeEntriesAndClose(new_entries); mHeaderMutex.unlock(); // unlock the mutex before calling again readHeaderCache(); // repeat with new entries file @@ -1250,7 +1451,7 @@ void LLTextureCache::readHeaderCache() } else { - writeEntriesAndClose(entries); + //entries are not changed, nothing here. } } } @@ -1259,6 +1460,29 @@ void LLTextureCache::readHeaderCache() ////////////////////////////////////////////////////////////////////////////// +//the header mutex is locked before calling this. +void LLTextureCache::clearCorruptedCache() +{ + llwarns << "the texture cache is corrupted, need to be cleared." << llendl ; + + closeHeaderEntriesFile();//close possible file handler + purgeAllTextures(false) ; //clear the cache. + + if (!mReadOnly) //regenerate the directory tree if not exists. + { + LLFile::mkdir(mTexturesDirName); + + const char* subdirs = "0123456789abcdef"; + for (S32 i=0; i<16; i++) + { + std::string dirname = mTexturesDirName + gDirUtilp->getDirDelimiter() + subdirs[i]; + LLFile::mkdir(dirname); + } + } + + return ; +} + void LLTextureCache::purgeAllTextures(bool purge_directories) { if (!mReadOnly) @@ -1269,6 +1493,7 @@ void LLTextureCache::purgeAllTextures(bool purge_directories) for (S32 i=0; i<16; i++) { std::string dirname = mTexturesDirName + delem + subdirs[i]; + llinfos << "Deleting files in directory: " << dirname << llendl; gDirUtilp->deleteFilesInDir(dirname,mask); if (purge_directories) { @@ -1277,19 +1502,23 @@ void LLTextureCache::purgeAllTextures(bool purge_directories) } if (purge_directories) { + gDirUtilp->deleteFilesInDir(mTexturesDirName, mask); LLFile::rmdir(mTexturesDirName); - } + } } mHeaderIDMap.clear(); mTexturesSizeMap.clear(); mTexturesSizeTotal = 0; mFreeList.clear(); mTexturesSizeTotal = 0; + mUpdatedEntryMap.clear(); // Info with 0 entries mHeaderEntriesInfo.mVersion = sHeaderCacheVersion; mHeaderEntriesInfo.mEntries = 0; writeEntriesHeader(); + + llinfos << "The entire texture cache is cleared." << llendl ; } void LLTextureCache::purgeTextures(bool validate) @@ -1314,7 +1543,6 @@ void LLTextureCache::purgeTextures(bool validate) U32 num_entries = openAndReadEntries(entries); if (!num_entries) { - writeEntriesAndClose(entries); return; // nothing to purge } @@ -1333,6 +1561,10 @@ void LLTextureCache::purgeTextures(bool validate) time_idx_set.insert(std::make_pair(entries[idx].mTime, idx)); // llinfos << "TIME: " << entries[idx].mTime << " TEX: " << entries[idx].mID << " IDX: " << idx << " Size: " << entries[idx].mImageSize << llendl; } + else + { + llerrs << "mTexturesSizeMap / mHeaderIDMap corrupted." << llendl ; + } } } @@ -1384,11 +1616,8 @@ void LLTextureCache::purgeTextures(bool validate) { purge_count++; LL_DEBUGS("TextureCache") << "PURGING: " << filename << LL_ENDL; - LLAPRFile::remove(filename); + removeEntry(idx, entries[idx], filename) ; cache_size -= entries[idx].mBodySize; - mTexturesSizeTotal -= entries[idx].mBodySize; - entries[idx].mBodySize = 0; - mTexturesSizeMap.erase(entries[idx].mID); } } @@ -1396,11 +1625,8 @@ void LLTextureCache::purgeTextures(bool validate) writeEntriesAndClose(entries); - if (!mThreaded) - { - // *FIX:Mani - watchdog back on. - LLAppViewer::instance()->resumeMainloopTimeout(); - } + // *FIX:Mani - watchdog back on. + LLAppViewer::instance()->resumeMainloopTimeout(); LL_INFOS("TextureCache") << "TEXTURE CACHE:" << " PURGED: " << purge_count @@ -1438,40 +1664,38 @@ LLTextureCacheWorker* LLTextureCache::getWriter(handle_t handle) // Called from work thread // Reads imagesize from the header, updates timestamp -S32 LLTextureCache::getHeaderCacheEntry(const LLUUID& id, S32& imagesize) +S32 LLTextureCache::getHeaderCacheEntry(const LLUUID& id, Entry& entry) { - LLMutexLock lock(&mHeaderMutex); - Entry entry; + LLMutexLock lock(&mHeaderMutex); S32 idx = openAndReadEntry(id, entry, false); if (idx >= 0) - { - imagesize = entry.mImageSize; - writeEntryAndClose(idx, entry); // updates time + { + updateEntryTimeStamp(idx, entry); // updates time } return idx; } // Writes imagesize to the header, updates timestamp -S32 LLTextureCache::setHeaderCacheEntry(const LLUUID& id, S32 imagesize) +S32 LLTextureCache::setHeaderCacheEntry(const LLUUID& id, Entry& entry, S32 imagesize, S32 datasize) { mHeaderMutex.lock(); - llassert_always(imagesize >= 0); - Entry entry; S32 idx = openAndReadEntry(id, entry, true); + mHeaderMutex.unlock(); + if (idx >= 0) { - entry.mImageSize = imagesize; - writeEntryAndClose(idx, entry); - mHeaderMutex.unlock(); + updateEntry(idx, entry, imagesize, datasize); } - else // retry + + if(idx < 0) // retry { - mHeaderMutex.unlock(); readHeaderCache(); // We couldn't write an entry, so refresh the LRU + mHeaderMutex.lock(); llassert_always(!mLRU.empty() || mHeaderEntriesInfo.mEntries < sCacheMaxEntries); mHeaderMutex.unlock(); - idx = setHeaderCacheEntry(id, imagesize); // assert above ensures no inf. recursion + + idx = setHeaderCacheEntry(id, entry, imagesize, datasize); // assert above ensures no inf. recursion } return idx; } @@ -1508,39 +1732,34 @@ LLTextureCache::handle_t LLTextureCache::readFromCache(const LLUUID& id, U32 pri return handle; } -// Return true if the handle is not valid, which is the case -// when the worker was already deleted or is scheduled for deletion. -// -// If the handle exists and a call to worker->complete() returns -// true or abort is true, then the handle is removed and the worker -// scheduled for deletion. + bool LLTextureCache::readComplete(handle_t handle, bool abort) { - lockWorkers(); // Needed for access to mReaders. - + lockWorkers(); handle_map_t::iterator iter = mReaders.find(handle); - bool handle_is_valid = iter != mReaders.end(); - llassert_always(handle_is_valid || abort); LLTextureCacheWorker* worker = NULL; - bool delete_worker = false; - - if (handle_is_valid) + bool complete = false; + if (iter != mReaders.end()) { worker = iter->second; - delete_worker = worker->complete() || abort; - if (delete_worker) + complete = worker->complete(); + + if(!complete && abort) { - mReaders.erase(handle); - handle_is_valid = false; + abortRequest(handle, true) ; } } - - unlockWorkers(); - - if (delete_worker) worker->scheduleDelete(); - - // Return false if the handle is (still) valid. - return !handle_is_valid; + if (worker && (complete || abort)) + { + mReaders.erase(iter); + unlockWorkers(); + worker->scheduleDelete(); + } + else + { + unlockWorkers(); + } + return (complete || abort); } LLTextureCache::handle_t LLTextureCache::writeToCache(const LLUUID& id, U32 priority, @@ -1573,20 +1792,20 @@ bool LLTextureCache::writeComplete(handle_t handle, bool abort) { lockWorkers(); handle_map_t::iterator iter = mWriters.find(handle); - llassert_always(iter != mWriters.end()); - LLTextureCacheWorker* worker = iter->second; - if (worker->complete() || abort) - { - mWriters.erase(handle); - unlockWorkers(); - worker->scheduleDelete(); - return true; - } - else + llassert(iter != mWriters.end()); + if (iter != mWriters.end()) { - unlockWorkers(); - return false; + LLTextureCacheWorker* worker = iter->second; + if (worker->complete() || abort) + { + mWriters.erase(handle); + unlockWorkers(); + worker->scheduleDelete(); + return true; + } } + unlockWorkers(); + return false; } void LLTextureCache::prioritizeWrite(handle_t handle) @@ -1605,38 +1824,56 @@ void LLTextureCache::addCompleted(Responder* responder, bool success) ////////////////////////////////////////////////////////////////////////////// -// Called from MAIN thread (endWork()) +//called after mHeaderMutex is locked. +void LLTextureCache::removeCachedTexture(const LLUUID& id) +{ + if(mTexturesSizeMap.find(id) != mTexturesSizeMap.end()) + { + mTexturesSizeTotal -= mTexturesSizeMap[id] ; + mTexturesSizeMap.erase(id); + } + mHeaderIDMap.erase(id); + LLAPRFile::remove(getTextureFileName(id)); +} -bool LLTextureCache::removeHeaderCacheEntry(const LLUUID& id) +//called after mHeaderMutex is locked. +void LLTextureCache::removeEntry(S32 idx, Entry& entry, std::string& filename) { - if (!mReadOnly) + if(idx >= 0) //valid entry { - LLMutexLock lock(&mHeaderMutex); - Entry entry; - S32 idx = openAndReadEntry(id, entry, false); - if (idx >= 0) - { - entry.mImageSize = -1; - entry.mBodySize = 0; - writeEntryAndClose(idx, entry); - mFreeList.insert(idx); - mHeaderIDMap.erase(id); - mTexturesSizeMap.erase(id); - return true; - } + entry.mImageSize = -1; + entry.mBodySize = 0; + mHeaderIDMap.erase(entry.mID); + mTexturesSizeMap.erase(entry.mID); + + mTexturesSizeTotal -= entry.mBodySize; + mFreeList.insert(idx); } - return false; + + LLAPRFile::remove(filename); } -void LLTextureCache::removeFromCache(const LLUUID& id) +bool LLTextureCache::removeFromCache(const LLUUID& id) { //llwarns << "Removing texture from cache: " << id << llendl; + bool ret = false ; if (!mReadOnly) { - removeHeaderCacheEntry(id); - LLMutexLock lock(&mHeaderMutex); - LLAPRFile::remove(getTextureFileName(id)); + lockHeaders() ; + + Entry entry; + S32 idx = openAndReadEntry(id, entry, false); + std::string tex_filename = getTextureFileName(id); + removeEntry(idx, entry, tex_filename) ; + if (idx >= 0) + { + writeEntryToHeaderImmediately(idx, entry); + ret = true; + } + + unlockHeaders() ; } + return ret ; } ////////////////////////////////////////////////////////////////////////////// diff --git a/linden/indra/newview/lltexturecache.h b/linden/indra/newview/lltexturecache.h index 45804c2..f80be00 100644 --- a/linden/indra/newview/lltexturecache.h +++ b/linden/indra/newview/lltexturecache.h @@ -40,6 +40,7 @@ #include "llworkerthread.h" +class LLImageFormatted; class LLTextureCacheWorker; class LLTextureCache : public LLWorkerThread @@ -58,10 +59,16 @@ private: }; struct Entry { - Entry() {} + Entry() : + mBodySize(0), + mImageSize(0), + mTime(0) + { + } Entry(const LLUUID& id, S32 imagesize, S32 bodysize, U32 time) : mID(id), mImageSize(imagesize), mBodySize(bodysize), mTime(time) {} void init(const LLUUID& id, U32 time) { mID = id, mImageSize = 0; mBodySize = 0; mTime = time; } + Entry& operator=(const Entry& entry) {mID = entry.mID, mImageSize = entry.mImageSize; mBodySize = entry.mBodySize; mTime = entry.mTime; return *this;} LLUUID mID; // 16 bytes S32 mImageSize; // total size of image if known S32 mBodySize; // size of body file in body cache @@ -103,7 +110,8 @@ public: /*virtual*/ S32 update(U32 max_time_ms); void purgeCache(ELLPath location); - S64 initCache(ELLPath location, S64 maxsize, BOOL read_only); + void setReadOnly(BOOL read_only) ; + S64 initCache(ELLPath location, S64 maxsize, BOOL disable_texture_cache); handle_t readFromCache(const std::string& local_filename, const LLUUID& id, U32 priority, S32 offset, S32 size, ReadResponder* responder); @@ -116,7 +124,7 @@ public: bool writeComplete(handle_t handle, bool abort = false); void prioritizeWrite(handle_t handle); - void removeFromCache(const LLUUID& id); + bool removeFromCache(const LLUUID& id); // For LLTextureCacheWorker::Responder LLTextureCacheWorker* getReader(handle_t handle); @@ -131,10 +139,11 @@ public: S64 getMaxUsage() { return sCacheMaxTexturesSize; } U32 getEntries() { return mHeaderEntriesInfo.mEntries; } U32 getMaxEntries() { return sCacheMaxEntries; }; + BOOL isInCache(const LLUUID& id) ; + BOOL isInLocal(const LLUUID& id) ; protected: // Accessed by LLTextureCacheWorker - bool updateTextureEntryList(const LLUUID& id, S32 size); std::string getLocalFileName(const LLUUID& id); std::string getTextureFileName(const LLUUID& id); void addCompleted(Responder* responder, bool success); @@ -145,6 +154,7 @@ protected: private: void setDirNames(ELLPath location); void readHeaderCache(); + void clearCorruptedCache(); void purgeAllTextures(bool purge_directories); void purgeTextures(bool validate); LLAPRFile* openHeaderEntriesFile(bool readonly, S32 offset); @@ -152,12 +162,20 @@ private: void readEntriesHeader(); void writeEntriesHeader(); S32 openAndReadEntry(const LLUUID& id, Entry& entry, bool create); - void writeEntryAndClose(S32 idx, Entry& entry); + bool updateEntry(S32& idx, Entry& entry, S32 new_image_size, S32 new_body_size); + void updateEntryTimeStamp(S32 idx, Entry& entry) ; U32 openAndReadEntries(std::vector& entries); void writeEntriesAndClose(const std::vector& entries); - S32 getHeaderCacheEntry(const LLUUID& id, S32& imagesize); - S32 setHeaderCacheEntry(const LLUUID& id, S32 imagesize); - bool removeHeaderCacheEntry(const LLUUID& id); + void readEntryFromHeaderImmediately(S32& idx, Entry& entry) ; + void writeEntryToHeaderImmediately(S32& idx, Entry& entry, bool write_header = false) ; + void removeEntry(S32 idx, Entry& entry, std::string& filename); + void removeCachedTexture(const LLUUID& id) ; + S32 getHeaderCacheEntry(const LLUUID& id, Entry& entry); + S32 setHeaderCacheEntry(const LLUUID& id, Entry& entry, S32 imagesize, S32 datasize); + void writeUpdatedEntries() ; + void updatedHeaderEntriesFile() ; + void lockHeaders() { mHeaderMutex.lock(); } + void unlockHeaders() { mHeaderMutex.unlock(); } private: // Internal @@ -194,6 +212,9 @@ private: S64 mTexturesSizeTotal; LLAtomic32 mDoPurge; + typedef std::map idx_entry_map_t; + idx_entry_map_t mUpdatedEntryMap; + // Statics static F32 sHeaderCacheVersion; static U32 sCacheMaxEntries; diff --git a/linden/indra/newview/lltexturefetch.cpp b/linden/indra/newview/lltexturefetch.cpp index 2184478..427e55a 100644 --- a/linden/indra/newview/lltexturefetch.cpp +++ b/linden/indra/newview/lltexturefetch.cpp @@ -1500,6 +1500,8 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image LLTextureFetch::~LLTextureFetch() { + clearDeleteList() ; + // ~LLQueuedThread() called here } -- cgit v1.1