More sanity checks in OpenJPEG decodes for #355

Patch by Carjay McGinnis from VWR-1815. This fixes decoding
lossless and small textures where the viewer's assumption
about compressed size breaks (i.e. discard level only half
there), and also encoding small lossy textures.

This relys on a patched OpenJPEG to work
(see openjpeg_top_corner_fix.diff from VWR-1815.)

1 files changed, 35 insertions, 10 deletions

diff --git a/linden/indra/llimagej2coj/llimagej2coj.cpp b/linden/indra/llimagej2coj/llimagej2coj.cpp
index b7a1b82..829dceb 100644
--- a/linden/indra/llimagej2coj/llimagej2coj.cpp
+++ b/linden/indra/llimagej2coj/llimagej2coj.cpp
@@ -152,8 +152,13 @@ BOOL LLImageJ2COJ::decodeImpl(LLImageJ2C &base, LLImageRaw &raw_image, F32 decod
         /* open a byte stream */
         cio = opj_cio_open((opj_common_ptr)dinfo, base.getData(), base.getDataSize());
 
-        /* decode the stream and fill the image structure */
-        image = opj_decode(dinfo, cio);
+        /* decode the stream and fill the image structure, also fill in an additional
+           structure to get the decoding result. This structure is a bit unusual in that
+           it is not received through opj, but still has some dynamically allocated fields
+           that need to be cleared up at the end by calling a destroy function. */
+        opj_codestream_info_t cinfo;
+        memset(&cinfo, 0, sizeof(opj_codestream_info_t));
+        image = opj_decode_with_info(dinfo, cio, &cinfo);
 
         /* close the byte stream */
         opj_cio_close(cio);
@@ -180,6 +185,7 @@ BOOL LLImageJ2COJ::decodeImpl(LLImageJ2C &base, LLImageRaw &raw_image, F32 decod
                 LL_DEBUGS("Openjpeg") << "ERROR -> decodeImpl: failed to decode image wrong number of components: " << img_components << LL_ENDL;
                 if (image)
                 {
+                        opj_destroy_cstr_info(&cinfo);
                         opj_image_destroy(image);
                 }
 
@@ -187,23 +193,40 @@ BOOL LLImageJ2COJ::decodeImpl(LLImageJ2C &base, LLImageRaw &raw_image, F32 decod
         }
 
         // sometimes we get bad data out of the cache - check to see if the decode succeeded
-        for (S32 i = 0; i < img_components; i++)
+        int decompdifference = 0;
+        if (cinfo.numdecompos) // sanity
         {
-                if (image->comps[i].factor != base.getRawDiscardLevel())
+                for (int comp = 0; comp < image->numcomps; comp++)
+                {       /* get maximum decomposition level difference, first field is from the COD header and the second
+                           is what is actually met in the codestream, NB: if everything was ok, this calculation will
+                           return what was set in the cp_reduce value! */
+                        decompdifference = std::max(decompdifference, cinfo.numdecompos[comp] - image->comps[comp].resno_decoded);
+                }
+                if (decompdifference < 0) // sanity
                 {
-                        // if we didn't get the discard level we're expecting, fail
-                        if (image) //anyway somthing odd with the image, better check than crash
-                                opj_image_destroy(image);
-                        base.mDecoding = FALSE;
-                        return TRUE;
+                        decompdifference = 0;
                 }
         }
         
+
+        /* if OpenJPEG failed to decode all requested decomposition levels
+           the difference will be greater than this level */
+        if (decompdifference > base.getRawDiscardLevel())
+        {
+                llwarns << "not enough data for requested discard level, setting mDecoding to FALSE, difference: " << (decompdifference - base.getRawDiscardLevel()) << llendl;
+                opj_destroy_cstr_info(&cinfo);
+                opj_image_destroy(image);
+                base.mDecoding = FALSE;
+                return TRUE;
+        }
+
         if(img_components <= first_channel)
         {
+                // sanity
                 LL_DEBUGS("Openjpeg") << "trying to decode more channels than are present in image: numcomps: " << img_components << " first_channel: " << first_channel << LL_ENDL;
                 if (image)
                 {
+                        opj_destroy_cstr_info(&cinfo);
                         opj_image_destroy(image);
                 }
                         
@@ -252,15 +275,17 @@ BOOL LLImageJ2COJ::decodeImpl(LLImageJ2C &base, LLImageRaw &raw_image, F32 decod
                 else // Some rare OpenJPEG versions have this bug.
                 {
                         llwarns << "ERROR -> decodeImpl: failed to decode image! (NULL comp data - OpenJPEG bug)" << llendl;
+                        opj_destroy_cstr_info(&cinfo);
                         opj_image_destroy(image);
 
                         return TRUE; // done
                 }
         }
 
-        /* free image data structure */
+        /* free opj data structures */
         if (image)
         {
+                opj_destroy_cstr_info(&cinfo);
                 opj_image_destroy(image);
         }