#4296 Crash at renderMorphMasks

3 files changed, 50 insertions, 19 deletions

diff --git a/indra/llappearance/lltexlayer.cpp b/indra/llappearance/lltexlayer.cpp
index aa48a2d621..b3800e6981 100644
--- a/indra/llappearance/lltexlayer.cpp
+++ b/indra/llappearance/lltexlayer.cpp
@@ -1293,7 +1293,7 @@ void LLTexLayer::renderMorphMasks(S32 x, S32 y, S32 width, S32 height, const LLC
 {
     if (!force_render && !hasMorph())
     {
-        LL_DEBUGS() << "skipping renderMorphMasks for " << getUUID() << LL_ENDL;
+        LL_DEBUGS("Morph") << "skipping renderMorphMasks for " << getUUID() << LL_ENDL;
         return;
     }
     LL_PROFILE_ZONE_SCOPED;
@@ -1325,7 +1325,7 @@ void LLTexLayer::renderMorphMasks(S32 x, S32 y, S32 width, S32 height, const LLC
         success &= param->render( x, y, width, height );
         if (!success && !force_render)
         {
-            LL_DEBUGS() << "Failed to render param " << param->getID() << " ; skipping morph mask." << LL_ENDL;
+            LL_DEBUGS("Morph") << "Failed to render param " << param->getID() << " ; skipping morph mask." << LL_ENDL;
             return;
         }
     }
@@ -1365,7 +1365,7 @@ void LLTexLayer::renderMorphMasks(S32 x, S32 y, S32 width, S32 height, const LLC
             }
             else
             {
-                LL_WARNS() << "Skipping rendering of " << getInfo()->mStaticImageFileName
+                LL_WARNS("Morph") << "Skipping rendering of " << getInfo()->mStaticImageFileName
                         << "; expected 1 or 4 components." << LL_ENDL;
             }
         }
@@ -1404,8 +1404,8 @@ void LLTexLayer::renderMorphMasks(S32 x, S32 y, S32 width, S32 height, const LLC
                 // We can get bad morph masks during login, on minimize, and occasional gl errors.
                 // We should only be doing this when we believe something has changed with respect to the user's appearance.
         {
-                       LL_DEBUGS("Avatar") << "gl alpha cache of morph mask not found, doing readback: " << getName() << LL_ENDL;
-                        // clear out a slot if we have filled our cache
+            LL_DEBUGS("Morph") << "gl alpha cache of morph mask not found, doing readback: " << getName() << LL_ENDL;
+            // clear out a slot if we have filled our cache
             S32 max_cache_entries = getTexLayerSet()->getAvatarAppearance()->isSelf() ? 4 : 1;
             while ((S32)mAlphaCache.size() >= max_cache_entries)
             {
@@ -1444,13 +1444,20 @@ void LLTexLayer::renderMorphMasks(S32 x, S32 y, S32 width, S32 height, const LLC
                     }
 
                     glGetTexImage(LLTexUnit::getInternalType(LLTexUnit::TT_TEXTURE), 0, GL_RGBA, GL_UNSIGNED_BYTE, temp);
-
-                    U8* alpha_cursor = alpha_data;
-                    U8* pixel        = temp;
-                    for (int i = 0; i < pixels; i++)
+                    GLenum error = glGetError();
+                    if (error != GL_NO_ERROR)
+                    {
+                        LL_INFOS("Morph") << "GL Error while reading back morph texture. Error code: " << error << LL_ENDL;
+                    }
+                    else
                     {
-                        *alpha_cursor++ = pixel[3];
-                        pixel += 4;
+                        U8* alpha_cursor = alpha_data;
+                        U8* pixel = temp;
+                        for (int i = 0; i < pixels; i++)
+                        {
+                            *alpha_cursor++ = pixel[3];
+                            pixel += 4;
+                        }
                     }
 
                     gGL.getTexUnit(0)->disable();
diff --git a/indra/llrender/llimagegl.cpp b/indra/llrender/llimagegl.cpp
index 3f8903ca09..1db36d91f9 100644
--- a/indra/llrender/llimagegl.cpp
+++ b/indra/llrender/llimagegl.cpp
@@ -1870,8 +1870,17 @@ bool LLImageGL::readBackRaw(S32 discard_level, LLImageRaw* imageraw, bool compre
         glGetTexLevelParameteriv(mTarget, gl_discard, GL_TEXTURE_COMPRESSED_IMAGE_SIZE, (GLint*)&glbytes);
         if(!imageraw->allocateDataSize(width, height, ncomponents, glbytes))
         {
-            LL_WARNS() << "Memory allocation failed for reading back texture. Size is: " << glbytes << LL_ENDL ;
-            LL_WARNS() << "width: " << width << "height: " << height << "components: " << ncomponents << LL_ENDL ;
+            constexpr S64 MAX_GL_BYTES = 2048 * 2048;
+            if (glbytes > 0 && glbytes <= MAX_GL_BYTES)
+            {
+                LLError::LLUserWarningMsg::showOutOfMemory();
+                LL_ERRS() << "Memory allocation failed for reading back texture. Data size: " << glbytes << LL_ENDL;
+            }
+            else
+            {
+                LL_WARNS() << "Memory allocation failed for reading back texture. Data size is: " << glbytes << LL_ENDL;
+                LL_WARNS() << "width: " << width << "height: " << height << "components: " << ncomponents << LL_ENDL;
+            }
             return false ;
         }
 
@@ -1882,8 +1891,18 @@ bool LLImageGL::readBackRaw(S32 discard_level, LLImageRaw* imageraw, bool compre
     {
         if(!imageraw->allocateDataSize(width, height, ncomponents))
         {
-            LL_WARNS() << "Memory allocation failed for reading back texture." << LL_ENDL ;
-            LL_WARNS() << "width: " << width << "height: " << height << "components: " << ncomponents << LL_ENDL ;
+            constexpr F32 MAX_IMAGE_SIZE = 2048 * 2048;
+            F32 size = (F32)width * (F32)height * (F32)ncomponents;
+            if (size > 0 && size <= MAX_IMAGE_SIZE)
+            {
+                LLError::LLUserWarningMsg::showOutOfMemory();
+                LL_ERRS() << "Memory allocation failed for reading back texture. Data size: " << size << LL_ENDL;
+            }
+            else
+            {
+                LL_WARNS() << "Memory allocation failed for reading back texture." << LL_ENDL;
+                LL_WARNS() << "width: " << width << "height: " << height << "components: " << ncomponents << LL_ENDL;
+            }
             return false ;
         }
 
diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp
index 9872233e98..f4b2fdfdf7 100644
--- a/indra/newview/llappviewer.cpp
+++ b/indra/newview/llappviewer.cpp
@@ -2245,10 +2245,7 @@ void errorCallback(LLError::ELevel level, const std::string &error_string)
 // Callback for LLError::LLUserWarningMsg
 void errorHandler(const std::string& title_string, const std::string& message_string, S32 code)
 {
-    if (!message_string.empty())
-    {
-        OSMessageBox(message_string, title_string.empty() ? LLTrans::getString("MBFatalError") : title_string, OSMB_OK);
-    }
+    // message is going to hang viewer, create marker first
     switch (code)
     {
     case LLError::LLUserWarningMsg::ERROR_OTHER:
@@ -2256,6 +2253,10 @@ void errorHandler(const std::string& title_string, const std::string& message_st
         break;
     case LLError::LLUserWarningMsg::ERROR_BAD_ALLOC:
         LLAppViewer::instance()->createErrorMarker(LAST_EXEC_BAD_ALLOC);
+        // When system run out of memory and errorHandler gets called from a thread,
+        // main thread might keep going while OSMessageBox freezes the caller.
+        // Todo: handle it better, but for now disconnect to avoid making things worse
+        gDisconnected = true;
         break;
     case LLError::LLUserWarningMsg::ERROR_MISSING_FILES:
         LLAppViewer::instance()->createErrorMarker(LAST_EXEC_MISSING_FILES);
@@ -2263,6 +2264,10 @@ void errorHandler(const std::string& title_string, const std::string& message_st
     default:
         break;
     }
+    if (!message_string.empty())
+    {
+        OSMessageBox(message_string, title_string.empty() ? LLTrans::getString("MBFatalError") : title_string, OSMB_OK);
+    }
 }
 
 void LLAppViewer::initLoggingAndGetLastDuration()