[gimp] plug-ins: fix #7429 Error loading PSD file: Unsupported or invalid layer mask height

[Jacob Boerema <jboerema src gnome org>]

commit b27bf5caa982597deb4cc3cd74d7674116950034
Author: Jacob Boerema <jgboerema gmail com>
Date:   Tue Oct 26 12:28:03 2021 -0400

    plug-ins: fix #7429 Error loading PSD file: Unsupported or invalid layer mask height
    
    Our psd plug-in checks for valid layer mask sizes. Apparently a psd
    layer mask where the rendered flag is set can have invalid dimensions
    0, 0, 0, -1.
    We will skip the sanity checks for rendered layers and at the same time
    move the debug statements up to have relevant debug info available even
    if our sanity checks stop further processing.

 plug-ins/file-psd/psd-load.c | 60 +++++++++++++++++++++++---------------------
 1 file changed, 32 insertions(+), 28 deletions(-)
---
diff --git a/plug-ins/file-psd/psd-load.c b/plug-ins/file-psd/psd-load.c
index 127eb6982f..f7c5487a13 100644
--- a/plug-ins/file-psd/psd-load.c
+++ b/plug-ins/file-psd/psd-load.c
@@ -870,34 +870,6 @@ read_layer_info (PSDimage      *img_a,
                     }
                 }
 
-              /* sanity checks */
-              if (lyr_a[lidx]->layer_mask.bottom < lyr_a[lidx]->layer_mask.top ||
-                  lyr_a[lidx]->layer_mask.bottom - lyr_a[lidx]->layer_mask.top > GIMP_MAX_IMAGE_SIZE)
-                {
-                  g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
-                               _("Unsupported or invalid layer mask height: %d"),
-                               lyr_a[lidx]->layer_mask.bottom - lyr_a[lidx]->layer_mask.top);
-                  return NULL;
-                }
-              if (lyr_a[lidx]->layer_mask.right < lyr_a[lidx]->layer_mask.left ||
-                  lyr_a[lidx]->layer_mask.right - lyr_a[lidx]->layer_mask.left > GIMP_MAX_IMAGE_SIZE)
-                {
-                  g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
-                               _("Unsupported or invalid layer mask width: %d"),
-                               lyr_a[lidx]->layer_mask.right - lyr_a[lidx]->layer_mask.left);
-                  return NULL;
-                }
-
-              if ((lyr_a[lidx]->layer_mask.right - lyr_a[lidx]->layer_mask.left) >
-                  G_MAXINT32 / MAX (lyr_a[lidx]->layer_mask.bottom - lyr_a[lidx]->layer_mask.top, 1))
-                {
-                  g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
-                               _("Unsupported or invalid layer mask size: %dx%d"),
-                               lyr_a[lidx]->layer_mask.right - lyr_a[lidx]->layer_mask.left,
-                               lyr_a[lidx]->layer_mask.bottom - lyr_a[lidx]->layer_mask.top);
-                  return NULL;
-                }
-
               IFDBG(2) g_debug ("Layer mask coords %d %d %d %d",
                                 lyr_a[lidx]->layer_mask.left,
                                 lyr_a[lidx]->layer_mask.top,
@@ -912,6 +884,38 @@ read_layer_info (PSDimage      *img_a,
                                 lyr_a[lidx]->layer_mask.flags,
                                 lyr_a[lidx]->layer_mask.extra_flags,
                                 lyr_a[lidx]->layer_mask.mask_params);
+
+              /* Rendered masks can have invalid dimensions: 0, 0, 0, -1 */
+              if (! lyr_a[lidx]->layer_mask.mask_flags.rendered)
+                {
+                  /* sanity checks */
+                  if (lyr_a[lidx]->layer_mask.bottom < lyr_a[lidx]->layer_mask.top ||
+                      lyr_a[lidx]->layer_mask.bottom - lyr_a[lidx]->layer_mask.top > GIMP_MAX_IMAGE_SIZE)
+                    {
+                      g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
+                                   _("Unsupported or invalid layer mask height: %d"),
+                                   lyr_a[lidx]->layer_mask.bottom - lyr_a[lidx]->layer_mask.top);
+                      return NULL;
+                    }
+                  if (lyr_a[lidx]->layer_mask.right < lyr_a[lidx]->layer_mask.left ||
+                      lyr_a[lidx]->layer_mask.right - lyr_a[lidx]->layer_mask.left > GIMP_MAX_IMAGE_SIZE)
+                    {
+                      g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
+                                   _("Unsupported or invalid layer mask width: %d"),
+                                   lyr_a[lidx]->layer_mask.right - lyr_a[lidx]->layer_mask.left);
+                      return NULL;
+                    }
+
+                  if ((lyr_a[lidx]->layer_mask.right - lyr_a[lidx]->layer_mask.left) >
+                      G_MAXINT32 / MAX (lyr_a[lidx]->layer_mask.bottom - lyr_a[lidx]->layer_mask.top, 1))
+                    {
+                      g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
+                                   _("Unsupported or invalid layer mask size: %dx%d"),
+                                   lyr_a[lidx]->layer_mask.right - lyr_a[lidx]->layer_mask.left,
+                                   lyr_a[lidx]->layer_mask.bottom - lyr_a[lidx]->layer_mask.top);
+                      return NULL;
+                    }
+                }
             }
 
           /* Layer blending ranges */           /* FIXME  */