[gimp] plug-ins: add extra layer groups when loading PSD images with clipping layers

[Jacob Boerema <jboerema src gnome org>]

commit e51d5d480d180cd8c4d2335edbe5f9a637beaa5d
Author: Jacob Boerema <jgboerema gmail com>
Date:   Tue Dec 14 12:00:33 2021 -0500

    plug-ins: add extra layer groups when loading PSD images with clipping layers
    
    Together with the previous commit this mostly fixes #5438.
    
    PhotoShop handles clipping layers in a different way than GIMP. The only
    way to have it look the same is by adding extra layer groups.
    PSD layers that have clipping set, combined with the first non clipping
    layer below it are grouped together in a new layer group.
    Doing this results in the same image as the PSD merged image unless there
    are other PSD elements in play that we don't handle yet.
    
    See e.g. the NSFW image in the mentioned issue where the purple hair color
    is a little darker than the merged image.

 plug-ins/file-psd/psd-load.c | 110 ++++++++++++++++++++++++++++++++++++++++++-
 plug-ins/file-psd/psd-save.c |   6 ++-
 plug-ins/file-psd/psd.h      |   1 +
 3 files changed, 113 insertions(+), 4 deletions(-)
---
diff --git a/plug-ins/file-psd/psd-load.c b/plug-ins/file-psd/psd-load.c
index cc768f8a43..a72fe688ce 100644
--- a/plug-ins/file-psd/psd-load.c
+++ b/plug-ins/file-psd/psd-load.c
@@ -85,6 +85,9 @@ static gint             add_merged_image           (GimpImage      *image,
                                                     GError        **error);
 
 /*  Local utility function prototypes  */
+static GimpLayer      * add_clipping_group         (GimpImage      *image,
+                                                    GimpLayer      *parent);
+
 static gchar          * get_psd_color_mode_name    (PSDColorMode    mode);
 
 static void             psd_to_gimp_color_map      (guchar         *map256);
@@ -1622,6 +1625,7 @@ add_layers (GimpImage     *image,
   PSDchannel          **lyr_chn;
   GArray               *parent_group_stack;
   GimpLayer            *parent_group = NULL;
+  GimpLayer            *clipping_group = NULL;
   guint16               alpha_chn;
   guint16               user_mask_chn;
   guint16               layer_channels;
@@ -1640,10 +1644,12 @@ add_layers (GimpImage     *image,
   GList                *selected_layers = NULL;
   gint                  lidx;                  /* Layer index */
   gint                  cidx;                  /* Channel index */
+  gint                  gidx;                  /* Clipping group start index */
   gboolean              alpha;
   gboolean              user_mask;
   gboolean              empty;
   gboolean              empty_mask;
+  gboolean              use_clipping_group;
   GeglBuffer           *buffer;
   GimpImageType         image_type;
   LayerModeInfo         mode_info;
@@ -1664,13 +1670,70 @@ add_layers (GimpImage     *image,
       return -1;
     }
 
+  IFDBG(3) g_debug ("Pre process layers...");
+  use_clipping_group = FALSE;
+  gidx = -1;
+  for (lidx = 0; lidx < img_a->num_layers; ++lidx)
+    {
+      if (lyr_a[lidx]->clipping == 1)
+        {
+          /* Photoshop handles layers with clipping differently than GIMP does.
+           * To correctly show these layers we need to make a new group
+           * starting with the first non-clipping layer and including all
+           * the clipping layers above it.
+           */
+          if (lidx > 0)
+            {
+              if (gidx == -1)
+                {
+                  use_clipping_group = TRUE;
+
+                  /* Looking at the results we should ignore layer groups */
+                  if (lyr_a[lidx-1]->group_type == 0)
+                    gidx = lidx - 1;
+                  else
+                    gidx = lidx;
+
+                  lyr_a[gidx]->clipping_group_type = 1; /* start clipping group */
+                  IFDBG(3) g_debug ("Layer: %s - start of clipping group", lyr_a[gidx]->name);
+                }
+              else if (lidx + 1 == img_a->num_layers && use_clipping_group)
+                {
+                  /* end clipping group at the top of the layer stack */
+                  lyr_a[lidx]->clipping_group_type = 2; /* end clipping group */
+                  IFDBG(3) g_debug ("Layer: %s - end of clipping group", lyr_a[lidx]->name);
+
+                  use_clipping_group = FALSE;
+                  gidx = -1;
+                }
+              else
+                {
+                  lyr_a[lidx]->clipping_group_type = 0;
+                }
+            }
+        }
+      else if (use_clipping_group)
+        {
+          /* end clipping group */
+          lyr_a[lidx-1]->clipping_group_type = 2;
+          IFDBG(3) g_debug ("Layer: %s - end of clipping group", lyr_a[lidx-1]->name);
+
+          use_clipping_group = FALSE;
+          gidx = -1;
+        }
+      else
+        {
+          lyr_a[lidx]->clipping_group_type = 0;
+        }
+    }
+
   /* set the root of the group hierarchy */
   parent_group_stack = g_array_new (FALSE, FALSE, sizeof (GimpLayer *));
   g_array_append_val (parent_group_stack, parent_group);
 
   for (lidx = 0; lidx < img_a->num_layers; ++lidx)
     {
-      IFDBG(2) g_debug ("Process Layer No %d.", lidx);
+      IFDBG(2) g_debug ("Process Layer No %d (%s).", lidx, lyr_a[lidx]->name);
 
       if (lyr_a[lidx]->drop)
         {
@@ -1875,6 +1938,9 @@ add_layers (GimpImage     *image,
               layer_channels++;
             }
 
+          IFDBG(4) g_debug ("Create the layer (group type: %d, clipping group type: %d)",
+                            lyr_a[lidx]->group_type, lyr_a[lidx]->clipping_group_type);
+
           /* Create the layer */
           if (lyr_a[lidx]->group_type != 0)
             {
@@ -1916,6 +1982,11 @@ add_layers (GimpImage     *image,
             }
           else
             {
+              if (lyr_a[lidx]->clipping_group_type == 1)
+                {
+                  clipping_group = add_clipping_group (image, parent_group);
+                }
+
               if (empty)
                 {
                   IFDBG(2) g_debug ("Create blank layer");
@@ -2196,7 +2267,20 @@ add_layers (GimpImage     *image,
                 if (lyr_a[lidx]->group_type == 0 || /* normal layer */
                     lyr_a[lidx]->group_type == 3    /* group layer end marker */)
                   {
-                    gimp_image_insert_layer (image, layer, parent_group, 0);
+                    if (clipping_group)
+                      {
+                        gimp_image_insert_layer (image, layer, clipping_group, 0);
+
+                        if (lyr_a[lidx]->clipping_group_type == 2)
+                          {
+                            /* End of our clipping group. */
+                            clipping_group = NULL;
+                          }
+                      }
+                    else
+                      {
+                        gimp_image_insert_layer (image, layer, parent_group, 0);
+                      }
                   }
             }
 
@@ -2650,6 +2734,28 @@ add_merged_image (GimpImage     *image,
 
 
 /* Local utility functions */
+static GimpLayer *
+add_clipping_group (GimpImage  *image,
+                    GimpLayer  *parent)
+{
+  GimpLayer * clipping_group = NULL;
+
+  /* We need to create a group because GIMP handles clipping and
+   * composition mode in a different manner than PS. */
+  IFDBG(2) g_debug ("Creating a layer group to handle PS transparency clipping correctly.");
+
+  clipping_group = gimp_layer_group_new (image);
+
+  gimp_item_set_name (GIMP_ITEM (clipping_group), "Group added by GIMP");
+  gimp_layer_set_blend_space (clipping_group, GIMP_LAYER_COLOR_SPACE_RGB_PERCEPTUAL);
+  gimp_layer_set_composite_space (clipping_group, GIMP_LAYER_COLOR_SPACE_RGB_PERCEPTUAL);
+  gimp_layer_set_composite_mode (clipping_group, GIMP_LAYER_COMPOSITE_UNION);
+
+  gimp_image_insert_layer (image, clipping_group, parent, 0);
+
+  return clipping_group;
+}
+
 static gchar *
 get_psd_color_mode_name (PSDColorMode mode)
 {
diff --git a/plug-ins/file-psd/psd-save.c b/plug-ins/file-psd/psd-save.c
index 7c4f2d4142..4419e75c1c 100644
--- a/plug-ins/file-psd/psd-save.c
+++ b/plug-ins/file-psd/psd-save.c
@@ -1169,8 +1169,10 @@ save_layer_and_mask (GOutputStream  *output,
       IFDBG(1) g_debug ("\t\tOpacity: %u", layerOpacity);
       write_gchar (output, layerOpacity, "Opacity");
 
-      /* Apparently this field is not used in GIMP */
-      write_gchar (output, 0, "Clipping");
+      if (gimp_layer_get_composite_mode (psd_layer->layer) == GIMP_LAYER_COMPOSITE_CLIP_TO_BACKDROP)
+        write_gchar (output, 1, "Clipping");
+      else
+        write_gchar (output, 0, "Clipping");
 
       flags = 0;
       if (gimp_layer_get_lock_alpha (psd_layer->layer)) flags |= 1;
diff --git a/plug-ins/file-psd/psd.h b/plug-ins/file-psd/psd.h
index 687cd6b00a..cd86019bf1 100644
--- a/plug-ins/file-psd/psd.h
+++ b/plug-ins/file-psd/psd.h
@@ -589,6 +589,7 @@ typedef struct
   gchar                 blend_mode[4];          /* Blend mode */
   guchar                opacity;                /* Opacity - 0 = transparent ... 255 = opaque */
   guchar                clipping;               /* Clipping */
+  guchar                clipping_group_type;    /* Used to track group needed for clipping (1 = group start, 
2 = group end) */
   guchar                flags;                  /* Layer flags */
   guchar                filler;                 /* Filler */
   guint64               extra_len;              /* Extra data length */