[gimp] app: fix "Luma Lighten/Darken only" layer modes.

[Jehan <jehanp src gnome org>]

commit 75e6f1062e9cb8bbdf476ed1551b2b8b12df001e
Author: Jehan <jehan girinstud io>
Date:   Tue Nov 10 15:16:07 2020 +0100

    app: fix "Luma Lighten/Darken only" layer modes.
    
    This is a continuation of #5888 as I realized that most layer modes were
    fixed with my commit b3fc24268a (and follow-up f40dc40cbc) but at least
    2 were still crashing GIMP: "Luma Lighten/Darken only" modes.
    
    There were 2 bugs here:
    
    * The first bug was that when gimp_operation_layer_mode_real_process()
      ran, gimp_operation_layer_mode_prepare() had not been run yet.
      prepare() is called before the process() of GeglOperation, but it
      would seem the process() of GimpOperationLayerMode on the other end
      happens before GeglOperation's prepare() is run. I am absolutely
      unsure if this is expected or not and have a hard time figuring out
      all the details of the C/C++ cohabitation.
      As a solution, I am moving out the fish caching (the needed part
      inside gimp_operation_layer_mode_real_process()) in its own function
      so that I can easily call it separately before inspecting the fishes.
    
    * The second issue was that some blend functions needed more than a
      GeglOperation alone. E.g. blend_function() for luma lighten
      gimp_operation_layer_mode_blend_luma_lighten_only() would call
      gegl_operation_get_source_space() which requires the node to exist.
      Similarly for the Luma darken only mode. So I keep both the node and
      operation around, and when finalizing, I free the node (which in turn
      frees the operation).
    
    Ell > if you are reading our commits, I would really appreciate your
    review (or fixes) of my code here! :)

 app/operations/layer-modes/gimp-layer-modes.c      |  27 +++++-
 .../layer-modes/gimpoperationlayermode.c           | 108 ++++++++++++++-------
 app/paint/gimppaintcore-loops.cc                   |   8 +-
 3 files changed, 94 insertions(+), 49 deletions(-)
---
diff --git a/app/operations/layer-modes/gimp-layer-modes.c b/app/operations/layer-modes/gimp-layer-modes.c
index 4f4da5a825..84bd5fe256 100644
--- a/app/operations/layer-modes/gimp-layer-modes.c
+++ b/app/operations/layer-modes/gimp-layer-modes.c
@@ -1131,7 +1131,16 @@ gimp_layer_modes_exit (void)
   gint i;
 
   for (i = 0; i < G_N_ELEMENTS (layer_mode_infos); i++)
-    g_clear_object (&ops[i]);
+    {
+      if (ops[i])
+        {
+          GeglNode *node;
+
+          node = ops[i]->node;
+          g_object_unref (node);
+          ops[i] = NULL;
+        }
+    }
 }
 
 static const GimpLayerModeInfo *
@@ -1305,11 +1314,19 @@ gimp_layer_mode_get_operation (GimpLayerMode mode)
                                   NULL);
 
       operation = gegl_node_get_gegl_operation (node);
-
-      g_object_ref (operation);
-      g_object_unref (node);
-
       ops[mode] = operation;
+
+      if (GIMP_IS_OPERATION_LAYER_MODE (operation))
+        {
+          GimpOperationLayerMode *layer_mode = GIMP_OPERATION_LAYER_MODE (operation);
+
+          layer_mode->layer_mode      = mode;
+          layer_mode->function        = GIMP_OPERATION_LAYER_MODE_GET_CLASS (operation)->process;
+          layer_mode->blend_function  = gimp_layer_mode_get_blend_function (mode);
+          layer_mode->blend_space     = gimp_layer_mode_get_blend_space (mode);
+          layer_mode->composite_space = gimp_layer_mode_get_composite_space (mode);
+          layer_mode->composite_mode  = gimp_layer_mode_get_paint_composite_mode (mode);
+        }
     }
 
   return ops[mode];
diff --git a/app/operations/layer-modes/gimpoperationlayermode.c 
b/app/operations/layer-modes/gimpoperationlayermode.c
index 4cbb0455c5..59749d7ec0 100644
--- a/app/operations/layer-modes/gimpoperationlayermode.c
+++ b/app/operations/layer-modes/gimpoperationlayermode.c
@@ -117,6 +117,9 @@ static gboolean        process_last_node                             (GeglOperat
                                                                       const GeglRectangle *roi,
                                                                       gint                 level);
 
+static void            gimp_operation_layer_mode_cache_fishes        (GimpOperationLayerMode *op,
+                                                                      const Babl             
*preferred_format);
+
 
 G_DEFINE_TYPE (GimpOperationLayerMode, gimp_operation_layer_mode,
                GEGL_TYPE_OPERATION_POINT_COMPOSER3)
@@ -341,48 +344,13 @@ gimp_operation_layer_mode_prepare (GeglOperation *operation)
 
   self->has_mask = mask_extent && ! gegl_rectangle_is_empty (mask_extent);
 
+  gimp_operation_layer_mode_cache_fishes (self, preferred_format);
+
   format = gimp_layer_mode_get_format (self->layer_mode,
                                        self->blend_space,
                                        self->composite_space,
                                        self->composite_mode,
                                        preferred_format);
-  if (self->cached_fish_format != format)
-    {
-      self->cached_fish_format = format;
-
-      self->space_fish
-        /* from */ [GIMP_LAYER_COLOR_SPACE_RGB_LINEAR     - 1]
-        /* to   */ [GIMP_LAYER_COLOR_SPACE_RGB_PERCEPTUAL - 1] =
-          babl_fish (babl_format_with_space ("RGBA float", format),
-                     babl_format_with_space ("R'G'B'A float", format));
-      self->space_fish
-        /* from */ [GIMP_LAYER_COLOR_SPACE_RGB_LINEAR     - 1]
-        /* to   */ [GIMP_LAYER_COLOR_SPACE_LAB            - 1] =
-          babl_fish (babl_format_with_space ("RGBA float", format),
-                     babl_format_with_space ("CIE Lab alpha float", format));
-
-      self->space_fish
-        /* from */ [GIMP_LAYER_COLOR_SPACE_RGB_PERCEPTUAL - 1]
-        /* to   */ [GIMP_LAYER_COLOR_SPACE_RGB_LINEAR     - 1] =
-          babl_fish (babl_format_with_space("R'G'B'A float", format),
-                     babl_format_with_space ( "RGBA float", format));
-      self->space_fish
-        /* from */ [GIMP_LAYER_COLOR_SPACE_RGB_PERCEPTUAL - 1]
-        /* to   */ [GIMP_LAYER_COLOR_SPACE_LAB            - 1] =
-          babl_fish (babl_format_with_space("R'G'B'A float", format),
-                     babl_format_with_space ( "CIE Lab alpha float", format));
-
-      self->space_fish
-        /* from */ [GIMP_LAYER_COLOR_SPACE_LAB            - 1]
-        /* to   */ [GIMP_LAYER_COLOR_SPACE_RGB_LINEAR     - 1] =
-          babl_fish (babl_format_with_space("CIE Lab alpha float", format),
-                     babl_format_with_space ( "RGBA float", format));
-      self->space_fish
-        /* from */ [GIMP_LAYER_COLOR_SPACE_LAB            - 1]
-        /* to   */ [GIMP_LAYER_COLOR_SPACE_RGB_PERCEPTUAL - 1] =
-          babl_fish (babl_format_with_space("CIE Lab alpha float", format),
-                     babl_format_with_space ( "R'G'B'A float", format));
-    }
 
   gegl_operation_set_format (operation, "input",  format);
   gegl_operation_set_format (operation, "output", format);
@@ -692,6 +660,10 @@ gimp_operation_layer_mode_real_process (GeglOperation       *operation,
       gimp_assert (composite_space >= 1 && composite_space < 4);
       gimp_assert (blend_space     >= 1 && blend_space     < 4);
 
+      /* Make sure the cache is set up from the start as the
+       * operation's prepare() method may have not been run yet.
+       */
+      gimp_operation_layer_mode_cache_fishes (layer_mode, NULL);
       composite_to_blend_fish = layer_mode->space_fish [composite_space - 1]
                                                        [blend_space     - 1];
 
@@ -901,6 +873,68 @@ process_last_node (GeglOperation       *operation,
   return TRUE;
 }
 
+static void
+gimp_operation_layer_mode_cache_fishes (GimpOperationLayerMode *op,
+                                        const Babl             *preferred_format)
+{
+  const Babl *format;
+
+  if (! preferred_format)
+    {
+      const GeglRectangle *input_extent;
+
+      input_extent = gegl_operation_source_get_bounding_box (GEGL_OPERATION (op), "input");
+
+      if (input_extent && ! gegl_rectangle_is_empty (input_extent))
+        preferred_format = gegl_operation_get_source_format (GEGL_OPERATION (op), "input");
+      else
+        preferred_format = gegl_operation_get_source_format (GEGL_OPERATION (op), "aux");
+    }
+
+  format = gimp_layer_mode_get_format (op->layer_mode,
+                                       op->blend_space,
+                                       op->composite_space,
+                                       op->composite_mode,
+                                       preferred_format);
+  if (op->cached_fish_format != format)
+    {
+      op->cached_fish_format = format;
+
+      op->space_fish
+        /* from */ [GIMP_LAYER_COLOR_SPACE_RGB_LINEAR     - 1]
+        /* to   */ [GIMP_LAYER_COLOR_SPACE_RGB_PERCEPTUAL - 1] =
+          babl_fish (babl_format_with_space ("RGBA float", format),
+                     babl_format_with_space ("R'G'B'A float", format));
+      op->space_fish
+        /* from */ [GIMP_LAYER_COLOR_SPACE_RGB_LINEAR     - 1]
+        /* to   */ [GIMP_LAYER_COLOR_SPACE_LAB            - 1] =
+          babl_fish (babl_format_with_space ("RGBA float", format),
+                     babl_format_with_space ("CIE Lab alpha float", format));
+
+      op->space_fish
+        /* from */ [GIMP_LAYER_COLOR_SPACE_RGB_PERCEPTUAL - 1]
+        /* to   */ [GIMP_LAYER_COLOR_SPACE_RGB_LINEAR     - 1] =
+          babl_fish (babl_format_with_space("R'G'B'A float", format),
+                     babl_format_with_space ( "RGBA float", format));
+      op->space_fish
+        /* from */ [GIMP_LAYER_COLOR_SPACE_RGB_PERCEPTUAL - 1]
+        /* to   */ [GIMP_LAYER_COLOR_SPACE_LAB            - 1] =
+          babl_fish (babl_format_with_space("R'G'B'A float", format),
+                     babl_format_with_space ( "CIE Lab alpha float", format));
+
+      op->space_fish
+        /* from */ [GIMP_LAYER_COLOR_SPACE_LAB            - 1]
+        /* to   */ [GIMP_LAYER_COLOR_SPACE_RGB_LINEAR     - 1] =
+          babl_fish (babl_format_with_space("CIE Lab alpha float", format),
+                     babl_format_with_space ( "RGBA float", format));
+      op->space_fish
+        /* from */ [GIMP_LAYER_COLOR_SPACE_LAB            - 1]
+        /* to   */ [GIMP_LAYER_COLOR_SPACE_RGB_PERCEPTUAL - 1] =
+          babl_fish (babl_format_with_space("CIE Lab alpha float", format),
+                     babl_format_with_space ( "R'G'B'A float", format));
+    }
+}
+
 
 /*  public functions  */
 
diff --git a/app/paint/gimppaintcore-loops.cc b/app/paint/gimppaintcore-loops.cc
index 664fe0476f..3211fc63cc 100644
--- a/app/paint/gimppaintcore-loops.cc
+++ b/app/paint/gimppaintcore-loops.cc
@@ -1884,13 +1884,7 @@ struct DoLayerBlend : Base
     Base (params)
   {
     layer_mode = GIMP_OPERATION_LAYER_MODE (gimp_layer_mode_get_operation (params->paint_mode));
-    layer_mode->layer_mode      = params->paint_mode;
-    layer_mode->opacity         = params->image_opacity;
-    layer_mode->function        = gimp_layer_mode_get_function (params->paint_mode);
-    layer_mode->blend_function  = gimp_layer_mode_get_blend_function (params->paint_mode);
-    layer_mode->blend_space     = gimp_layer_mode_get_blend_space (params->paint_mode);
-    layer_mode->composite_space = gimp_layer_mode_get_composite_space (params->paint_mode);
-    layer_mode->composite_mode  = gimp_layer_mode_get_paint_composite_mode (params->paint_mode);
+    layer_mode->opacity = params->image_opacity;
 
     iterator_format = gimp_layer_mode_get_format (params->paint_mode,
                                                   layer_mode->blend_space,