[gimp] app: disregard composite space in non-union alpha-only layer modes

[Ell <ell src gnome org>]

commit a5962e4049461d2f7b29ab840e58ad4fab2b430c
Author: Ell <ell_se yahoo com>
Date:   Fri May 24 01:33:53 2019 -0400

    app: disregard composite space in non-union alpha-only layer modes
    
    In gimp_layer_mode_get_format(), disregard the requested composite
    space when selecting the format, if the input layer mode is alpha-
    only, and the requested composite mode is not UNION, since, in this
    case, the layer mode doesn't combine the layer/backdrop colors, and
    rather only modifies the alpha of one of them.  This allows us to
    use the preferred format, avoiding gamma conversion.
    
    This particularly improves the performance of the Eraser tool in
    perceptual images.

 app/core/gimpfilloptions.c                         |  2 ++
 app/gegl/gimp-gegl-nodes.c                         |  3 ++-
 app/operations/layer-modes/gimp-layer-modes.c      | 31 +++++++++++++++++++---
 app/operations/layer-modes/gimp-layer-modes.h      |  3 ++-
 .../layer-modes/gimpoperationlayermode.c           |  3 ++-
 app/paint/gimpbrushcore.c                          |  8 ++++--
 app/paint/gimpink.c                                |  8 ++++--
 app/paint/gimppaintcore-loops.cc                   |  3 ++-
 8 files changed, 49 insertions(+), 12 deletions(-)
---
diff --git a/app/core/gimpfilloptions.c b/app/core/gimpfilloptions.c
index fc87149e98..d10b6e4572 100644
--- a/app/core/gimpfilloptions.c
+++ b/app/core/gimpfilloptions.c
@@ -473,6 +473,8 @@ gimp_fill_options_get_format (GimpFillOptions *options,
   return gimp_layer_mode_get_format (gimp_context_get_paint_mode (context),
                                      GIMP_LAYER_COLOR_SPACE_AUTO,
                                      GIMP_LAYER_COLOR_SPACE_AUTO,
+                                     gimp_layer_mode_get_paint_composite_mode (
+                                       gimp_context_get_paint_mode (context)),
                                      gimp_drawable_get_format (drawable));
 }
 
diff --git a/app/gegl/gimp-gegl-nodes.c b/app/gegl/gimp-gegl-nodes.c
index aeaaee2942..197109d8be 100644
--- a/app/gegl/gimp-gegl-nodes.c
+++ b/app/gegl/gimp-gegl-nodes.c
@@ -58,8 +58,9 @@ gimp_gegl_create_flatten_node (const GimpRGB       *background,
                                "value",     c,
                                "format",    gimp_layer_mode_get_format (
                                               GIMP_LAYER_MODE_NORMAL,
-                                              composite_space,
                                               GIMP_LAYER_COLOR_SPACE_AUTO,
+                                              composite_space,
+                                              GIMP_LAYER_COMPOSITE_AUTO,
                                               NULL),
                                NULL);
   g_object_unref (c);
diff --git a/app/operations/layer-modes/gimp-layer-modes.c b/app/operations/layer-modes/gimp-layer-modes.c
index 2737bed5b8..b59b30506a 100644
--- a/app/operations/layer-modes/gimp-layer-modes.c
+++ b/app/operations/layer-modes/gimp-layer-modes.c
@@ -1423,10 +1423,11 @@ gimp_layer_mode_get_for_group (GimpLayerMode       old_mode,
 }
 
 const Babl *
-gimp_layer_mode_get_format (GimpLayerMode        mode,
-                            GimpLayerColorSpace  composite_space,
-                            GimpLayerColorSpace  blend_space,
-                            const Babl          *preferred_format)
+gimp_layer_mode_get_format (GimpLayerMode           mode,
+                            GimpLayerColorSpace     blend_space,
+                            GimpLayerColorSpace     composite_space,
+                            GimpLayerCompositeMode  composite_mode,
+                            const Babl             *preferred_format)
 {
   /* for now, all modes perform i/o in the composite space. */
   (void) mode;
@@ -1435,6 +1436,28 @@ gimp_layer_mode_get_format (GimpLayerMode        mode,
   if (composite_space == GIMP_LAYER_COLOR_SPACE_AUTO)
     composite_space = gimp_layer_mode_get_composite_space (mode);
 
+  if (composite_mode == GIMP_LAYER_COMPOSITE_AUTO)
+    composite_mode = gimp_layer_mode_get_composite_mode (mode);
+
+  if (gimp_layer_mode_is_alpha_only (mode))
+    {
+      switch (composite_mode)
+        {
+        case GIMP_LAYER_COMPOSITE_AUTO:
+        case GIMP_LAYER_COMPOSITE_UNION:
+          break;
+
+        case GIMP_LAYER_COMPOSITE_CLIP_TO_BACKDROP:
+        case GIMP_LAYER_COMPOSITE_CLIP_TO_LAYER:
+        case GIMP_LAYER_COMPOSITE_INTERSECTION:
+          /* alpha-only layer modes don't combine colors in non-union composite
+           * modes, hence we can disregard the composite space.
+           */
+          composite_space = GIMP_LAYER_COLOR_SPACE_AUTO;
+          break;
+        }
+    }
+
   switch (composite_space)
     {
     case GIMP_LAYER_COLOR_SPACE_AUTO:
diff --git a/app/operations/layer-modes/gimp-layer-modes.h b/app/operations/layer-modes/gimp-layer-modes.h
index fa340c1d8f..30c5e152a4 100644
--- a/app/operations/layer-modes/gimp-layer-modes.h
+++ b/app/operations/layer-modes/gimp-layer-modes.h
@@ -61,8 +61,9 @@ gboolean                   gimp_layer_mode_get_for_group              (GimpLayer
                                                                        GimpLayerMode          *new_mode);
 
 const Babl               * gimp_layer_mode_get_format                 (GimpLayerMode           mode,
-                                                                       GimpLayerColorSpace     
composite_space,
                                                                        GimpLayerColorSpace     blend_space,
+                                                                       GimpLayerColorSpace     
composite_space,
+                                                                       GimpLayerCompositeMode  
composite_mode,
                                                                        const Babl             
*preferred_format);
 
 GimpLayerCompositeRegion   gimp_layer_mode_get_included_region        (GimpLayerMode           mode,
diff --git a/app/operations/layer-modes/gimpoperationlayermode.c 
b/app/operations/layer-modes/gimpoperationlayermode.c
index 05df9b9253..2ec62493a2 100644
--- a/app/operations/layer-modes/gimpoperationlayermode.c
+++ b/app/operations/layer-modes/gimpoperationlayermode.c
@@ -330,8 +330,9 @@ gimp_operation_layer_mode_prepare (GeglOperation *operation)
     }
 
   format = gimp_layer_mode_get_format (self->layer_mode,
-                                       self->composite_space,
                                        self->blend_space,
+                                       self->composite_space,
+                                       self->composite_mode,
                                        preferred_format);
   if (self->cached_fish_format != format)
     {
diff --git a/app/paint/gimpbrushcore.c b/app/paint/gimpbrushcore.c
index 6f67da979b..d1fed9a5d5 100644
--- a/app/paint/gimpbrushcore.c
+++ b/app/paint/gimpbrushcore.c
@@ -816,12 +816,16 @@ gimp_brush_core_get_paint_buffer (GimpPaintCore    *paint_core,
   /*  configure the canvas buffer  */
   if ((x2 - x1) && (y2 - y1))
     {
-      GimpTempBuf *temp_buf;
-      const Babl  *format;
+      GimpTempBuf            *temp_buf;
+      const Babl             *format;
+      GimpLayerCompositeMode  composite_mode;
+
+      composite_mode = gimp_layer_mode_get_paint_composite_mode (paint_mode);
 
       format = gimp_layer_mode_get_format (paint_mode,
                                            GIMP_LAYER_COLOR_SPACE_AUTO,
                                            GIMP_LAYER_COLOR_SPACE_AUTO,
+                                           composite_mode,
                                            gimp_drawable_get_format (drawable));
 
       if (paint_core->paint_buffer                                       &&
diff --git a/app/paint/gimpink.c b/app/paint/gimpink.c
index 08c86d99e5..644d16a0ce 100644
--- a/app/paint/gimpink.c
+++ b/app/paint/gimpink.c
@@ -258,12 +258,16 @@ gimp_ink_get_paint_buffer (GimpPaintCore    *paint_core,
   /*  configure the canvas buffer  */
   if ((x2 - x1) && (y2 - y1))
     {
-      GimpTempBuf *temp_buf;
-      const Babl  *format;
+      GimpTempBuf            *temp_buf;
+      const Babl             *format;
+      GimpLayerCompositeMode  composite_mode;
+
+      composite_mode = gimp_layer_mode_get_paint_composite_mode (paint_mode);
 
       format = gimp_layer_mode_get_format (paint_mode,
                                            GIMP_LAYER_COLOR_SPACE_AUTO,
                                            GIMP_LAYER_COLOR_SPACE_AUTO,
+                                           composite_mode,
                                            gimp_drawable_get_format (drawable));
 
       temp_buf = gimp_temp_buf_new ((x2 - x1), (y2 - y1),
diff --git a/app/paint/gimppaintcore-loops.cc b/app/paint/gimppaintcore-loops.cc
index 182f7f45d2..f17f8c42eb 100644
--- a/app/paint/gimppaintcore-loops.cc
+++ b/app/paint/gimppaintcore-loops.cc
@@ -1888,8 +1888,9 @@ struct DoLayerBlend : Base
     layer_mode.real_composite_mode = layer_mode.composite_mode;
 
     iterator_format = gimp_layer_mode_get_format (params->paint_mode,
-                                                  layer_mode.composite_space,
                                                   layer_mode.blend_space,
+                                                  layer_mode.composite_space,
+                                                  layer_mode.composite_mode,
                                                   gimp_temp_buf_get_format (params->paint_buf));
 
     g_return_if_fail (gimp_temp_buf_get_format (params->paint_buf) == iterator_format);