[gimp] app: change behavior of REPLACE mode for fully-transparent pixels

[Ell <ell src gnome org>]

commit 27e8f452b34d4b4a570e8c9c3469b3b13e54dd61
Author: Ell <ell_se yahoo com>
Date:   Thu Feb 14 10:31:29 2019 -0500

    app: change behavior of REPLACE mode for fully-transparent pixels
    
    When the result of compositing has an alpha value of 0, the
    corresponding color value is not mathematically defined.
    Currently, all out layer modes opt to preserve the destination's
    color value in this case.  However, REPLACE mode is different
    enough to warrant a different behavior:
    
    Unlike the other layer modes, when the compositing opacity
    approaches 0 or 1, the output color value approaches the
    destination or source color values, respectively, regardless of the
    output alpha value.  When the opacity doesn't approach 0 or 1, the
    output color value generally doesn't approach a limit as the output
    alpha value approaches 0, however, when both the destination and
    source alpha values are equal, the output color value is always a
    simple linear interpolation between the destination and source
    color values, according to the opacity.  In other words, this means
    that it's reasonable to simply use the above linear interpolation
    for the output color value, whenever the output alpha value is 0.
    
    Since filters are commonly combined with the input using REPALCE
    mode with full opacity, this has the effect that filters may now
    modify the color values of fully-transparent pixels.  This is
    generally desirable, IMO, especially for point filters.  Indeed,
    painting with REPLACE mode (i.e., with tools that use
    gimp_paint_core_replace()) behaved excatly as described above, and
    had this property, before we switched gimp_paint_core_replace() to
    use the common compositing code; this created a discrepancy between
    painting and applying filters, which is now gone.
    
    A side effect of this change is that we can now turn gimp:replace
    into a NOP when the opacity is 100% and there's no mask, which
    avoids the compositing step when applying filters.  We could
    previously only apply this optimization to PASS_THROUGH mode, which
    is a subclass of REPLACE mode.
    
    Note that the discussion above concerns the UNION composite mode,
    which is the only mode we currently use REPLACE in.  We modify the
    rest of the composite modes to match the new behavior:
    CLIP_TO_BACKDROP always preserves the color values of the
    destionation, CLIP_TO_LAYER always preserves the color values of
    the source, and INTERSECTION always produces fully-zeroed pixels.

 .../layer-modes/gimpoperationpassthrough.c         | 51 ------------
 app/operations/layer-modes/gimpoperationreplace.c  | 93 ++++++++++++++--------
 2 files changed, 58 insertions(+), 86 deletions(-)
---
diff --git a/app/operations/layer-modes/gimpoperationpassthrough.c 
b/app/operations/layer-modes/gimpoperationpassthrough.c
index fab9dc7d74..bc7d328a8f 100644
--- a/app/operations/layer-modes/gimpoperationpassthrough.c
+++ b/app/operations/layer-modes/gimpoperationpassthrough.c
@@ -25,22 +25,12 @@
 
 #include "../operations-types.h"
 
-#include "gimp-layer-modes.h"
 #include "gimpoperationpassthrough.h"
 
 
-static gboolean   gimp_operation_pass_through_parent_process (GeglOperation        *operation,
-                                                              GeglOperationContext *context,
-                                                              const gchar          *output_prop,
-                                                              const GeglRectangle  *result,
-                                                              gint                  level);
-
-
 G_DEFINE_TYPE (GimpOperationPassThrough, gimp_operation_pass_through,
                GIMP_TYPE_OPERATION_REPLACE)
 
-#define parent_class gimp_operation_pass_through_parent_class
-
 
 static void
 gimp_operation_pass_through_class_init (GimpOperationPassThroughClass *klass)
@@ -53,8 +43,6 @@ gimp_operation_pass_through_class_init (GimpOperationPassThroughClass *klass)
                                  "description", "GIMP pass through mode operation",
                                  NULL);
 
-  operation_class->process              = gimp_operation_pass_through_parent_process;
-
   /* don't use REPLACE mode's specialized get_affected_region(); PASS_THROUGH
    * behaves like an ordinary layer mode here.
    */
@@ -65,42 +53,3 @@ static void
 gimp_operation_pass_through_init (GimpOperationPassThrough *self)
 {
 }
-
-static gboolean
-gimp_operation_pass_through_parent_process (GeglOperation        *operation,
-                                            GeglOperationContext *context,
-                                            const gchar          *output_prop,
-                                            const GeglRectangle  *result,
-                                            gint                  level)
-{
-  GimpOperationLayerMode *layer_mode = (gpointer) operation;
-
-  /* if the layer's opacity is 100%, it has no mask, and its composite mode
-   * contains "aux" (the latter should always be the case for pass through
-   * mode,) we can just pass "aux" directly as output.  note that the same
-   * optimization would more generally apply to REPLACE mode, save for the fact
-   * that when both the backdrop and the layer have a pixel with 0% alpha, we
-   * want to maintain the color value of the backdrop, not the layer; since,
-   * for pass through groups, the layer is already composited against the
-   * backdrop, such pixels will have the same color value for both the backdrop
-   * and the layer.
-   */
-  if (layer_mode->opacity == 1.0                            &&
-      ! gegl_operation_context_get_object (context, "aux2") &&
-      (gimp_layer_mode_get_included_region (layer_mode->layer_mode,
-                                            layer_mode->real_composite_mode) &
-       GIMP_LAYER_COMPOSITE_REGION_SOURCE))
-    {
-      GObject *aux;
-
-      aux = gegl_operation_context_get_object (context, "aux");
-
-      gegl_operation_context_set_object (context, "output", aux);
-
-      return TRUE;
-    }
-
-  return GEGL_OPERATION_CLASS (parent_class)->process (operation, context,
-                                                       output_prop, result,
-                                                       level);
-}
diff --git a/app/operations/layer-modes/gimpoperationreplace.c 
b/app/operations/layer-modes/gimpoperationreplace.c
index d033be20b3..f730298daf 100644
--- a/app/operations/layer-modes/gimpoperationreplace.c
+++ b/app/operations/layer-modes/gimpoperationreplace.c
@@ -24,9 +24,16 @@
 
 #include "../operations-types.h"
 
+#include "gimp-layer-modes.h"
 #include "gimpoperationreplace.h"
 
 
+static gboolean                   gimp_operation_replace_parent_process      (GeglOperation        *op,
+                                                                              GeglOperationContext *context,
+                                                                              const gchar          
*output_prop,
+                                                                              const GeglRectangle  *result,
+                                                                              gint                  level);
+
 static gboolean                   gimp_operation_replace_process             (GeglOperation          *op,
                                                                               void                   *in,
                                                                               void                   *layer,
@@ -41,6 +48,8 @@ static GimpLayerCompositeRegion   gimp_operation_replace_get_affected_region (Gi
 G_DEFINE_TYPE (GimpOperationReplace, gimp_operation_replace,
                GIMP_TYPE_OPERATION_LAYER_MODE)
 
+#define parent_class gimp_operation_replace_parent_class
+
 
 static void
 gimp_operation_replace_class_init (GimpOperationReplaceClass *klass)
@@ -53,6 +62,8 @@ gimp_operation_replace_class_init (GimpOperationReplaceClass *klass)
                                  "description", "GIMP replace mode operation",
                                  NULL);
 
+  operation_class->process              = gimp_operation_replace_parent_process;
+
   layer_mode_class->process             = gimp_operation_replace_process;
   layer_mode_class->get_affected_region = gimp_operation_replace_get_affected_region;
 }
@@ -62,6 +73,41 @@ gimp_operation_replace_init (GimpOperationReplace *self)
 {
 }
 
+static gboolean
+gimp_operation_replace_parent_process (GeglOperation        *op,
+                                       GeglOperationContext *context,
+                                       const gchar          *output_prop,
+                                       const GeglRectangle  *result,
+                                       gint                  level)
+{
+  GimpOperationLayerMode *layer_mode = (gpointer) op;
+
+  /* if the layer's opacity is 100%, it has no mask, and its composite mode
+   * contains "aux" (the latter should always be the case in practice,
+   * currently,) we can just pass "aux" directly as output.
+   */
+  if (layer_mode->opacity == 1.0                            &&
+      ! gegl_operation_context_get_object (context, "aux2") &&
+      (gimp_layer_mode_get_included_region (layer_mode->layer_mode,
+                                            layer_mode->real_composite_mode) &
+       GIMP_LAYER_COMPOSITE_REGION_SOURCE))
+    {
+      GObject *aux;
+
+      aux = gegl_operation_context_get_object (context, "aux");
+
+      gegl_operation_context_set_object (context, "output", aux);
+
+      return TRUE;
+    }
+  /* the opposite case, where the opacity is 0%, is handled by
+   * GimpOperationLayerMode.
+   */
+
+  return GEGL_OPERATION_CLASS (parent_class)->process (op, context, output_prop,
+                                                       result, level);
+}
+
 static gboolean
 gimp_operation_replace_process (GeglOperation       *op,
                                 void                *in_p,
@@ -88,6 +134,7 @@ gimp_operation_replace_process (GeglOperation       *op,
         {
           gfloat opacity_value = opacity;
           gfloat new_alpha;
+          gfloat ratio;
           gint   b;
 
           if (has_mask)
@@ -95,18 +142,13 @@ gimp_operation_replace_process (GeglOperation       *op,
 
           new_alpha = (layer[ALPHA] - in[ALPHA]) * opacity_value + in[ALPHA];
 
+          ratio = opacity_value;
+
           if (new_alpha)
-            {
-              gfloat ratio = opacity_value * layer[ALPHA] / new_alpha;
-
-              for (b = RED; b < ALPHA; b++)
-                out[b] = (layer[b] - in[b]) * ratio + in[b];
-            }
-          else
-            {
-              for (b = RED; b < ALPHA; b++)
-                out[b] = in[b];
-            }
+            ratio *= layer[ALPHA] / new_alpha;
+
+          for (b = RED; b < ALPHA; b++)
+            out[b] = (layer[b] - in[b]) * ratio + in[b];
 
           out[ALPHA] = new_alpha;
 
@@ -136,8 +178,8 @@ gimp_operation_replace_process (GeglOperation       *op,
 
           out[ALPHA] = new_alpha;
 
-          in    += 4;
-          out   += 4;
+          in  += 4;
+          out += 4;
 
           if (has_mask)
             mask++;
@@ -156,16 +198,8 @@ gimp_operation_replace_process (GeglOperation       *op,
 
           new_alpha = layer[ALPHA] * opacity_value;
 
-          if (new_alpha)
-            {
-              for (b = RED; b < ALPHA; b++)
-                out[b] = layer[b];
-            }
-          else
-            {
-              for (b = RED; b < ALPHA; b++)
-                out[b] = in[b];
-            }
+          for (b = RED; b < ALPHA; b++)
+            out[b] = layer[b];
 
           out[ALPHA] = new_alpha;
 
@@ -179,18 +213,7 @@ gimp_operation_replace_process (GeglOperation       *op,
       break;
 
     case GIMP_LAYER_COMPOSITE_INTERSECTION:
-      while (samples--)
-        {
-          gint b;
-
-          for (b = RED; b < ALPHA; b++)
-            out[b] = in[b];
-
-          out[ALPHA] = 0.0f;
-
-          in  += 4;
-          out += 4;
-        }
+      memset (out, 0, 4 * samples * sizeof (gfloat));
       break;
     }