app: change behavior of REPLACE mode for fully-transparent pixels

When the result of compositing has an alpha value of 0, the
corresponding color value is not mathematically defined.
Currently, all out layer modes opt to preserve the destination's
color value in this case.  However, REPLACE mode is different
enough to warrant a different behavior:

Unlike the other layer modes, when the compositing opacity
approaches 0 or 1, the output color value approaches the
destination or source color values, respectively, regardless of the
output alpha value.  When the opacity doesn't approach 0 or 1, the
output color value generally doesn't approach a limit as the output
alpha value approaches 0, however, when both the destination and
source alpha values are equal, the output color value is always a
simple linear interpolation between the destination and source
color values, according to the opacity.  In other words, this means
that it's reasonable to simply use the above linear interpolation
for the output color value, whenever the output alpha value is 0.

Since filters are commonly combined with the input using REPALCE
mode with full opacity, this has the effect that filters may now
modify the color values of fully-transparent pixels.  This is
generally desirable, IMO, especially for point filters.  Indeed,
painting with REPLACE mode (i.e., with tools that use
gimp_paint_core_replace()) behaved excatly as described above, and
had this property, before we switched gimp_paint_core_replace() to
use the common compositing code; this created a discrepancy between
painting and applying filters, which is now gone.

A side effect of this change is that we can now turn gimp:replace
into a NOP when the opacity is 100% and there's no mask, which
avoids the compositing step when applying filters.  We could
previously only apply this optimization to PASS_THROUGH mode, which
is a subclass of REPLACE mode.

Note that the discussion above concerns the UNION composite mode,
which is the only mode we currently use REPLACE in.  We modify the
rest of the composite modes to match the new behavior:
CLIP_TO_BACKDROP always preserves the color values of the
destionation, CLIP_TO_LAYER always preserves the color values of
the source, and INTERSECTION always produces fully-zeroed pixels.

app/operations/layer-modes/gimpoperationpassthrough.c

@@ -25,22 +25,12 @@
 
 #include "../operations-types.h"
 
-#include "gimp-layer-modes.h"
 #include "gimpoperationpassthrough.h"
 
 
-static gboolean   gimp_operation_pass_through_parent_process (GeglOperation        *operation,
-                                                              GeglOperationContext *context,
-                                                              const gchar          *output_prop,
-                                                              const GeglRectangle  *result,
-                                                              gint                  level);
-
-
 G_DEFINE_TYPE (GimpOperationPassThrough, gimp_operation_pass_through,
                GIMP_TYPE_OPERATION_REPLACE)
 
-#define parent_class gimp_operation_pass_through_parent_class
-
 
 static void
 gimp_operation_pass_through_class_init (GimpOperationPassThroughClass *klass)
@@ -53,8 +43,6 @@ gimp_operation_pass_through_class_init (GimpOperationPassThroughClass *klass)
                                  "description", "GIMP pass through mode operation",
                                  NULL);
 
-  operation_class->process              = gimp_operation_pass_through_parent_process;
-
   /* don't use REPLACE mode's specialized get_affected_region(); PASS_THROUGH
    * behaves like an ordinary layer mode here.
    */
@@ -65,42 +53,3 @@ static void
 gimp_operation_pass_through_init (GimpOperationPassThrough *self)
 {
 }
-
-static gboolean
-gimp_operation_pass_through_parent_process (GeglOperation        *operation,
-                                            GeglOperationContext *context,
-                                            const gchar          *output_prop,
-                                            const GeglRectangle  *result,
-                                            gint                  level)
-{
-  GimpOperationLayerMode *layer_mode = (gpointer) operation;
-
-  /* if the layer's opacity is 100%, it has no mask, and its composite mode
-   * contains "aux" (the latter should always be the case for pass through
-   * mode,) we can just pass "aux" directly as output.  note that the same
-   * optimization would more generally apply to REPLACE mode, save for the fact
-   * that when both the backdrop and the layer have a pixel with 0% alpha, we
-   * want to maintain the color value of the backdrop, not the layer; since,
-   * for pass through groups, the layer is already composited against the
-   * backdrop, such pixels will have the same color value for both the backdrop
-   * and the layer.
-   */
-  if (layer_mode->opacity == 1.0                            &&
-      ! gegl_operation_context_get_object (context, "aux2") &&
-      (gimp_layer_mode_get_included_region (layer_mode->layer_mode,
-                                            layer_mode->real_composite_mode) &
-       GIMP_LAYER_COMPOSITE_REGION_SOURCE))
-    {
-      GObject *aux;
-
-      aux = gegl_operation_context_get_object (context, "aux");
-
-      gegl_operation_context_set_object (context, "output", aux);
-
-      return TRUE;
-    }
-
-  return GEGL_OPERATION_CLASS (parent_class)->process (operation, context,
-                                                       output_prop, result,
-                                                       level);
-}

app/operations/layer-modes/gimpoperationreplace.c

@@ -24,9 +24,16 @@
 
 #include "../operations-types.h"
 
+#include "gimp-layer-modes.h"
 #include "gimpoperationreplace.h"
 
 
+static gboolean                   gimp_operation_replace_parent_process      (GeglOperation        *op,
+                                                                              GeglOperationContext *context,
+                                                                              const gchar          *output_prop,
+                                                                              const GeglRectangle  *result,
+                                                                              gint                  level);
+
 static gboolean                   gimp_operation_replace_process             (GeglOperation          *op,
                                                                               void                   *in,
                                                                               void                   *layer,
@@ -41,6 +48,8 @@ static GimpLayerCompositeRegion   gimp_operation_replace_get_affected_region (Gi
 G_DEFINE_TYPE (GimpOperationReplace, gimp_operation_replace,
                GIMP_TYPE_OPERATION_LAYER_MODE)
 
+#define parent_class gimp_operation_replace_parent_class
+
 
 static void
 gimp_operation_replace_class_init (GimpOperationReplaceClass *klass)
@@ -53,6 +62,8 @@ gimp_operation_replace_class_init (GimpOperationReplaceClass *klass)
                                  "description", "GIMP replace mode operation",
                                  NULL);
 
+  operation_class->process              = gimp_operation_replace_parent_process;
+
   layer_mode_class->process             = gimp_operation_replace_process;
   layer_mode_class->get_affected_region = gimp_operation_replace_get_affected_region;
 }
@@ -62,6 +73,41 @@ gimp_operation_replace_init (GimpOperationReplace *self)
 {
 }
 
+static gboolean
+gimp_operation_replace_parent_process (GeglOperation        *op,
+                                       GeglOperationContext *context,
+                                       const gchar          *output_prop,
+                                       const GeglRectangle  *result,
+                                       gint                  level)
+{
+  GimpOperationLayerMode *layer_mode = (gpointer) op;
+
+  /* if the layer's opacity is 100%, it has no mask, and its composite mode
+   * contains "aux" (the latter should always be the case in practice,
+   * currently,) we can just pass "aux" directly as output.
+   */
+  if (layer_mode->opacity == 1.0                            &&
+      ! gegl_operation_context_get_object (context, "aux2") &&
+      (gimp_layer_mode_get_included_region (layer_mode->layer_mode,
+                                            layer_mode->real_composite_mode) &
+       GIMP_LAYER_COMPOSITE_REGION_SOURCE))
+    {
+      GObject *aux;
+
+      aux = gegl_operation_context_get_object (context, "aux");
+
+      gegl_operation_context_set_object (context, "output", aux);
+
+      return TRUE;
+    }
+  /* the opposite case, where the opacity is 0%, is handled by
+   * GimpOperationLayerMode.
+   */
+
+  return GEGL_OPERATION_CLASS (parent_class)->process (op, context, output_prop,
+                                                       result, level);
+}
+
 static gboolean
 gimp_operation_replace_process (GeglOperation       *op,
                                 void                *in_p,
@@ -88,6 +134,7 @@ gimp_operation_replace_process (GeglOperation       *op,
         {
           gfloat opacity_value = opacity;
           gfloat new_alpha;
+          gfloat ratio;
           gint   b;
 
           if (has_mask)
@@ -95,18 +142,13 @@ gimp_operation_replace_process (GeglOperation       *op,
 
           new_alpha = (layer[ALPHA] - in[ALPHA]) * opacity_value + in[ALPHA];
 
+          ratio = opacity_value;
+
           if (new_alpha)
-            {
+            ratio *= layer[ALPHA] / new_alpha;
-              gfloat ratio = opacity_value * layer[ALPHA] / new_alpha;
 
           for (b = RED; b < ALPHA; b++)
             out[b] = (layer[b] - in[b]) * ratio + in[b];
-            }
-          else
-            {
-              for (b = RED; b < ALPHA; b++)
-                out[b] = in[b];
-            }
 
           out[ALPHA] = new_alpha;
 
@@ -156,16 +198,8 @@ gimp_operation_replace_process (GeglOperation       *op,
 
           new_alpha = layer[ALPHA] * opacity_value;
 
-          if (new_alpha)
-            {
           for (b = RED; b < ALPHA; b++)
             out[b] = layer[b];
-            }
-          else
-            {
-              for (b = RED; b < ALPHA; b++)
-                out[b] = in[b];
-            }
 
           out[ALPHA] = new_alpha;
 
@@ -179,18 +213,7 @@ gimp_operation_replace_process (GeglOperation       *op,
       break;
 
     case GIMP_LAYER_COMPOSITE_INTERSECTION:
-      while (samples--)
+      memset (out, 0, 4 * samples * sizeof (gfloat));
-        {
-          gint b;
-
-          for (b = RED; b < ALPHA; b++)
-            out[b] = in[b];
-
-          out[ALPHA] = 0.0f;
-
-          in  += 4;
-          out += 4;
-        }
       break;
     }