From 70b7316ebc49b4b5e1681b45f7c67d734905e42b Mon Sep 17 00:00:00 2001
From: Ell <ell_se@yahoo.com>
Date: Sat, 1 Dec 2018 05:31:37 -0500
Subject: [PATCH] app: in Luminance mode, replace VLAs with gimp-scratch

In the Luminance layer-mode, use the scratch allocator for
allocating temporary buffers, instead of using VLAs.
GimpOperationLayerMode already allocates data on the stack,
calculated as not to overflow the stack on any platform, so having
any of its descendants also allocate big buffers on the stack is
risky.
---
 .../gimpoperationlayermode-blend.c            | 31 ++++++++++++-------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/app/operations/layer-modes/gimpoperationlayermode-blend.c b/app/operations/layer-modes/gimpoperationlayermode-blend.c
index 5ab6a15ff9..c3fa09f70c 100644
--- a/app/operations/layer-modes/gimpoperationlayermode-blend.c
+++ b/app/operations/layer-modes/gimpoperationlayermode-blend.c
@@ -32,6 +32,8 @@
 
 #include "../operations-types.h"
 
+#include "core/gimp-scratch.h"
+
 #include "gimpoperationlayermode-blend.h"
 
 
@@ -866,20 +868,23 @@ gimp_operation_layer_mode_blend_luminance (const gfloat *in,
                                            gfloat       *comp,
                                            gint          samples)
 {
-  gfloat layer_Y[samples], *layer_Y_p;
-  gfloat in_Y[samples],    *in_Y_p;
+  gfloat *scratch;
+  gfloat *in_Y;
+  gfloat *layer_Y;
+
+  scratch = gimp_scratch_new (gfloat, 2 * samples);
+
+  in_Y    = scratch;
+  layer_Y = scratch + samples;
 
-  babl_process (babl_fish ("RGBA float", "Y float"), layer, layer_Y, samples);
   babl_process (babl_fish ("RGBA float", "Y float"), in,    in_Y,    samples);
-
-  layer_Y_p = &layer_Y[0];
-  in_Y_p    = &in_Y[0];
+  babl_process (babl_fish ("RGBA float", "Y float"), layer, layer_Y, samples);
 
   while (samples--)
     {
       if (layer[ALPHA] != 0.0f && in[ALPHA] != 0.0f)
         {
-          gfloat ratio = safe_div (layer_Y_p[0], in_Y_p[0]);
+          gfloat ratio = safe_div (layer_Y[0], in_Y[0]);
           gint   c;
 
           for (c = 0; c < 3; c ++)
@@ -888,12 +893,14 @@ gimp_operation_layer_mode_blend_luminance (const gfloat *in,
 
       comp[ALPHA] = layer[ALPHA];
 
-      comp      += 4;
-      in        += 4;
-      layer     += 4;
-      in_Y_p    ++;
-      layer_Y_p ++;
+      comp    += 4;
+      in      += 4;
+      layer   += 4;
+      in_Y    ++;
+      layer_Y ++;
     }
+
+  gimp_scratch_free (scratch);
 }
 
 void