8233255: Better Swing Buttons

Reviewed-by: alitvinov, prr, ahgross, rhalade

src/java.desktop/unix/native/libawt_xawt/awt/gtk2_interface.c

 /*
- * Copyright (c) 2005, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1012,45 +1012,46 @@ static gint gtk2_copy_image(gint *dst, gint width, gint height)
     black = (*fp_gdk_pixbuf_get_pixels)(gtk2_black_pixbuf);
     stride = (*fp_gdk_pixbuf_get_rowstride)(gtk2_black_pixbuf);
     padding = stride - width * 4;
+    if (padding >= 0 && stride > 0) {
+        for (i = 0; i < height; i++) {
+            for (j = 0; j < width; j++) {
+                int r1 = *white++;
+                int r2 = *black++;
+                int alpha = 0xff + r2 - r1;
+
+                switch (alpha) {
+                    case 0:       /* transparent pixel */
+                        r = g = b = 0;
+                        black += 3;
+                        white += 3;
+                        is_opaque = FALSE;
+                        break;
+
+                    case 0xff:    /* opaque pixel */
+                        r = r2;
+                        g = *black++;
+                        b = *black++;
+                        black++;
+                        white += 3;
+                        break;
+
+                    default:      /* translucent pixel */
+                        r = 0xff * r2 / alpha;
+                        g = 0xff * *black++ / alpha;
+                        b = 0xff * *black++ / alpha;
+                        black++;
+                        white += 3;
+                        is_opaque = FALSE;
+                        is_bitmask = FALSE;
+                        break;
+                }
 
-    for (i = 0; i < height; i++) {
-        for (j = 0; j < width; j++) {
-            int r1 = *white++;
-            int r2 = *black++;
-            int alpha = 0xff + r2 - r1;
-
-            switch (alpha) {
-                case 0:       /* transparent pixel */
-                    r = g = b = 0;
-                    black += 3;
-                    white += 3;
-                    is_opaque = FALSE;
-                    break;
-
-                case 0xff:    /* opaque pixel */
-                    r = r2;
-                    g = *black++;
-                    b = *black++;
-                    black++;
-                    white += 3;
-                    break;
-
-                default:      /* translucent pixel */
-                    r = 0xff * r2 / alpha;
-                    g = 0xff * *black++ / alpha;
-                    b = 0xff * *black++ / alpha;
-                    black++;
-                    white += 3;
-                    is_opaque = FALSE;
-                    is_bitmask = FALSE;
-                    break;
+                *dst++ = (alpha << 24 | r << 16 | g << 8 | b);
             }
 
-            *dst++ = (alpha << 24 | r << 16 | g << 8 | b);
+            white += padding;
+            black += padding;
         }
-
-        white += padding;
-        black += padding;
     }
     return is_opaque ? java_awt_Transparency_OPAQUE :
                        (is_bitmask ? java_awt_Transparency_BITMASK :

src/java.desktop/unix/native/libawt_xawt/awt/gtk3_interface.c

 /*
- * Copyright (c) 2005, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -347,8 +347,10 @@ GtkApi* gtk3_load(JNIEnv *env, const char* lib_name)
 
         fp_cairo_image_surface_create = dl_symbol("cairo_image_surface_create");
         fp_cairo_surface_destroy = dl_symbol("cairo_surface_destroy");
+        fp_cairo_surface_status = dl_symbol("cairo_surface_status");
         fp_cairo_create = dl_symbol("cairo_create");
         fp_cairo_destroy = dl_symbol("cairo_destroy");
+        fp_cairo_status = dl_symbol("cairo_status");
         fp_cairo_fill = dl_symbol("cairo_fill");
         fp_cairo_rectangle = dl_symbol("cairo_rectangle");
         fp_cairo_set_source_rgb = dl_symbol("cairo_set_source_rgb");
@@ -775,6 +777,9 @@ static void gtk3_init_painting(JNIEnv *env, gint width, gint height)
     }
 
     cr = fp_cairo_create(surface);
+    if (fp_cairo_surface_status(surface) || fp_cairo_status(cr)) {
+        JNU_ThrowOutOfMemoryError(env, "The surface size is too big");
+    }
 }
 
 /*
@@ -797,16 +802,17 @@ static gint gtk3_copy_image(gint *dst, gint width, gint height)
     data = (*fp_cairo_image_surface_get_data)(surface);
     stride = (*fp_cairo_image_surface_get_stride)(surface);
     padding = stride - width * 4;
-
-    for (i = 0; i < height; i++) {
-        for (j = 0; j < width; j++) {
-            int r = *data++;
-            int g = *data++;
-            int b = *data++;
-            int a = *data++;
-            *dst++ = (a << 24 | b << 16 | g << 8 | r);
+    if (stride > 0 && padding >= 0) {
+        for (i = 0; i < height; i++) {
+            for (j = 0; j < width; j++) {
+                int r = *data++;
+                int g = *data++;
+                int b = *data++;
+                int a = *data++;
+                *dst++ = (a << 24 | b << 16 | g << 8 | r);
+            }
+            data += padding;
         }
-        data += padding;
     }
     return java_awt_Transparency_TRANSLUCENT;
 }

src/java.desktop/unix/native/libawt_xawt/awt/gtk3_interface.h

@@ -132,6 +132,48 @@ typedef enum {
     CAIRO_FORMAT_RGB16_565 = 4
 } cairo_format_t;
 
+typedef enum _cairo_status {
+    CAIRO_STATUS_SUCCESS = 0,
+
+    CAIRO_STATUS_NO_MEMORY,
+    CAIRO_STATUS_INVALID_RESTORE,
+    CAIRO_STATUS_INVALID_POP_GROUP,
+    CAIRO_STATUS_NO_CURRENT_POINT,
+    CAIRO_STATUS_INVALID_MATRIX,
+    CAIRO_STATUS_INVALID_STATUS,
+    CAIRO_STATUS_NULL_POINTER,
+    CAIRO_STATUS_INVALID_STRING,
+    CAIRO_STATUS_INVALID_PATH_DATA,
+    CAIRO_STATUS_READ_ERROR,
+    CAIRO_STATUS_WRITE_ERROR,
+    CAIRO_STATUS_SURFACE_FINISHED,
+    CAIRO_STATUS_SURFACE_TYPE_MISMATCH,
+    CAIRO_STATUS_PATTERN_TYPE_MISMATCH,
+    CAIRO_STATUS_INVALID_CONTENT,
+    CAIRO_STATUS_INVALID_FORMAT,
+    CAIRO_STATUS_INVALID_VISUAL,
+    CAIRO_STATUS_FILE_NOT_FOUND,
+    CAIRO_STATUS_INVALID_DASH,
+    CAIRO_STATUS_INVALID_DSC_COMMENT,
+    CAIRO_STATUS_INVALID_INDEX,
+    CAIRO_STATUS_CLIP_NOT_REPRESENTABLE,
+    CAIRO_STATUS_TEMP_FILE_ERROR,
+    CAIRO_STATUS_INVALID_STRIDE,
+    CAIRO_STATUS_FONT_TYPE_MISMATCH,
+    CAIRO_STATUS_USER_FONT_IMMUTABLE,
+    CAIRO_STATUS_USER_FONT_ERROR,
+    CAIRO_STATUS_NEGATIVE_COUNT,
+    CAIRO_STATUS_INVALID_CLUSTERS,
+    CAIRO_STATUS_INVALID_SLANT,
+    CAIRO_STATUS_INVALID_WEIGHT,
+    CAIRO_STATUS_INVALID_SIZE,
+    CAIRO_STATUS_USER_FONT_NOT_IMPLEMENTED,
+    CAIRO_STATUS_DEVICE_TYPE_MISMATCH,
+    CAIRO_STATUS_DEVICE_ERROR,
+
+    CAIRO_STATUS_LAST_STATUS
+} cairo_status_t;
+
 /* We define all structure pointers to be void* */
 typedef void GdkPixbuf;
 typedef void GMainContext;
@@ -363,8 +405,10 @@ static void (*fp_g_strfreev)(gchar **str_array);
 static cairo_surface_t* (*fp_cairo_image_surface_create)(cairo_format_t format,
                                int width, int height);
 static void (*fp_cairo_surface_destroy)(cairo_surface_t *surface);
+static cairo_status_t (*fp_cairo_surface_status)(cairo_surface_t *surface);
 static cairo_t* (*fp_cairo_create)(cairo_surface_t *target);
 static void (*fp_cairo_destroy)(cairo_t *cr);
+static cairo_status_t (*fp_cairo_status)(cairo_t *cr);
 static void (*fp_cairo_fill)(cairo_t *cr);
 static void (*fp_cairo_surface_flush)(cairo_surface_t *surface);
 static void (*fp_cairo_rectangle)(cairo_t *cr, double x, double y, double width,

src/java.desktop/unix/native/libawt_xawt/awt/swing_GTKEngine.c

 /*
- * Copyright (c) 2005, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,10 +23,11 @@
  * questions.
  */
 
-#include <stdlib.h>
-#include <string.h>
 #include "gtk_interface.h"
 #include "com_sun_java_swing_plaf_gtk_GTKEngine.h"
+#include <jni_util.h>
+#include <stdlib.h>
+#include <string.h>
 
 /* Static buffer for conversion from java.lang.String to UTF-8 */
 static char conversionBuffer[(CONV_BUFFER_SIZE - 1) * 3 + 1];
@@ -309,6 +310,11 @@ JNIEXPORT void JNICALL
 Java_com_sun_java_swing_plaf_gtk_GTKEngine_nativeStartPainting(
         JNIEnv *env, jobject this, jint w, jint h)
 {
+    if (w > 0x7FFF || h > 0x7FFF || (uintptr_t)4 * w * h > 0x7FFFFFFFL) {
+        // Same limitation as in X11SurfaceData.c
+        JNU_ThrowOutOfMemoryError(env, "Can't create offscreen surface");
+        return;
+    }
     gtk->gdk_threads_enter();
     gtk->init_painting(env, w, h);
     gtk->gdk_threads_leave();