8357584: [XWayland] [OL10] Robot.mousePress() is delivered to wrong place

Reviewed-by: lkostyra, arapte

Author: Alexander Zvegintsev

modules/javafx.graphics/src/main/java/com/sun/glass/ui/gtk/GtkRobot.java

@@ -25,6 +25,7 @@
 package com.sun.glass.ui.gtk;
 
 import com.sun.glass.ui.gtk.screencast.ScreencastHelper;
+import com.sun.glass.ui.gtk.screencast.XdgDesktopPortal;
 import javafx.scene.input.KeyCode;
 import javafx.scene.input.MouseButton;
 import javafx.scene.paint.Color;
@@ -35,20 +36,6 @@
 
 final class GtkRobot extends GlassRobot {
 
-    private static final String screenshotMethod;
-    private static final String METHOD_GTK = "gtk";
-    private static final String METHOD_SCREENCAST = "dbusScreencast";
-
-    static {
-        String waylandDisplay = System.getenv("WAYLAND_DISPLAY");
-        boolean isOnWayland = waylandDisplay != null && !waylandDisplay.isBlank();
-
-        String method =
-            System.getProperty("javafx.robot.screenshotMethod",
-                isOnWayland ? METHOD_SCREENCAST : METHOD_GTK);
-        screenshotMethod = method;
-    }
-
     @Override
     public void create() {
         // no-op
@@ -62,15 +49,23 @@ public void destroy() {
     @Override
     public void keyPress(KeyCode code) {
         Application.checkEventThread();
-        _keyPress(code.getCode());
+        if (XdgDesktopPortal.isRemoteDesktop() && ScreencastHelper.isAvailable()) {
+            ScreencastHelper.remoteDesktopKey(true, code.getCode());
+        } else {
+            _keyPress(code.getCode());
+        }
     }
 
     protected native void _keyPress(int code);
 
     @Override
     public void keyRelease(KeyCode code) {
         Application.checkEventThread();
-        _keyRelease(code.getCode());
+        if (XdgDesktopPortal.isRemoteDesktop() && ScreencastHelper.isAvailable()) {
+            ScreencastHelper.remoteDesktopKey(false, code.getCode());
+        } else {
+            _keyRelease(code.getCode());
+        }
     }
 
     protected native void _keyRelease(int code);
@@ -81,28 +76,45 @@ public void keyRelease(KeyCode code) {
     public void mouseMove(double x, double y) {
         Application.checkEventThread();
         _mouseMove((int) x, (int) y);
+        if (XdgDesktopPortal.isRemoteDesktop() && ScreencastHelper.isAvailable()) {
+            // We still call _mouseMove on purpose to change the mouse position
+            // within the XWayland server so that we can retrieve it later.
+            ScreencastHelper.remoteDesktopMouseMove((int) x, (int) y);
+        }
     }
 
     @Override
     public void mousePress(MouseButton... buttons) {
         Application.checkEventThread();
-        _mousePress(GlassRobot.convertToRobotMouseButton(buttons));
+        if (XdgDesktopPortal.isRemoteDesktop() && ScreencastHelper.isAvailable()) {
+            ScreencastHelper.remoteDesktopMouseButton(true, GlassRobot.convertToRobotMouseButton(buttons));
+        } else {
+            _mousePress(GlassRobot.convertToRobotMouseButton(buttons));
+        }
     }
 
     protected native void _mousePress(int button);
 
     @Override
     public void mouseRelease(MouseButton... buttons) {
         Application.checkEventThread();
-        _mouseRelease(GlassRobot.convertToRobotMouseButton(buttons));
+        if (XdgDesktopPortal.isRemoteDesktop() && ScreencastHelper.isAvailable()) {
+            ScreencastHelper.remoteDesktopMouseButton(false, GlassRobot.convertToRobotMouseButton(buttons));
+        } else {
+            _mouseRelease(GlassRobot.convertToRobotMouseButton(buttons));
+        }
     }
 
     protected native void _mouseRelease(int buttons);
 
     @Override
     public void mouseWheel(int wheelAmt) {
         Application.checkEventThread();
-        _mouseWheel(wheelAmt);
+        if (XdgDesktopPortal.isRemoteDesktop() && ScreencastHelper.isAvailable()) {
+            ScreencastHelper.remoteDesktopMouseWheel(wheelAmt);
+        } else {
+            _mouseWheel(wheelAmt);
+        }
     }
 
     protected native void _mouseWheel(int wheelAmt);
@@ -130,7 +142,8 @@ public Color getPixelColor(double x, double y) {
         x = (int) Math.floor((x + 0.5) * mainScreen.getPlatformScaleX());
         y = (int) Math.floor((y + 0.5) * mainScreen.getPlatformScaleY());
         int[] result = new int[1];
-        if (METHOD_SCREENCAST.equals(screenshotMethod)) {
+        if ((XdgDesktopPortal.isScreencast()
+                || XdgDesktopPortal.isRemoteDesktop()) && ScreencastHelper.isAvailable()) {
             ScreencastHelper.getRGBPixels((int) x, (int) y, 1, 1, result);
         } else {
             _getScreenCapture((int) x, (int) y, 1, 1, result);
@@ -143,7 +156,8 @@ public Color getPixelColor(double x, double y) {
     @Override
     public void getScreenCapture(int x, int y, int width, int height, int[] data, boolean scaleToFit) {
         Application.checkEventThread();
-        if (METHOD_SCREENCAST.equals(screenshotMethod)) {
+        if ((XdgDesktopPortal.isScreencast()
+                || XdgDesktopPortal.isRemoteDesktop()) && ScreencastHelper.isAvailable()) {
             ScreencastHelper.getRGBPixels(x, y, width, height, data);
         } else {
             _getScreenCapture(x, y, width, height, data);

modules/javafx.graphics/src/main/java/com/sun/glass/ui/gtk/screencast/ScreencastHelper.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,24 +32,27 @@
 import java.util.Set;
 import java.util.Timer;
 import java.util.TimerTask;
+import java.util.function.Function;
 import java.util.stream.IntStream;
-import java.util.function.Supplier;
 
 /**
  * Helper class for grabbing pixels from the screen using the
  * <a href="https://flatpak.github.io/xdg-desktop-portal/#gdbus-org.freedesktop.portal.ScreenCast">
  * org.freedesktop.portal.ScreenCast API</a>
  */
 
-public class ScreencastHelper {
+public final class ScreencastHelper {
 
     static final boolean SCREENCAST_DEBUG;
     private static final boolean IS_NATIVE_LOADED;
 
-
     private static final int ERROR = -1;
     private static final int DENIED = -11;
     private static final int OUT_OF_BOUNDS = -12;
+    private static final int NO_STREAMS = -13;
+
+    private static final int XDG_METHOD_SCREENCAST = 0;
+    private static final int XDG_METHOD_REMOTE_DESKTOP = 1;
 
     private static final int DELAY_BEFORE_SESSION_CLOSE = 2000;
 
@@ -58,26 +61,37 @@ public class ScreencastHelper {
             = new Timer("auto-close screencast session", true);
 
 
-    private ScreencastHelper() {
-    }
+    private ScreencastHelper() {}
 
     static {
-        boolean isDebugEnabled =
-                ((Supplier<Boolean>) () -> {
-                    final String str =
-                            System.getProperty("javafx.robot.screenshotDebug", "false");
-                    return "true".equalsIgnoreCase(str);
-                }).get();
-        SCREENCAST_DEBUG = isDebugEnabled;
-
-        IS_NATIVE_LOADED = loadPipewire(SCREENCAST_DEBUG);
+        SCREENCAST_DEBUG = Boolean.getBoolean("javafx.robot.screenshotDebug");
+
+        boolean loadFailed = false;
+
+        boolean shouldLoadNative = XdgDesktopPortal.isRemoteDesktop()
+                || XdgDesktopPortal.isScreencast();
+
+        int methodId = XdgDesktopPortal.isScreencast()
+                ? XDG_METHOD_SCREENCAST
+                : XDG_METHOD_REMOTE_DESKTOP;
+
+        if (!(shouldLoadNative && loadPipewire(methodId, SCREENCAST_DEBUG))) {
+
+            System.err.println(
+                    "Could not load native libraries for ScreencastHelper"
+            );
+
+            loadFailed = true;
+        }
+
+        IS_NATIVE_LOADED = !loadFailed;
     }
 
     public static boolean isAvailable() {
         return IS_NATIVE_LOADED;
     }
 
-    private static native boolean loadPipewire(boolean screencastDebug);
+    private static native boolean loadPipewire(int method, boolean isDebug);
 
     private static native int getRGBPixelsImpl(
             int x, int y, int width, int height,
@@ -196,21 +210,62 @@ public static synchronized void getRGBPixels(
         debugReturnValue(retVal);
     }
 
-    private static void debugReturnValue(int retVal) {
+    private static boolean debugReturnValue(int retVal) {
         if (retVal == DENIED) {
-            // user explicitly denied the capture, no more tries.
             if (SCREENCAST_DEBUG) {
-                System.err.println("Screen Capture in the selected area was not allowed");
+                System.err.println("robot action: access denied by user.");
             }
         } else if (retVal == ERROR) {
             if (SCREENCAST_DEBUG) {
-                System.err.println("Screen capture failed.");
+                System.err.println("robot action: failed.");
             }
         } else if (retVal == OUT_OF_BOUNDS) {
             if (SCREENCAST_DEBUG) {
                 System.err.println(
                         "Token does not provide access to requested area.");
             }
+        } else if (retVal == NO_STREAMS) {
+            if (SCREENCAST_DEBUG) {
+                System.err.println("robot action: no streams available");
+            }
         }
+        return retVal != ERROR;
     }
+
+    private static void performWithToken(Function<String, Integer> func) {
+        if (!XdgDesktopPortal.isRemoteDesktop() || !IS_NATIVE_LOADED) return;
+
+        timerCloseSessionRestart();
+
+        for (TokenItem tokenItem : TokenStorage.getTokens(getSystemScreensBounds())) {
+            int retVal = func.apply(tokenItem.token);
+
+            if (retVal >= 0 || !debugReturnValue(retVal)) {
+                return;
+            }
+        }
+
+        debugReturnValue(func.apply(null));
+    }
+
+    public static synchronized void remoteDesktopMouseMove(int x, int y) {
+        performWithToken((token) -> remoteDesktopMouseMoveImpl(x, y, token));
+    }
+
+    public static synchronized void remoteDesktopMouseButton(boolean isPress, int buttons) {
+        performWithToken((token) -> remoteDesktopMouseButtonImpl(isPress, buttons, token));
+    }
+
+    public static synchronized void remoteDesktopMouseWheel(int wheel) {
+        performWithToken((token) -> remoteDesktopMouseWheelImpl(wheel, token));
+    }
+
+    public static synchronized void remoteDesktopKey(boolean isPress, int key) {
+        performWithToken((token) -> remoteDesktopKeyImpl(isPress, key, token));
+    }
+
+    private static synchronized native int remoteDesktopMouseMoveImpl(int x, int y, String token);
+    private static synchronized native int remoteDesktopMouseButtonImpl(boolean isPress, int buttons, String token);
+    private static synchronized native int remoteDesktopMouseWheelImpl(int wheelAmt, String token);
+    private static synchronized native int remoteDesktopKeyImpl(boolean isPress, int key, String token);
 }