diff --git a/docs/site/content/docs/cli/computer-use.mdx b/docs/site/content/docs/cli/computer-use.mdx index 47ed98dbe08..be075f766f0 100644 --- a/docs/site/content/docs/cli/computer-use.mdx +++ b/docs/site/content/docs/cli/computer-use.mdx @@ -103,7 +103,7 @@ On Linux and Windows, action payloads also pass briefly through a local operatio On Wayland, screenshots work on KDE Plasma, Hyprland and Sway. On Hyprland and Sway they need `grim`, GdkPixbuf, and `hyprctl` or `swaymsg`; Orca finds the target window by process ID and title and captures its region, so overlapping windows can show in the image. -On KDE Plasma, Orca asks KWin where the window is, takes a screenshot of the desktop through `xdg-desktop-portal` (with `xdg-desktop-portal-kde`), and crops it to the window. The portal saves that screenshot in your Pictures folder; Orca reads it and deletes it. The first screenshot makes the desktop ask whether apps may take screenshots. Answer the prompt, then run `get-app-state` again. If you deny it, screenshots keep failing until you allow them again in the desktop's permission settings. A KDE window that another window covers returns a screenshot error instead of an image of the other window. +On KDE Plasma, Orca asks KWin where the window is, takes a screenshot of the desktop through `xdg-desktop-portal` (with `xdg-desktop-portal-kde`), and crops it to the window. The portal saves that screenshot in your Pictures folder; Orca reads it and deletes it. The first screenshot makes the desktop ask whether apps may take screenshots. Answer the prompt, then run `get-app-state` again. If the prompt is answered after `get-app-state` gave up waiting, the portal may leave that one screenshot in Pictures. If you deny it, screenshots keep failing until you allow them again in the desktop's permission settings. A KDE window that another window covers returns a screenshot error instead of an image of the other window. GNOME doesn't tell apps where windows are, so it returns a screenshot error. On any Wayland desktop, when the window is minimized or on another workspace, matches more than one window, or has a different size than its accessibility bounds, `get-app-state` returns the accessibility tree with a screenshot error. diff --git a/native/computer-use-linux/runtime.py b/native/computer-use-linux/runtime.py index bf645d90cdf..b276ecd05dc 100644 --- a/native/computer-use-linux/runtime.py +++ b/native/computer-use-linux/runtime.py @@ -652,7 +652,7 @@ KWIN_LAYOUT_SCRIPT = """(function (target) { caption: window.caption, captionNormal: window.captionNormal === undefined ? null : window.captionNormal, stackingOrder: window.stackingOrder, - shown: !window.minimized && !window.hidden && !window.deleted + shown: !window.minimized && !window.hidden && !window.deleted && !window.shade && onCurrentDesktop(window) && onCurrentActivity(window), ownedPopup: window.popupWindow === true || window.transient === true, client: window.clientGeometry === undefined ? null : rect(window.clientGeometry), @@ -958,14 +958,16 @@ def take_portal_file(uri, not_before): with os.fdopen(descriptor, "rb") as handle: info = os.fstat(descriptor) # Why: the portal saves into the user's folders; only a file it just wrote for this call is ours to read and delete. - if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_mtime < not_before - 1 - or not 0 < info.st_size <= MAX_WAYLAND_PNG_BYTES): - raise ValueError("the desktop portal's screenshot file is not a new image file of this user") - data = handle.read(MAX_WAYLAND_PNG_BYTES + 1) - current = attempt(lambda: os.stat(path, follow_symlinks=False)) - if current is not None and (current.st_dev, current.st_ino) == (info.st_dev, info.st_ino): - attempt(lambda: os.unlink(path)) - return data + if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_mtime < not_before - 1: + raise ValueError("the desktop portal's screenshot file is not a new file of this user") + try: + if not 0 < info.st_size <= MAX_WAYLAND_PNG_BYTES: + raise ValueError("the desktop portal's screenshot exceeds the size limit") + return handle.read(MAX_WAYLAND_PNG_BYTES + 1) + finally: + current = attempt(lambda: os.stat(path, follow_symlinks=False)) + if current is not None and (current.st_dev, current.st_ino) == (info.st_dev, info.st_ino): + attempt(lambda: os.unlink(path)) def portal_screenshot_png(bus): @@ -992,7 +994,8 @@ def portal_screenshot_png(bus): if not pump_until(mine, PORTAL_ANSWER_SECONDS): attempt(lambda: bus_call(bus, PORTAL_SERVICE, handle, "org.freedesktop.portal.Request", "Close")) raise ValueError("the desktop portal did not answer; if the desktop is asking whether apps may take " - "screenshots, answer that prompt, then retry") + "screenshots, wait for the person to answer that prompt before retrying (allowing it may " + "leave one screenshot in their Pictures folder)") response, results = mine() finally: bus.signal_unsubscribe(subscription) diff --git a/native/computer-use-linux/runtime_render_test.py b/native/computer-use-linux/runtime_render_test.py index d9f9c119d2a..5c31d6126f4 100644 --- a/native/computer-use-linux/runtime_render_test.py +++ b/native/computer-use-linux/runtime_render_test.py @@ -332,6 +332,7 @@ class KWinScreenshotTest(unittest.TestCase): self.assertEqual(self.pixbuf_calls, []) +@unittest.skipIf(sys.platform == "win32", "the Linux bridge reads portal files with POSIX flags") class PortalFileTest(unittest.TestCase): def setUp(self): self.directory = Path(tempfile.mkdtemp())