From 508440075d7a441dacd5f016ea861034ecb8baeb Mon Sep 17 00:00:00 2001
From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com>
Date: Thu, 12 Nov 2026 13:45:03 +0000
Subject: [PATCH] fix(computer-use): capture KDE Wayland windows through the
desktop portal
KWin is the only process on KDE Wayland that knows where a window is and
what covers it, so a one-shot KWin script reports window geometry and
stacking. Orca crops the xdg-desktop-portal screenshot to that window and
returns a screenshot error when the window is hidden, covered, or moves
during capture, or when the portal prompt is unanswered or denied.
---
docs/site/content/docs/cli/computer-use.mdx | 6 +-
native/computer-use-linux/runtime.py | 373 +++++++++++++++++-
.../computer-use-linux/runtime_render_test.py | 123 ++++++
3 files changed, 482 insertions(+), 20 deletions(-)
diff --git a/docs/site/content/docs/cli/computer-use.mdx b/docs/site/content/docs/cli/computer-use.mdx
index 115a0fc79b2..47ed98dbe08 100644
--- a/docs/site/content/docs/cli/computer-use.mdx
+++ b/docs/site/content/docs/cli/computer-use.mdx
@@ -101,7 +101,11 @@ On Linux and Windows, action payloads also pass briefly through a local operatio
`get-app-state` returns an accessibility tree and, by default, a screenshot. With `--json`, the image bytes are written to disk and the path is returned in `screenshot.path` rather than embedded in the response. Pass `--no-screenshot` when pixels aren't needed (faster, smaller payload). Pass `--restore-window` to bring a hidden or minimized window into view before capture.
-On Wayland, screenshots work on Hyprland and Sway and need `grim`, GdkPixbuf, and `hyprctl` or `swaymsg`. Orca finds the target window by process ID and title and captures its region, so overlapping windows can show in the image. When the window is on another workspace, matches more than one window, has a different size than its accessibility bounds (for example, drawn shadows), or runs on another compositor such as GNOME or KDE, `get-app-state` returns the accessibility tree with a screenshot error.
+On Wayland, screenshots work on KDE Plasma, Hyprland and Sway. On Hyprland and Sway they need `grim`, GdkPixbuf, and `hyprctl` or `swaymsg`; Orca finds the target window by process ID and title and captures its region, so overlapping windows can show in the image.
+
+On KDE Plasma, Orca asks KWin where the window is, takes a screenshot of the desktop through `xdg-desktop-portal` (with `xdg-desktop-portal-kde`), and crops it to the window. The portal saves that screenshot in your Pictures folder; Orca reads it and deletes it. The first screenshot makes the desktop ask whether apps may take screenshots. Answer the prompt, then run `get-app-state` again. If you deny it, screenshots keep failing until you allow them again in the desktop's permission settings. A KDE window that another window covers returns a screenshot error instead of an image of the other window.
+
+GNOME doesn't tell apps where windows are, so it returns a screenshot error. On any Wayland desktop, when the window is minimized or on another workspace, matches more than one window, or has a different size than its accessibility bounds, `get-app-state` returns the accessibility tree with a screenshot error.
## Use it from an agent
diff --git a/native/computer-use-linux/runtime.py b/native/computer-use-linux/runtime.py
index 22851711576..d6e1e612d49 100644
--- a/native/computer-use-linux/runtime.py
+++ b/native/computer-use-linux/runtime.py
@@ -15,10 +15,12 @@ import os
import re
import shutil
import signal
+import stat
import subprocess
import sys
import tempfile
import time
+import urllib.parse
import uuid
from dataclasses import dataclass
@@ -43,6 +45,12 @@ except (ImportError, ValueError):
GdkPixbuf = None
from gi.repository import Atspi
+try:
+ # Why separate: KDE Wayland capture talks to KWin and the desktop portal over the session bus.
+ from gi.repository import Gio, GLib
+except ImportError:
+ Gio = GLib = None
+
try:
gi.require_version("GdkX11", "3.0")
from gi.repository import GdkX11
@@ -595,6 +603,72 @@ def rendered_element_index(line, depth):
return int(match.group(1)) if match else None
+KWIN_SERVICE = "org.kde.KWin"
+PORTAL_SERVICE = "org.freedesktop.portal.Desktop"
+DBUS_DAEMON = ("org.freedesktop.DBus", "/org/freedesktop/DBus", "org.freedesktop.DBus")
+PERMISSION_STORE = ("org.freedesktop.impl.portal.PermissionStore", "/org/freedesktop/impl/portal/PermissionStore",
+ "org.freedesktop.impl.portal.PermissionStore")
+KWIN_REPORT_XML = (""
+ "")
+KWIN_QUERY_SECONDS = 2
+PORTAL_ANSWER_SECONDS = 10
+MAX_WAYLAND_PNG_BYTES = 64_000_000
+UNSUPPORTED_WAYLAND_SCREENSHOT = ("Wayland screenshots are supported on KDE Plasma, Hyprland and Sway; this desktop "
+ "does not tell apps where windows are. Use --no-screenshot or an X11 session")
+# One-shot KWin script: KWin is the only process on KDE Wayland that knows where windows are and what covers them.
+KWIN_LAYOUT_SCRIPT = """(function (target) {
+ function rect(value) {
+ return value ? [value.x, value.y, value.width, value.height] : null;
+ }
+ function onCurrentDesktop(window) {
+ if (window.onAllDesktops) {
+ return true;
+ }
+ var current = workspace.currentDesktopForScreen && window.output
+ ? workspace.currentDesktopForScreen(window.output) : workspace.currentDesktop;
+ if (typeof current === "number") {
+ return window.desktop === current;
+ }
+ var desktops = window.desktops || [];
+ for (var i = 0; i < desktops.length; i++) {
+ if (desktops[i].id === current.id) {
+ return true;
+ }
+ }
+ return false;
+ }
+ function onCurrentActivity(window) {
+ var activities = window.activities || [];
+ return activities.length === 0 || activities.indexOf(workspace.currentActivity) >= 0;
+ }
+ var report;
+ try {
+ var windows = workspace.windowList ? workspace.windowList() : workspace.clientList();
+ report = {workspace: rect(workspace.virtualScreenGeometry), windows: []};
+ for (var i = 0; i < windows.length; i++) {
+ var window = windows[i];
+ report.windows.push({
+ pid: window.pid,
+ caption: window.caption,
+ captionNormal: window.captionNormal === undefined ? null : window.captionNormal,
+ stackingOrder: window.stackingOrder,
+ shown: !window.minimized && !window.hidden && !window.deleted
+ && onCurrentDesktop(window) && onCurrentActivity(window),
+ desktopWindow: window.desktopWindow === true,
+ ownedPopup: window.popupWindow === true || window.transient === true,
+ client: window.clientGeometry === undefined ? null : rect(window.clientGeometry),
+ buffer: rect(window.bufferGeometry),
+ frame: rect(window.frameGeometry)
+ });
+ }
+ } catch (error) {
+ report = {error: String(error)};
+ }
+ callDBus(target[0], target[1], target[2], "Report", JSON.stringify(report));
+})(__ORCA_REPORT_TARGET__);
+"""
+
+
def wayland_compositor():
if os.environ.get("HYPRLAND_INSTANCE_SIGNATURE") and shutil.which("hyprctl"):
return "hyprland"
@@ -608,6 +682,37 @@ def wayland_session():
return os.environ.get("XDG_SESSION_TYPE", "").lower() == "wayland" or wayland_compositor() is not None
+def session_bus():
+ return Gio.bus_get_sync(Gio.BusType.SESSION, None) if Gio is not None else None
+
+
+def bus_call(bus, service, path, interface, method, parameters=None, reply_type=None, timeout_ms=2000):
+ return bus.call_sync(service, path, interface, method, parameters,
+ GLib.VariantType.new(reply_type) if reply_type else None,
+ Gio.DBusCallFlags.NONE, timeout_ms, None)
+
+
+def bus_name_owner(bus, name):
+ return attempt(lambda: bus_call(bus, *DBUS_DAEMON, "GetNameOwner", GLib.Variant("(s)", (name,)), "(s)").unpack()[0])
+
+
+def wayland_screenshot_backend():
+ compositor = wayland_compositor()
+ if compositor is not None:
+ return compositor
+ bus = attempt(session_bus)
+ return "kwin" if bus is not None and bus_name_owner(bus, KWIN_SERVICE) else None
+
+
+def wayland_screenshot_ready():
+ backend = wayland_screenshot_backend()
+ if backend != "kwin":
+ return backend is not None and bool(shutil.which("grim"))
+ bus = session_bus()
+ activatable = attempt(lambda: bus_call(bus, *DBUS_DAEMON, "ListActivatableNames", None, "(as)").unpack()[0], [])
+ return bus_name_owner(bus, PORTAL_SERVICE) is not None or PORTAL_SERVICE in activatable
+
+
def screenshot_failure(message):
return {"error": {"code": "screenshot_failed", "message": message}}
@@ -654,6 +759,15 @@ def sway_windows(pid, title):
return [[rect.get(key) for key in ("x", "y", "width", "height")]], matches[0].get("visible") is True
+def finite_rect(values):
+ if (not isinstance(values, list) or len(values) != 4
+ or any(isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value)
+ for value in values)
+ or values[2] <= 0 or values[3] <= 0):
+ return None
+ return Rect(*(float(value) for value in values))
+
+
def wayland_window_rect(compositor, app, window):
try:
lookup = hyprland_windows if compositor == "hyprland" else sway_windows
@@ -665,31 +779,40 @@ def wayland_window_rect(compositor, app, window):
# Why: grim captures a screen region, so a window off-workspace or behind a group tab would return other content.
if not visible:
return None, "the window is not visible on any output; switch to its workspace"
- values = rects[0]
- if (len(values) != 4 or any(not isinstance(value, (int, float)) or not math.isfinite(value) for value in values)
- or values[2] <= 0 or values[3] <= 0):
+ rect = finite_rect(rects[0])
+ if rect is None:
return None, "the compositor reported invalid window geometry"
- return Rect(*values), None
+ return rect, None
+
+
+def same_size(rect, bounds):
+ return bounds is not None and abs(bounds.width - rect.width) <= 2 and abs(bounds.height - rect.height) <= 2
+
+
+def pixel_box(rect):
+ x, y = math.floor(rect.x), math.floor(rect.y)
+ return x, y, math.ceil(rect.x + rect.width) - x, math.ceil(rect.y + rect.height) - y
def capture_wayland_png(bounds, app, window):
- compositor = wayland_compositor()
- if compositor is None:
- return screenshot_failure("Wayland screenshots are supported on Hyprland and Sway only; use --no-screenshot or an X11 session")
+ backend = wayland_screenshot_backend()
+ if backend is None:
+ return screenshot_failure(UNSUPPORTED_WAYLAND_SCREENSHOT)
+ if app is None or window is None:
+ return screenshot_failure("Wayland screenshots need a target window")
+ if backend == "kwin":
+ return capture_kwin_png(bounds, app, window)
grim = shutil.which("grim")
if grim is None or GdkPixbuf is None:
return screenshot_failure("Wayland screenshots require grim and GdkPixbuf; install them or use --no-screenshot")
- if app is None or window is None:
- return screenshot_failure("Wayland screenshots need a target window")
# AT-SPI often reports (0, 0) on Wayland, which would capture an unrelated desktop region.
- rect, error = wayland_window_rect(compositor, app, window)
+ rect, error = wayland_window_rect(backend, app, window)
if rect is None:
return screenshot_failure(f"Wayland screenshot skipped: {error}")
# Why: element frames are AT-SPI coordinates; a crop of another size (e.g. CSD shadows) would misplace them.
- if bounds is None or abs(bounds.width - rect.width) > 2 or abs(bounds.height - rect.height) > 2:
+ if not same_size(rect, bounds):
return screenshot_failure("Wayland screenshot skipped: the compositor window size differs from the accessibility bounds")
- x, y = math.floor(rect.x), math.floor(rect.y)
- width, height = math.ceil(rect.x + rect.width) - x, math.ceil(rect.y + rect.height) - y
+ x, y, width, height = pixel_box(rect)
if width > 16384 or height > 16384 or width * height > 67_108_864:
return screenshot_failure("Wayland window exceeds the screenshot dimension limit")
try:
@@ -698,13 +821,10 @@ def capture_wayland_png(bounds, app, window):
result = subprocess.run([grim, "-s", "1", "-g", f"{x},{y} {width}x{height}", "-t", "png", "-"],
stdout=output, stderr=subprocess.DEVNULL, timeout=5, check=False)
size = output.tell()
- if result.returncode != 0 or not 0 < size <= 64_000_000:
+ if result.returncode != 0 or not 0 < size <= MAX_WAYLAND_PNG_BYTES:
return screenshot_failure("grim could not capture the window; check compositor screencopy support and screen-capture access")
output.seek(0)
- loader = GdkPixbuf.PixbufLoader.new_with_type("png")
- loader.write(output.read())
- loader.close()
- pixbuf = loader.get_pixbuf()
+ pixbuf = decode_png(output.read())
if pixbuf is None or pixbuf.get_width() != width or pixbuf.get_height() != height:
return screenshot_failure("grim returned an image with unexpected window dimensions")
return bounded_png_payload(pixbuf)
@@ -712,6 +832,221 @@ def capture_wayland_png(bounds, app, window):
return screenshot_failure("Wayland screenshot failed: " + str(error)[:200])
+def decode_png(data):
+ loader = GdkPixbuf.PixbufLoader.new_with_type("png")
+ loader.write(data)
+ loader.close()
+ return loader.get_pixbuf()
+
+
+def pump_until(done, seconds):
+ deadline = time.monotonic() + seconds
+ context = GLib.MainContext.default()
+ # Why: a periodic wake-up lets the deadline end a blocking iteration when no D-Bus message arrives.
+ tick = GLib.timeout_add(20, lambda: True)
+ try:
+ while not done() and time.monotonic() < deadline:
+ context.iteration(True)
+ finally:
+ GLib.source_remove(tick)
+ return done()
+
+
+def run_kwin_script(bus, script_id):
+ try:
+ bus_call(bus, KWIN_SERVICE, f"/Scripting/Script{script_id}", "org.kde.kwin.Script", "run")
+ except GLib.Error:
+ # Why: Plasma 5 exports loaded scripts at / instead of /Scripting/Script.
+ bus_call(bus, KWIN_SERVICE, f"/{script_id}", "org.kde.kwin.Script", "run")
+
+
+def kwin_layout(bus):
+ kwin = bus_name_owner(bus, KWIN_SERVICE)
+ if kwin is None:
+ raise ValueError("KWin is not on the session bus")
+ token = uuid.uuid4().hex
+ path = "/dev/orca/ComputerUse/KWinReport/" + token
+ reports = []
+
+ def on_call(_connection, sender, _path, _interface, method, parameters, invocation):
+ if sender == kwin and method == "Report":
+ reports.append(parameters.unpack()[0])
+ invocation.return_value(None)
+
+ interface = Gio.DBusNodeInfo.new_for_xml(KWIN_REPORT_XML).interfaces[0]
+ registration = bus.register_object(path, interface, on_call, None, None)
+ plugin = "orca-computer-use-" + token
+ descriptor, script_path = tempfile.mkstemp(prefix="orca-kwin-", suffix=".js", dir=os.environ.get("XDG_RUNTIME_DIR"))
+ try:
+ target = json.dumps([bus.get_unique_name(), path, "dev.orca.ComputerUse.KWinReport"])
+ with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+ handle.write(KWIN_LAYOUT_SCRIPT.replace("__ORCA_REPORT_TARGET__", target))
+ script_id = bus_call(bus, KWIN_SERVICE, "/Scripting", "org.kde.kwin.Scripting", "loadScript",
+ GLib.Variant("(ss)", (script_path, plugin)), "(i)").unpack()[0]
+ if script_id < 0:
+ raise ValueError("KWin refused the window layout script")
+ run_kwin_script(bus, script_id)
+ if not pump_until(lambda: reports, KWIN_QUERY_SECONDS):
+ raise ValueError("KWin did not report window geometry")
+ finally:
+ attempt(lambda: bus_call(bus, KWIN_SERVICE, "/Scripting", "org.kde.kwin.Scripting", "unloadScript",
+ GLib.Variant("(s)", (plugin,)), "(b)"))
+ bus.unregister_object(registration)
+ attempt(lambda: os.unlink(script_path))
+ report = json.loads(reports[0])
+ if not isinstance(report, dict) or report.get("error"):
+ raise ValueError("KWin could not list windows: " + str(report.get("error") if isinstance(report, dict) else report)[:120])
+ return report
+
+
+def overlaps(a, b):
+ return a.x < b.x + b.width and b.x < a.x + a.width and a.y < b.y + b.height and b.y < a.y + a.height
+
+
+def kwin_window_rect(report, pid, title, bounds):
+ windows = [item for item in report.get("windows") or [] if isinstance(item, dict)]
+ workspace = finite_rect(report.get("workspace"))
+ matches = [item for item in windows if item.get("pid") == pid and title in (item.get("captionNormal"), item.get("caption"))]
+ if len(matches) != 1:
+ return None, "no single KWin window matches this app and window title"
+ target = matches[0]
+ if target.get("shown") is not True:
+ return None, "the window is minimized, hidden, or on another virtual desktop or activity"
+ # Why: Wayland AT-SPI frames are window-local; the KWin rectangle with the toolkit's size is their origin.
+ rect = next((rect for rect in (finite_rect(target.get(key)) for key in ("client", "buffer", "frame"))
+ if rect is not None and same_size(rect, bounds)), None)
+ if rect is None:
+ return None, "the KWin window size differs from the accessibility bounds"
+ if workspace is None or not overlaps(workspace, rect):
+ return None, "the window is not on any screen"
+ order = target.get("stackingOrder")
+ if isinstance(order, bool) or not isinstance(order, (int, float)):
+ return None, "KWin reported no stacking order for the window"
+ for other in windows:
+ if other is target or other.get("shown") is not True or other.get("desktopWindow") is True:
+ continue
+ # The app's own menus and dialogs are part of what it shows.
+ if other.get("pid") == pid and other.get("ownedPopup") is True:
+ continue
+ above = other.get("stackingOrder")
+ frame = finite_rect(other.get("frame"))
+ if (isinstance(above, bool) or not isinstance(above, (int, float)) or above > order) and frame and overlaps(frame, rect):
+ return None, "another window covers part of it; bring it to the front and retry"
+ return (rect, workspace), None
+
+
+def screenshot_permissions(bus):
+ return attempt(lambda: bus_call(bus, *PERMISSION_STORE, "Lookup", GLib.Variant("(ss)", ("screenshot", "screenshot")),
+ "(a{sas}v)").unpack()[0])
+
+
+def take_portal_file(uri, not_before):
+ parsed = urllib.parse.urlsplit(uri) if isinstance(uri, str) else None
+ if parsed is None or parsed.scheme != "file" or parsed.netloc not in ("", "localhost"):
+ raise ValueError("the desktop portal returned no local screenshot file")
+ path = urllib.parse.unquote(parsed.path)
+ descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC)
+ with os.fdopen(descriptor, "rb") as handle:
+ info = os.fstat(descriptor)
+ # Why: the portal saves into the user's folders; only a file it just wrote for this call is ours to read and delete.
+ if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_mtime < not_before - 1
+ or not 0 < info.st_size <= MAX_WAYLAND_PNG_BYTES):
+ raise ValueError("the desktop portal's screenshot file is not a new image file of this user")
+ data = handle.read(MAX_WAYLAND_PNG_BYTES + 1)
+ current = attempt(lambda: os.stat(path, follow_symlinks=False))
+ if current is not None and (current.st_dev, current.st_ino) == (info.st_dev, info.st_ino):
+ attempt(lambda: os.unlink(path))
+ return data
+
+
+def portal_screenshot_png(bus):
+ permissions = screenshot_permissions(bus)
+ token = "orca_" + uuid.uuid4().hex
+ answers = []
+ subscription = bus.signal_subscribe(None, "org.freedesktop.portal.Request", "Response", None, None,
+ Gio.DBusSignalFlags.NONE,
+ lambda _c, sender, path, _i, _s, parameters: answers.append((sender, path, parameters.unpack())))
+ started = time.time()
+ try:
+ options = {"handle_token": GLib.Variant("s", token), "interactive": GLib.Variant("b", False)}
+ try:
+ handle = bus_call(bus, PORTAL_SERVICE, "/org/freedesktop/portal/desktop", "org.freedesktop.portal.Screenshot",
+ "Screenshot", GLib.Variant("(sa{sv})", ("", options)), "(o)", 5000).unpack()[0]
+ except GLib.Error as error:
+ if Gio.DBusError.get_remote_error(error) in ("org.freedesktop.DBus.Error.ServiceUnknown",
+ "org.freedesktop.DBus.Error.UnknownInterface",
+ "org.freedesktop.DBus.Error.UnknownMethod"):
+ raise ValueError("KDE Wayland screenshots need xdg-desktop-portal and xdg-desktop-portal-kde; install them or use --no-screenshot")
+ raise
+ portal = bus_name_owner(bus, PORTAL_SERVICE)
+ mine = lambda: next((answer for sender, path, answer in answers if path == handle and sender == portal), None)
+ if not pump_until(mine, PORTAL_ANSWER_SECONDS):
+ attempt(lambda: bus_call(bus, PORTAL_SERVICE, handle, "org.freedesktop.portal.Request", "Close"))
+ raise ValueError("the desktop portal did not answer; if the desktop is asking whether apps may take "
+ "screenshots, answer that prompt, then retry")
+ response, results = mine()
+ finally:
+ bus.signal_unsubscribe(subscription)
+ if response != 0:
+ raise ValueError("the desktop declined the screenshot; if a screenshot prompt was denied, allow screenshots "
+ "for this app again in the desktop's permission settings, then retry")
+ data = take_portal_file(results.get("uri"), started)
+ # Why: the first call shows a consent prompt that can still be fading out in this image.
+ if screenshot_permissions(bus) != permissions:
+ raise ValueError("screenshots were just allowed for this app; retry to capture the window")
+ return data
+
+
+def crop_workspace_png(data, rect, workspace):
+ image = decode_png(data)
+ if image is None:
+ raise ValueError("the desktop portal's screenshot is not a readable PNG")
+ # Why: KWin renders the whole workspace at its largest output scale; any other size means an unknown layout.
+ factor = image.get_width() / workspace.width
+ if abs(workspace.height * factor - image.get_height()) > 1:
+ raise ValueError("the desktop portal's screenshot does not match KWin's screen layout")
+ x, y, width, height = pixel_box(rect)
+ left, top = round((x - workspace.x) * factor), round((y - workspace.y) * factor)
+ right, bottom = round((x + width - workspace.x) * factor), round((y + height - workspace.y) * factor)
+ shown_left, shown_top = max(left, 0), max(top, 0)
+ shown_right, shown_bottom = min(right, image.get_width()), min(bottom, image.get_height())
+ if shown_right <= shown_left or shown_bottom <= shown_top:
+ raise ValueError("the window lies outside the desktop portal's screenshot")
+ if (shown_left, shown_top, shown_right, shown_bottom) == (left, top, right, bottom):
+ window = image.new_subpixbuf(left, top, right - left, bottom - top)
+ else:
+ # Why: a window partly off screen keeps its full size, transparent where no screen shows it, so frames stay aligned.
+ window = GdkPixbuf.Pixbuf.new(GdkPixbuf.Colorspace.RGB, True, 8, right - left, bottom - top)
+ window.fill(0)
+ image.copy_area(shown_left, shown_top, shown_right - shown_left, shown_bottom - shown_top,
+ window, shown_left - left, shown_top - top)
+ # Logical size keeps screenshot pixels aligned with window-local accessibility frames.
+ if (right - left, bottom - top) != (width, height):
+ window = window.scale_simple(width, height, GdkPixbuf.InterpType.BILINEAR)
+ return window
+
+
+def capture_kwin_png(bounds, app, window):
+ if GdkPixbuf is None:
+ return screenshot_failure("KDE Wayland screenshots require GdkPixbuf; install it or use --no-screenshot")
+ pid, title = pid_of(app), name_of(window)
+ try:
+ bus = session_bus()
+ layout, error = kwin_window_rect(kwin_layout(bus), pid, title, bounds)
+ if layout is None:
+ return screenshot_failure(f"Wayland screenshot skipped: {error}")
+ data = portal_screenshot_png(bus)
+ # Why: the portal may wait on a consent prompt; a window that moved meanwhile would misalign the crop.
+ after, _ = kwin_window_rect(kwin_layout(bus), pid, title, bounds)
+ if after != layout:
+ return screenshot_failure("Wayland screenshot skipped: the window moved, resized or was covered while capturing; retry")
+ return bounded_png_payload(crop_workspace_png(data, *layout))
+ except ValueError as error:
+ return screenshot_failure("Wayland screenshot skipped: " + str(error)[:300])
+ except Exception as error:
+ return screenshot_failure("Wayland screenshot failed: " + str(error)[:200])
+
+
def capture_png(rect, app=None, window=None):
if wayland_session():
return capture_wayland_png(rect, app, window)
@@ -905,7 +1240,7 @@ def handshake_response():
has_hotkey = shutil.which("xdotool") is not None and not is_wayland
has_clipboard = any(shutil.which(command) for command in ("wl-copy", "xclip", "xsel"))
has_screenshot = GdkPixbuf is not None and (
- bool(shutil.which("grim") and wayland_compositor()) if wayland_session() else Gdk is not None
+ attempt(wayland_screenshot_ready, False) if wayland_session() else Gdk is not None
)
return {
"platform": "linux",
diff --git a/native/computer-use-linux/runtime_render_test.py b/native/computer-use-linux/runtime_render_test.py
index a511c8b4c72..3be7bb8fe30 100644
--- a/native/computer-use-linux/runtime_render_test.py
+++ b/native/computer-use-linux/runtime_render_test.py
@@ -1,7 +1,10 @@
from contextlib import ExitStack, nullcontext
import importlib.util
import json
+import shutil
import sys
+import tempfile
+import time
import types
import unittest
from pathlib import Path
@@ -232,6 +235,126 @@ class WaylandScreenshotTest(unittest.TestCase):
self.assertIsNone(hidden)
self.assertIsNotNone(hidden_error)
+class FakePixbuf:
+ def __init__(self, width, height, calls):
+ self.width, self.height, self.calls = width, height, calls
+
+ def get_width(self):
+ return self.width
+
+ def get_height(self):
+ return self.height
+
+ def new_subpixbuf(self, x, y, width, height):
+ self.calls.append(("crop", x, y, width, height))
+ return FakePixbuf(width, height, self.calls)
+
+ def fill(self, _):
+ self.calls.append(("fill", self.width, self.height))
+
+ def copy_area(self, x, y, width, height, _, to_x, to_y):
+ self.calls.append(("copy", x, y, width, height, to_x, to_y))
+
+ def scale_simple(self, width, height, _):
+ self.calls.append(("scale", width, height))
+ return FakePixbuf(width, height, self.calls)
+
+ def save_to_bufferv(self, *_):
+ return True, b"png"
+
+
+class KWinScreenshotTest(unittest.TestCase):
+ def setUp(self):
+ self.app = types.SimpleNamespace(get_process_id=lambda: 42)
+ self.window = types.SimpleNamespace(get_name=lambda: "Editor")
+ self.target = {"pid": 42, "caption": "Editor", "captionNormal": "Editor", "stackingOrder": 3, "shown": True,
+ "client": [100, 50, 400, 300], "buffer": [90, 40, 420, 320], "frame": [100, 20, 400, 330]}
+ self.reports = []
+ self.portal_calls = []
+ self.pixbuf_calls = []
+ self.context = ExitStack()
+ self.addCleanup(self.context.close)
+ for name, value in (
+ ("wayland_session", lambda: True),
+ ("wayland_screenshot_backend", lambda: "kwin"),
+ ("session_bus", lambda: "bus"),
+ ("kwin_layout", lambda _: self.reports.pop(0) if len(self.reports) > 1 else self.reports[0]),
+ ("portal_screenshot_png", self.portal),
+ ("decode_png", lambda _: FakePixbuf(5120, 1600, self.pixbuf_calls)),
+ ("GdkPixbuf", types.SimpleNamespace(
+ InterpType=types.SimpleNamespace(BILINEAR=0), Colorspace=types.SimpleNamespace(RGB=0),
+ Pixbuf=types.SimpleNamespace(new=lambda _c, _a, _b, width, height: FakePixbuf(width, height, self.pixbuf_calls)))),
+ ):
+ self.context.enter_context(patch.object(runtime, name, value))
+
+ def portal(self, _):
+ self.portal_calls.append(True)
+ return b"desktop"
+
+ def capture(self, *windows, bounds=(400, 300)):
+ self.reports = [{"workspace": [-1280, 0, 2560, 800], "windows": list(windows)}]
+ return runtime.capture_png(runtime.Rect(0, 0, *bounds), self.app, self.window)
+
+ def test_windows_kwin_cannot_place_never_reach_the_portal(self):
+ cover = {**self.target, "pid": 7, "caption": "Other", "captionNormal": "Other", "stackingOrder": 4,
+ "frame": [450, 300, 200, 200]}
+ for windows in ([], [self.target, dict(self.target)], [{**self.target, "shown": False}],
+ [{**self.target, "pid": 43}], [{**self.target, "client": [100, 50, 500, 300]}],
+ [{**self.target, "client": [1300, 50, 400, 300]}], [self.target, cover]):
+ with self.subTest(windows=windows):
+ self.portal_calls.clear()
+ result = self.capture(*windows)
+ self.assertEqual(result["error"]["code"], "screenshot_failed")
+ self.assertEqual(self.portal_calls, [])
+
+ def test_crop_maps_the_matching_kwin_rectangle_into_the_scaled_workspace_image(self):
+ below = {**self.target, "pid": 7, "caption": "Other", "stackingOrder": 1, "frame": [0, 0, 800, 800]}
+ menu = {**self.target, "caption": "Menu", "captionNormal": "Menu", "stackingOrder": 5, "ownedPopup": True,
+ "frame": [150, 80, 100, 200]}
+ result = self.capture(self.target, below, menu)
+ self.assertNotIn("error", result)
+ self.assertEqual(self.pixbuf_calls, [("crop", 2760, 100, 800, 600), ("scale", 400, 300)])
+ self.pixbuf_calls.clear()
+ # Client-side decorations: the toolkit reports the buffer (shadow included), not the content rectangle.
+ self.capture(self.target, bounds=(420, 320))
+ self.assertEqual(self.pixbuf_calls, [("crop", 2740, 80, 840, 640), ("scale", 420, 320)])
+ self.pixbuf_calls.clear()
+ # Partly left of the leftmost screen: full window size, only the shown part copied at its offset.
+ self.capture({**self.target, "client": [-1380, 50, 400, 300]})
+ self.assertEqual(self.pixbuf_calls, [("fill", 800, 600), ("copy", 0, 100, 600, 600, 200, 0), ("scale", 400, 300)])
+
+ def test_window_that_moves_while_the_portal_answers_is_not_cropped(self):
+ moved = {"workspace": [-1280, 0, 2560, 800], "windows": [{**self.target, "client": [120, 50, 400, 300]}]}
+ self.reports = [{"workspace": [-1280, 0, 2560, 800], "windows": [self.target]}, moved]
+ result = runtime.capture_png(runtime.Rect(0, 0, 400, 300), self.app, self.window)
+ self.assertEqual(result["error"]["code"], "screenshot_failed")
+ self.assertEqual(self.pixbuf_calls, [])
+
+
+class PortalFileTest(unittest.TestCase):
+ def setUp(self):
+ self.directory = Path(tempfile.mkdtemp())
+ self.addCleanup(shutil.rmtree, self.directory)
+ self.file = self.directory / "Screenshot 1.png"
+ self.file.write_bytes(b"png")
+
+ def test_reads_and_removes_only_a_new_file_the_portal_named(self):
+ uri = self.file.as_uri()
+ self.assertEqual(runtime.take_portal_file(uri, time.time()), b"png")
+ self.assertFalse(self.file.exists())
+
+ self.file.write_bytes(b"png")
+ with self.assertRaises(ValueError):
+ runtime.take_portal_file(uri, time.time() + 60)
+ self.assertTrue(self.file.exists())
+
+ link = self.directory / "link.png"
+ link.symlink_to(self.file)
+ with self.assertRaises(OSError):
+ runtime.take_portal_file(link.as_uri(), time.time())
+ self.assertTrue(link.is_symlink() and self.file.exists())
+
+
class RuntimeRenderTest(unittest.TestCase):
def test_focus_search_keeps_transient_accessible_wrappers_alive(self):