fix(computer-use): clean up oversized portal files and skip shaded KDE windows

This commit is contained in:
Kelvin Amoaba
2026-11-12 14:18:26 +00:00
parent 30ec0dfec5
commit dced80ca21
3 changed files with 15 additions and 11 deletions
+1 -1
View File
@@ -103,7 +103,7 @@ On Linux and Windows, action payloads also pass briefly through a local operatio
On Wayland, screenshots work on KDE Plasma, Hyprland and Sway. On Hyprland and Sway they need `grim`, GdkPixbuf, and `hyprctl` or `swaymsg`; Orca finds the target window by process ID and title and captures its region, so overlapping windows can show in the image.
On KDE Plasma, Orca asks KWin where the window is, takes a screenshot of the desktop through `xdg-desktop-portal` (with `xdg-desktop-portal-kde`), and crops it to the window. The portal saves that screenshot in your Pictures folder; Orca reads it and deletes it. The first screenshot makes the desktop ask whether apps may take screenshots. Answer the prompt, then run `get-app-state` again. If you deny it, screenshots keep failing until you allow them again in the desktop's permission settings. A KDE window that another window covers returns a screenshot error instead of an image of the other window.
On KDE Plasma, Orca asks KWin where the window is, takes a screenshot of the desktop through `xdg-desktop-portal` (with `xdg-desktop-portal-kde`), and crops it to the window. The portal saves that screenshot in your Pictures folder; Orca reads it and deletes it. The first screenshot makes the desktop ask whether apps may take screenshots. Answer the prompt, then run `get-app-state` again. If the prompt is answered after `get-app-state` gave up waiting, the portal may leave that one screenshot in Pictures. If you deny it, screenshots keep failing until you allow them again in the desktop's permission settings. A KDE window that another window covers returns a screenshot error instead of an image of the other window.
GNOME doesn't tell apps where windows are, so it returns a screenshot error. On any Wayland desktop, when the window is minimized or on another workspace, matches more than one window, or has a different size than its accessibility bounds, `get-app-state` returns the accessibility tree with a screenshot error.
+13 -10
View File
@@ -652,7 +652,7 @@ KWIN_LAYOUT_SCRIPT = """(function (target) {
caption: window.caption,
captionNormal: window.captionNormal === undefined ? null : window.captionNormal,
stackingOrder: window.stackingOrder,
shown: !window.minimized && !window.hidden && !window.deleted
shown: !window.minimized && !window.hidden && !window.deleted && !window.shade
&& onCurrentDesktop(window) && onCurrentActivity(window),
ownedPopup: window.popupWindow === true || window.transient === true,
client: window.clientGeometry === undefined ? null : rect(window.clientGeometry),
@@ -958,14 +958,16 @@ def take_portal_file(uri, not_before):
with os.fdopen(descriptor, "rb") as handle:
info = os.fstat(descriptor)
# Why: the portal saves into the user's folders; only a file it just wrote for this call is ours to read and delete.
if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_mtime < not_before - 1
or not 0 < info.st_size <= MAX_WAYLAND_PNG_BYTES):
raise ValueError("the desktop portal's screenshot file is not a new image file of this user")
data = handle.read(MAX_WAYLAND_PNG_BYTES + 1)
current = attempt(lambda: os.stat(path, follow_symlinks=False))
if current is not None and (current.st_dev, current.st_ino) == (info.st_dev, info.st_ino):
attempt(lambda: os.unlink(path))
return data
if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_mtime < not_before - 1:
raise ValueError("the desktop portal's screenshot file is not a new file of this user")
try:
if not 0 < info.st_size <= MAX_WAYLAND_PNG_BYTES:
raise ValueError("the desktop portal's screenshot exceeds the size limit")
return handle.read(MAX_WAYLAND_PNG_BYTES + 1)
finally:
current = attempt(lambda: os.stat(path, follow_symlinks=False))
if current is not None and (current.st_dev, current.st_ino) == (info.st_dev, info.st_ino):
attempt(lambda: os.unlink(path))
def portal_screenshot_png(bus):
@@ -992,7 +994,8 @@ def portal_screenshot_png(bus):
if not pump_until(mine, PORTAL_ANSWER_SECONDS):
attempt(lambda: bus_call(bus, PORTAL_SERVICE, handle, "org.freedesktop.portal.Request", "Close"))
raise ValueError("the desktop portal did not answer; if the desktop is asking whether apps may take "
"screenshots, answer that prompt, then retry")
"screenshots, wait for the person to answer that prompt before retrying (allowing it may "
"leave one screenshot in their Pictures folder)")
response, results = mine()
finally:
bus.signal_unsubscribe(subscription)
@@ -332,6 +332,7 @@ class KWinScreenshotTest(unittest.TestCase):
self.assertEqual(self.pixbuf_calls, [])
@unittest.skipIf(sys.platform == "win32", "the Linux bridge reads portal files with POSIX flags")
class PortalFileTest(unittest.TestCase):
def setUp(self):
self.directory = Path(tempfile.mkdtemp())