diff --git a/docs/site/content/docs/cli/computer-use.mdx b/docs/site/content/docs/cli/computer-use.mdx index ce971148bd6..6d4c319cc57 100644 --- a/docs/site/content/docs/cli/computer-use.mdx +++ b/docs/site/content/docs/cli/computer-use.mdx @@ -128,7 +128,7 @@ On Linux and Windows, action payloads also pass briefly through a local operatio ## Screenshots -`get-app-state` returns an accessibility tree and, by default, a screenshot. With `--json`, the image bytes are written to disk and the path is returned in `screenshot.path` rather than embedded in the response. Pass `--no-screenshot` when pixels aren't needed (faster, smaller payload). On macOS, a window that is minimized or hidden has no pixels, so the screenshot fails while the accessibility tree still comes back; a window on another Space is captured. +`get-app-state` returns an accessibility tree and, by default, a screenshot. With `--json`, the image bytes are written to disk and the path is returned in `screenshot.path` rather than embedded in the response. Pass `--no-screenshot` when pixels aren't needed (faster, smaller payload). To keep pixels but send fewer bytes, `--screenshot-scale 1` caps the image at one pixel per window point (Retina captures are otherwise two), `--screenshot-max-edge ` caps its longer side, and `--screenshot-format jpeg` encodes it as JPEG; `screenshot.format` and `screenshot.scale` say what came back. On macOS, a window that is minimized or hidden has no pixels, so the screenshot fails while the accessibility tree still comes back; a window on another Space is captured. On Wayland, screenshots work on KDE Plasma, Hyprland and Sway. On Hyprland and Sway they need `grim`, GdkPixbuf, and `hyprctl` or `swaymsg`; Orca finds the target window by process ID and title and captures its region, so overlapping windows can show in the image. diff --git a/native/computer-use-linux/runtime.py b/native/computer-use-linux/runtime.py index 3c92f8b7d0f..127ad8f6135 100644 --- a/native/computer-use-linux/runtime.py +++ b/native/computer-use-linux/runtime.py @@ -63,6 +63,7 @@ TEXT_LIMIT = 500 MAX_SCREENSHOT_PNG_BYTES = 900_000 MAX_SCREENSHOT_EDGE = 1280 MIN_SCREENSHOT_SCALE = 0.25 +JPEG_SCREENSHOT_QUALITY = 80 SCREENSHOT_SCALE_STEP = 0.85 BLOCKED_APP_FRAGMENTS = ( "1password", @@ -1054,14 +1055,14 @@ def within_capture_limit(width, height): return width <= 16384 and height <= 16384 and width * height <= 67_108_864 -def capture_wayland_png(bounds, app, window): +def capture_wayland_png(bounds, request, app, window): backend = wayland_screenshot_backend() if backend is None: return screenshot_failure(UNSUPPORTED_WAYLAND_SCREENSHOT) if app is None or window is None: return screenshot_failure("Wayland screenshots need a target window") if backend == "kwin": - return capture_kwin_png(bounds, app, window) + return capture_kwin_png(bounds, request, app, window) grim = shutil.which("grim") if grim is None or GdkPixbuf is None: return screenshot_failure("Wayland screenshots require grim and GdkPixbuf; install them or use --no-screenshot") @@ -1087,7 +1088,7 @@ def capture_wayland_png(bounds, app, window): pixbuf = decode_png(output.read()) if pixbuf is None or pixbuf.get_width() != width or pixbuf.get_height() != height: return screenshot_failure("grim returned an image with unexpected window dimensions") - return bounded_png_payload(pixbuf) + return bounded_screenshot_payload(pixbuf, request) except Exception as error: return screenshot_failure("Wayland screenshot failed: " + str(error)[:200]) @@ -1296,7 +1297,7 @@ def crop_workspace_png(data, rect, workspace): return window -def capture_kwin_png(bounds, app, window): +def capture_kwin_png(bounds, request, app, window): if GdkPixbuf is None: return screenshot_failure("KDE Wayland screenshots require GdkPixbuf; install it or use --no-screenshot") pid, title = pid_of(app), name_of(window) @@ -1310,16 +1311,16 @@ def capture_kwin_png(bounds, app, window): after, _ = kwin_window_rect(kwin_layout(bus), pid, title, bounds) if after != layout: return screenshot_failure("Wayland screenshot skipped: the window moved, resized or was covered while capturing; retry") - return bounded_png_payload(crop_workspace_png(data, *layout)) + return bounded_screenshot_payload(crop_workspace_png(data, *layout), request) except ValueError as error: return screenshot_failure("Wayland screenshot skipped: " + str(error)[:300]) except Exception as error: return screenshot_failure("Wayland screenshot failed: " + str(error)[:200]) -def capture_png(rect, app=None, window=None): +def capture_png(rect, request, app=None, window=None): if wayland_session(): - return capture_wayland_png(rect, app, window) + return capture_wayland_png(rect, request, app, window) if Gdk is None or GdkPixbuf is None or rect is None: return None screen = Gdk.Screen.get_default() @@ -1329,50 +1330,58 @@ def capture_png(rect, app=None, window=None): pixbuf = Gdk.pixbuf_get_from_window(root, round(rect.x), round(rect.y), max(1, round(rect.width)), max(1, round(rect.height))) if pixbuf is None: return None - return bounded_png_payload(pixbuf) + return bounded_screenshot_payload(pixbuf, request) -def png_bytes(pixbuf): - ok, data = pixbuf.save_to_bufferv("png", [], []) +def screenshot_request_of(operation): + """The size and format the caller asked for; a value the RPC layer would refuse is ignored.""" + def positive(key): + value = operation.get(key) + return value if isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and value > 0 else None + return { + "format": "jpeg" if operation.get("screenshotFormat") == "jpeg" else "png", + "maxScale": positive("screenshotScale"), + "maxEdge": positive("screenshotMaxEdge"), + } + + +def image_bytes(pixbuf, image_format): + if image_format == "jpeg": + ok, data = pixbuf.save_to_bufferv("jpeg", ["quality"], [str(JPEG_SCREENSHOT_QUALITY)]) + else: + ok, data = pixbuf.save_to_bufferv("png", [], []) return bytes(data) if ok else None -def screenshot_payload(data, width, height, original_width): +def screenshot_payload(data, image_format, width, height, original_width): return { "base64": base64.b64encode(data).decode("ascii"), + "format": image_format, "width": width, "height": height, "scale": width / max(1, original_width), } -def bounded_png_payload(pixbuf): +def bounded_screenshot_payload(pixbuf, request): original_width = max(1, pixbuf.get_width()) original_height = max(1, pixbuf.get_height()) - data = png_bytes(pixbuf) - if data is None: - return None - if len(data) <= MAX_SCREENSHOT_PNG_BYTES: - return screenshot_payload(data, original_width, original_height, original_width) - - # Why: screenshots are sent through JSON/stdout; matching macOS bounds keeps - # large or high-DPI windows from multiplying native, base64, and Node memory. - scale = min(1.0, MAX_SCREENSHOT_EDGE / max(original_width, original_height)) - while scale >= MIN_SCREENSHOT_SCALE: - width = max(1, round(original_width * scale)) - height = max(1, round(original_height * scale)) - if width == original_width and height == original_height: - scale *= SCREENSHOT_SCALE_STEP + image_format = request["format"] + # Captures here are one pixel per window coordinate, so the scale cap is the resize factor itself. + requested = min(1.0, request["maxScale"] or 1.0) + if request["maxEdge"]: + requested = min(requested, request["maxEdge"] / max(original_width, original_height)) + tried = None + for scale in screenshot_scales(requested, original_width, original_height): + width, height = pixel_size(original_width, scale), pixel_size(original_height, scale) + if (width, height) == tried: continue - scaled = pixbuf.scale_simple(width, height, GdkPixbuf.InterpType.BILINEAR) - if scaled is None: - scale *= SCREENSHOT_SCALE_STEP - continue - candidate = png_bytes(scaled) - if candidate is not None: - if len(candidate) <= MAX_SCREENSHOT_PNG_BYTES: - return screenshot_payload(candidate, width, height, original_width) - scale *= SCREENSHOT_SCALE_STEP + tried = (width, height) + scaled = pixbuf if (width, height) == (original_width, original_height) else pixbuf.scale_simple( + width, height, GdkPixbuf.InterpType.BILINEAR) + candidate = image_bytes(scaled, image_format) if scaled is not None else None + if candidate is not None and len(candidate) <= MAX_SCREENSHOT_PNG_BYTES: + return screenshot_payload(candidate, image_format, width, height, original_width) return { "error": { @@ -1382,6 +1391,22 @@ def bounded_png_payload(pixbuf): } +def pixel_size(length, scale): + """Rounds down, so a requested scale or edge is a ceiling; the epsilon keeps an exact product such as 0.5 x 200 whole.""" + return max(1, math.floor(length * scale + 1e-9)) + + +def screenshot_scales(requested, original_width, original_height): + """The requested size first, then smaller sizes while the encoded image is over the payload cap.""" + yield requested + # Why: screenshots are sent through JSON/stdout; matching macOS bounds keeps + # large or high-DPI windows from multiplying native, base64, and Node memory. + scale = min(requested, MAX_SCREENSHOT_EDGE / max(original_width, original_height)) + while scale >= MIN_SCREENSHOT_SCALE: + yield scale + scale *= SCREENSHOT_SCALE_STEP + + def first_descendant(root, predicate, max_nodes=MAX_NODES, max_depth=MAX_DEPTH): visited_count = 0 stack = [(root, 0)] @@ -1447,7 +1472,7 @@ def window_json(app, index, window): } -def make_snapshot(query, include_screenshot, window_id=None, window_index=None, restore=False, root=None): +def make_snapshot(query, screenshot_request, window_id=None, window_index=None, restore=False, root=None): app = find_app(query) window_index, window = choose_window(app, window_id, window_index) if restore: @@ -1472,8 +1497,8 @@ def make_snapshot(query, include_screenshot, window_id=None, window_index=None, selection = selected_text(window) app_info = app_json(app) window_title = name_of(window) - screenshot = capture_png(bounds, app, window) if include_screenshot else None - if include_screenshot and bounds is None and placement.window_coords: + screenshot = capture_png(bounds, screenshot_request, app, window) if screenshot_request is not None else None + if screenshot_request is not None and bounds is None and placement.window_coords: screenshot = screenshot_failure("GTK 4 screenshot skipped: Orca can't tell where this window sits on screen; " "use --no-screenshot and trust the tree") geometry = GeometryReader() @@ -1488,7 +1513,8 @@ def make_snapshot(query, include_screenshot, window_id=None, window_index=None, "windowId": None, "windowIndex": window_index, "windowBounds": bounds.to_json() if bounds else None, - "screenshotPngBase64": screenshot.get("base64") if screenshot else None, + "screenshotBase64": screenshot.get("base64") if screenshot else None, + "screenshotFormat": screenshot.get("format") if screenshot else None, "screenshotWidth": screenshot.get("width") if screenshot else None, "screenshotHeight": screenshot.get("height") if screenshot else None, "screenshotScale": screenshot.get("scale") if screenshot else None, @@ -2506,7 +2532,7 @@ def set_value(node, value): def run_operation(operation): _x11_windows_by_pid.clear() tool = operation.get("tool") - include_screenshot = not bool(operation.get("noScreenshot")) + screenshot_request = None if operation.get("noScreenshot") else screenshot_request_of(operation) if tool == "handshake": return {"ok": True, "capabilities": handshake_response()} if tool == "list_apps": @@ -2518,7 +2544,7 @@ def run_operation(operation): "ok": True, "snapshot": make_snapshot( operation.get("app", ""), - include_screenshot, + screenshot_request, operation.get("windowId"), operation.get("windowIndex"), bool(operation.get("restoreWindow")), @@ -2639,7 +2665,7 @@ def run_operation(operation): try: snapshot = make_snapshot( operation.get("app", ""), - include_screenshot, + screenshot_request, operation.get("windowId"), operation.get("windowIndex"), ) @@ -2647,7 +2673,7 @@ def run_operation(operation): if operation.get("windowId") is None and operation.get("windowIndex") is None: raise action.setdefault("verification", {"state": "unverified", "reason": "window_changed"}) - snapshot = make_snapshot(operation.get("app", ""), include_screenshot, None, None) + snapshot = make_snapshot(operation.get("app", ""), screenshot_request, None, None) # Why measured: synthetic input needs the window in front, and an app can activate itself after an AT-SPI action. took_foreground = ((not was_active[0] and has_state(window, Atspi.StateType.ACTIVE)) diff --git a/native/computer-use-linux/runtime_render_test.py b/native/computer-use-linux/runtime_render_test.py index adcb208cafb..8f258640349 100644 --- a/native/computer-use-linux/runtime_render_test.py +++ b/native/computer-use-linux/runtime_render_test.py @@ -55,5 +55,39 @@ class PortalFileTest(unittest.TestCase): self.assertTrue(link.is_symlink() and self.file.exists()) +class FakePixbuf: + def __init__(self, width, height): + self.width, self.height = width, height + + def get_width(self): + return self.width + + def get_height(self): + return self.height + + def scale_simple(self, width, height, _interp): + return FakePixbuf(width, height) + + def save_to_bufferv(self, image_format, keys, values): + return True, f"{image_format}:{self.width}x{self.height}:{dict(zip(keys, values))}".encode() + + +class ScreenshotSizingTest(unittest.TestCase): + def setUp(self): + runtime.GdkPixbuf.InterpType = types.SimpleNamespace(BILINEAR="bilinear") + + def payload(self, **operation): + return runtime.bounded_screenshot_payload(FakePixbuf(400, 200), runtime.screenshot_request_of(operation)) + + def test_default_request_keeps_the_full_capture_as_png(self): + payload = self.payload() + self.assertEqual((payload["format"], payload["width"], payload["height"], payload["scale"]), ("png", 400, 200, 1)) + + def test_requested_scale_edge_and_jpeg_shrink_and_reencode_the_capture(self): + payload = self.payload(screenshotFormat="jpeg", screenshotScale=0.5, screenshotMaxEdge=100) + self.assertEqual((payload["format"], payload["width"], payload["height"]), ("jpeg", 100, 50)) + self.assertEqual(runtime.base64.b64decode(payload["base64"]), b"jpeg:100x50:{'quality': '80'}") + + if __name__ == "__main__": unittest.main() diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift index 1186e089be2..3f23b79adac 100644 --- a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift @@ -218,6 +218,7 @@ struct Snapshot { struct ScreenshotPayload { let data: String + let format: ScreenshotFormat let width: Int let height: Int let scale: Double @@ -513,6 +514,7 @@ final class Provider { let snapshot = try buildSnapshot( app: app, includeScreenshot: params["noScreenshot"]?.bool != true, + screenshotOptions: screenshotOptions(params), windowId: subtree.map(\.base.windowId) ?? windowId, windowIndex: subtree == nil ? windowIndex : nil, subtree: subtree, @@ -957,6 +959,7 @@ final class Provider { private func buildSnapshot( app: AppDescriptor, includeScreenshot: Bool, + screenshotOptions: ScreenshotOptions, windowId: CGWindowID?, windowIndex: Int?, subtree: (root: ElementRecord, base: Snapshot)?, @@ -1043,7 +1046,7 @@ final class Provider { elements = kept.merging(renderer.records) { _, walked in walked } elementOrder = subtree.base.elementOrder.filter { kept[$0] != nil } + indexes.order } - let screenshot = includeScreenshot ? capture.screenshotPayload() : nil + let screenshot = includeScreenshot ? capture.screenshotPayload(screenshotOptions) : nil let screenshotStatus: ScreenshotStatus = if screenshot != nil { .captured } else if includeScreenshot && !canCaptureScreenshot { @@ -1084,7 +1087,7 @@ final class Provider { if let payload = snapshot.screenshot { screenshot = [ "data": payload.data, - "format": "png", + "format": payload.format.rawValue, "width": payload.width, "height": payload.height, "scale": payload.scale, @@ -2232,6 +2235,15 @@ private func requestedWindowIndex(_ params: [String: JSONValue]) throws -> Int? return index } +/// The RPC layer validates these, so a value it would refuse is ignored here rather than failing the observe. +private func screenshotOptions(_ params: [String: JSONValue]) -> ScreenshotOptions { + ScreenshotOptions( + format: params["screenshotFormat"]?.string.flatMap(ScreenshotFormat.init(rawValue:)) ?? .png, + maxScale: params["screenshotScale"]?.number.flatMap { $0.isFinite && $0 > 0 ? $0 : nil }, + maxEdge: params["screenshotMaxEdge"]?.number.flatMap { $0 >= 1 ? boundedInteger($0, as: Int.self) : nil } + ) +} + private func usableWindow(_ element: AXUIElement) -> Bool { stringAttribute(element, kAXRoleAttribute as String) == kAXWindowRole as String && boolAttribute(element, kAXMinimizedAttribute as String) != true @@ -3412,13 +3424,16 @@ private struct WindowCapture { ) } - func screenshotPayload() -> ScreenshotPayload? { - guard let image, let bounded = boundedPngData(image.image) else { return nil } + func screenshotPayload(_ options: ScreenshotOptions) -> ScreenshotPayload? { + guard let image, let encoded = ScreenshotEncoding.encode(image.image, windowWidth: bounds.width, options: options) else { + return nil + } return ScreenshotPayload( - data: bounded.data.base64EncodedString(), - width: bounded.width, - height: bounded.height, - scale: Double(bounded.width) / max(Double(bounds.width), 1) + data: encoded.data.base64EncodedString(), + format: encoded.format, + width: encoded.width, + height: encoded.height, + scale: Double(encoded.width) / max(Double(bounds.width), 1) ) } @@ -3482,48 +3497,6 @@ private enum BlockingAsync { } } -private struct BoundedPNG { - let data: Data - let width: Int - let height: Int -} - -private func boundedPngData(_ image: CGImage) -> BoundedPNG? { - let rep = NSBitmapImageRep(cgImage: image) - guard let data = rep.representation(using: .png, properties: [:]) else { return nil } - if data.count <= 900_000 { - return BoundedPNG(data: data, width: image.width, height: image.height) - } - var scale = min(1, 1280 / CGFloat(max(image.width, image.height))) - var best = BoundedPNG(data: data, width: image.width, height: image.height) - while scale >= 0.25 { - guard let resized = resizePng(image, scale: scale) else { - break - } - best = resized - if resized.data.count <= 900_000 { - return resized - } - scale *= 0.85 - } - return best -} - -private func resizePng(_ image: CGImage, scale: CGFloat) -> BoundedPNG? { - let width = max(1, Int(CGFloat(image.width) * scale)) - let height = max(1, Int(CGFloat(image.height) * scale)) - guard let context = CGContext(data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 0, space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else { - return nil - } - context.interpolationQuality = .medium - context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height)) - guard let resized = context.makeImage() else { return nil } - guard let data = NSBitmapImageRep(cgImage: resized).representation(using: .png, properties: [:]) else { - return nil - } - return BoundedPNG(data: data, width: width, height: height) -} - private enum Input { static func click( at point: CGPoint, diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/ScreenshotEncoding.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/ScreenshotEncoding.swift new file mode 100644 index 00000000000..0f08f7c372d --- /dev/null +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/ScreenshotEncoding.swift @@ -0,0 +1,117 @@ +import CoreGraphics +import Foundation +import ImageIO +import UniformTypeIdentifiers + +public enum ScreenshotFormat: String { + case png + case jpeg +} + +/// What the caller asked for; `maxScale` is image pixels per window point, the unit `screenshot.scale` reports. +public struct ScreenshotOptions: Equatable { + public var format: ScreenshotFormat + public var maxScale: Double? + public var maxEdge: Int? + + public init(format: ScreenshotFormat = .png, maxScale: Double? = nil, maxEdge: Int? = nil) { + self.format = format + self.maxScale = maxScale + self.maxEdge = maxEdge + } +} + +public struct EncodedScreenshot { + public let data: Data + public let format: ScreenshotFormat + public let width: Int + public let height: Int +} + +public enum ScreenshotEncoding { + static let maxBytes = 900_000 + static let fallbackEdge: CGFloat = 1280 + static let minScale: CGFloat = 0.25 + static let scaleStep: CGFloat = 0.85 + static let jpegQuality: CGFloat = 0.8 + + /// Applies the requested scale and edge caps, then shrinks further while the image is over the payload cap. + public static func encode(_ image: CGImage, windowWidth: CGFloat, options: ScreenshotOptions) -> EncodedScreenshot? { + let edge = CGFloat(max(image.width, image.height, 1)) + var requested: CGFloat = 1 + if let maxScale = options.maxScale { + requested = min(requested, CGFloat(maxScale) * max(windowWidth, 1) / CGFloat(max(image.width, 1))) + } + if let maxEdge = options.maxEdge { + requested = min(requested, CGFloat(maxEdge) / edge) + } + guard let first = encode(image, scale: requested, format: options.format) else { return nil } + if first.data.count <= maxBytes { + return first + } + var scale = min(requested, fallbackEdge / edge) + var best = first + while scale >= minScale { + if pixelSize(image.width, scale) == best.width && pixelSize(image.height, scale) == best.height { + scale *= scaleStep + continue + } + guard let resized = encode(image, scale: scale, format: options.format) else { + break + } + best = resized + if resized.data.count <= maxBytes { + return resized + } + scale *= scaleStep + } + return best + } + + private static func encode(_ image: CGImage, scale: CGFloat, format: ScreenshotFormat) -> EncodedScreenshot? { + let width = pixelSize(image.width, scale) + let height = pixelSize(image.height, scale) + // Why: JPEG has no alpha, so transparent window corners would turn black without an opaque white backing. + let source = width == image.width && height == image.height && format == .png + ? image + : redraw(image, width: width, height: height, opaque: format == .jpeg) + guard let source else { return nil } + let data = NSMutableData() + let type = format == .png ? UTType.png : UTType.jpeg + guard let destination = CGImageDestinationCreateWithData(data, type.identifier as CFString, 1, nil) else { + return nil + } + let properties: [CFString: Any] = format == .jpeg ? [kCGImageDestinationLossyCompressionQuality: jpegQuality] : [:] + CGImageDestinationAddImage(destination, source, properties as CFDictionary) + guard CGImageDestinationFinalize(destination) else { return nil } + return EncodedScreenshot(data: data as Data, format: format, width: width, height: height) + } + + /// Rounds down, so a requested scale or edge is a ceiling; the epsilon keeps an exact product such as 0.5 × 200 whole. + private static func pixelSize(_ length: Int, _ scale: CGFloat) -> Int { + max(1, Int((CGFloat(length) * scale + 1e-9).rounded(.down))) + } + + private static func redraw(_ image: CGImage, width: Int, height: Int, opaque: Bool) -> CGImage? { + let alpha = opaque ? CGImageAlphaInfo.noneSkipLast : CGImageAlphaInfo.premultipliedLast + guard let context = CGContext( + data: nil, + width: width, + height: height, + bitsPerComponent: 8, + bytesPerRow: 0, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: alpha.rawValue + ) else { + return nil + } + let rect = CGRect(x: 0, y: 0, width: width, height: height) + if opaque { + context.setFillColor(CGColor(gray: 1, alpha: 1)) + context.fill(rect) + } + context.interpolationQuality = .medium + context.draw(image, in: rect) + return context.makeImage() + } +} diff --git a/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/ScreenshotEncodingTests.swift b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/ScreenshotEncodingTests.swift new file mode 100644 index 00000000000..87c5c3a7320 --- /dev/null +++ b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/ScreenshotEncodingTests.swift @@ -0,0 +1,41 @@ +import CoreGraphics +import XCTest +@testable import OrcaComputerUseMacOSCore + +final class ScreenshotEncodingTests: XCTestCase { + // A 200×100 pixel capture of a 100-point-wide window, as on a Retina display. + private let retinaCapture: CGImage = { + let context = CGContext( + data: nil, + width: 200, + height: 100, + bitsPerComponent: 8, + bytesPerRow: 0, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue + )! + context.setFillColor(CGColor(red: 0.2, green: 0.4, blue: 0.8, alpha: 1)) + context.fill(CGRect(x: 0, y: 0, width: 200, height: 100)) + return context.makeImage()! + }() + + func testDefaultsKeepTheFullCaptureAsPNG() throws { + let encoded = try XCTUnwrap(ScreenshotEncoding.encode(retinaCapture, windowWidth: 100, options: ScreenshotOptions())) + XCTAssertEqual([encoded.width, encoded.height], [200, 100]) + XCTAssertEqual(Array(encoded.data.prefix(4)), [0x89, 0x50, 0x4E, 0x47]) + } + + func testScaleOneGivesOnePixelPerWindowPointAsJPEG() throws { + let options = ScreenshotOptions(format: .jpeg, maxScale: 1) + let encoded = try XCTUnwrap(ScreenshotEncoding.encode(retinaCapture, windowWidth: 100, options: options)) + XCTAssertEqual([encoded.width, encoded.height], [100, 50]) + XCTAssertEqual(Array(encoded.data.prefix(3)), [0xFF, 0xD8, 0xFF]) + } + + func testMaxEdgeCapsTheLongerSideWithoutUpscaling() throws { + let capped = try XCTUnwrap(ScreenshotEncoding.encode(retinaCapture, windowWidth: 100, options: ScreenshotOptions(maxEdge: 80))) + XCTAssertEqual([capped.width, capped.height], [80, 40]) + let loose = try XCTUnwrap(ScreenshotEncoding.encode(retinaCapture, windowWidth: 100, options: ScreenshotOptions(maxScale: 4, maxEdge: 4000))) + XCTAssertEqual([loose.width, loose.height], [200, 100]) + } +} diff --git a/native/computer-use-windows/runtime.ps1 b/native/computer-use-windows/runtime.ps1 index d791dc0814f..90504088a97 100644 --- a/native/computer-use-windows/runtime.ps1 +++ b/native/computer-use-windows/runtime.ps1 @@ -607,6 +607,7 @@ $EditReadbackLimit = 1048576 $MaxScreenshotPngBytes = 900000 $MaxScreenshotEdge = 1280 $MinScreenshotScale = 0.25 +$JpegScreenshotQuality = 80 $ScreenshotScaleStep = 0.85 # Windows' own hung-window threshold, so a briefly busy app is not reported as hung; still far under the request timeout. $WindowResponseTimeoutMilliseconds = 5000 @@ -1463,20 +1464,45 @@ function Get-OrcaRenderedElementIndex([string]$Line, [int]$Depth) { $null } -function ConvertTo-OrcaPngBytes([System.Drawing.Image]$Image) { +function ConvertTo-OrcaImageBytes([System.Drawing.Image]$Image, [string]$Format) { $stream = $null + $parameters = $null try { $stream = New-Object System.IO.MemoryStream - $Image.Save($stream, [System.Drawing.Imaging.ImageFormat]::Png) + if ($Format -eq "jpeg") { + $codec = [System.Drawing.Imaging.ImageCodecInfo]::GetImageEncoders() | + Where-Object { $_.FormatID -eq [System.Drawing.Imaging.ImageFormat]::Jpeg.Guid } | + Select-Object -First 1 + $parameters = New-Object System.Drawing.Imaging.EncoderParameters 1 + $parameters.Param[0] = New-Object System.Drawing.Imaging.EncoderParameter ([System.Drawing.Imaging.Encoder]::Quality), ([long]$JpegScreenshotQuality) + $Image.Save($stream, $codec, $parameters) + } else { + $Image.Save($stream, [System.Drawing.Imaging.ImageFormat]::Png) + } return ,$stream.ToArray() } finally { + if ($null -ne $parameters) { $parameters.Dispose() } if ($null -ne $stream) { $stream.Dispose() } } } -function New-OrcaScreenshotPayload([byte[]]$Bytes, [int]$Width, [int]$Height, [double]$Scale) { +# The size and format the caller asked for; a value the RPC layer would refuse is ignored. +function Get-OrcaScreenshotRequest($Operation) { + $request = [pscustomobject]@{ format = "png"; maxScale = $null; maxEdge = $null } + if ($Operation.screenshotFormat -eq "jpeg") { $request.format = "jpeg" } + foreach ($pair in @(@("screenshotScale", "maxScale"), @("screenshotMaxEdge", "maxEdge"))) { + $value = $Operation.($pair[0]) + if (($value -is [int] -or $value -is [long] -or $value -is [double] -or $value -is [decimal]) -and [double]$value -gt 0 -and -not [double]::IsInfinity([double]$value)) { + $request.($pair[1]) = [double]$value + } + } + $request +} + +function New-OrcaScreenshotPayload([byte[]]$Bytes, [string]$Format, [int]$Width, [int]$Height, [double]$Scale) { [pscustomobject]@{ base64 = [Convert]::ToBase64String($Bytes) + format = $Format width = $Width height = $Height scale = $Scale @@ -1500,37 +1526,47 @@ function Resize-OrcaBitmap([System.Drawing.Bitmap]$Source, [int]$Width, [int]$He } } -function Get-OrcaBoundedScreenshotPayload([System.Drawing.Bitmap]$Bitmap) { +function Get-OrcaBoundedScreenshotPayload([System.Drawing.Bitmap]$Bitmap, $Request) { $originalWidth = [int][Math]::Max(1, $Bitmap.Width) $originalHeight = [int][Math]::Max(1, $Bitmap.Height) - $pngBytes = ConvertTo-OrcaPngBytes $Bitmap - if ($pngBytes.Length -le $MaxScreenshotPngBytes) { - return New-OrcaScreenshotPayload $pngBytes $originalWidth $originalHeight 1.0 + $longEdge = [double][Math]::Max($originalWidth, $originalHeight) + # Captures here are one pixel per window coordinate, so the scale cap is the resize factor itself. + $requested = 1.0 + if ($null -ne $Request.maxScale) { $requested = [Math]::Min($requested, $Request.maxScale) } + if ($null -ne $Request.maxEdge) { $requested = [Math]::Min($requested, $Request.maxEdge / $longEdge) } + + # Why: screenshots cross process boundaries as base64 in JSON; after the requested size, cap noisy + # large-window payloads to match the macOS provider's memory bounds. + $scales = New-Object System.Collections.Generic.List[double] + $scales.Add($requested) + $scale = [Math]::Min($requested, $MaxScreenshotEdge / $longEdge) + while ($scale -ge $MinScreenshotScale) { + $scales.Add($scale) + $scale *= $ScreenshotScaleStep } - # Why: screenshots cross process boundaries as PNG base64 in JSON; cap noisy - # large-window payloads to match the macOS provider's memory bounds. - $scale = [Math]::Min(1.0, $MaxScreenshotEdge / [double][Math]::Max($originalWidth, $originalHeight)) - while ($scale -ge $MinScreenshotScale) { - $width = [int][Math]::Max(1, [Math]::Round($originalWidth * $scale)) - $height = [int][Math]::Max(1, [Math]::Round($originalHeight * $scale)) - if ($width -eq $originalWidth -and $height -eq $originalHeight) { - $scale *= $ScreenshotScaleStep - continue - } + $tried = $null + foreach ($candidateScale in $scales) { + # Rounds down, so a requested scale or edge is a ceiling; the epsilon keeps an exact product such as 0.5 x 200 whole. + $width = [int][Math]::Max(1, [Math]::Floor($originalWidth * $candidateScale + 1e-9)) + $height = [int][Math]::Max(1, [Math]::Floor($originalHeight * $candidateScale + 1e-9)) + if ("${width}x${height}" -eq $tried) { continue } + $tried = "${width}x${height}" $resized = $null try { - $resized = Resize-OrcaBitmap $Bitmap $width $height - $candidateBytes = ConvertTo-OrcaPngBytes $resized + $image = $Bitmap + if ($width -ne $originalWidth -or $height -ne $originalHeight) { + $resized = Resize-OrcaBitmap $Bitmap $width $height + $image = $resized + } + $candidateBytes = ConvertTo-OrcaImageBytes $image $Request.format if ($candidateBytes.Length -le $MaxScreenshotPngBytes) { - return New-OrcaScreenshotPayload $candidateBytes $width $height ($width / [double]$originalWidth) + return New-OrcaScreenshotPayload $candidateBytes $Request.format $width $height ($width / [double]$originalWidth) } } finally { if ($null -ne $resized) { $resized.Dispose() } } - - $scale *= $ScreenshotScaleStep } [pscustomobject]@{ @@ -1541,8 +1577,8 @@ function Get-OrcaBoundedScreenshotPayload([System.Drawing.Bitmap]$Bitmap) { } } -function Get-OrcaScreenshot([bool]$IncludeScreenshot, $WindowFrame, [IntPtr]$WindowHandle) { - if (-not $IncludeScreenshot -or $null -eq $WindowFrame) { return $null } +function Get-OrcaScreenshot($ScreenshotRequest, $WindowFrame, [IntPtr]$WindowHandle) { + if ($null -eq $ScreenshotRequest -or $null -eq $WindowFrame) { return $null } $bitmap = $null $graphics = $null try { @@ -1555,7 +1591,7 @@ function Get-OrcaScreenshot([bool]$IncludeScreenshot, $WindowFrame, [IntPtr]$Win $graphics = [System.Drawing.Graphics]::FromImage($bitmap) $graphics.CopyFromScreen([int][Math]::Round($WindowFrame.x), [int][Math]::Round($WindowFrame.y), 0, 0, $bitmap.Size) } - Get-OrcaBoundedScreenshotPayload $bitmap + Get-OrcaBoundedScreenshotPayload $bitmap $ScreenshotRequest } catch { $null } finally { @@ -1564,7 +1600,7 @@ function Get-OrcaScreenshot([bool]$IncludeScreenshot, $WindowFrame, [IntPtr]$Win } } -function New-OrcaSnapshot([string]$Query, [bool]$IncludeScreenshot, $WindowId = $null, $WindowIndex = $null, [bool]$RestoreWindow = $false, $TreeRoot = $null) { +function New-OrcaSnapshot([string]$Query, $ScreenshotRequest, $WindowId = $null, $WindowIndex = $null, [bool]$RestoreWindow = $false, $TreeRoot = $null) { $process = Find-OrcaProcess $Query Assert-OrcaWindowTarget $process $WindowId $WindowIndex # Root first: its hang probe must run before ShowWindow, which blocks on a hung window. @@ -1580,7 +1616,7 @@ function New-OrcaSnapshot([string]$Query, [bool]$IncludeScreenshot, $WindowId = } } $tree = Render-OrcaTree $walkRoot $windowFrame (Test-OrcaBrowserProcess $process) - $screenshot = Get-OrcaScreenshot $IncludeScreenshot $windowFrame ([IntPtr]$process.MainWindowHandle) + $screenshot = Get-OrcaScreenshot $ScreenshotRequest $windowFrame ([IntPtr]$process.MainWindowHandle) [pscustomobject]@{ snapshotId = [guid]::NewGuid().ToString() @@ -1588,7 +1624,8 @@ function New-OrcaSnapshot([string]$Query, [bool]$IncludeScreenshot, $WindowId = windowTitle = $process.MainWindowTitle windowId = Get-OrcaWindowId $process windowBounds = $windowFrame - screenshotPngBase64 = if ($null -ne $screenshot) { $screenshot.base64 } else { $null } + screenshotBase64 = if ($null -ne $screenshot) { $screenshot.base64 } else { $null } + screenshotFormat = if ($null -ne $screenshot) { $screenshot.format } else { $null } screenshotWidth = if ($null -ne $screenshot) { $screenshot.width } else { $null } screenshotHeight = if ($null -ne $screenshot) { $screenshot.height } else { $null } screenshotScale = if ($null -ne $screenshot) { $screenshot.scale } else { $null } @@ -2423,7 +2460,7 @@ function Send-OrcaPasteText([IntPtr]$WindowHandle, [string]$Text) { } function Invoke-OrcaOperation($Operation) { - $includeScreenshot = -not [bool]$Operation.noScreenshot + $screenshotRequest = if ([bool]$Operation.noScreenshot) { $null } else { Get-OrcaScreenshotRequest $Operation } if ($Operation.tool -eq "handshake") { return [pscustomobject]@{ ok = $true; capabilities = Get-OrcaHandshake } } @@ -2438,7 +2475,7 @@ function Invoke-OrcaOperation($Operation) { return Invoke-OrcaLaunchApp $Operation } if ($Operation.tool -eq "get_app_state") { - return [pscustomobject]@{ ok = $true; snapshot = New-OrcaSnapshot $Operation.app $includeScreenshot $Operation.windowId $Operation.windowIndex ([bool]$Operation.restoreWindow) $Operation.root } + return [pscustomobject]@{ ok = $true; snapshot = New-OrcaSnapshot $Operation.app $screenshotRequest $Operation.windowId $Operation.windowIndex ([bool]$Operation.restoreWindow) $Operation.root } } $process = Find-OrcaProcess $Operation.app @@ -2610,14 +2647,14 @@ function Invoke-OrcaOperation($Operation) { try { try { - $snapshot = New-OrcaSnapshot $Operation.app $includeScreenshot $Operation.windowId $Operation.windowIndex + $snapshot = New-OrcaSnapshot $Operation.app $screenshotRequest $Operation.windowId $Operation.windowIndex } catch { if ($_.Exception.Message -like "window_not_responding:*") { throw } if ($null -eq $Operation.windowId -and $null -eq $Operation.windowIndex) { throw } if ($null -eq $action.verification) { $action | Add-Member -NotePropertyName verification -NotePropertyValue ([pscustomobject]@{ state = "unverified"; reason = "window_changed" }) } - $snapshot = New-OrcaSnapshot $Operation.app $includeScreenshot $null $null + $snapshot = New-OrcaSnapshot $Operation.app $screenshotRequest $null $null } } catch { # Why: the input was already sent, so the pre-action "nothing was sent" wording would invite a duplicate retry. diff --git a/skill-guides/orca-computer-use.md b/skill-guides/orca-computer-use.md index 46a22dee8b7..44bd9e17a31 100644 --- a/skill-guides/orca-computer-use.md +++ b/skill-guides/orca-computer-use.md @@ -173,6 +173,8 @@ images. Use the tree for indexes/actions and the screenshot for visual confirmation; failed capture usually means hidden, minimized, off-screen, or permission-blocked. +For smaller screenshots on any action or `get-app-state`, add `--screenshot-scale 1` (at most one pixel per window point, which halves a Retina capture), `--screenshot-max-edge ` and `--screenshot-format jpeg`. The result's `screenshot.format` and `screenshot.scale` say what came back; older runtimes ignore these flags and return the full PNG. + Coordinates passed to `click`, `scroll`, and `drag` are window-local action coordinates. If the screenshot reports `scale` other than `1`, convert visual screenshot pixels before acting: ```text diff --git a/src/cli/computer/computer-action-flags.ts b/src/cli/computer/computer-action-flags.ts index 61ed5ecba26..695b096a8c9 100644 --- a/src/cli/computer/computer-action-flags.ts +++ b/src/cli/computer/computer-action-flags.ts @@ -35,24 +35,45 @@ export function getComputerWindowTargetFlags(flags: Map): { noScreenshot?: boolean + screenshotFormat?: 'png' | 'jpeg' + screenshotScale?: number + screenshotMaxEdge?: number restoreWindow?: boolean windowId?: number windowIndex?: number } { return { noScreenshot: flags.has('no-screenshot') ? true : undefined, + ...getComputerScreenshotFlags(flags), ...(flags.has('restore-window') ? { restoreWindow: true } : {}), ...getComputerWindowTargetFlags(flags) } } -export function getComputerActionObserveFlags(flags: Map): { - noScreenshot?: boolean - restoreWindow?: boolean - treeDiff?: boolean - windowId?: number - windowIndex?: number +function getComputerScreenshotFlags(flags: Map): { + screenshotFormat?: 'png' | 'jpeg' + screenshotScale?: number + screenshotMaxEdge?: number } { + const format = getOptionalStringFlag(flags, 'screenshot-format') + if (format !== undefined && format !== 'png' && format !== 'jpeg') { + throw new RuntimeClientError('invalid_argument', '--screenshot-format must be png or jpeg') + } + const scale = getOptionalNumberFlag(flags, 'screenshot-scale') + if (scale !== undefined && !(scale > 0)) { + throw new RuntimeClientError('invalid_argument', '--screenshot-scale must be greater than 0') + } + const maxEdge = getOptionalPositiveIntegerFlag(flags, 'screenshot-max-edge') + return { + ...(format !== undefined ? { screenshotFormat: format } : {}), + ...(scale !== undefined ? { screenshotScale: scale } : {}), + ...(maxEdge !== undefined ? { screenshotMaxEdge: maxEdge } : {}) + } +} + +export function getComputerActionObserveFlags( + flags: Map +): ReturnType & { treeDiff?: boolean } { return { ...getComputerObserveFlags(flags), ...(flags.has('full-tree') ? {} : { treeDiff: true }) diff --git a/src/cli/computer/computer-format.ts b/src/cli/computer/computer-format.ts index c7cd1b31427..4003b46f64d 100644 --- a/src/cli/computer/computer-format.ts +++ b/src/cli/computer/computer-format.ts @@ -70,7 +70,7 @@ export function prepareComputerCliJsonResult( COMPUTER_SCREENSHOT_TEMP_DIR, response.id, screenshot.data, - screenshot.format === 'png' ? 'png' : 'img' + screenshot.format === 'jpeg' ? 'jpg' : screenshot.format === 'png' ? 'png' : 'img' ) // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only the screenshot field is rewritten; every other TResult field is spread through unchanged. return { diff --git a/src/cli/discovery/flag-help-text.ts b/src/cli/discovery/flag-help-text.ts index e04f2956b86..4e51f4bab89 100644 --- a/src/cli/discovery/flag-help-text.ts +++ b/src/cli/discovery/flag-help-text.ts @@ -62,6 +62,10 @@ export const FLAG_HELP_TEXT: Record = { name: '--name Name for the new worktree or automation', 'no-parent': '--no-parent Force no parent lineage for unrelated work', 'no-screenshot': '--no-screenshot Skip screenshot capture after the operation', + 'screenshot-format': '--screenshot-format Screenshot image format (default png)', + 'screenshot-scale': + '--screenshot-scale At most n screenshot pixels per window point; 1 drops Retina detail', + 'screenshot-max-edge': '--screenshot-max-edge Shrink the screenshot so its longer side fits', pages: '--pages Number of scroll pages', 'parent-worktree': '--parent-worktree Parent worktree selector such as identity:, id:::, folder:, name:, branch:, issue:, path:, or active/current', diff --git a/src/cli/skills/bundled-skill-guides.ts b/src/cli/skills/bundled-skill-guides.ts index 1d45d95705f..5e3b3ef9c6c 100644 --- a/src/cli/skills/bundled-skill-guides.ts +++ b/src/cli/skills/bundled-skill-guides.ts @@ -1197,6 +1197,8 @@ const ORCA_COMPUTER_USE_MARKDOWN = [ "", "Use the tree for indexes/actions and the screenshot for visual confirmation; failed capture usually means hidden, minimized, off-screen, or permission-blocked.", "", + "For smaller screenshots on any action or `get-app-state`, add `--screenshot-scale 1` (at most one pixel per window point, which halves a Retina capture), `--screenshot-max-edge ` and `--screenshot-format jpeg`. The result's `screenshot.format` and `screenshot.scale` say what came back; older runtimes ignore these flags and return the full PNG.", + "", "Coordinates passed to `click`, `scroll`, and `drag` are window-local action coordinates. If the screenshot reports `scale` other than `1`, convert visual screenshot pixels before acting:", "", "```text", diff --git a/src/cli/specs/computer.ts b/src/cli/specs/computer.ts index 4960c022ff5..8815dd19926 100644 --- a/src/cli/specs/computer.ts +++ b/src/cli/specs/computer.ts @@ -11,7 +11,10 @@ const COMPUTER_OBSERVE_FLAGS = [ ...COMPUTER_FLAGS, ...COMPUTER_WINDOW_TARGET_FLAGS, 'restore-window', - 'no-screenshot' + 'no-screenshot', + 'screenshot-format', + 'screenshot-scale', + 'screenshot-max-edge' ] const COMPUTER_ACTION_FLAGS = [...COMPUTER_OBSERVE_FLAGS, 'full-tree'] @@ -56,7 +59,7 @@ export const COMPUTER_COMMAND_SPECS: CommandSpec[] = [ path: ['computer', 'get-app-state'], summary: 'Capture a compact accessibility snapshot of an app', usage: - 'orca computer get-app-state --app [--window-id | --window-index ] [--worktree | --session ] [--element-index ] [--query ] [--max-elements ] [--restore-window] [--no-screenshot] [--json]', + 'orca computer get-app-state --app [--window-id | --window-index ] [--worktree | --session ] [--element-index ] [--query ] [--max-elements ] [--restore-window] [--no-screenshot] [--screenshot-format ] [--screenshot-scale ] [--screenshot-max-edge ] [--json]', allowedFlags: [...COMPUTER_OBSERVE_FLAGS, 'element-index', 'query', 'max-elements'] }, { diff --git a/src/main/computer/desktop-script/desktop-script-provider-client.ts b/src/main/computer/desktop-script/desktop-script-provider-client.ts index 5cd004b6399..afd1bba2ee1 100644 --- a/src/main/computer/desktop-script/desktop-script-provider-client.ts +++ b/src/main/computer/desktop-script/desktop-script-provider-client.ts @@ -25,6 +25,7 @@ import { launchBridgeApp, listBridgeApps } from './desktop-script-apps' import { optionalNumberParam, optionalStringParam, + screenshotRequestParams, stringParam } from './desktop-script-provider-params' import { @@ -130,7 +131,7 @@ export class DesktopScriptProviderClient { app, windowId: optionalNumberParam(params, 'windowId'), windowIndex: optionalNumberParam(params, 'windowIndex'), - noScreenshot: params.noScreenshot === true, + ...screenshotRequestParams(params), restoreWindow: params.restoreWindow === true }) return this.rememberAndRender(app, response, params.noScreenshot === true, params) @@ -176,7 +177,7 @@ export class DesktopScriptProviderClient { value: optionalStringParam(params, 'value'), windowBounds: null, ...actionWindowTarget, - noScreenshot: params.noScreenshot === true, + ...screenshotRequestParams(params), restoreWindow: params.restoreWindow === true }) const action = verifyDesktopAction( diff --git a/src/main/computer/desktop-script/desktop-script-provider-params.ts b/src/main/computer/desktop-script/desktop-script-provider-params.ts index a1f655ea040..5b4d01d30a6 100644 --- a/src/main/computer/desktop-script/desktop-script-provider-params.ts +++ b/src/main/computer/desktop-script/desktop-script-provider-params.ts @@ -1,4 +1,5 @@ import { RuntimeClientError } from '../runtime-client-error' +import type { BridgeRequest } from './desktop-script-provider-types' export function stringParam(params: Record, key: string): string { const value = params[key] @@ -23,3 +24,17 @@ export function optionalNumberParam( const value = params[key] return typeof value === 'number' && Number.isFinite(value) ? value : undefined } + +export function screenshotRequestParams( + params: Record +): Pick< + BridgeRequest, + 'noScreenshot' | 'screenshotFormat' | 'screenshotScale' | 'screenshotMaxEdge' +> { + return { + noScreenshot: params.noScreenshot === true, + screenshotFormat: params.screenshotFormat === 'jpeg' ? 'jpeg' : undefined, + screenshotScale: optionalNumberParam(params, 'screenshotScale'), + screenshotMaxEdge: optionalNumberParam(params, 'screenshotMaxEdge') + } +} diff --git a/src/main/computer/desktop-script/desktop-script-provider-test-harness.ts b/src/main/computer/desktop-script/desktop-script-provider-test-harness.ts index 1da3b0d261a..37493345818 100644 --- a/src/main/computer/desktop-script/desktop-script-provider-test-harness.ts +++ b/src/main/computer/desktop-script/desktop-script-provider-test-harness.ts @@ -100,7 +100,7 @@ export function sampleBridgeSnapshot(name: string, value: string) { windowTitle: name, windowId: 99, windowBounds: { x: 10, y: 20, width: 300, height: 200 }, - screenshotPngBase64: 'iVBORw0KGgo=', + screenshotBase64: 'iVBORw0KGgo=', coordinateSpace: 'window', truncation: { truncated: false, maxNodes: 1200, maxDepth: 64, maxDepthReached: false }, treeLines: [`0 text entry area, Value: ${value}`], diff --git a/src/main/computer/desktop-script/desktop-script-provider-types.ts b/src/main/computer/desktop-script/desktop-script-provider-types.ts index cf6c3542195..548be045a0c 100644 --- a/src/main/computer/desktop-script/desktop-script-provider-types.ts +++ b/src/main/computer/desktop-script/desktop-script-provider-types.ts @@ -59,7 +59,8 @@ export type BridgeSnapshot = { windowId?: number | null windowIndex?: number | null windowBounds?: BridgeFrame | null - screenshotPngBase64?: string | null + screenshotBase64?: string | null + screenshotFormat?: 'png' | 'jpeg' | null screenshotWidth?: number | null screenshotHeight?: number | null screenshotScale?: number | null @@ -162,6 +163,9 @@ export type BridgeRequest = { windowId?: number windowIndex?: number noScreenshot?: boolean + screenshotFormat?: 'jpeg' + screenshotScale?: number + screenshotMaxEdge?: number restoreWindow?: boolean /** launch_app: let the started app take the foreground. */ activate?: boolean diff --git a/src/main/computer/desktop-script/snapshots/desktop-script-snapshot-rendering.ts b/src/main/computer/desktop-script/snapshots/desktop-script-snapshot-rendering.ts index d65969152ea..9b6cfb63966 100644 --- a/src/main/computer/desktop-script/snapshots/desktop-script-snapshot-rendering.ts +++ b/src/main/computer/desktop-script/snapshots/desktop-script-snapshot-rendering.ts @@ -15,10 +15,10 @@ export function renderSnapshot( const screenshotHeight = positiveRoundedNumber(snapshot.screenshotHeight) ?? Math.max(1, Math.round(bounds?.height ?? 1)) const screenshotScale = positiveNumber(snapshot.screenshotScale) ?? 1 - const screenshot = snapshot.screenshotPngBase64 + const screenshot = snapshot.screenshotBase64 ? { - data: snapshot.screenshotPngBase64, - format: 'png' as const, + data: snapshot.screenshotBase64, + format: snapshot.screenshotFormat === 'jpeg' ? ('jpeg' as const) : ('png' as const), width: screenshotWidth, height: screenshotHeight, scale: screenshotScale @@ -97,7 +97,7 @@ function desktopScreenshotFailureStatus( } export function snapshotWithoutScreenshot(snapshot: BridgeSnapshot): BridgeSnapshot { - return snapshot.screenshotPngBase64 ? { ...snapshot, screenshotPngBase64: null } : snapshot + return snapshot.screenshotBase64 ? { ...snapshot, screenshotBase64: null } : snapshot } function normalizedFocusedElementId(snapshot: BridgeSnapshot): number | null { diff --git a/src/main/computer/desktop-script/snapshots/desktop-script-subtree-snapshot.ts b/src/main/computer/desktop-script/snapshots/desktop-script-subtree-snapshot.ts index 4d1b8407314..7fbcac114b7 100644 --- a/src/main/computer/desktop-script/snapshots/desktop-script-subtree-snapshot.ts +++ b/src/main/computer/desktop-script/snapshots/desktop-script-subtree-snapshot.ts @@ -2,7 +2,7 @@ import type { ComputerSnapshotResult } from '../../../../shared/computer-use-run import { renumberComputerTreeLine } from '../../../../shared/computer-use-tree-text' import { RuntimeClientError } from '../../runtime-client-error' import { elementParam } from '../actions/desktop-script-action' -import { optionalNumberParam } from '../desktop-script-provider-params' +import { optionalNumberParam, screenshotRequestParams } from '../desktop-script-provider-params' import type { BridgeElement, BridgeRequest, @@ -35,7 +35,7 @@ export async function readSubtreeSnapshot( root: elementParam(base, rootElementIndex), windowId: base.windowId ?? undefined, windowIndex: base.windowIndex ?? undefined, - noScreenshot: params.noScreenshot === true, + ...screenshotRequestParams(params), restoreWindow: params.restoreWindow === true }) if (!response.snapshot) { diff --git a/src/shared/computer-use-runtime-types.ts b/src/shared/computer-use-runtime-types.ts index 736d93405a9..d9be85cad40 100644 --- a/src/shared/computer-use-runtime-types.ts +++ b/src/shared/computer-use-runtime-types.ts @@ -80,7 +80,7 @@ export type ComputerSnapshotData = { export type ComputerScreenshotData = { data?: string - format: 'png' + format: 'png' | 'jpeg' width: number height: number scale: number diff --git a/src/shared/rpc-contract/computer/computer-schemas-params.ts b/src/shared/rpc-contract/computer/computer-schemas-params.ts index 134488ae17e..1dfe117a894 100644 --- a/src/shared/rpc-contract/computer/computer-schemas-params.ts +++ b/src/shared/rpc-contract/computer/computer-schemas-params.ts @@ -24,6 +24,10 @@ export const ComputerTarget = z.object({ export const ComputerObserveTargetBase = ComputerTarget.extend({ noScreenshot: OptionalBoolean, + // Older runtimes strip these and return the full-size PNG; the result's screenshot says what it is. + screenshotFormat: z.enum(['png', 'jpeg']).optional(), + screenshotScale: z.number().positive().finite().optional(), + screenshotMaxEdge: OptionalPositiveInt, restoreWindow: OptionalBoolean, // Older runtimes strip this and return the full tree, so callers need no capability check. treeDiff: OptionalBoolean,