Files
orca/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/ElementActionability.swift
T

156 lines
6.8 KiB
Swift

import CoreGraphics
/// The accessibility reads an element action needs before it sends input, so the policy runs against a fake tree in tests.
public protocol ElementActionabilityAccessibility {
associatedtype Element
/// AXEnabled; nil when the element does not publish it.
func isEnabled(_ element: Element) -> Bool?
/// Frame in global top-left screen points.
func frame(of element: Element) -> CGRect?
func role(of element: Element) -> String?
func parent(of element: Element) -> Element?
func isSame(_ lhs: Element, _ rhs: Element) -> Bool
/// The target app's own topmost element at a screen point; nil when the app cannot say.
func element(at point: CGPoint) -> Element?
/// Asks the app to scroll the element into view; false when the element does not offer that.
func scrollToVisible(_ element: Element) throws -> Bool
/// Pauses before the next check; false once the wait budget is spent.
func waitBeforeRetry() -> Bool
}
/// Checks that an element can take an action before any input is sent: it is enabled and, for pointer
/// input, it is in view (scrolled there if needed) and is what the app hit-tests under the click point.
public enum ElementActionability {
public enum PointerTarget<Element> {
case ready(CGPoint)
case notEnabled
case noFrame
/// Nothing of the element shows inside its window and scroll areas.
case outOfView(scrolled: Bool)
/// Another element is on top at every point tried.
case covered(by: Element)
}
static let maxAncestorDepth = 64
/// Waits briefly for a disabled element to enable; an element that publishes no AXEnabled counts as enabled.
public static func waitUntilEnabled<AX: ElementActionabilityAccessibility>(_ element: AX.Element, ax: AX) -> Bool {
while ax.isEnabled(element) == false {
guard ax.waitBeforeRetry() else { return false }
}
return true
}
/// Where a pointer click on the element lands, re-checked until it holds or the wait budget runs out.
public static func pointerTarget<AX: ElementActionabilityAccessibility>(
_ element: AX.Element,
windowFrame: CGRect,
ax: AX
) throws -> PointerTarget<AX.Element> {
var scrolled = false
var scrollTried = false
var settledFrame: CGRect?
var last: PointerTarget<AX.Element> = .noFrame
repeat {
guard ax.isEnabled(element) != false else {
last = .notEnabled
continue
}
guard let frame = ax.frame(of: element), frame.width > 0, frame.height > 0 else {
return .noFrame
}
// Why: a scroll can animate, and a click mid-animation lands where the element used to be.
if scrolled, settledFrame.map({ !framesMatch($0, frame) }) ?? true {
settledFrame = frame
last = .outOfView(scrolled: true)
continue
}
let (clip, ancestors) = clipRect(of: element, windowFrame: windowFrame, ax: ax)
let visible = frame.intersection(clip)
// Why: an element bigger than its viewport cannot be brought fully into view, so it is clicked where it shows.
let fits = frame.width <= clip.width && frame.height <= clip.height
if !scrollTried, visible.isEmpty || (fits && !visible.contains(CGPoint(x: frame.midX, y: frame.midY))) {
scrollTried = true
if try ax.scrollToVisible(element) {
scrolled = true
last = .outOfView(scrolled: true)
continue
}
}
guard !visible.isEmpty else {
last = .outOfView(scrolled: scrolled)
continue
}
last = hitTarget(element, visible: visible, ancestors: ancestors, ax: ax)
if case .ready = last { return last }
} while ax.waitBeforeRetry()
return last
}
/// The part of the screen the element can show in (its window and every scroll area it sits in), plus its ancestors.
static func clipRect<AX: ElementActionabilityAccessibility>(
of element: AX.Element,
windowFrame: CGRect,
ax: AX
) -> (CGRect, [AX.Element]) {
var visible = windowFrame
var ancestors: [AX.Element] = []
var current = element
for _ in 0..<maxAncestorDepth {
guard let parent = ax.parent(of: current) else { break }
ancestors.append(parent)
let role = ax.role(of: parent)
if role == "AXScrollArea", let clip = ax.frame(of: parent) {
visible = visible.intersection(clip)
}
if role == "AXWindow" { break }
current = parent
}
return (visible, ancestors)
}
/// Accepts a point whose hit is the element, inside it, or one of its ancestors (the element is
/// see-through to hit tests there); anything else is a different element covering it.
static func hitTarget<AX: ElementActionabilityAccessibility>(
_ element: AX.Element,
visible: CGRect,
ancestors: [AX.Element],
ax: AX
) -> PointerTarget<AX.Element> {
var coveredBy: AX.Element?
for point in candidatePoints(in: visible) {
guard let hit = ax.element(at: point) else { return .ready(point) }
if isSelfOrDescendant(hit, of: element, ax: ax) || ancestors.contains(where: { ax.isSame($0, hit) }) {
return .ready(point)
}
coveredBy = coveredBy ?? hit
}
return coveredBy.map { .covered(by: $0) } ?? .noFrame
}
/// The centre first, then the middle of each quadrant, so a partly covered element is still reachable.
static func candidatePoints(in rect: CGRect) -> [CGPoint] {
let center = CGPoint(x: rect.midX, y: rect.midY)
let quarters = [(0.25, 0.25), (0.75, 0.25), (0.25, 0.75), (0.75, 0.75)].map {
CGPoint(x: rect.minX + rect.width * $0.0, y: rect.minY + rect.height * $0.1)
}
return [center] + quarters
}
private static func isSelfOrDescendant<AX: ElementActionabilityAccessibility>(_ hit: AX.Element, of element: AX.Element, ax: AX) -> Bool {
var current = hit
for _ in 0..<maxAncestorDepth {
if ax.isSame(current, element) { return true }
if ax.role(of: current) == "AXWindow" { return false }
guard let parent = ax.parent(of: current) else { return false }
current = parent
}
return false
}
private static func framesMatch(_ lhs: CGRect, _ rhs: CGRect) -> Bool {
abs(lhs.minX - rhs.minX) < 0.5 && abs(lhs.minY - rhs.minY) < 0.5
&& abs(lhs.width - rhs.width) < 0.5 && abs(lhs.height - rhs.height) < 0.5
}
}