fix(computer-macos): allow final click to dismiss target (#15169)

native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift

@@ -1177,10 +1177,29 @@ private func isTargetWindowFocused(_ snapshot: Snapshot) -> Bool {
     return !intersection.isNull && intersection.area >= min(frame.area, snapshot.windowBounds.area) * 0.75
 }
 
+private enum AXElementProbe {
+    case value(AXUIElement)
+    case absent
+    case unavailable
+}
+
+private func copyElementProbe(_ element: AXUIElement, _ attribute: String) -> AXElementProbe {
+    var value: CFTypeRef?
+    switch AXUIElementCopyAttributeValue(element, attribute as CFString, &value) {
+    case .success:
+        guard let value else { return .unavailable }
+        return .value(value as! AXUIElement)
+    case .noValue:
+        return .absent
+    default:
+        return .unavailable
+    }
+}
+
 private func currentSyntheticClickRecipient(
     snapshot: Snapshot,
     point: CGPoint
-) -> SyntheticMouseClickDelivery.Recipient? {
+) -> SyntheticMouseClickDelivery.RecipientObservation {
     let target = syntheticClickRecipient(pid: snapshot.app.pid, windowId: snapshot.windowId)
     var cachedTargetCandidates: [WindowCandidate]?
     func targetCandidates() -> [WindowCandidate] {
@@ -1189,39 +1208,60 @@ private func currentSyntheticClickRecipient(
         cachedTargetCandidates = candidates
         return candidates
     }
-    if let focused = focusedSyntheticClickRecipient(
+    switch focusedSyntheticClickRecipient(
         targetPID: snapshot.app.pid,
         targetCandidates: targetCandidates
     ) {
-        guard focused == target else { return focused }
-    } else {
-        guard NSWorkspace.shared.frontmostApplication?.processIdentifier == snapshot.app.pid else {
-            return nil
+    case let .focused(focused):
+        guard focused == target else { return .focused(focused) }
+        switch hitTestSyntheticClickRecipient(
+            at: point,
+            targetPID: snapshot.app.pid,
+            targetCandidates: targetCandidates
+        ) {
+        case let .focused(recipient):
+            return .focused(recipient)
+        case .dismissed, .unavailable:
+            return .unavailable
         }
+    case .dismissed:
+        return .dismissed
+    case .unavailable:
+        return .unavailable
     }
-    return hitTestSyntheticClickRecipient(
-        at: point,
-        targetPID: snapshot.app.pid,
-        targetCandidates: targetCandidates
-    )
 }
 
 private func focusedSyntheticClickRecipient(
     targetPID: pid_t,
     targetCandidates: () -> [WindowCandidate]
-) -> SyntheticMouseClickDelivery.Recipient? {
+) -> SyntheticMouseClickDelivery.RecipientObservation {
     let systemWide = AXUIElementCreateSystemWide()
-    guard let focusedApp = copyElement(systemWide, kAXFocusedApplicationAttribute as String),
-          let ownerPID = pidAttribute(focusedApp),
-          let focusedWindow = copyElement(systemWide, kAXFocusedWindowAttribute as String) ??
-            copyElement(focusedApp, kAXFocusedWindowAttribute as String)
-    else {
-        return nil
+    let focusedApp: AXUIElement
+    switch copyElementProbe(systemWide, kAXFocusedApplicationAttribute as String) {
+    case let .value(value):
+        focusedApp = value
+    case .absent:
+        return .dismissed
+    case .unavailable:
+        return .unavailable
     }
+    guard let ownerPID = pidAttribute(focusedApp) else { return .unavailable }
+
+    let focusedWindow: AXUIElement
+    switch copyElementProbe(focusedApp, kAXFocusedWindowAttribute as String) {
+    case let .value(value):
+        focusedWindow = value
+    case .absent:
+        guard ownerPID == targetPID else { return .unavailable }
+        return .dismissed
+    case .unavailable:
+        return .unavailable
+    }
+
     if let windowId = windowNumber(focusedWindow) {
-        return syntheticClickRecipient(pid: ownerPID, windowId: windowId)
+        return .focused(syntheticClickRecipient(pid: ownerPID, windowId: windowId))
     }
-    guard ownerPID == targetPID else { return nil }
+    guard ownerPID == targetPID else { return .unavailable }
     guard let frame = absoluteFrame(focusedWindow),
           let candidate = SyntheticMouseClickDelivery.uniqueWindowCandidate(
             from: targetCandidates(),
@@ -1229,35 +1269,38 @@ private func focusedSyntheticClickRecipient(
               windowFramesMatch($0.bounds, frame)
             }
           )
-    else {
-        return nil
-    }
-    return syntheticClickRecipient(pid: ownerPID, windowId: candidate.windowId)
+    else { return .unavailable }
+    return .focused(syntheticClickRecipient(pid: ownerPID, windowId: candidate.windowId))
 }
 
 private func hitTestSyntheticClickRecipient(
     at point: CGPoint,
     targetPID: pid_t,
     targetCandidates: () -> [WindowCandidate]
-) -> SyntheticMouseClickDelivery.Recipient? {
+) -> SyntheticMouseClickDelivery.RecipientObservation {
     let systemWide = AXUIElementCreateSystemWide()
     var hitElement: AXUIElement?
-    guard AXUIElementCopyElementAtPosition(
+    switch AXUIElementCopyElementAtPosition(
         systemWide,
         Float(point.x),
         Float(point.y),
         &hitElement
-    ) == .success,
-          let hitElement,
+    ) {
+    case .success:
+        break
+    case .noValue:
+        return .dismissed
+    default:
+        return .unavailable
+    }
+    guard let hitElement,
           let ownerPID = pidAttribute(hitElement),
           let window = containingWindow(hitElement)
-    else {
-        return nil
-    }
+    else { return .unavailable }
     if let windowId = windowNumber(window) {
-        return syntheticClickRecipient(pid: ownerPID, windowId: windowId)
+        return .focused(syntheticClickRecipient(pid: ownerPID, windowId: windowId))
     }
-    guard ownerPID == targetPID else { return nil }
+    guard ownerPID == targetPID else { return .unavailable }
     guard let frame = absoluteFrame(window),
           let candidate = SyntheticMouseClickDelivery.uniqueWindowCandidate(
             from: targetCandidates(),
@@ -1265,10 +1308,8 @@ private func hitTestSyntheticClickRecipient(
               windowFramesMatch($0.bounds, frame)
             }
           )
-    else {
-        return nil
-    }
-    return syntheticClickRecipient(pid: ownerPID, windowId: candidate.windowId)
+    else { return .unavailable }
+    return .focused(syntheticClickRecipient(pid: ownerPID, windowId: candidate.windowId))
 }
 
 private func containingWindow(_ element: AXUIElement) -> AXUIElement? {
@@ -2534,7 +2575,7 @@ private enum Input {
             try SyntheticMouseClickDelivery.deliver(
                 clickCount: count,
                 target: target,
-                currentRecipient: {
+                currentObservation: {
                     currentSyntheticClickRecipient(snapshot: targetWindow, point: point)
                 },
                 makeEvent: { step in

native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/SyntheticMouseClickDelivery.swift

@@ -18,6 +18,17 @@ public enum SyntheticMouseClickDelivery {
         }
     }
 
+    public enum RecipientObservation: Equatable, Sendable {
+        case focused(Recipient)
+        case dismissed
+        case unavailable
+
+        var recipient: Recipient? {
+            guard case let .focused(recipient) = self else { return nil }
+            return recipient
+        }
+    }
+
     public enum FenceFailure: Error, Equatable {
         case recipientChanged(expected: Recipient, actual: Recipient?, deliveredPresses: Int)
     }
@@ -69,19 +80,20 @@ public enum SyntheticMouseClickDelivery {
     public static func deliver<Event>(
         clickCount: Int,
         target: Recipient,
-        currentRecipient: () -> Recipient?,
+        currentObservation: () -> RecipientObservation,
         makeEvent: (Step) throws -> Event,
         post: (Event) -> Void,
         pause: (UInt32) -> Void
     ) throws {
         post(try makeEvent(.move))
         pause(interEventPauseMicroseconds)
-        for pressIndex in 1...min(max(clickCount, 1), maxClickCount) {
-            let beforeDown = currentRecipient()
-            guard beforeDown == target else {
+        let pressCount = min(max(clickCount, 1), maxClickCount)
+        for pressIndex in 1...pressCount {
+            let beforeDown = currentObservation()
+            guard beforeDown == .focused(target) else {
                 throw FenceFailure.recipientChanged(
                     expected: target,
-                    actual: beforeDown,
+                    actual: beforeDown.recipient,
                     deliveredPresses: pressIndex - 1
                 )
             }
@@ -90,11 +102,13 @@ public enum SyntheticMouseClickDelivery {
             post(down)
             pause(interEventPauseMicroseconds)
             post(up)
-            let afterUp = currentRecipient()
-            guard afterUp == target else {
+            let afterUp = currentObservation()
+            // A final mouse-up may dismiss the target, but an unavailable probe is unsafe.
+            let finalDismissal = pressIndex == pressCount && afterUp == .dismissed
+            guard afterUp == .focused(target) || finalDismissal else {
                 throw FenceFailure.recipientChanged(
                     expected: target,
-                    actual: afterUp,
+                    actual: afterUp.recipient,
                     deliveredPresses: pressIndex
                 )
             }

native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/SyntheticMouseClickDeliveryTests.swift

@@ -82,7 +82,7 @@ final class SyntheticMouseClickDeliveryTests: XCTestCase {
             try SyntheticMouseClickDelivery.deliver(
                 clickCount: 1,
                 target: target,
-                currentRecipient: { intruder },
+                currentObservation: { .focused(intruder) },
                 makeEvent: { $0 },
                 post: { posted.append($0) },
                 pause: { _ in }
@@ -96,19 +96,61 @@ final class SyntheticMouseClickDeliveryTests: XCTestCase {
         XCTAssertEqual(posted, [.move])
     }
 
-    func testRecipientChangeAfterMouseUpStopsUntilStateIsVerified() throws {
+    func testFinalMouseUpMayDismissTheTargetWithoutFailingTheClick() throws {
         let target = SyntheticMouseClickDelivery.Recipient(ownerPID: 41, windowID: 101)
-        let intruder = SyntheticMouseClickDelivery.Recipient(ownerPID: 52, windowID: 202)
-        var recipients = [target, intruder, target, target]
-        var firstAttempt: [SyntheticMouseClickDelivery.Step] = []
+        var observations: [SyntheticMouseClickDelivery.RecipientObservation] = [.focused(target), .dismissed]
+        var posted: [SyntheticMouseClickDelivery.Step] = []
+
+        try SyntheticMouseClickDelivery.deliver(
+            clickCount: 1,
+            target: target,
+            currentObservation: { observations.removeFirst() },
+            makeEvent: { $0 },
+            post: { posted.append($0) },
+            pause: { _ in }
+        )
+
+        XCTAssertEqual(posted, [.move, .buttonDown(pressIndex: 1), .buttonUp(pressIndex: 1)])
+    }
+
+    func testFinalUnavailableObservationRemainsFailClosed() {
+        let target = SyntheticMouseClickDelivery.Recipient(ownerPID: 41, windowID: 101)
+        var observations: [SyntheticMouseClickDelivery.RecipientObservation] = [.focused(target), .unavailable]
+        var posted: [SyntheticMouseClickDelivery.Step] = []
 
         XCTAssertThrowsError(
             try SyntheticMouseClickDelivery.deliver(
                 clickCount: 1,
                 target: target,
-                currentRecipient: { recipients.removeFirst() },
+                currentObservation: { observations.removeFirst() },
                 makeEvent: { $0 },
-                post: { firstAttempt.append($0) },
+                post: { posted.append($0) },
+                pause: { _ in }
+            )
+        ) { error in
+            XCTAssertEqual(
+                error as? SyntheticMouseClickDelivery.FenceFailure,
+                .recipientChanged(expected: target, actual: nil, deliveredPresses: 1)
+            )
+        }
+        XCTAssertEqual(posted, [.move, .buttonDown(pressIndex: 1), .buttonUp(pressIndex: 1)])
+    }
+
+    func testFinalMouseUpRejectsADifferentFocusedRecipient() {
+        let target = SyntheticMouseClickDelivery.Recipient(ownerPID: 41, windowID: 101)
+        let intruder = SyntheticMouseClickDelivery.Recipient(ownerPID: 52, windowID: 202)
+        var recipients: [SyntheticMouseClickDelivery.Recipient?] = [target, intruder]
+        var posted: [SyntheticMouseClickDelivery.Step] = []
+
+        XCTAssertThrowsError(
+            try SyntheticMouseClickDelivery.deliver(
+                clickCount: 1,
+                target: target,
+                currentObservation: {
+                    recipients.removeFirst().map(SyntheticMouseClickDelivery.RecipientObservation.focused) ?? .dismissed
+                },
+                makeEvent: { $0 },
+                post: { posted.append($0) },
                 pause: { _ in }
             )
         ) { error in
@@ -117,34 +159,19 @@ final class SyntheticMouseClickDeliveryTests: XCTestCase {
                 .recipientChanged(expected: target, actual: intruder, deliveredPresses: 1)
             )
         }
-        XCTAssertEqual(firstAttempt, [
-            .move,
-            .buttonDown(pressIndex: 1),
-            .buttonUp(pressIndex: 1),
-        ])
-
-        var retry: [SyntheticMouseClickDelivery.Step] = []
-        try SyntheticMouseClickDelivery.deliver(
-            clickCount: 1,
-            target: target,
-            currentRecipient: { recipients.removeFirst() },
-            makeEvent: { $0 },
-            post: { retry.append($0) },
-            pause: { _ in }
-        )
-        XCTAssertEqual(retry, [.move, .buttonDown(pressIndex: 1), .buttonUp(pressIndex: 1)])
+        XCTAssertEqual(posted, [.move, .buttonDown(pressIndex: 1), .buttonUp(pressIndex: 1)])
     }
 
-    func testMouseUpPostsBeforeSecondRecipientCheck() throws {
+    func testMouseUpPostsBeforeRecipientCheckForTheNextPress() throws {
         let target = SyntheticMouseClickDelivery.Recipient(ownerPID: 41, windowID: 101)
         var trace: [String] = []
 
         try SyntheticMouseClickDelivery.deliver(
-            clickCount: 1,
+            clickCount: 2,
             target: target,
-            currentRecipient: {
+            currentObservation: {
                 trace.append("recipient")
-                return target
+                return .focused(target)
             },
             makeEvent: { $0 },
             post: {
@@ -160,7 +187,9 @@ final class SyntheticMouseClickDeliveryTests: XCTestCase {
             pause: { _ in }
         )
 
-        XCTAssertEqual(trace, ["move", "recipient", "down", "up", "recipient"])
+        XCTAssertEqual(trace, [
+            "move", "recipient", "down", "up", "recipient", "recipient", "down", "up", "recipient"
+        ])
     }
 
     func testRecipientChangeBeforeLaterPressReportsCompletedPresses() {
@@ -173,7 +202,9 @@ final class SyntheticMouseClickDeliveryTests: XCTestCase {
             try SyntheticMouseClickDelivery.deliver(
                 clickCount: 2,
                 target: target,
-                currentRecipient: { recipients.removeFirst() },
+                currentObservation: {
+                    .focused(recipients.removeFirst())
+                },
                 makeEvent: { $0 },
                 post: { posted.append($0) },
                 pause: { _ in }
@@ -191,7 +222,66 @@ final class SyntheticMouseClickDeliveryTests: XCTestCase {
         ])
     }
 
-    func testMultiClickRevalidatesBeforeEveryPressAndAfterEveryRelease() throws {
+    func testRecipientChangeAfterIntermediateMouseUpStopsBeforeNextPress() {
+        let target = SyntheticMouseClickDelivery.Recipient(ownerPID: 41, windowID: 101)
+        let intruder = SyntheticMouseClickDelivery.Recipient(ownerPID: 52, windowID: 202)
+        var recipients: [SyntheticMouseClickDelivery.Recipient?] = [target, intruder]
+        var posted: [SyntheticMouseClickDelivery.Step] = []
+
+        XCTAssertThrowsError(
+            try SyntheticMouseClickDelivery.deliver(
+                clickCount: 2,
+                target: target,
+                currentObservation: {
+                    recipients.removeFirst().map(SyntheticMouseClickDelivery.RecipientObservation.focused) ?? .dismissed
+                },
+                makeEvent: { $0 },
+                post: { posted.append($0) },
+                pause: { _ in }
+            )
+        ) { error in
+            XCTAssertEqual(
+                error as? SyntheticMouseClickDelivery.FenceFailure,
+                .recipientChanged(expected: target, actual: intruder, deliveredPresses: 1)
+            )
+        }
+        XCTAssertEqual(posted, [
+            .move,
+            .buttonDown(pressIndex: 1),
+            .buttonUp(pressIndex: 1),
+        ])
+    }
+
+    func testIntermediateMouseUpDismissalStopsBeforeNextPress() {
+        let target = SyntheticMouseClickDelivery.Recipient(ownerPID: 41, windowID: 101)
+        var recipients: [SyntheticMouseClickDelivery.Recipient?] = [target, nil]
+        var posted: [SyntheticMouseClickDelivery.Step] = []
+
+        XCTAssertThrowsError(
+            try SyntheticMouseClickDelivery.deliver(
+                clickCount: 2,
+                target: target,
+                currentObservation: {
+                    recipients.removeFirst().map(SyntheticMouseClickDelivery.RecipientObservation.focused) ?? .dismissed
+                },
+                makeEvent: { $0 },
+                post: { posted.append($0) },
+                pause: { _ in }
+            )
+        ) { error in
+            XCTAssertEqual(
+                error as? SyntheticMouseClickDelivery.FenceFailure,
+                .recipientChanged(expected: target, actual: nil, deliveredPresses: 1)
+            )
+        }
+        XCTAssertEqual(posted, [
+            .move,
+            .buttonDown(pressIndex: 1),
+            .buttonUp(pressIndex: 1),
+        ])
+    }
+
+    func testMultiClickRevalidatesBeforeEveryPressAndBetweenReleases() throws {
         let target = SyntheticMouseClickDelivery.Recipient(ownerPID: 41, windowID: 101)
         var validationCount = 0
         var posted: [SyntheticMouseClickDelivery.Step] = []
@@ -199,9 +289,9 @@ final class SyntheticMouseClickDeliveryTests: XCTestCase {
         try SyntheticMouseClickDelivery.deliver(
             clickCount: 2,
             target: target,
-            currentRecipient: {
+            currentObservation: {
                 validationCount += 1
-                return target
+                return .focused(target)
             },
             makeEvent: { $0 },
             post: { posted.append($0) },
@@ -220,7 +310,7 @@ final class SyntheticMouseClickDeliveryTests: XCTestCase {
         XCTAssertThrowsError(try SyntheticMouseClickDelivery.deliver(
             clickCount: 1,
             target: target,
-            currentRecipient: { target },
+            currentObservation: { .focused(target) },
             makeEvent: { step in
                 if case .buttonUp = step { throw PreparationFailure.mouseUp }
                 return step