diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift index 6e1d92adcce..324dba707da 100644 --- a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift @@ -1177,10 +1177,29 @@ private func isTargetWindowFocused(_ snapshot: Snapshot) -> Bool { return !intersection.isNull && intersection.area >= min(frame.area, snapshot.windowBounds.area) * 0.75 } +private enum AXElementProbe { + case value(AXUIElement) + case absent + case unavailable +} + +private func copyElementProbe(_ element: AXUIElement, _ attribute: String) -> AXElementProbe { + var value: CFTypeRef? + switch AXUIElementCopyAttributeValue(element, attribute as CFString, &value) { + case .success: + guard let value else { return .unavailable } + return .value(value as! AXUIElement) + case .noValue: + return .absent + default: + return .unavailable + } +} + private func currentSyntheticClickRecipient( snapshot: Snapshot, point: CGPoint -) -> SyntheticMouseClickDelivery.Recipient? { +) -> SyntheticMouseClickDelivery.RecipientObservation { let target = syntheticClickRecipient(pid: snapshot.app.pid, windowId: snapshot.windowId) var cachedTargetCandidates: [WindowCandidate]? func targetCandidates() -> [WindowCandidate] { @@ -1189,39 +1208,60 @@ private func currentSyntheticClickRecipient( cachedTargetCandidates = candidates return candidates } - if let focused = focusedSyntheticClickRecipient( + switch focusedSyntheticClickRecipient( targetPID: snapshot.app.pid, targetCandidates: targetCandidates ) { - guard focused == target else { return focused } - } else { - guard NSWorkspace.shared.frontmostApplication?.processIdentifier == snapshot.app.pid else { - return nil + case let .focused(focused): + guard focused == target else { return .focused(focused) } + switch hitTestSyntheticClickRecipient( + at: point, + targetPID: snapshot.app.pid, + targetCandidates: targetCandidates + ) { + case let .focused(recipient): + return .focused(recipient) + case .dismissed, .unavailable: + return .unavailable } + case .dismissed: + return .dismissed + case .unavailable: + return .unavailable } - return hitTestSyntheticClickRecipient( - at: point, - targetPID: snapshot.app.pid, - targetCandidates: targetCandidates - ) } private func focusedSyntheticClickRecipient( targetPID: pid_t, targetCandidates: () -> [WindowCandidate] -) -> SyntheticMouseClickDelivery.Recipient? { +) -> SyntheticMouseClickDelivery.RecipientObservation { let systemWide = AXUIElementCreateSystemWide() - guard let focusedApp = copyElement(systemWide, kAXFocusedApplicationAttribute as String), - let ownerPID = pidAttribute(focusedApp), - let focusedWindow = copyElement(systemWide, kAXFocusedWindowAttribute as String) ?? - copyElement(focusedApp, kAXFocusedWindowAttribute as String) - else { - return nil + let focusedApp: AXUIElement + switch copyElementProbe(systemWide, kAXFocusedApplicationAttribute as String) { + case let .value(value): + focusedApp = value + case .absent: + return .dismissed + case .unavailable: + return .unavailable } + guard let ownerPID = pidAttribute(focusedApp) else { return .unavailable } + + let focusedWindow: AXUIElement + switch copyElementProbe(focusedApp, kAXFocusedWindowAttribute as String) { + case let .value(value): + focusedWindow = value + case .absent: + guard ownerPID == targetPID else { return .unavailable } + return .dismissed + case .unavailable: + return .unavailable + } + if let windowId = windowNumber(focusedWindow) { - return syntheticClickRecipient(pid: ownerPID, windowId: windowId) + return .focused(syntheticClickRecipient(pid: ownerPID, windowId: windowId)) } - guard ownerPID == targetPID else { return nil } + guard ownerPID == targetPID else { return .unavailable } guard let frame = absoluteFrame(focusedWindow), let candidate = SyntheticMouseClickDelivery.uniqueWindowCandidate( from: targetCandidates(), @@ -1229,35 +1269,38 @@ private func focusedSyntheticClickRecipient( windowFramesMatch($0.bounds, frame) } ) - else { - return nil - } - return syntheticClickRecipient(pid: ownerPID, windowId: candidate.windowId) + else { return .unavailable } + return .focused(syntheticClickRecipient(pid: ownerPID, windowId: candidate.windowId)) } private func hitTestSyntheticClickRecipient( at point: CGPoint, targetPID: pid_t, targetCandidates: () -> [WindowCandidate] -) -> SyntheticMouseClickDelivery.Recipient? { +) -> SyntheticMouseClickDelivery.RecipientObservation { let systemWide = AXUIElementCreateSystemWide() var hitElement: AXUIElement? - guard AXUIElementCopyElementAtPosition( + switch AXUIElementCopyElementAtPosition( systemWide, Float(point.x), Float(point.y), &hitElement - ) == .success, - let hitElement, + ) { + case .success: + break + case .noValue: + return .dismissed + default: + return .unavailable + } + guard let hitElement, let ownerPID = pidAttribute(hitElement), let window = containingWindow(hitElement) - else { - return nil - } + else { return .unavailable } if let windowId = windowNumber(window) { - return syntheticClickRecipient(pid: ownerPID, windowId: windowId) + return .focused(syntheticClickRecipient(pid: ownerPID, windowId: windowId)) } - guard ownerPID == targetPID else { return nil } + guard ownerPID == targetPID else { return .unavailable } guard let frame = absoluteFrame(window), let candidate = SyntheticMouseClickDelivery.uniqueWindowCandidate( from: targetCandidates(), @@ -1265,10 +1308,8 @@ private func hitTestSyntheticClickRecipient( windowFramesMatch($0.bounds, frame) } ) - else { - return nil - } - return syntheticClickRecipient(pid: ownerPID, windowId: candidate.windowId) + else { return .unavailable } + return .focused(syntheticClickRecipient(pid: ownerPID, windowId: candidate.windowId)) } private func containingWindow(_ element: AXUIElement) -> AXUIElement? { @@ -2534,7 +2575,7 @@ private enum Input { try SyntheticMouseClickDelivery.deliver( clickCount: count, target: target, - currentRecipient: { + currentObservation: { currentSyntheticClickRecipient(snapshot: targetWindow, point: point) }, makeEvent: { step in diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/SyntheticMouseClickDelivery.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/SyntheticMouseClickDelivery.swift index 13aa6e15ced..e9ba429dfe2 100644 --- a/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/SyntheticMouseClickDelivery.swift +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/SyntheticMouseClickDelivery.swift @@ -18,6 +18,17 @@ public enum SyntheticMouseClickDelivery { } } + public enum RecipientObservation: Equatable, Sendable { + case focused(Recipient) + case dismissed + case unavailable + + var recipient: Recipient? { + guard case let .focused(recipient) = self else { return nil } + return recipient + } + } + public enum FenceFailure: Error, Equatable { case recipientChanged(expected: Recipient, actual: Recipient?, deliveredPresses: Int) } @@ -69,19 +80,20 @@ public enum SyntheticMouseClickDelivery { public static func deliver( clickCount: Int, target: Recipient, - currentRecipient: () -> Recipient?, + currentObservation: () -> RecipientObservation, makeEvent: (Step) throws -> Event, post: (Event) -> Void, pause: (UInt32) -> Void ) throws { post(try makeEvent(.move)) pause(interEventPauseMicroseconds) - for pressIndex in 1...min(max(clickCount, 1), maxClickCount) { - let beforeDown = currentRecipient() - guard beforeDown == target else { + let pressCount = min(max(clickCount, 1), maxClickCount) + for pressIndex in 1...pressCount { + let beforeDown = currentObservation() + guard beforeDown == .focused(target) else { throw FenceFailure.recipientChanged( expected: target, - actual: beforeDown, + actual: beforeDown.recipient, deliveredPresses: pressIndex - 1 ) } @@ -90,11 +102,13 @@ public enum SyntheticMouseClickDelivery { post(down) pause(interEventPauseMicroseconds) post(up) - let afterUp = currentRecipient() - guard afterUp == target else { + let afterUp = currentObservation() + // A final mouse-up may dismiss the target, but an unavailable probe is unsafe. + let finalDismissal = pressIndex == pressCount && afterUp == .dismissed + guard afterUp == .focused(target) || finalDismissal else { throw FenceFailure.recipientChanged( expected: target, - actual: afterUp, + actual: afterUp.recipient, deliveredPresses: pressIndex ) } diff --git a/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/SyntheticMouseClickDeliveryTests.swift b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/SyntheticMouseClickDeliveryTests.swift index 5870e740880..d040b5d8f26 100644 --- a/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/SyntheticMouseClickDeliveryTests.swift +++ b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/SyntheticMouseClickDeliveryTests.swift @@ -82,7 +82,7 @@ final class SyntheticMouseClickDeliveryTests: XCTestCase { try SyntheticMouseClickDelivery.deliver( clickCount: 1, target: target, - currentRecipient: { intruder }, + currentObservation: { .focused(intruder) }, makeEvent: { $0 }, post: { posted.append($0) }, pause: { _ in } @@ -96,19 +96,61 @@ final class SyntheticMouseClickDeliveryTests: XCTestCase { XCTAssertEqual(posted, [.move]) } - func testRecipientChangeAfterMouseUpStopsUntilStateIsVerified() throws { + func testFinalMouseUpMayDismissTheTargetWithoutFailingTheClick() throws { let target = SyntheticMouseClickDelivery.Recipient(ownerPID: 41, windowID: 101) - let intruder = SyntheticMouseClickDelivery.Recipient(ownerPID: 52, windowID: 202) - var recipients = [target, intruder, target, target] - var firstAttempt: [SyntheticMouseClickDelivery.Step] = [] + var observations: [SyntheticMouseClickDelivery.RecipientObservation] = [.focused(target), .dismissed] + var posted: [SyntheticMouseClickDelivery.Step] = [] + + try SyntheticMouseClickDelivery.deliver( + clickCount: 1, + target: target, + currentObservation: { observations.removeFirst() }, + makeEvent: { $0 }, + post: { posted.append($0) }, + pause: { _ in } + ) + + XCTAssertEqual(posted, [.move, .buttonDown(pressIndex: 1), .buttonUp(pressIndex: 1)]) + } + + func testFinalUnavailableObservationRemainsFailClosed() { + let target = SyntheticMouseClickDelivery.Recipient(ownerPID: 41, windowID: 101) + var observations: [SyntheticMouseClickDelivery.RecipientObservation] = [.focused(target), .unavailable] + var posted: [SyntheticMouseClickDelivery.Step] = [] XCTAssertThrowsError( try SyntheticMouseClickDelivery.deliver( clickCount: 1, target: target, - currentRecipient: { recipients.removeFirst() }, + currentObservation: { observations.removeFirst() }, makeEvent: { $0 }, - post: { firstAttempt.append($0) }, + post: { posted.append($0) }, + pause: { _ in } + ) + ) { error in + XCTAssertEqual( + error as? SyntheticMouseClickDelivery.FenceFailure, + .recipientChanged(expected: target, actual: nil, deliveredPresses: 1) + ) + } + XCTAssertEqual(posted, [.move, .buttonDown(pressIndex: 1), .buttonUp(pressIndex: 1)]) + } + + func testFinalMouseUpRejectsADifferentFocusedRecipient() { + let target = SyntheticMouseClickDelivery.Recipient(ownerPID: 41, windowID: 101) + let intruder = SyntheticMouseClickDelivery.Recipient(ownerPID: 52, windowID: 202) + var recipients: [SyntheticMouseClickDelivery.Recipient?] = [target, intruder] + var posted: [SyntheticMouseClickDelivery.Step] = [] + + XCTAssertThrowsError( + try SyntheticMouseClickDelivery.deliver( + clickCount: 1, + target: target, + currentObservation: { + recipients.removeFirst().map(SyntheticMouseClickDelivery.RecipientObservation.focused) ?? .dismissed + }, + makeEvent: { $0 }, + post: { posted.append($0) }, pause: { _ in } ) ) { error in @@ -117,34 +159,19 @@ final class SyntheticMouseClickDeliveryTests: XCTestCase { .recipientChanged(expected: target, actual: intruder, deliveredPresses: 1) ) } - XCTAssertEqual(firstAttempt, [ - .move, - .buttonDown(pressIndex: 1), - .buttonUp(pressIndex: 1), - ]) - - var retry: [SyntheticMouseClickDelivery.Step] = [] - try SyntheticMouseClickDelivery.deliver( - clickCount: 1, - target: target, - currentRecipient: { recipients.removeFirst() }, - makeEvent: { $0 }, - post: { retry.append($0) }, - pause: { _ in } - ) - XCTAssertEqual(retry, [.move, .buttonDown(pressIndex: 1), .buttonUp(pressIndex: 1)]) + XCTAssertEqual(posted, [.move, .buttonDown(pressIndex: 1), .buttonUp(pressIndex: 1)]) } - func testMouseUpPostsBeforeSecondRecipientCheck() throws { + func testMouseUpPostsBeforeRecipientCheckForTheNextPress() throws { let target = SyntheticMouseClickDelivery.Recipient(ownerPID: 41, windowID: 101) var trace: [String] = [] try SyntheticMouseClickDelivery.deliver( - clickCount: 1, + clickCount: 2, target: target, - currentRecipient: { + currentObservation: { trace.append("recipient") - return target + return .focused(target) }, makeEvent: { $0 }, post: { @@ -160,7 +187,9 @@ final class SyntheticMouseClickDeliveryTests: XCTestCase { pause: { _ in } ) - XCTAssertEqual(trace, ["move", "recipient", "down", "up", "recipient"]) + XCTAssertEqual(trace, [ + "move", "recipient", "down", "up", "recipient", "recipient", "down", "up", "recipient" + ]) } func testRecipientChangeBeforeLaterPressReportsCompletedPresses() { @@ -173,7 +202,9 @@ final class SyntheticMouseClickDeliveryTests: XCTestCase { try SyntheticMouseClickDelivery.deliver( clickCount: 2, target: target, - currentRecipient: { recipients.removeFirst() }, + currentObservation: { + .focused(recipients.removeFirst()) + }, makeEvent: { $0 }, post: { posted.append($0) }, pause: { _ in } @@ -191,7 +222,66 @@ final class SyntheticMouseClickDeliveryTests: XCTestCase { ]) } - func testMultiClickRevalidatesBeforeEveryPressAndAfterEveryRelease() throws { + func testRecipientChangeAfterIntermediateMouseUpStopsBeforeNextPress() { + let target = SyntheticMouseClickDelivery.Recipient(ownerPID: 41, windowID: 101) + let intruder = SyntheticMouseClickDelivery.Recipient(ownerPID: 52, windowID: 202) + var recipients: [SyntheticMouseClickDelivery.Recipient?] = [target, intruder] + var posted: [SyntheticMouseClickDelivery.Step] = [] + + XCTAssertThrowsError( + try SyntheticMouseClickDelivery.deliver( + clickCount: 2, + target: target, + currentObservation: { + recipients.removeFirst().map(SyntheticMouseClickDelivery.RecipientObservation.focused) ?? .dismissed + }, + makeEvent: { $0 }, + post: { posted.append($0) }, + pause: { _ in } + ) + ) { error in + XCTAssertEqual( + error as? SyntheticMouseClickDelivery.FenceFailure, + .recipientChanged(expected: target, actual: intruder, deliveredPresses: 1) + ) + } + XCTAssertEqual(posted, [ + .move, + .buttonDown(pressIndex: 1), + .buttonUp(pressIndex: 1), + ]) + } + + func testIntermediateMouseUpDismissalStopsBeforeNextPress() { + let target = SyntheticMouseClickDelivery.Recipient(ownerPID: 41, windowID: 101) + var recipients: [SyntheticMouseClickDelivery.Recipient?] = [target, nil] + var posted: [SyntheticMouseClickDelivery.Step] = [] + + XCTAssertThrowsError( + try SyntheticMouseClickDelivery.deliver( + clickCount: 2, + target: target, + currentObservation: { + recipients.removeFirst().map(SyntheticMouseClickDelivery.RecipientObservation.focused) ?? .dismissed + }, + makeEvent: { $0 }, + post: { posted.append($0) }, + pause: { _ in } + ) + ) { error in + XCTAssertEqual( + error as? SyntheticMouseClickDelivery.FenceFailure, + .recipientChanged(expected: target, actual: nil, deliveredPresses: 1) + ) + } + XCTAssertEqual(posted, [ + .move, + .buttonDown(pressIndex: 1), + .buttonUp(pressIndex: 1), + ]) + } + + func testMultiClickRevalidatesBeforeEveryPressAndBetweenReleases() throws { let target = SyntheticMouseClickDelivery.Recipient(ownerPID: 41, windowID: 101) var validationCount = 0 var posted: [SyntheticMouseClickDelivery.Step] = [] @@ -199,9 +289,9 @@ final class SyntheticMouseClickDeliveryTests: XCTestCase { try SyntheticMouseClickDelivery.deliver( clickCount: 2, target: target, - currentRecipient: { + currentObservation: { validationCount += 1 - return target + return .focused(target) }, makeEvent: { $0 }, post: { posted.append($0) }, @@ -220,7 +310,7 @@ final class SyntheticMouseClickDeliveryTests: XCTestCase { XCTAssertThrowsError(try SyntheticMouseClickDelivery.deliver( clickCount: 1, target: target, - currentRecipient: { target }, + currentObservation: { .focused(target) }, makeEvent: { step in if case .buttonUp = step { throw PreparationFailure.mouseUp } return step