From 01b926828f0a83f1a7486dfbe919b510a52151cf Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Wed, 29 Jul 2026 18:08:34 -0700 Subject: [PATCH] fix(computer-use): pace modified Windows multiclicks --- .../computer-use-modifier-safety.test.mjs | 11 +++++++++- native/computer-use-windows/runtime.ps1 | 22 +++++++++---------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/config/scripts/computer-use-modifier-safety.test.mjs b/config/scripts/computer-use-modifier-safety.test.mjs index 052b995b3ac..79c739a7ec3 100644 --- a/config/scripts/computer-use-modifier-safety.test.mjs +++ b/config/scripts/computer-use-modifier-safety.test.mjs @@ -24,16 +24,25 @@ describe('computer-use modifier safety', () => { expect(clickInput).not.toContain('down: true') }) - it('submits the complete modified click in one Windows SendInput batch', () => { + it('submits each modified Windows click in a closed, timed SendInput batch', () => { const windows = source('native/computer-use-windows/runtime.ps1') const modifiedClick = windows.slice( windows.indexOf('public static void SendModifiedClick'), windows.indexOf('private static INPUT KeyboardInput') ) + const mouseClick = windows.slice( + windows.indexOf('function Send-OrcaMouseClick'), + windows.indexOf('function Send-OrcaDrag') + ) expect(modifiedClick).toContain('SendInput((uint)values.Length, values') expect(modifiedClick).toContain('SendInput((uint)releaseValues.Length, releaseValues') expect(modifiedClick).toContain('if (sent != (uint)values.Length)') + expect(modifiedClick).not.toContain('int count') + expect(mouseClick).toMatch( + /for \(\$i = 0; \$i -lt \$clickCount; \$i\+\+\) \{\s+\[OrcaDesktopWin32\]::SendModifiedClick\(/ + ) + expect(mouseClick).toContain('if ($i + 1 -lt $clickCount) { Start-Sleep -Milliseconds 35 }') expect(windows).not.toContain('keybd_event') }) diff --git a/native/computer-use-windows/runtime.ps1 b/native/computer-use-windows/runtime.ps1 index c6355b3a1c0..96ab0e9b2aa 100644 --- a/native/computer-use-windows/runtime.ps1 +++ b/native/computer-use-windows/runtime.ps1 @@ -94,7 +94,7 @@ public static class OrcaDesktopWin32 { [DllImport("user32.dll")] public static extern uint SendInput(uint count, INPUT[] inputs, int size); - public static void SendModifiedClick(byte[] modifiers, uint mouseDown, uint mouseUp, int count) { + public static void SendModifiedClick(byte[] modifiers, uint mouseDown, uint mouseUp) { const uint keyboardInput = 1; const uint mouseInput = 0; const uint keyUp = 0x0002; @@ -102,10 +102,8 @@ public static class OrcaDesktopWin32 { foreach (var modifier in modifiers) { inputs.Add(KeyboardInput(keyboardInput, modifier, 0)); } - for (var index = 0; index < count; index++) { - inputs.Add(MouseInput(mouseInput, mouseDown)); - inputs.Add(MouseInput(mouseInput, mouseUp)); - } + inputs.Add(MouseInput(mouseInput, mouseDown)); + inputs.Add(MouseInput(mouseInput, mouseUp)); for (var index = modifiers.Length - 1; index >= 0; index--) { inputs.Add(KeyboardInput(keyboardInput, modifiers[index], keyUp)); } @@ -1016,12 +1014,14 @@ function Send-OrcaMouseClick([IntPtr]$WindowHandle, [int]$ScreenX, [int]$ScreenY } return } - [OrcaDesktopWin32]::SendModifiedClick( - [byte[]]$modifierKeys, - [uint32]$down, - [uint32]$up, - $clickCount - ) + for ($i = 0; $i -lt $clickCount; $i++) { + [OrcaDesktopWin32]::SendModifiedClick( + [byte[]]$modifierKeys, + [uint32]$down, + [uint32]$up + ) + if ($i + 1 -lt $clickCount) { Start-Sleep -Milliseconds 35 } + } } function Send-OrcaDrag([IntPtr]$WindowHandle, $From, $To) {