From 52c749207a006853aab4f88b40cbfd9651237b75 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Wed, 29 Jul 2026 19:21:18 -0700 Subject: [PATCH] fix(mobile): route external mouse click and drag to the terminal The terminal WebView suppresses mousedown/click at capture so xterm's own mouse handling stays inert (its onData bytes are dropped by the mobile bridge). That left hardware mouse clicks and drags with no path at all: touch taps reached mouse-aware TUIs and drove selection, while a Bluetooth mouse or trackpad click did nothing (#8818; wheel half landed in #11247). Add a pointer-event router on the terminal surface (pointerType 'mouse', left button only) that mirrors touch semantics: - plain click: same pipeline as a touch tap (links/file paths first, then tracking-mode press+release reports, else keyboard focus), and a click on an active selection dismisses it like touch does - drag with mouse tracking: press at the anchor, per-cell motion reports (drag/any modes), release on pointerup or pointercancel - drag without tracking: character-anchored selection reusing the touch handle-drag plumbing (edge scroll, handles, copy pill) Widen the RN gesture-input grammar to pass left-drag motion reports (SGR button 32, default-encoding byte 64) through the existing validation and rate limiting. Mock server: echo the subscribe viewport and serialize scrollback so the session screen leaves the resubscribe loop, serve the session-tabs subscribe stream, and add a MOCK_TUI=1 mouse-tracking scenario plus a [SEND] byte log - the rig used to reproduce and verify this fix on an Android emulator. Fixes #8818 --- mobile/scripts/mock-server-rpc-handlers.ts | 47 +-- .../mock-server-session-tabs-fixture.ts | 24 +- mobile/scripts/mock-server-terminal-stream.ts | 83 ++++ .../terminal/terminal-gesture-input.test.ts | 16 + mobile/src/terminal/terminal-gesture-input.ts | 6 +- mobile/src/terminal/terminal-webview-html.ts | 6 + ...minal-webview-mouse-click-drag-injected.ts | 183 +++++++++ .../terminal-webview-mouse-click-drag.test.ts | 354 ++++++++++++++++++ 8 files changed, 668 insertions(+), 51 deletions(-) create mode 100644 mobile/scripts/mock-server-terminal-stream.ts create mode 100644 mobile/src/terminal/terminal-webview-mouse-click-drag-injected.ts create mode 100644 mobile/src/terminal/terminal-webview-mouse-click-drag.test.ts diff --git a/mobile/scripts/mock-server-rpc-handlers.ts b/mobile/scripts/mock-server-rpc-handlers.ts index c5aebf054b4..25546c1076e 100644 --- a/mobile/scripts/mock-server-rpc-handlers.ts +++ b/mobile/scripts/mock-server-rpc-handlers.ts @@ -13,11 +13,7 @@ import { handleMockGitRequest } from './mock-server-git-state' import { handleMockAccountRequest } from './mock-server-account-rpc' import { handleMockNativeChatRequest } from './mock-server-native-chat-scenario' import { handleMockSessionTabsRequest } from './mock-server-session-tabs-fixture' -import { - createMockTerminals, - FAKE_SCROLLBACK, - STREAMING_CHUNKS -} from './mock-server-terminal-fixtures' +import { handleMockTerminalRequest } from './mock-server-terminal-stream' import { createMockRepos, createMockWorktrees, readScenarioNumber } from './mobile-lag-scenario' const MOCK_REPO_COUNT = readScenarioNumber('MOCK_REPO_COUNT', 2) @@ -126,7 +122,8 @@ export function handleRequest( handleMockFilePreviewRequest(request, respond, success, error) || handleMockAccountRequest(request, respond, success, error) || handleMockNativeChatRequest(request, respond, success, error, ws) || - handleMockSessionTabsRequest(request, respond, success, terminalListWorktreeId) + handleMockSessionTabsRequest(request, respond, success, terminalListWorktreeId) || + handleMockTerminalRequest(request, respond, success, ws, terminalListWorktreeId) ) { return } @@ -279,44 +276,6 @@ export function handleRequest( break } - case 'terminal.list': { - const terminals = createMockTerminals(terminalListWorktreeId(request.params?.worktree)) - respond( - success(request.id, { - terminals, - totalCount: terminals.length, - truncated: false - }) - ) - break - } - - case 'terminal.subscribe': { - respond(success(request.id, { type: 'scrollback', lines: FAKE_SCROLLBACK, truncated: false })) - - let chunkIndex = 0 - const interval = setInterval(() => { - if (chunkIndex >= STREAMING_CHUNKS.length || ws.readyState !== ws.OPEN) { - clearInterval(interval) - if (ws.readyState === ws.OPEN) { - respond(success(request.id, { type: 'end' })) - } - return - } - respond(success(request.id, { type: 'data', chunk: STREAMING_CHUNKS[chunkIndex] }, true)) - chunkIndex++ - }, 500) - break - } - - case 'terminal.send': - respond(success(request.id, { send: { handle: 'term-1', ok: true } })) - break - - case 'terminal.unsubscribe': - respond(success(request.id, { unsubscribed: true })) - break - case 'files.open': case 'files.openDiff': respond( diff --git a/mobile/scripts/mock-server-session-tabs-fixture.ts b/mobile/scripts/mock-server-session-tabs-fixture.ts index fbc94c2829e..f4a5bd1da1c 100644 --- a/mobile/scripts/mock-server-session-tabs-fixture.ts +++ b/mobile/scripts/mock-server-session-tabs-fixture.ts @@ -60,10 +60,24 @@ export function handleMockSessionTabsRequest( // absent or `id:`-prefixed selector means. resolveWorktreeId: (selector: unknown) => string | undefined ): boolean { - if (request.method !== 'session.tabs.list') { - return false + if (request.method === 'session.tabs.list') { + const worktreeId = resolveWorktreeId(request.params?.worktree) ?? 'mock' + respond(success(request.id, createMockSessionTabs(worktreeId))) + return true } - const worktreeId = resolveWorktreeId(request.params?.worktree) ?? 'mock' - respond(success(request.id, createMockSessionTabs(worktreeId))) - return true + if (request.method === 'session.tabs.subscribe') { + // Without a live stream the client's health loop keeps invalidating list + // fetches mid-flight (barrier bump on the failed probe), so the session + // screen never leaves 'Loading tabs'. snapshot then updated => 'live'. + const worktreeId = resolveWorktreeId(request.params?.worktree) ?? 'mock' + const snapshot = createMockSessionTabs(worktreeId) + respond(success(request.id, { type: 'snapshot', ...snapshot }, true)) + respond(success(request.id, { type: 'updated', ...snapshot }, true)) + return true + } + if (request.method === 'session.tabs.unsubscribe') { + respond(success(request.id, { unsubscribed: true })) + return true + } + return false } diff --git a/mobile/scripts/mock-server-terminal-stream.ts b/mobile/scripts/mock-server-terminal-stream.ts new file mode 100644 index 00000000000..a1e6c9bd86b --- /dev/null +++ b/mobile/scripts/mock-server-terminal-stream.ts @@ -0,0 +1,83 @@ +import type { WebSocket } from 'ws' +import type { RpcRequest, RpcResponse } from './mock-server-rpc-handlers' +import { + createMockTerminals, + FAKE_SCROLLBACK, + STREAMING_CHUNKS +} from './mock-server-terminal-fixtures' + +/** Terminal list/stream/input backend for the mock server. Returns false for + * methods it does not own. */ +export function handleMockTerminalRequest( + request: RpcRequest, + respond: (response: RpcResponse) => void, + success: (id: string, result: unknown, streaming?: boolean) => RpcResponse, + ws: WebSocket, + // Shared with `session.tabs.list` so both surfaces agree on which worktree an + // absent or `id:`-prefixed selector means. + resolveWorktreeId: (selector: unknown) => string | undefined +): boolean { + switch (request.method) { + case 'terminal.list': { + const terminals = createMockTerminals(resolveWorktreeId(request.params?.worktree)) + respond( + success(request.id, { + terminals, + totalCount: terminals.length, + truncated: false + }) + ) + return true + } + + case 'terminal.subscribe': { + // Why: the client resubscribes until scrollback echoes its viewport dims; + // the legacy `lines` shape left the session screen in that loop forever. + const viewport = request.params?.viewport as { cols?: number; rows?: number } | undefined + // MOCK_TUI=1 arms SGR drag mouse tracking (1002/1006) inside the scrollback + // itself so every xterm re-init re-enters the mode - used by mouse/touch + // input repros (#8818). + const tuiPreamble = + process.env.MOCK_TUI === '1' + ? '\x1b[?1002h\x1b[?1006h[mock] mouse tracking ON (1002/1006)\r\n' + : '' + respond( + success(request.id, { + type: 'scrollback', + cols: viewport?.cols ?? 80, + rows: viewport?.rows ?? 24, + serialized: FAKE_SCROLLBACK.replace(/\n/g, '\r\n') + tuiPreamble, + truncated: false + }) + ) + + let chunkIndex = 0 + const interval = setInterval(() => { + if (chunkIndex >= STREAMING_CHUNKS.length || ws.readyState !== ws.OPEN) { + // Why: no `end` event - a live terminal stream stays open, and `end` + // makes the client tear the subscription down and blank the pane. + clearInterval(interval) + return + } + respond(success(request.id, { type: 'data', chunk: STREAMING_CHUNKS[chunkIndex] }, true)) + chunkIndex++ + }, 500) + return true + } + + case 'terminal.send': + // Input-routing repros (#8818) assert on the exact bytes reaching the host. + console.log( + `[SEND] terminal=${String(request.params?.terminal)} text=${JSON.stringify(request.params?.text)}` + ) + respond(success(request.id, { send: { handle: 'term-1', ok: true } })) + return true + + case 'terminal.unsubscribe': + respond(success(request.id, { unsubscribed: true })) + return true + + default: + return false + } +} diff --git a/mobile/src/terminal/terminal-gesture-input.test.ts b/mobile/src/terminal/terminal-gesture-input.test.ts index a120d569db9..a920f37a7a3 100644 --- a/mobile/src/terminal/terminal-gesture-input.test.ts +++ b/mobile/src/terminal/terminal-gesture-input.test.ts @@ -27,6 +27,22 @@ describe('isTerminalGestureInput', () => { expect(isTerminalGestureInput(`${ESC}[<0;9999;9999M${ESC}[<0;9999;9999m`)).toBe(true) }) + it('accepts SGR left-drag motion sequences but rejects a motion release', () => { + expect(isTerminalGestureInput(`${ESC}[<32;10;5M${ESC}[<32;11;5M`)).toBe(true) + expect(countTerminalGestureInputSequences(`${ESC}[<32;10;5M${ESC}[<32;11;5M`)).toBe(2) + expect(isTerminalGestureInput(`${ESC}[<0;10;5M${ESC}[<32;11;5M${ESC}[<0;11;5m`)).toBe(true) + expect(isTerminalGestureInput(`${ESC}[<32;10;5m`)).toBe(false) + }) + + it('accepts default-encoding left-drag motion sequences', () => { + expect(isTerminalGestureInput(`${ESC}[M${String.fromCharCode(64, 43, 38)}`)).toBe(true) + expect( + isTerminalGestureInput( + `${ESC}[M${String.fromCharCode(32, 33, 33)}${ESC}[M${String.fromCharCode(64, 34, 33)}${ESC}[M${String.fromCharCode(35, 34, 33)}` + ) + ).toBe(true) + }) + it('accepts repeated default mouse wheel sequences', () => { expect( isTerminalGestureInput( diff --git a/mobile/src/terminal/terminal-gesture-input.ts b/mobile/src/terminal/terminal-gesture-input.ts index 6b84c4824fe..6af8a2367f9 100644 --- a/mobile/src/terminal/terminal-gesture-input.ts +++ b/mobile/src/terminal/terminal-gesture-input.ts @@ -1,8 +1,9 @@ const ESC = '\x1b' const MAX_TERMINAL_GESTURE_INPUT_LENGTH = 2048 const MAX_TERMINAL_GESTURE_INPUT_SEQUENCES = 32 +// Buttons: 0 left press/release, 32 left-drag motion, 64/65 wheel. const SGR_MOUSE_GESTURE_SEQUENCE_RE = new RegExp( - `^${ESC}\\[<(0|64|65);([0-9]{1,4});([0-9]{1,4})([Mm])$` + `^${ESC}\\[<(0|32|64|65);([0-9]{1,4});([0-9]{1,4})([Mm])$` ) function isDefaultMouseGestureSequence(bytes: string, offset: number): number | null { @@ -12,8 +13,9 @@ function isDefaultMouseGestureSequence(bytes: string, offset: number): number | const button = bytes.charCodeAt(offset + 3) const col = bytes.charCodeAt(offset + 4) const row = bytes.charCodeAt(offset + 5) + // Buttons: 32 left press, 35 release, 64 left-drag motion, 96/97 wheel. if ( - (button === 32 || button === 35 || button === 96 || button === 97) && + (button === 32 || button === 35 || button === 64 || button === 96 || button === 97) && col >= 33 && col <= 126 && row >= 33 && diff --git a/mobile/src/terminal/terminal-webview-html.ts b/mobile/src/terminal/terminal-webview-html.ts index 11579955926..6885915cecb 100644 --- a/mobile/src/terminal/terminal-webview-html.ts +++ b/mobile/src/terminal/terminal-webview-html.ts @@ -11,6 +11,7 @@ import { TERMINAL_WEBVIEW_THEME_JS } from './terminal-webview-theme-injected' import { TERMINAL_QUERY_REPLY_JS } from './terminal-webview-query-reply-injected' import { URL_TAP_WEBVIEW_JS } from './terminal-webview-url-tap' import { TERMINAL_WEBGL_RECOVERY_JS } from './terminal-webview-webgl-recovery-injected' +import { TERMINAL_MOUSE_CLICK_DRAG_JS } from './terminal-webview-mouse-click-drag-injected' import { TERMINAL_WHEEL_SCROLL_JS } from './terminal-webview-wheel-scroll-injected' const DEFAULT_TERMINAL_THEME: RuntimeMobileTerminalTheme['theme'] = { @@ -1640,6 +1641,10 @@ ${TERMINAL_WEBGL_RECOVERY_JS} // terminal-webview-wheel-scroll-injected.ts (extracted for max-lines). ${TERMINAL_WHEEL_SCROLL_JS} + // External mouse click/drag: see + // terminal-webview-mouse-click-drag-injected.ts (extracted for max-lines). + ${TERMINAL_MOUSE_CLICK_DRAG_JS} + btnCopy.addEventListener('click', function(e) { e.preventDefault(); e.stopPropagation(); @@ -1697,6 +1702,7 @@ ${TERMINAL_WEBGL_RECOVERY_JS} targetSurface.addEventListener('click', function(e) { e.preventDefault(); e.stopPropagation(); }, true); attachSurfaceWheelHandler(targetSurface); + attachSurfaceMouseClickDragHandler(targetSurface); targetSurface.addEventListener('touchstart', function(e) { if (dispatcherShouldBlockSurface()) return; diff --git a/mobile/src/terminal/terminal-webview-mouse-click-drag-injected.ts b/mobile/src/terminal/terminal-webview-mouse-click-drag-injected.ts new file mode 100644 index 00000000000..d7eb7ccb593 --- /dev/null +++ b/mobile/src/terminal/terminal-webview-mouse-click-drag-injected.ts @@ -0,0 +1,183 @@ +// Indirect-pointer (external mouse / trackpad) click and drag for the terminal +// surface, injected into XTERM_HTML. Extracted from terminal-webview-html.ts to +// keep that file within its max-lines budget. Companion to +// terminal-webview-wheel-scroll-injected.ts, which owns the wheel half (#11247); +// this owns the click/drag half of #8818. Closes over host-IIFE state/functions: +// term, ESC, sel, selMode, selectionOverlay, TAP_SLOP, getMouseTrackingMode, +// viewportToCell, viewportToMouseReportCell, isSafeSgrMouseCoordinate, +// sgrMouseMode, sgrMousePixelsMode, notify, notifyTerminalSurfaceTap, +// cancelSelect, applyXtermSelection, repositionOverlay, handleDragMove, +// stopEdgeScroll, and dispatcherShouldBlockSurface. +// +// Why pointer events: a hardware mouse on Android/iPadOS raises pointer events +// with pointerType 'mouse' and NO touch events, while a finger raises +// pointerType 'touch' plus the touch events the document dispatcher owns. The +// capture-phase mousedown/click suppression in attachSurfaceEventHandlers stays: +// it is what keeps xterm's own mouse handling inert (its onData output is +// dropped by the mobile bridge), and pointer events are unaffected by it. +export const TERMINAL_MOUSE_CLICK_DRAG_JS = ` + var mouseGesture = null; + + // One report per transition, built with the same encoding ladder as + // buildMouseClickInput: SGR pixels (1016) > SGR (1006) > default. Returns '' + // when the mode does not report this transition (x10 has no release, only + // drag/any report motion) or the cell is not encodable. + function buildMouseButtonReport(kind, clientX, clientY) { + var mouseTrackingMode = getMouseTrackingMode(); + if (mouseTrackingMode === 'none') return ''; + if (kind === 'motion' && mouseTrackingMode !== 'drag' && mouseTrackingMode !== 'any') return ''; + if (kind === 'release' && mouseTrackingMode === 'x10') return ''; + var cell = viewportToMouseReportCell(clientX, clientY); + if (!cell) return ''; + var sgrButton = kind === 'motion' ? 32 : 0; + var sgrFinal = kind === 'release' ? 'm' : 'M'; + if (sgrMousePixelsMode) { + if (!isSafeSgrMouseCoordinate(cell.x) || !isSafeSgrMouseCoordinate(cell.y)) return ''; + return ESC + '[<' + sgrButton + ';' + cell.x + ';' + cell.y + sgrFinal; + } + if (sgrMouseMode) { + // Why: xterm increments zero-based mouse cells before encoding reports. + var sgrCol = cell.col + 1; + var sgrRow = cell.row + 1; + if (!isSafeSgrMouseCoordinate(sgrCol) || !isSafeSgrMouseCoordinate(sgrRow)) return ''; + return ESC + '[<' + sgrButton + ';' + sgrCol + ';' + sgrRow + sgrFinal; + } + var button = kind === 'motion' ? 64 : kind === 'release' ? 35 : 32; + var col = cell.col + 1 + 32; + var row = cell.row + 1 + 32; + // Why: non-SGR mouse bytes above ASCII are not preserved reliably through + // the mobile JSON/RPC string path; drop instead of corrupting input. + if (col > 126 || row > 126) return ''; + return ESC + '[M' + String.fromCharCode(button) + String.fromCharCode(col) + String.fromCharCode(row); + } + + function mouseReportCellKey(clientX, clientY) { + var cell = viewportToMouseReportCell(clientX, clientY); + return cell ? cell.col + ',' + cell.row : null; + } + + function abandonMouseGesture() { + var gesture = mouseGesture; + mouseGesture = null; + if (!gesture) return; + if (gesture.mode === 'tracking') { + // Why: the press report already went to the TUI; a lost pointer must not + // leave the button latched down on the far side. + var release = buildMouseButtonReport('release', gesture.lastX, gesture.lastY); + if (release) notify({ type: 'terminal-input', bytes: release }); + } else if (gesture.mode === 'selecting') { + if (sel) sel.activeHandle = null; + stopEdgeScroll(); + } + } + + function beginMouseDrag(gesture) { + gesture.moved = true; + if (getMouseTrackingMode() !== 'none') { + gesture.mode = 'tracking'; + gesture.lastCellKey = mouseReportCellKey(gesture.startX, gesture.startY); + var press = buildMouseButtonReport('press', gesture.startX, gesture.startY); + if (press) notify({ type: 'terminal-input', bytes: press }); + return; + } + var anchor = viewportToCell(gesture.startX, gesture.startY); + if (!anchor) { + gesture.mode = 'cancelled'; + return; + } + // Why: mouse drags select character-anchored ranges like desktop terminals, + // not the word-seeded long-press selection; reuse the touch handle-drag + // plumbing (edge scroll included) by acting as a live 'end' handle. + gesture.mode = 'selecting'; + selMode = 'select'; + sel = { anchor: anchor, focus: anchor, activeHandle: 'end' }; + selectionOverlay.classList.add('active'); + notify({ type: 'set-select-mode', enabled: true }); + applyXtermSelection(); + repositionOverlay(); + } + + function attachSurfaceMouseClickDragHandler(targetSurface) { + targetSurface.addEventListener('pointerdown', function(e) { + if (e.pointerType !== 'mouse' || e.button !== 0) return; + if (dispatcherShouldBlockSurface() || !term) return; + mouseGesture = { + startX: e.clientX, startY: e.clientY, + lastX: e.clientX, lastY: e.clientY, + lastCellKey: null, + moved: false, + mode: 'pending', + dismissedSelection: false + }; + if (selMode === 'select') { + // Why: touch parity — pressing outside the pill dismisses the current + // selection; the same press may still start a new drag selection. + cancelSelect(); + mouseGesture.dismissedSelection = true; + } + }, true); + + targetSurface.addEventListener('pointermove', function(e) { + var gesture = mouseGesture; + if (e.pointerType !== 'mouse' || !gesture || gesture.mode === 'cancelled') return; + if (!term || (e.buttons & 1) === 0) return; + gesture.lastX = e.clientX; + gesture.lastY = e.clientY; + if (!gesture.moved) { + var dx = Math.abs(e.clientX - gesture.startX); + var dy = Math.abs(e.clientY - gesture.startY); + if (dx + dy <= TAP_SLOP) return; + beginMouseDrag(gesture); + } + if (gesture.mode === 'tracking') { + // Why: one motion report per cell keeps drags bounded by grid size, not + // by pointermove cadence, so the RN rate limiter is never the bottleneck. + var cellKey = mouseReportCellKey(e.clientX, e.clientY); + if (cellKey && cellKey !== gesture.lastCellKey) { + gesture.lastCellKey = cellKey; + var motion = buildMouseButtonReport('motion', e.clientX, e.clientY); + if (motion) notify({ type: 'terminal-input', bytes: motion }); + } + } else if (gesture.mode === 'selecting') { + handleDragMove('end', e.clientX, e.clientY); + } + }, true); + + targetSurface.addEventListener('pointerup', function(e) { + var gesture = mouseGesture; + if (e.pointerType !== 'mouse' || !gesture || e.button !== 0) return; + mouseGesture = null; + if (gesture.mode === 'cancelled' || !term) return; + if (gesture.mode === 'tracking') { + var release = buildMouseButtonReport('release', e.clientX, e.clientY); + if (release) notify({ type: 'terminal-input', bytes: release }); + return; + } + if (gesture.mode === 'selecting') { + if (sel) sel.activeHandle = null; + stopEdgeScroll(); + repositionOverlay(); + return; + } + if (dispatcherShouldBlockSurface()) return; + // Why: a dismissing tap only clears the selection (touch parity); it must + // not also open a link or focus the keyboard underneath. + if (gesture.dismissedSelection) return; + // Plain click == touch tap: links and file paths win, then tracking-mode + // click reports, then keyboard focus. + notifyTerminalSurfaceTap(e.clientX, e.clientY); + }, true); + + targetSurface.addEventListener('pointercancel', function(e) { + if (e.pointerType !== 'mouse') return; + abandonMouseGesture(); + }, true); + + // Why: Android input injection can pair a mouse-flavored pointerdown with + // real touch events (SOURCE_MOUSE + TOOL_TYPE_FINGER). If touch arrives, + // the document touch dispatcher owns the gesture. + targetSurface.addEventListener('touchstart', function() { + if (mouseGesture) abandonMouseGesture(); + }, true); + } +` diff --git a/mobile/src/terminal/terminal-webview-mouse-click-drag.test.ts b/mobile/src/terminal/terminal-webview-mouse-click-drag.test.ts new file mode 100644 index 00000000000..2df4bf24ce8 --- /dev/null +++ b/mobile/src/terminal/terminal-webview-mouse-click-drag.test.ts @@ -0,0 +1,354 @@ +// @vitest-environment happy-dom +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { XTERM_HTML } from './terminal-webview-html' + +function iifeSource(): string { + const start = XTERM_HTML.indexOf('(function() {') + const end = XTERM_HTML.lastIndexOf('})();') + return XTERM_HTML.slice(start, end + '})();'.length) +} + +function bodyMarkup(): string { + const start = XTERM_HTML.indexOf('
') + ''.length + const end = XTERM_HTML.indexOf('