From 515bf2d6bb6934ddb057762b55495d1c908014b9 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Tue, 14 Jul 2026 16:00:25 -0700 Subject: [PATCH 01/15] Fix restored terminals rendering blank on mobile (#8768) Co-authored-by: Orca --- src/main/daemon/daemon-pty-adapter.test.ts | 6 + src/main/daemon/daemon-pty-adapter.ts | 22 +- src/main/ipc/pty.test.ts | 76 +++- src/main/ipc/pty.ts | 34 +- src/main/providers/types.ts | 6 + src/main/runtime/orca-runtime.test.ts | 395 ++++++++++++++++++ src/main/runtime/orca-runtime.ts | 222 +++++++++- ...erminal-provider-snapshot-sequence.test.ts | 187 +++++++++ ...rminal-kitty-keyboard-mode-tracker.test.ts | 3 + .../terminal-kitty-keyboard-mode-tracker.ts | 12 + 10 files changed, 954 insertions(+), 9 deletions(-) create mode 100644 src/main/runtime/rpc/terminal-provider-snapshot-sequence.test.ts diff --git a/src/main/daemon/daemon-pty-adapter.test.ts b/src/main/daemon/daemon-pty-adapter.test.ts index 37896ab8336..1b3b5fb0b7e 100644 --- a/src/main/daemon/daemon-pty-adapter.test.ts +++ b/src/main/daemon/daemon-pty-adapter.test.ts @@ -132,6 +132,7 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => { const result = await adapter.spawn({ cols: 80, rows: 24 }) expect(result.id).toBeDefined() expect(typeof result.id).toBe('string') + expect(result.providerSequence).toEqual({ value: 0, generation: 'reset' }) }) it('uses worktreeId as session prefix when provided', async () => { @@ -506,6 +507,10 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => { expect(second.launchAgent).toBe('droid') expect(second.snapshot).toBeDefined() expect(second.snapshot).toContain('hello from shell') + expect(second.providerSequence).toEqual({ + value: 'hello from shell\r\n'.length, + generation: 'continued' + }) }) it('includes rehydrateSequences in snapshot when terminal modes are active', async () => { @@ -528,6 +533,7 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => { expect(result.id).toBe('brand-new') expect(result.isReattach).toBeUndefined() expect(result.snapshot).toBeUndefined() + expect(result.providerSequence).toEqual({ value: 0, generation: 'reset' }) }) }) diff --git a/src/main/daemon/daemon-pty-adapter.ts b/src/main/daemon/daemon-pty-adapter.ts index 0fc07729383..151969928be 100644 --- a/src/main/daemon/daemon-pty-adapter.ts +++ b/src/main/daemon/daemon-pty-adapter.ts @@ -47,6 +47,17 @@ function getRecoveredHistorySeed(restoreInfo: ColdRestoreInfo): string | null { : restoreInfo.rehydrateSequences + restoreInfo.snapshotAnsi } +function providerSequenceForSpawn( + result: CreateOrAttachResult +): PtySpawnResult['providerSequence'] { + if (result.isNew) { + return { value: 0, generation: 'reset' } + } + return typeof result.snapshot?.outputSequence === 'number' + ? { value: result.snapshot.outputSequence, generation: 'continued' } + : undefined +} + export type DaemonPtyAdapterOptions = { socketPath: string tokenPath: string @@ -322,6 +333,7 @@ export class DaemonPtyAdapter implements IPtyProvider { const wasAlreadyManaged = this.activeSessionIds.has(sessionId) this.activeSessionIds.add(sessionId) + const providerSequence = providerSequenceForSpawn(result) // Cold restore: daemon created a new session but disk history shows // an unclean shutdown → return saved scrollback so the renderer can @@ -353,10 +365,16 @@ export class DaemonPtyAdapter implements IPtyProvider { pid, ...launchIdentity(), coldRestore, + ...(providerSequence ? { providerSequence } : {}), ...(!result.isNew ? { isReattach: true } : {}) } } - return { id: sessionId, pid, ...launchIdentity() } + return { + id: sessionId, + pid, + ...launchIdentity(), + ...(providerSequence ? { providerSequence } : {}) + } } if (this.historyManager && result.isNew) { @@ -394,6 +412,7 @@ export class DaemonPtyAdapter implements IPtyProvider { id: sessionId, pid, ...launchIdentity(), + ...(providerSequence ? { providerSequence } : {}), ...(isReattach ? { isReattach: true } : {}) } } @@ -415,6 +434,7 @@ export class DaemonPtyAdapter implements IPtyProvider { snapshot: snapshotPayload, snapshotCols: result.snapshot.cols, snapshotRows: result.snapshot.rows, + ...(providerSequence ? { providerSequence } : {}), ...(typeof kittyKeyboardFlags === 'number' && kittyKeyboardFlags > 0 ? { snapshotKittyKeyboardFlags: kittyKeyboardFlags } : {}), diff --git a/src/main/ipc/pty.test.ts b/src/main/ipc/pty.test.ts index 40efbe69908..6adb7e5cf69 100644 --- a/src/main/ipc/pty.test.ts +++ b/src/main/ipc/pty.test.ts @@ -10506,6 +10506,44 @@ describe('registerPtyHandlers', () => { expect(mockProc.proc.write).not.toHaveBeenCalled() }) + it('synchronizes runtime output sequencing from a provider reattach snapshot', async () => { + setLocalPtyProvider({ + spawn: vi.fn(async () => ({ + id: 'pty-restored', + isReattach: true, + providerSequence: { value: 900, generation: 'continued' as const } + })), + write: vi.fn(), + resize: vi.fn(), + kill: vi.fn(), + shutdown: vi.fn(), + onData: vi.fn(() => vi.fn()), + onExit: vi.fn(() => vi.fn()), + listProcesses: vi.fn(async () => []), + getForegroundProcess: vi.fn(async () => null) + } as never) + const runtime = { + setPtyController: vi.fn(), + noteTerminalSpawnCommand: vi.fn(), + getPtyOutputSequence: vi.fn().mockReturnValue(7), + synchronizePtyOutputSequenceFromProvider: vi.fn(), + onPtySpawned: vi.fn(), + onPtyData: vi.fn(), + onPtyExit: vi.fn(), + createPreAllocatedTerminalHandle: vi.fn(() => null), + preAllocateHandleForPty: vi.fn() + } + registerPtyHandlers(mainWindow as never, runtime as never) + + await handlers.get('pty:spawn')!(null, { cols: 80, rows: 24 }) + + expect(runtime.synchronizePtyOutputSequenceFromProvider).toHaveBeenCalledWith( + 'pty-restored', + { value: 900, generation: 'continued' }, + 7 + ) + }) + it('seeds headless terminal state with cold-restore cwd metadata', async () => { const oscLinks = [{ row: 0, startCol: 0, endCol: 8, uri: 'https://example.com/restored' }] const coldRestore = { @@ -10543,7 +10581,7 @@ describe('registerPtyHandlers', () => { 'pty-cold-restore', 'restored history\r\n', undefined, - { cwd: '/projects/restored', oscLinks } + { cwd: '/projects/restored', oscLinks, preferProviderIfExisting: true } ) }) @@ -11360,6 +11398,42 @@ describe('registerPtyHandlers', () => { }) }) + describe('provider buffer snapshot dispatch', () => { + it('exposes daemon history when no renderer pane is mounted', async () => { + const provider = installObservableDaemonTestProvider() + provider.getBufferSnapshot.mockResolvedValue({ + data: 'restored screen\r\n', + scrollbackAnsi: 'restored history\r\n', + cols: 120, + rows: 40, + cwd: '/projects/restored', + seq: 900, + source: 'headless' + }) + const runtime = { setPtyController: vi.fn() } + handlers.clear() + registerPtyHandlers(mainWindow as never, runtime as never) + const controller = runtime.setPtyController.mock.calls[0]?.[0] as { + serializeProviderBuffer(ptyId: string, opts?: { scrollbackRows?: number }): Promise + } + + await expect( + controller.serializeProviderBuffer('daemon-restored', { scrollbackRows: 5000 }) + ).resolves.toEqual({ + data: 'restored screen\r\n', + scrollbackAnsi: 'restored history\r\n', + cols: 120, + rows: 40, + cwd: '/projects/restored', + seq: 900, + source: 'headless' + }) + expect(provider.getBufferSnapshot).toHaveBeenCalledWith('daemon-restored', { + scrollbackRows: 5000 + }) + }) + }) + describe('main buffer snapshot dispatch', () => { it('returns a hidden-output recovery snapshot with clamped scrollback', async () => { const runtime = { diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index 67db1b690cd..ffaa3a3990d 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -3146,7 +3146,18 @@ export function registerPtyHandlers( if (args.preAllocatedHandle) { trustedTerminalHandleEnv.add(args.preAllocatedHandle) } + const expectedPtyId = effectiveSessionAppId ?? sessionId + const sequenceBeforeProviderSpawn = expectedPtyId + ? (runtime?.getPtyOutputSequence?.(expectedPtyId) ?? 0) + : 0 result = await provider.spawn(spawnOptions) + if (result.providerSequence) { + runtime?.synchronizePtyOutputSequenceFromProvider?.( + result.id, + result.providerSequence, + sequenceBeforeProviderSpawn + ) + } } catch (err) { const rawMessage = err instanceof Error ? err.message : String(err) const spawnError = normalizeNodePtySpawnError(err) @@ -3487,6 +3498,15 @@ export function registerPtyHandlers( // state and dimensions before live TUI chunks can render correctly. return requestSerializedBuffer(ptyId, opts) }, + serializeProviderBuffer: async (ptyId, opts) => { + try { + // Why: restored daemon PTYs can be live while their desktop pane stays + // unmounted; query the provider model so phone-local navigation works. + return (await getProviderForPty(ptyId).getBufferSnapshot?.(ptyId, opts)) ?? null + } catch { + return null + } + }, hasRendererSerializer: (ptyId) => { // Why: the runtime needs a synchronous probe so it can decide whether to // skip the daemon-snapshot seed (the renderer will hydrate it) or run the @@ -4058,7 +4078,18 @@ export function registerPtyHandlers( ) } spawnTiming.mark('options') + const expectedPtyId = effectiveSessionAppId ?? effectiveSessionId + const sequenceBeforeProviderSpawn = expectedPtyId + ? (runtime?.getPtyOutputSequence?.(expectedPtyId) ?? 0) + : 0 result = await provider.spawn(spawnOptions) + if (result.providerSequence) { + runtime?.synchronizePtyOutputSequenceFromProvider?.( + result.id, + result.providerSequence, + sequenceBeforeProviderSpawn + ) + } spawnTiming.mark('provider_spawn') } catch (err) { // Why: a failed spawn must not leave a stale hidden mark on a session @@ -4293,7 +4324,8 @@ export function registerPtyHandlers( ) { runtime.seedHeadlessTerminal(result.id, result.coldRestore.scrollback, seedSize, { cwd: result.coldRestore.cwd, - oscLinks: result.coldRestore.oscLinks + oscLinks: result.coldRestore.oscLinks, + preferProviderIfExisting: true }) } } diff --git a/src/main/providers/types.ts b/src/main/providers/types.ts index f6e4042f482..ab6012cb7b9 100644 --- a/src/main/providers/types.ts +++ b/src/main/providers/types.ts @@ -117,6 +117,12 @@ export type PtySpawnResult = { * writing the snapshot so ANSI cursor positions land correctly. */ snapshotCols?: number snapshotRows?: number + /** Provider sequence at the attach boundary. `reset` starts a new provider + * generation; `continued` resumes the existing absolute domain. */ + providerSequence?: { + value: number + generation: 'continued' | 'reset' + } /** Kitty keyboard flags persisted in the daemon snapshot, threaded so the * re-seeded runtime emulator answers hidden `CSI ? u` with the real flags * (terminal-query-authority.md §kitty). Never replayed into a renderer diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index 2122e17d9a2..994328da875 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -7117,6 +7117,153 @@ describe('OrcaRuntimeService', () => { expect(runtime.getPtyOutputSequence('pty-1')).toBe(0) }) + it('aligns a restored session and pre-response bytes to the provider sequence', async () => { + const { runtime } = createSideEffectRuntime() + + // Simulate a stream-socket byte winning the race with the spawn response. + runtime.onPtyData('pty-restored', 'queued', 100) + expect( + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-restored', + { value: 900, generation: 'continued' }, + 0 + ) + ).toBe(906) + runtime.onPtyData('pty-restored', 'fresh', 101) + + expect(runtime.getPtyOutputSequence('pty-restored')).toBe(911) + await expect(runtime.serializeMainTerminalBuffer('pty-restored')).resolves.toMatchObject({ + data: expect.stringContaining('queuedfresh'), + seq: 911, + source: 'headless' + }) + }) + + it('does not double a fresh daemon sequence on same-main reattach', () => { + const { runtime } = createSideEffectRuntime() + + expect( + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-fresh', + { value: 0, generation: 'reset' }, + 0 + ) + ).toBe(0) + runtime.onPtyData('pty-fresh', 'fresh output', 100) + const sequenceBeforeReattach = runtime.getPtyOutputSequence('pty-fresh') + + expect( + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-fresh', + { value: sequenceBeforeReattach, generation: 'continued' }, + sequenceBeforeReattach + ) + ).toBe(sequenceBeforeReattach) + }) + + it('does not jump ahead of delayed bytes covered by a reattach snapshot', () => { + const { runtime } = createSideEffectRuntime() + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-delayed', + { value: 0, generation: 'reset' }, + 0 + ) + runtime.onPtyData('pty-delayed', 'before', 100) + const sequenceBeforeReattach = runtime.getPtyOutputSequence('pty-delayed') + const delayedCoveredBytes = 'queued' + + expect( + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-delayed', + { + value: sequenceBeforeReattach + delayedCoveredBytes.length, + generation: 'continued' + }, + sequenceBeforeReattach + ) + ).toBe(sequenceBeforeReattach) + runtime.onPtyData('pty-delayed', delayedCoveredBytes, 101) + + expect(runtime.getPtyOutputSequence('pty-delayed')).toBe( + sequenceBeforeReattach + delayedCoveredBytes.length + ) + }) + + it('retains bytes emitted before a fresh daemon spawn resolves', async () => { + const { runtime } = createSideEffectRuntime() + const earlyOutput = '\x1b]0;Fresh shell\x07early prompt' + runtime.onPtyData('pty-fresh', earlyOutput, 100) + + expect( + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-fresh', + { value: 0, generation: 'reset' }, + 0 + ) + ).toBe(earlyOutput.length) + + await expect(runtime.serializeMainTerminalBuffer('pty-fresh')).resolves.toMatchObject({ + data: expect.stringContaining('early prompt'), + lastTitle: 'Fresh shell', + seq: earlyOutput.length + }) + }) + + it('drops stale headless state without rewinding the runtime sequence', async () => { + const { runtime } = createSideEffectRuntime() + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-restarted', + { value: 0, generation: 'reset' }, + 0 + ) + runtime.onPtyData('pty-restarted', 'old generation', 100) + const sequenceBeforeRespawn = runtime.getPtyOutputSequence('pty-restarted') + + expect( + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-restarted', + { value: 0, generation: 'reset' }, + sequenceBeforeRespawn + ) + ).toBe(sequenceBeforeRespawn) + runtime.onPtyData('pty-restarted', 'new generation', 101) + + await expect(runtime.serializeMainTerminalBuffer('pty-restarted')).resolves.toMatchObject({ + data: expect.not.stringContaining('old generation'), + seq: sequenceBeforeRespawn + 'new generation'.length + }) + }) + + it('keeps active listener sequences monotonic across a daemon reset', () => { + const { runtime } = createSideEffectRuntime() + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-reset', + { value: 0, generation: 'reset' }, + 0 + ) + runtime.onPtyData('pty-reset', 'old', 100) + const sequenceBeforeRespawn = runtime.getPtyOutputSequence('pty-reset') + const observedSequences: number[] = [] + runtime.subscribeToTerminalData('pty-reset', (_data, meta) => { + if (typeof meta?.seq === 'number') { + observedSequences.push(meta.seq) + } + }) + + runtime.onPtyData('pty-reset', 'early', 101) + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-reset', + { value: 0, generation: 'reset' }, + sequenceBeforeRespawn + ) + runtime.onPtyData('pty-reset', 'later', 102) + + expect(observedSequences).toEqual([ + sequenceBeforeRespawn + 'early'.length, + sequenceBeforeRespawn + 'early'.length + 'later'.length + ]) + }) + it('carries the synthetic permission BEL as a bell fact', () => { const { runtime, batches } = createSideEffectRuntime() syncSinglePty(runtime) @@ -7217,6 +7364,254 @@ describe('OrcaRuntimeService', () => { expect(snapshot?.lastTitle).toBe('⠋ Cursor Agent') }) + it('falls back to the provider snapshot for a restored PTY with no mounted renderer', async () => { + const { runtime } = createSideEffectRuntime() + const serializeBuffer = vi.fn() + const serializeProviderBuffer = vi.fn().mockResolvedValue({ + data: 'restored screen\r\n', + scrollbackAnsi: 'restored history\r\n', + cols: 120, + rows: 40, + cwd: '/projects/restored', + seq: 900, + source: 'headless' + }) + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + serializeBuffer, + serializeProviderBuffer, + hasRendererSerializer: () => false + }) + + const snapshot = await runtime.serializeTerminalBuffer('pty-restored', { + scrollbackRows: 5000 + }) + + expect(serializeBuffer).not.toHaveBeenCalled() + expect(serializeProviderBuffer).toHaveBeenCalledWith('pty-restored', { + scrollbackRows: 5000 + }) + expect(snapshot).toEqual({ + data: 'restored screen\r\n', + scrollbackAnsi: 'restored history\r\n', + cols: 120, + rows: 40, + cwd: '/projects/restored', + seq: 900, + source: 'headless' + }) + }) + + it('does not let pre-response bytes hide restored provider history', async () => { + const { runtime } = createSideEffectRuntime() + const serializeProviderBuffer = vi.fn().mockResolvedValue({ + data: 'restored history\r\nqueued', + cols: 80, + rows: 24, + seq: 906, + source: 'headless' + }) + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + serializeProviderBuffer, + hasRendererSerializer: () => false + }) + runtime.onPtyData('pty-restored', 'queued', 100) + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-restored', + { value: 900, generation: 'continued' }, + 0 + ) + + const snapshot = await runtime.serializeTerminalBuffer('pty-restored', { + scrollbackRows: 5000 + }) + + expect(snapshot?.data).toContain('restored history') + expect(serializeProviderBuffer).toHaveBeenCalledOnce() + }) + + it('keeps restored provider history authoritative after later live output', async () => { + const { runtime } = createSideEffectRuntime() + const serializeProviderBuffer = vi.fn().mockResolvedValue({ + data: 'restored history\r\nlater output', + cols: 80, + rows: 24, + seq: 912, + source: 'headless' + }) + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + serializeProviderBuffer, + hasRendererSerializer: () => false + }) + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-restored', + { value: 900, generation: 'continued' }, + 0 + ) + runtime.onPtyData('pty-restored', 'later output', 100) + + const snapshot = await runtime.serializeTerminalBuffer('pty-restored') + + expect(snapshot?.data).toContain('restored history') + expect(serializeProviderBuffer).toHaveBeenCalledOnce() + }) + + it('uses provider alternate-screen state while a partial model is unsafe', async () => { + const { runtime } = createSideEffectRuntime() + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + serializeProviderBuffer: vi.fn().mockResolvedValue({ + data: 'restored tui', + cols: 80, + rows: 24, + seq: 903, + source: 'headless', + alternateScreen: true + }), + hasRendererSerializer: () => false + }) + runtime.onPtyData('pty-tui', 'tui', 100) + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-tui', + { value: 900, generation: 'continued' }, + 0 + ) + + await runtime.serializeTerminalBuffer('pty-tui') + + expect(runtime.isTerminalAlternateScreen('pty-tui')).toBe(true) + }) + + it('tracks live alternate-screen transitions after a provider snapshot', async () => { + const { runtime } = createSideEffectRuntime() + const serializeProviderBuffer = vi.fn().mockResolvedValue({ + data: 'restored tui', + cols: 80, + rows: 24, + seq: 900, + source: 'headless', + alternateScreen: true + }) + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + serializeProviderBuffer, + hasRendererSerializer: () => false + }) + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-tui', + { value: 900, generation: 'continued' }, + 0 + ) + await runtime.serializeTerminalBuffer('pty-tui') + expect(runtime.isTerminalAlternateScreen('pty-tui')).toBe(true) + + runtime.onPtyData('pty-tui', '\x1b[?1049l', 100) + expect(runtime.isTerminalAlternateScreen('pty-tui')).toBe(false) + runtime.onPtyData('pty-tui', '\x1b[?1049h', 101) + expect(runtime.isTerminalAlternateScreen('pty-tui')).toBe(true) + }) + + it('keeps mode transitions that race a provider snapshot response', async () => { + const { runtime } = createSideEffectRuntime() + let resolveProviderSnapshot: + | ((snapshot: { + data: string + cols: number + rows: number + seq: number + source: 'headless' + alternateScreen: boolean + }) => void) + | undefined + const serializeProviderBuffer = vi.fn( + () => + new Promise<{ + data: string + cols: number + rows: number + seq: number + source: 'headless' + alternateScreen: boolean + }>((resolve) => { + resolveProviderSnapshot = resolve + }) + ) + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + serializeProviderBuffer, + hasRendererSerializer: () => false + }) + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-tui-race', + { value: 900, generation: 'continued' }, + 0 + ) + + const snapshotPromise = runtime.serializeTerminalBuffer('pty-tui-race') + await vi.waitFor(() => expect(resolveProviderSnapshot).toBeDefined()) + runtime.onPtyData('pty-tui-race', '\x1b[?1049l', 100) + resolveProviderSnapshot?.({ + data: 'captured alt screen', + cols: 80, + rows: 24, + seq: 900, + source: 'headless', + alternateScreen: true + }) + await snapshotPromise + + expect(runtime.isTerminalAlternateScreen('pty-tui-race')).toBe(false) + }) + + it('translates reset provider snapshots without retaining the old title', async () => { + const { runtime } = createSideEffectRuntime() + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-replaced', + { value: 0, generation: 'reset' }, + 0 + ) + runtime.onPtyData('pty-replaced', '\x1b]0;Old process\x07old', 100) + const sequenceBeforeRespawn = runtime.getPtyOutputSequence('pty-replaced') + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + serializeProviderBuffer: vi.fn().mockResolvedValue({ + data: 'new process', + cols: 80, + rows: 24, + seq: 'new process'.length, + source: 'headless', + lastTitle: 'New process' + }), + hasRendererSerializer: () => false + }) + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-replaced', + { value: 0, generation: 'reset' }, + sequenceBeforeRespawn + ) + + await expect(runtime.serializeTerminalBuffer('pty-replaced')).resolves.toMatchObject({ + lastTitle: 'New process', + seq: sequenceBeforeRespawn + 'new process'.length + }) + }) + it('prefers the tracked title over the headless emulator lastTitle', async () => { const { runtime } = createSideEffectRuntime() syncSinglePty(runtime) diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 6a193a404e5..6d8c041e019 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -25,6 +25,7 @@ import type { TerminalSideEffectFact } from '../../shared/terminal-side-effect-facts' import type { TerminalGitHubPRLink } from '../../shared/terminal-github-pr-link-detector' +import { TerminalKittyKeyboardModeTracker } from '../../shared/terminal-kitty-keyboard-mode-tracker' import { AGENT_STATUS_STALE_AFTER_MS, isFreshNonDoneAgentStatus, @@ -351,6 +352,7 @@ import { serveSimStateWatcher } from '../emulator/serve-sim-state-watcher' import type { EmulatorBridge } from '../emulator/emulator-bridge' import { RuntimeFileCommands } from './orca-runtime-files' import { RuntimeGitCommands } from './orca-runtime-git' +import type { PtyProviderBufferSnapshot } from '../providers/types' import { ClaudeAgentTeamsService } from './claude-agent-teams-service' import type { AgentTeamsTmuxCompatRequest, @@ -1179,6 +1181,8 @@ type RuntimeHeadlessTerminal = { type HeadlessSeedMetadata = { cwd?: string | null oscLinks?: TerminalOscLinkRange[] + /** Cold restore history must outrank a model that only saw new-generation bytes. */ + preferProviderIfExisting?: boolean /** Persisted kitty flags from the daemon snapshot, re-applied to the fresh * emulator so hidden `CSI ? u` answers the real flags instead of ?0u * (terminal-query-authority.md §kitty). */ @@ -1219,6 +1223,11 @@ type RuntimePtyController = { ptyId: string, opts?: { scrollbackRows?: number; altScreenForcesZeroRows?: boolean } ): Promise<{ data: string; cols: number; rows: number; lastTitle?: string } | null> + /** Authoritative provider-owned snapshot for restored PTYs with no mounted renderer. */ + serializeProviderBuffer?( + ptyId: string, + opts?: { scrollbackRows?: number } + ): Promise // Why: synchronous probe used by maybeHydrateHeadlessFromRenderer to skip // hydration when no renderer is authoritative for this PTY. See // docs/mobile-prefer-renderer-scrollback.md. @@ -2244,6 +2253,14 @@ export class OrcaRuntimeService { private titleObservationSequence = 0 private headlessTerminals = new Map() private ptyOutputSequenceById = new Map() + private providerSequenceInitializedPtys = new Set() + private providerSequenceOffsetByPtyId = new Map() + private providerSnapshotPreferredPtys = new Set() + private providerModeTrackersByPtyId = new Map() + private providerModeSnapshotScansByPtyId = new Map< + string, + Set + >() private recentPtyPathCandidatesById = new Map() // Why: OSC 9999 status can span PTY chunks. Keeping parser state in the // runtime lets hidden/model-owned terminals observe agent state without a @@ -5680,6 +5697,10 @@ export class OrcaRuntimeService { onPtyData(ptyId: string, data: string, at: number, sequenceChars = data.length): number { const outputSequence = (this.ptyOutputSequenceById.get(ptyId) ?? 0) + sequenceChars this.ptyOutputSequenceById.set(ptyId, outputSequence) + this.providerModeTrackersByPtyId.get(ptyId)?.scan(data) + for (const tracker of this.providerModeSnapshotScansByPtyId.get(ptyId) ?? []) { + tracker.scan(data) + } const osc7Metadata = this.recordOsc7MetadataForPty(ptyId, data) const cwd = osc7Metadata.cwd const cwdChanged = osc7Metadata.cwdChanged @@ -6404,6 +6425,29 @@ export class OrcaRuntimeService { this.ptyTitleTrackersByPtyId.delete(ptyId) } + private resetTrackedTerminalStateForProviderGeneration(ptyId: string): void { + // Why: a replacement daemon session can reuse the PTY id, but title/parser + // state from the prior process must not bleed into its snapshots or chunks. + this.disposePtyTitleTracker(ptyId) + this.oscTitleScanTailByPtyId.delete(ptyId) + this.osc7ScanTailByPtyId.delete(ptyId) + this.agentStatusOscProcessorsByPtyId.delete(ptyId) + const pty = this.ptysById.get(ptyId) + if (pty) { + pty.lastOscTitle = null + pty.lastOscTitleAt = null + pty.lastAgentStatus = null + pty.managementTitle = null + pty.managementTitleAt = null + } + for (const leaf of this.getLeavesForPty(ptyId)) { + leaf.lastOscTitle = null + leaf.lastOscTitleAt = null + leaf.lastAgentStatus = null + } + this.clearAgentRowSnapshotsForPty(ptyId) + } + private setTerminalSideEffectConsumerAvailable(available: boolean): void { const nextAvailable = available && this.onTerminalSideEffects !== null if (nextAvailable === this.terminalSideEffectConsumerAvailable) { @@ -6594,6 +6638,80 @@ export class OrcaRuntimeService { return this.ptyOutputSequenceById.get(ptyId) ?? 0 } + synchronizePtyOutputSequenceFromProvider( + ptyId: string, + providerSequence: { value: number; generation: 'continued' | 'reset' }, + runtimeSequenceAtSpawnStart = 0 + ): number { + if ( + !Number.isFinite(providerSequence.value) || + providerSequence.value < 0 || + !Number.isFinite(runtimeSequenceAtSpawnStart) || + runtimeSequenceAtSpawnStart < 0 + ) { + return this.getPtyOutputSequence(ptyId) + } + const baseline = Math.floor(providerSequence.value) + const currentSequence = this.getPtyOutputSequence(ptyId) + const sequenceAtSpawnStart = Math.min(currentSequence, Math.floor(runtimeSequenceAtSpawnStart)) + const postSpawnSequence = currentSequence - sequenceAtSpawnStart + const wasInitialized = this.providerSequenceInitializedPtys.has(ptyId) + const replacesExistingRuntimeGeneration = wasInitialized || sequenceAtSpawnStart > 0 + const providerOffset = + providerSequence.generation === 'reset' + ? sequenceAtSpawnStart + : (this.providerSequenceOffsetByPtyId.get(ptyId) ?? 0) + const providerBaseline = providerOffset + baseline + + if (providerSequence.generation === 'reset') { + // Why: daemon respawn/cold restore starts a new absolute domain. Old + // emulator state cannot remain authoritative over the replacement. + if (replacesExistingRuntimeGeneration) { + this.disposeHeadlessTerminal(ptyId) + } + this.providerModeTrackersByPtyId.delete(ptyId) + if (replacesExistingRuntimeGeneration && postSpawnSequence === 0) { + this.resetTrackedTerminalStateForProviderGeneration(ptyId) + } + } + + const synchronizedSequence = + providerSequence.generation === 'reset' + ? currentSequence + : wasInitialized + ? currentSequence + : providerBaseline + postSpawnSequence + this.ptyOutputSequenceById.set(ptyId, synchronizedSequence) + this.providerSequenceInitializedPtys.add(ptyId) + this.providerSequenceOffsetByPtyId.set(ptyId, providerOffset) + + const snapshotMayCoverMissingState = + (providerSequence.generation === 'continued' && !wasInitialized) || + (postSpawnSequence > 0 && + providerSequence.generation === 'reset' && + replacesExistingRuntimeGeneration) || + (providerSequence.generation === 'continued' && + wasInitialized && + providerBaseline > currentSequence) + if (snapshotMayCoverMissingState) { + // Why: bytes can cross the control/stream sockets around attach. Until a + // full renderer/provider snapshot is available, a partial model is unsafe. + this.providerSnapshotPreferredPtys.add(ptyId) + } else if (providerSequence.generation === 'reset') { + this.providerSnapshotPreferredPtys.delete(ptyId) + } + + const headless = this.headlessTerminals.get(ptyId) + if (headless && !wasInitialized && providerSequence.generation === 'continued') { + // Why: daemon bytes can reach main just before spawn resolves. Queue the + // baseline behind those writes so their emulator sequence is rebased too. + headless.writeChain = headless.writeChain.then(() => { + headless.outputSequence = synchronizedSequence + }) + } + return synchronizedSequence + } + subscribeToTerminalData( ptyId: string, listener: (data: string, meta?: { seq?: number; rawLength?: number; cwd?: string }) => void @@ -6766,7 +6884,8 @@ export class OrcaRuntimeService { // Why: hidden-output recovery is initiated by the desktop renderer. If the // runtime has not built headless state yet, the mounted xterm is still the // best available state and avoids a false "snapshot unavailable" result. - return this.serializeRendererTerminalBuffer(ptyId, opts) + const rendererSnapshot = await this.serializeRendererTerminalBuffer(ptyId, opts) + return rendererSnapshot ?? this.serializeProviderTerminalBuffer(ptyId, opts) } async clearTerminalBuffer(handle: string): Promise<{ handle: string; cleared: boolean }> { @@ -6791,10 +6910,17 @@ export class OrcaRuntimeService { // scrollback to mobile so it rewraps at the new cols, but alternate-screen // TUIs (vim, Claude Code) own their repaint and have no scrollback — for // those the mobile client just resizes xterm geometry and consumes the - // TUI's own redraw, so the resize re-stream must be skipped. Returns false - // when there is no headless emulator (resize falls back to geometry-only). + // TUI's own redraw, so the resize re-stream must be skipped. Provider state + // covers restored PTYs whose main-side emulator is only a partial suffix. isTerminalAlternateScreen(ptyId: string): boolean { - return this.headlessTerminals.get(ptyId)?.emulator.isAlternateScreen ?? false + if (this.providerSnapshotPreferredPtys.has(ptyId)) { + return this.providerModeTrackersByPtyId.get(ptyId)?.isAlternateScreen ?? false + } + return ( + this.headlessTerminals.get(ptyId)?.emulator.isAlternateScreen ?? + this.providerModeTrackersByPtyId.get(ptyId)?.isAlternateScreen ?? + false + ) } // Why: daemon-backed PTYs that the runtime adopted after an Orca relaunch @@ -6818,10 +6944,14 @@ export class OrcaRuntimeService { if (existing) { // Why: emulator already has live data — re-seeding would duplicate // every byte. The seed is only valid when the emulator is fresh. + if (metadata.preferProviderIfExisting) { + this.providerSnapshotPreferredPtys.add(ptyId) + } return } const dims = size ?? this.getTerminalSize(ptyId) ?? { cols: 80, rows: 24 } const state = this.createPtyHeadlessTerminalState(ptyId, dims) + state.outputSequence = this.getPtyOutputSequence(ptyId) this.headlessTerminals.set(ptyId, state) this.recordOsc7MetadataForPty(ptyId, data) this.recordRecentPtyOutputForPathProvenance(ptyId, data) @@ -6843,6 +6973,7 @@ export class OrcaRuntimeService { if (metadata.oscLinks !== undefined) { state.emulator.setRestoredOscLinks(metadata.oscLinks) } + this.providerSnapshotPreferredPtys.delete(ptyId) }) .catch(() => { // Seeding is best-effort; live data will continue to populate the @@ -6860,7 +6991,8 @@ export class OrcaRuntimeService { if (this.headlessHydrationState.has(ptyId)) { return } - if (this.headlessTerminals.has(ptyId)) { + const providerSnapshotPreferred = this.providerSnapshotPreferredPtys.has(ptyId) + if (this.headlessTerminals.has(ptyId) && !providerSnapshotPreferred) { // Daemon-snapshot seed already populated the emulator — skip hydration. this.headlessHydrationState.set(ptyId, 'done') return @@ -6875,11 +7007,18 @@ export class OrcaRuntimeService { return } + if (providerSnapshotPreferred) { + // Why: a stream byte can create a partial model before restored history + // arrives. A mounted renderer snapshot can safely replace that model. + this.disposeHeadlessTerminal(ptyId) + } + this.headlessHydrationState.set(ptyId, 'pending') const dims = this.getTerminalSize(ptyId) ?? { cols: 80, rows: 24 } // Why: hydration writes below never set forwardQueryReplies (main-side // replay guard) — renderer-buffer snapshots can embed stale queries. const state = this.createPtyHeadlessTerminalState(ptyId, dims) + state.outputSequence = this.getPtyOutputSequence(ptyId) this.headlessTerminals.set(ptyId, state) // Why: append the seed work to writeChain so live writes queued by @@ -6918,6 +7057,7 @@ export class OrcaRuntimeService { state.emulator.setLastTitle(seedTitle) this.applySeededAgentStatus(ptyId, seedTitle) } + this.providerSnapshotPreferredPtys.delete(ptyId) } catch { // Hydration is best-effort. Live writes continue via the same // writeChain that this catch-arm leaves intact. @@ -7110,12 +7250,26 @@ export class OrcaRuntimeService { alternateScreen?: boolean pendingEscapeTailAnsi?: string } | null> { + if (this.providerSnapshotPreferredPtys.has(ptyId)) { + // Why: pre-attach stream bytes only form a suffix of restored state. A + // sequenced provider snapshot safely reconciles live bytes; renderer is + // the fallback when an older provider cannot expose that boundary. + const providerSnapshot = await this.serializeProviderTerminalBuffer(ptyId, opts) + if (providerSnapshot) { + return providerSnapshot + } + const rendererSnapshot = await this.serializeRendererTerminalBuffer(ptyId, opts) + if (rendererSnapshot) { + return rendererSnapshot + } + } const headlessSnapshot = await this.serializeHeadlessTerminalBuffer(ptyId, opts) if (headlessSnapshot) { return headlessSnapshot } - return this.serializeRendererTerminalBuffer(ptyId, opts) + const rendererSnapshot = await this.serializeRendererTerminalBuffer(ptyId, opts) + return rendererSnapshot ?? this.serializeProviderTerminalBuffer(ptyId, opts) } private async serializeRendererTerminalBuffer( @@ -7130,6 +7284,9 @@ export class OrcaRuntimeService { source?: 'renderer' oscLinks?: TerminalOscLinkRange[] } | null> { + if (this.ptyController?.hasRendererSerializer?.(ptyId) === false) { + return null + } let rendererSnapshot: { data: string cols: number @@ -7160,6 +7317,49 @@ export class OrcaRuntimeService { : null } + private async serializeProviderTerminalBuffer( + ptyId: string, + opts: { scrollbackRows?: number } = {} + ): Promise { + const liveModeTracker = new TerminalKittyKeyboardModeTracker() + let liveModeTrackers = this.providerModeSnapshotScansByPtyId.get(ptyId) + if (!liveModeTrackers) { + liveModeTrackers = new Set() + this.providerModeSnapshotScansByPtyId.set(ptyId, liveModeTrackers) + } + liveModeTrackers.add(liveModeTracker) + try { + // Why: daemon PTYs survive an app relaunch before any renderer mounts. + // Mobile still needs their retained history without navigating desktop. + const snapshot = await this.ptyController?.serializeProviderBuffer?.(ptyId, opts) + if (typeof snapshot?.alternateScreen === 'boolean') { + const modeTracker = new TerminalKittyKeyboardModeTracker() + if (snapshot.alternateScreen) { + modeTracker.scan('\x1b[?1049h') + } + if (liveModeTracker.hasObservedAlternateScreenSwitch) { + modeTracker.scan(liveModeTracker.isAlternateScreen ? '\x1b[?1049h' : '\x1b[?1049l') + } + this.providerModeTrackersByPtyId.set(ptyId, modeTracker) + } + if (!snapshot) { + return null + } + const providerOffset = this.providerSequenceOffsetByPtyId.get(ptyId) ?? 0 + return this.preferTrackedLastTitle(ptyId, { + ...snapshot, + seq: providerOffset + snapshot.seq + }) + } catch { + return null + } finally { + liveModeTrackers.delete(liveModeTracker) + if (liveModeTrackers.size === 0) { + this.providerModeSnapshotScansByPtyId.delete(ptyId) + } + } + } + private async withVisibleSnapshotFallback( ptyId: string, read: RuntimeTerminalRead, @@ -8245,6 +8445,11 @@ export class OrcaRuntimeService { this.clearWaitBlockedCheckState(ptyId) this.recentPtyPathCandidatesById.delete(ptyId) this.ptyOutputSequenceById.delete(ptyId) + this.providerSequenceInitializedPtys.delete(ptyId) + this.providerSequenceOffsetByPtyId.delete(ptyId) + this.providerSnapshotPreferredPtys.delete(ptyId) + this.providerModeTrackersByPtyId.delete(ptyId) + this.providerModeSnapshotScansByPtyId.delete(ptyId) this.agentStatusOscProcessorsByPtyId.delete(ptyId) this.terminalSpawnCommandsByPtyId.delete(ptyId) this.disposePtyTitleTracker(ptyId) @@ -20469,6 +20674,11 @@ export class OrcaRuntimeService { this.clearWaitBlockedCheckState(ptyId) this.recentPtyPathCandidatesById.delete(ptyId) this.ptyOutputSequenceById.delete(ptyId) + this.providerSequenceInitializedPtys.delete(ptyId) + this.providerSequenceOffsetByPtyId.delete(ptyId) + this.providerSnapshotPreferredPtys.delete(ptyId) + this.providerModeTrackersByPtyId.delete(ptyId) + this.providerModeSnapshotScansByPtyId.delete(ptyId) this.agentStatusOscProcessorsByPtyId.delete(ptyId) this.terminalSpawnCommandsByPtyId.delete(ptyId) this.disposePtyTitleTracker(ptyId) diff --git a/src/main/runtime/rpc/terminal-provider-snapshot-sequence.test.ts b/src/main/runtime/rpc/terminal-provider-snapshot-sequence.test.ts new file mode 100644 index 00000000000..5d04ef24389 --- /dev/null +++ b/src/main/runtime/rpc/terminal-provider-snapshot-sequence.test.ts @@ -0,0 +1,187 @@ +import { expect, it, vi } from 'vitest' +import { RpcDispatcher } from './dispatcher' +import type { RpcRequest } from './core' +import { TERMINAL_METHODS } from './methods/terminal' +import type { OrcaRuntimeService } from '../orca-runtime' +import type { RuntimeTerminalWait } from '../../../shared/runtime-types' +import { + TerminalStreamOpcode, + decodeTerminalStreamFrame, + decodeTerminalStreamText +} from '../../../shared/terminal-stream-protocol' + +const request: RpcRequest = { + id: 'req-provider-sequence', + authToken: 'tok', + method: 'terminal.subscribe', + params: { + terminal: 'terminal-restored', + client: { id: 'phone-1', type: 'mobile' }, + capabilities: { terminalBinaryStream: 1 } + } +} + +it('replays post-capture output in the provider snapshot sequence domain', async () => { + const binaryFrames: Uint8Array[] = [] + const cleanups = new Map void>() + let dataListener: + | ((data: string, meta?: { seq?: number; rawLength?: number }) => void) + | undefined + let resolveSnapshot: + | ((snapshot: { data: string; cols: number; rows: number; seq: number }) => void) + | undefined + const runtime = { + getRuntimeId: () => 'test-runtime', + registerRemoteTerminalViewSubscriber: () => () => {}, + resolveLeafForHandle: vi.fn().mockReturnValue({ ptyId: 'pty-restored' }), + handleMobileSubscribe: vi.fn().mockResolvedValue(true), + handleMobileUnsubscribe: vi.fn(), + subscribeToTerminalData: vi.fn((_ptyId, listener) => { + dataListener = listener + return vi.fn() + }), + readTerminal: vi.fn().mockResolvedValue({ tail: [], truncated: false }), + serializeTerminalBuffer: vi.fn( + () => + new Promise<{ data: string; cols: number; rows: number; seq: number }>((resolve) => { + resolveSnapshot = resolve + }) + ), + getTerminalSize: vi.fn().mockReturnValue({ cols: 80, rows: 24 }), + getMobileDisplayMode: vi.fn().mockReturnValue('auto'), + getLayout: vi.fn().mockReturnValue({ seq: 1 }), + isTerminalAlternateScreen: vi.fn().mockReturnValue(false), + subscribeToTerminalResize: vi.fn().mockReturnValue(vi.fn()), + subscribeToFitOverrideChanges: vi.fn().mockReturnValue(vi.fn()), + registerSubscriptionCleanup: vi.fn((id: string, cleanup: () => void) => { + cleanups.set(id, cleanup) + }), + cleanupSubscription: vi.fn((id: string) => { + cleanups.get(id)?.() + cleanups.delete(id) + }), + waitForTerminal: vi.fn(() => new Promise(() => {})) + } as unknown as OrcaRuntimeService + const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS }) + + const dispatchPromise = dispatcher.dispatchStreaming(request, vi.fn(), { + connectionId: 'conn-restored', + sendBinary: (bytes) => { + binaryFrames.push(bytes) + } + }) + + await vi.waitFor(() => expect(resolveSnapshot).toBeDefined()) + resolveSnapshot?.({ data: 'restored snapshot', cols: 80, rows: 24, seq: 900 }) + // Why: provider attach anchoring makes this first new chunk continue at + // 905 instead of restarting at 5, so snapshot reconciliation keeps it. + dataListener?.('fresh', { seq: 905, rawLength: 5 }) + + await vi.waitFor(() => + expect( + binaryFrames.some((bytes) => { + const frame = decodeTerminalStreamFrame(bytes) + return ( + frame?.opcode === TerminalStreamOpcode.Output && + decodeTerminalStreamText(frame.payload) === 'fresh' + ) + }) + ).toBe(true) + ) + + runtime.cleanupSubscription('terminal-restored:phone-1') + await dispatchPromise +}) + +it('keeps provider-backed alternate-screen resizes geometry-only', async () => { + const binaryFrames: Uint8Array[] = [] + const cleanups = new Map void>() + let resizeListener: + | ((event: { + cols: number + rows: number + displayMode: string + reason: string + seq: number + }) => void) + | undefined + const serializeTerminalBuffer = vi + .fn() + .mockResolvedValue({ data: 'restored tui', cols: 80, rows: 24 }) + const runtime = { + getRuntimeId: () => 'test-runtime', + registerRemoteTerminalViewSubscriber: () => () => {}, + resolveLeafForHandle: vi.fn().mockReturnValue({ ptyId: 'pty-tui' }), + readTerminal: vi.fn().mockResolvedValue({ tail: [], truncated: false }), + serializeTerminalBuffer, + getTerminalSize: vi.fn().mockReturnValue({ cols: 80, rows: 24 }), + getMobileDisplayMode: vi.fn().mockReturnValue('auto'), + getLayout: vi.fn().mockReturnValue({ seq: 1 }), + isTerminalAlternateScreen: vi.fn().mockReturnValue(true), + handleMobileSubscribe: vi.fn().mockResolvedValue(undefined), + handleMobileUnsubscribe: vi.fn(), + subscribeToTerminalData: vi.fn().mockReturnValue(vi.fn()), + subscribeToTerminalResize: vi.fn((_ptyId, listener) => { + resizeListener = listener + return vi.fn() + }), + subscribeToFitOverrideChanges: vi.fn().mockReturnValue(vi.fn()), + registerSubscriptionCleanup: vi.fn((id: string, cleanup: () => void) => { + cleanups.set(id, cleanup) + }), + cleanupSubscription: vi.fn((id: string) => { + const cleanup = cleanups.get(id) + cleanups.delete(id) + cleanup?.() + }), + waitForTerminal: vi.fn(() => new Promise(() => {})), + sendTerminal: vi.fn().mockResolvedValue({ accepted: true }), + updateMobileViewport: vi.fn().mockResolvedValue({ updated: true, applied: true }) + } as unknown as OrcaRuntimeService + const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS }) + const dispatchPromise = dispatcher.dispatchStreaming( + { + ...request, + id: 'req-provider-alt-screen', + params: { + terminal: 'terminal-tui', + client: { id: 'phone-1', type: 'mobile' }, + capabilities: { terminalBinaryStream: 1 } + } + }, + vi.fn(), + { + connectionId: 'conn-tui-resize', + sendBinary: (bytes) => { + binaryFrames.push(bytes) + } + } + ) + + await vi.waitFor(() => expect(resizeListener).toBeDefined()) + binaryFrames.splice(0) + resizeListener?.({ + cols: 100, + rows: 30, + displayMode: 'auto', + reason: 'apply-layout', + seq: 2 + }) + await vi.waitFor(() => + expect( + binaryFrames.some( + (frame) => decodeTerminalStreamFrame(frame)?.opcode === TerminalStreamOpcode.Resized + ) + ).toBe(true) + ) + + expect(serializeTerminalBuffer).toHaveBeenCalledOnce() + expect( + binaryFrames.some( + (frame) => decodeTerminalStreamFrame(frame)?.opcode === TerminalStreamOpcode.SnapshotStart + ) + ).toBe(false) + + runtime.cleanupSubscription('terminal-tui:phone-1') + await dispatchPromise +}) diff --git a/src/shared/terminal-kitty-keyboard-mode-tracker.test.ts b/src/shared/terminal-kitty-keyboard-mode-tracker.test.ts index 6d88f56a070..9ad2cb4d77a 100644 --- a/src/shared/terminal-kitty-keyboard-mode-tracker.test.ts +++ b/src/shared/terminal-kitty-keyboard-mode-tracker.test.ts @@ -60,10 +60,13 @@ describe('TerminalKittyKeyboardModeTracker', () => { tracker.scan('\x1b[>1u') expect(tracker.flags).toBe(1) tracker.scan('\x1b[?1049h') + expect(tracker.hasObservedAlternateScreenSwitch).toBe(true) + expect(tracker.isAlternateScreen).toBe(true) expect(tracker.flags).toBe(0) tracker.scan('\x1b[>2u') expect(tracker.flags).toBe(2) tracker.scan('\x1b[?1049l') + expect(tracker.isAlternateScreen).toBe(false) expect(tracker.flags).toBe(1) }) diff --git a/src/shared/terminal-kitty-keyboard-mode-tracker.ts b/src/shared/terminal-kitty-keyboard-mode-tracker.ts index a0b7b0d1f2b..2b501a2c9ff 100644 --- a/src/shared/terminal-kitty-keyboard-mode-tracker.ts +++ b/src/shared/terminal-kitty-keyboard-mode-tracker.ts @@ -30,12 +30,21 @@ export class TerminalKittyKeyboardModeTracker { private mainStack: number[] = [] private altStack: number[] = [] private alternateScreenActive = false + private alternateScreenSwitchObserved = false /** Current effective kitty keyboard flags (0 = protocol inactive). */ get flags(): number { return this.currentFlags } + get isAlternateScreen(): boolean { + return this.alternateScreenActive + } + + get hasObservedAlternateScreenSwitch(): boolean { + return this.alternateScreenSwitchObserved + } + reset(): void { this.scanTail = '' this.currentFlags = 0 @@ -44,6 +53,7 @@ export class TerminalKittyKeyboardModeTracker { this.mainStack = [] this.altStack = [] this.alternateScreenActive = false + this.alternateScreenSwitchObserved = false } scan(data: string): void { @@ -77,6 +87,7 @@ export class TerminalKittyKeyboardModeTracker { const tail = this.scanTail this.reset() this.scanTail = tail + this.alternateScreenSwitchObserved = true continue } if (match[0].endsWith('!p')) { @@ -108,6 +119,7 @@ export class TerminalKittyKeyboardModeTracker { if (param !== 47 && param !== 1047 && param !== 1049) { continue } + this.alternateScreenSwitchObserved = true // Why: xterm swaps the current flags with the inactive screen's slot on // every 47/1047/1049 transition, without an already-active guard — // mirror it exactly so this state matches what the renderer encodes. From 0a0b400ada741982eeb574820fa8d8b3a3b7b1a2 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Tue, 14 Jul 2026 16:16:30 -0700 Subject: [PATCH 02/15] Prepare mobile 0.0.30 (#8779) Co-authored-by: Orca --- mobile/app.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/app.json b/mobile/app.json index e2d6a248f85..cb740d1f545 100644 --- a/mobile/app.json +++ b/mobile/app.json @@ -2,7 +2,7 @@ "expo": { "name": "Orca", "slug": "orca-mobile", - "version": "0.0.27", + "version": "0.0.30", "orientation": "default", "icon": "./assets/icon.png", "userInterfaceStyle": "automatic", From dfabd85513e42ff8f337a6d2651827ae192a5485 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 14 Jul 2026 16:35:31 -0700 Subject: [PATCH 03/15] Stop usage chips flashing and back off retries for failing providers (#8772) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Stop usage chips flashing and back off retries for failing providers Two behaviors made the status bar unusable when any provider was persistently failing (bad auth, unsupported plan): 1. Every refetch repainted all providers as 'fetching', so a settled error chip flashed to a loading "…" chip and back on every cycle. withFetchingStatus now keeps settled states (ok/error/unavailable) visible until the new result lands; only providers with no settled state (first load, explicit account-switch clear) show loading. 2. Error providers on the fast activation-retry lane (claude/codex/ grok) were retried every 30s on any focus/show/restore event — forever. Repeated hits drove Claude's tight-budget usage endpoint into 429s, flipping the chip between 'Limited' and its actual error. Retries now back off exponentially per consecutive applied failure (30s, 60s, 120s, … capped at the 15-minute poll cadence) and reset on success or account/target switch. * Count full fetches as failure-lane retries and keep Grok pane refresh feedback - Stamp failing providers' activation-retry clocks when a stale-driven full fetch runs, so the individual failure lane does not fire a redundant retry right after the full fetch already retried them. - GrokUsagePane: manual refresh spinner/disable is now renderer-local, since settled snapshots no longer repaint as 'fetching' during refetches. - Strengthen the backoff-reset test so it distinguishes a reset streak from a stale retry timestamp (CodeRabbit), and add a regression test for the full-fetch retry stamping. --- src/main/rate-limits/service.test.ts | 180 +++++++++++++++++- src/main/rate-limits/service.ts | 89 ++++++++- .../src/components/stats/GrokUsagePane.tsx | 17 +- 3 files changed, 282 insertions(+), 4 deletions(-) diff --git a/src/main/rate-limits/service.test.ts b/src/main/rate-limits/service.test.ts index 261a94f36be..9bd66cb806a 100644 --- a/src/main/rate-limits/service.test.ts +++ b/src/main/rate-limits/service.test.ts @@ -120,6 +120,19 @@ function unavailableProvider( } } +// Why: beforeEach caches snapshot objects whose updatedAt is pinned at suite +// start, so after 5 fake minutes every healthy provider looks stale and every +// activation degrades to a full fetch. Backoff tests that reason about the +// individual retry lane need healthy providers minted fresh at fetch time. +function mockFreshBackgroundProviderFetches(): void { + vi.mocked(fetchCodexRateLimits).mockImplementation(async () => okProvider('codex', 24)) + vi.mocked(fetchGeminiRateLimits).mockImplementation(async () => okProvider('gemini', 0)) + vi.mocked(fetchOpenCodeGoRateLimits).mockImplementation(async () => okProvider('opencode-go', 0)) + vi.mocked(fetchKimiRateLimits).mockImplementation(async () => okProvider('kimi', 0)) + vi.mocked(fetchMiniMaxRateLimits).mockImplementation(async () => okProvider('minimax', 0)) + vi.mocked(fetchGrokRateLimits).mockImplementation(async () => unavailableProvider('grok')) +} + function serviceInternals(service: RateLimitService): { fetchAll: () => Promise } { return service as unknown as { fetchAll: () => Promise } } @@ -393,7 +406,7 @@ describe('RateLimitService', () => { } }) - it('throttles repeated active-window retries while Claude is still failing', async () => { + it('backs off repeated active-window retries while Claude is still failing', async () => { vi.useFakeTimers() try { vi.mocked(fetchClaudeRateLimits).mockResolvedValue(errorProvider('claude', 'still failing')) @@ -407,6 +420,7 @@ describe('RateLimitService', () => { await vi.advanceTimersByTimeAsync(1000) expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(1) + // First activation recovers immediately (retry timestamps start at 0). window.emit('focus') await vi.advanceTimersByTimeAsync(0) expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(2) @@ -415,11 +429,175 @@ describe('RateLimitService', () => { await vi.advanceTimersByTimeAsync(0) expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(2) + // Two consecutive failures: the retry window doubled to 60s, so an + // activation at +30s must not hammer the endpoint again. await vi.advanceTimersByTimeAsync(30 * 1000) window.emit('restore') await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(2) + + await vi.advanceTimersByTimeAsync(30 * 1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(3) + // Three consecutive failures: 120s window. + await vi.advanceTimersByTimeAsync(60 * 1000) + window.emit('show') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(3) + + await vi.advanceTimersByTimeAsync(60 * 1000) + window.emit('restore') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(4) + + service.stop() + } finally { + vi.useRealTimers() + } + }) + + it('resets the retry backoff once Claude recovers', async () => { + vi.useFakeTimers() + try { + vi.mocked(fetchClaudeRateLimits) + .mockResolvedValueOnce(errorProvider('claude', 'still failing')) + .mockResolvedValueOnce(errorProvider('claude', 'still failing')) + .mockResolvedValueOnce(okProvider('claude', 12)) + .mockResolvedValue(errorProvider('claude', 'failing again')) + mockFreshBackgroundProviderFetches() + + const service = new RateLimitService() + const window = new FakeRateLimitWindow() + service.attach(asRateLimitWindow(window)) + service.start({ fetchImmediately: false }) + + await vi.advanceTimersByTimeAsync(1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(2) + + // Recovery fetch succeeds and must clear the failure streak. + await vi.advanceTimersByTimeAsync(60 * 1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(3) + expect(service.getState().claude?.status).toBe('ok') + + // The stale-ok snapshot forces a full refresh, which fails again. + await vi.advanceTimersByTimeAsync(5 * 60 * 1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(4) + expect(service.getState().claude?.status).toBe('error') + + // Consume the stale active-retry timestamp so the next windows measure + // the post-recovery streak rather than time elapsed before recovery. + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(5) + + // Post-recovery streak is 2 (not the pre-recovery 4): the window must be + // 60s, so +30s stays throttled and +60s retries. + await vi.advanceTimersByTimeAsync(30 * 1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(5) + + await vi.advanceTimersByTimeAsync(30 * 1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(6) + + service.stop() + } finally { + vi.useRealTimers() + } + }) + + it('counts a stale-driven full fetch as the failing provider retry attempt', async () => { + vi.useFakeTimers() + try { + vi.mocked(fetchClaudeRateLimits).mockImplementation(async () => + errorProvider('claude', 'still failing') + ) + mockFreshBackgroundProviderFetches() + + const service = new RateLimitService() + const window = new FakeRateLimitWindow() + service.attach(asRateLimitWindow(window)) + service.start({ fetchImmediately: false }) + + await vi.advanceTimersByTimeAsync(1000) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(1) + + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(2) + + // Healthy providers go stale after 5 minutes, so this activation runs a + // full fetch that also retries failing Claude (streak now 3 → 120s). + await vi.advanceTimersByTimeAsync(5 * 60 * 1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(3) + + // Why: the full fetch was itself a retry. An activation moments later + // must not fire the individual failure lane ahead of the backoff window. + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(3) + + // Once the 120s window elapses, the individual retry lane fires again. + await vi.advanceTimersByTimeAsync(120 * 1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(4) + + service.stop() + } finally { + vi.useRealTimers() + } + }) + + it('keeps a settled error chip settled during background refetches instead of flashing fetching', async () => { + vi.useFakeTimers() + try { + const secondClaude = deferred() + vi.mocked(fetchClaudeRateLimits) + .mockResolvedValueOnce(errorProvider('claude', 'still failing')) + .mockImplementationOnce(() => secondClaude.promise) + vi.mocked(fetchCodexRateLimits).mockResolvedValue(okProvider('codex', 24)) + + const service = new RateLimitService() + const window = new FakeRateLimitWindow() + const claudeStatuses: string[] = [] + service.onStateChange((state) => { + if (state.claude) { + claudeStatuses.push(state.claude.status) + } + }) + service.attach(asRateLimitWindow(window)) + service.start({ fetchImmediately: false }) + + await vi.advanceTimersByTimeAsync(1000) + expect(service.getState().claude?.status).toBe('error') + + // Why: the retry must not repaint the settled error chip as a loading + // "…" chip while the refetch is in flight — that is the flash users see + // every cycle when a provider is stuck failing. + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(service.getState().claude?.status).toBe('error') + + secondClaude.resolve(okProvider('claude', 12)) + await vi.advanceTimersByTimeAsync(0) + expect(service.getState().claude?.status).toBe('ok') + + const statusesAfterFirstSettle = claudeStatuses.slice(claudeStatuses.indexOf('error')) + expect(statusesAfterFirstSettle).not.toContain('fetching') + service.stop() } finally { vi.useRealTimers() diff --git a/src/main/rate-limits/service.ts b/src/main/rate-limits/service.ts index cd301cfe516..84fb016fec8 100644 --- a/src/main/rate-limits/service.ts +++ b/src/main/rate-limits/service.ts @@ -77,6 +77,12 @@ const MIN_POLL_MS = 30 * 1000 // 30 seconds — renderer input should never crea const MAX_POLL_MS = 2_147_483_647 // Max safe setInterval delay before Node clamps back to 1ms. const MIN_REFETCH_MS = 5 * 60 * 1000 // 5 minutes — debounce resume/manual refresh bursts const ACTIVE_FAILURE_REFETCH_MS = MIN_POLL_MS +// Why: a persistent failure (bad auth, unsupported plan) retried at the 30s +// floor hammers provider endpoints — Claude's tight-budget usage endpoint +// starts returning 429s — without ever recovering. Back off per consecutive +// failure, capped at the background poll cadence. +const MAX_ACTIVE_FAILURE_REFETCH_MS = DEFAULT_POLL_MS +const MAX_ACTIVE_FAILURE_STREAK = 8 // Why: these providers have a dedicated fetch cycle, so an activation retry can // refresh just the failing one. Providers without one force a full fetchAll, so // their error retries stay on the 5-minute cadence to protect Claude's budget. @@ -152,6 +158,18 @@ export class RateLimitService { grok: 0, antigravity: 0 } + // Why: consecutive applied failures per provider drive exponential backoff of + // the fast activation-retry lane; reset on any successful/unavailable result. + private activeFailureStreakByProvider: Record = { + claude: 0, + codex: 0, + gemini: 0, + 'opencode-go': 0, + kimi: 0, + minimax: 0, + grok: 0, + antigravity: 0 + } private mainWindow: BrowserWindow | null = null private detachWindowListeners: (() => void) | null = null private isFetching = false @@ -358,6 +376,8 @@ export class RateLimitService { } this.codexFetchTarget = nextTarget this.codexFetchGeneration += 1 + // Why: a new account/target starts with a clean retry schedule. + this.activeFailureStreakByProvider.codex = 0 this.inactiveCodexAccountsGeneration += 1 this.pruneInactiveCodexState() this.lastInactiveCodexFetchAt = 0 @@ -377,6 +397,7 @@ export class RateLimitService { const targetChanged = !this.isSameCodexTarget(this.codexFetchTarget, nextTarget) this.codexFetchTarget = nextTarget this.codexFetchGeneration += 1 + this.activeFailureStreakByProvider.codex = 0 this.updateState({ ...this.state, codex: this.withFetchingStatus(targetChanged ? null : this.state.codex, 'codex') @@ -426,6 +447,8 @@ export class RateLimitService { this.inactiveClaudeAccountsGeneration += 1 this.pruneInactiveClaudeState() this.claudeFetchGeneration += 1 + // Why: a new account/target starts with a clean retry schedule. + this.activeFailureStreakByProvider.claude = 0 this.lastInactiveClaudeFetchAt = 0 this.updateState({ ...this.state, @@ -440,6 +463,7 @@ export class RateLimitService { const targetChanged = !this.isSameClaudeTarget(this.claudeFetchTarget, nextTarget) this.claudeFetchTarget = nextTarget this.claudeFetchGeneration += 1 + this.activeFailureStreakByProvider.claude = 0 this.updateState({ ...this.state, claude: this.withFetchingStatus(targetChanged ? null : this.state.claude, 'claude') @@ -764,7 +788,11 @@ export class RateLimitService { if (limits.status === 'error') { const lastRetryAt = this.lastActiveFailureRetryAtByProvider[provider] const throttleMs = INDIVIDUALLY_REFRESHABLE_PROVIDERS.has(provider) - ? ACTIVE_FAILURE_REFETCH_MS + ? Math.min( + ACTIVE_FAILURE_REFETCH_MS * + 2 ** Math.max(0, this.activeFailureStreakByProvider[provider] - 1), + MAX_ACTIVE_FAILURE_REFETCH_MS + ) : MIN_REFETCH_MS if (now - lastRetryAt >= throttleMs) { retryableFailures.push(provider) @@ -783,6 +811,18 @@ export class RateLimitService { return } if (plan.kind === 'full') { + // Why: a full fetch retries failing providers too. Restart their retry + // clocks so the individual failure lane doesn't fire again right after, + // ahead of its backoff window. Skip when a fetch is already in flight — + // fetchAll would no-op and the throttle must not be consumed for free. + if (!this.isFetching) { + const now = Date.now() + for (const { provider, limits } of this.getActiveProviderState()) { + if (limits?.status === 'error') { + this.lastActiveFailureRetryAtByProvider[provider] = now + } + } + } await this.fetchAll() return } @@ -1242,6 +1282,22 @@ export class RateLimitService { } } + private trackActiveFailureStreak( + provider: ActiveRateLimitProvider, + fresh: ProviderRateLimits + ): void { + if (fresh.status === 'error') { + this.activeFailureStreakByProvider[provider] = Math.min( + this.activeFailureStreakByProvider[provider] + 1, + MAX_ACTIVE_FAILURE_STREAK + ) + return + } + if (fresh.status === 'ok' || fresh.status === 'unavailable') { + this.activeFailureStreakByProvider[provider] = 0 + } + } + private withFetchingStatus( current: ProviderRateLimits | null, provider: @@ -1264,6 +1320,13 @@ export class RateLimitService { status: 'fetching' } } + // Why: repainting a settled chip as "fetching" on every background refetch + // makes the status bar flash "…" → error each retry cycle when a provider + // is persistently failing. Keep the settled state visible until the new + // result lands; only providers with no settled state show a loading chip. + if (current.status === 'ok' || current.status === 'error' || current.status === 'unavailable') { + return current + } return { ...current, status: 'fetching' } } @@ -1481,6 +1544,22 @@ export class RateLimitService { const shouldApplyOpencode = opencodeGeneration === this.opencodeFetchGeneration const shouldApplyMiniMax = miniMaxGeneration === this.minimaxFetchGeneration + if (shouldApplyClaude) { + this.trackActiveFailureStreak('claude', claude) + } + if (shouldApplyCodex) { + this.trackActiveFailureStreak('codex', codex) + } + this.trackActiveFailureStreak('gemini', gemini) + this.trackActiveFailureStreak('antigravity', antigravity) + if (shouldApplyOpencode) { + this.trackActiveFailureStreak('opencode-go', opencodeGo) + } + this.trackActiveFailureStreak('kimi', kimi) + if (shouldApplyMiniMax) { + this.trackActiveFailureStreak('minimax', miniMax) + } + // Why: account switches can race in-flight Codex fetches. Only apply a // Codex result if both the selected-account provenance and the request // generation still match, otherwise an old account could overwrite the @@ -1523,6 +1602,7 @@ export class RateLimitService { error: grokResult.reason instanceof Error ? grokResult.reason.message : 'Unknown error', status: 'error' } satisfies ProviderRateLimits) + this.trackActiveFailureStreak('grok', grok) this.updateState({ ...this.state, grok: this.applyStalePolicy(grok, previousState.grok) @@ -1575,6 +1655,9 @@ export class RateLimitService { const shouldApplyCodex = codexGeneration === this.codexFetchGeneration && codexProvenance === latestCodexProvenance + if (shouldApplyCodex) { + this.trackActiveFailureStreak('codex', codex) + } this.updateState({ ...this.state, codex: shouldApplyCodex ? this.applyStalePolicy(codex, previousState.codex) : this.state.codex @@ -1630,6 +1713,9 @@ export class RateLimitService { claudeProvenance === latestClaudeProvenance && this.isSameClaudeTarget(claudeTarget, this.claudeFetchTarget) + if (shouldApplyClaude) { + this.trackActiveFailureStreak('claude', claude) + } this.updateState({ ...this.state, claude: shouldApplyClaude @@ -1669,6 +1755,7 @@ export class RateLimitService { return } + this.trackActiveFailureStreak('grok', grok) this.updateState({ ...this.state, grok: this.applyStalePolicy(grok, previousState.grok) diff --git a/src/renderer/src/components/stats/GrokUsagePane.tsx b/src/renderer/src/components/stats/GrokUsagePane.tsx index 8e51049e3d4..08600621552 100644 --- a/src/renderer/src/components/stats/GrokUsagePane.tsx +++ b/src/renderer/src/components/stats/GrokUsagePane.tsx @@ -1,3 +1,4 @@ +import { useState } from 'react' import { CalendarClock, ExternalLink, RefreshCw, Sparkles } from 'lucide-react' import { translate } from '@/i18n/i18n' import { useAppStore } from '../../store' @@ -13,6 +14,18 @@ export function GrokUsagePane(): React.JSX.Element { const openSettingsPage = useAppStore((s) => s.openSettingsPage) const openSettingsTarget = useAppStore((s) => s.openSettingsTarget) const recordFeatureInteraction = useAppStore((s) => s.recordFeatureInteraction) + // Why: settled snapshots keep their status during refetches (no 'fetching' + // repaint), so manual-refresh feedback must be renderer-local, matching the + // StatusBar refresh button. + const [isRefreshing, setIsRefreshing] = useState(false) + + const handleRefresh = (): void => { + if (isRefreshing) { + return + } + setIsRefreshing(true) + void refreshGrokRateLimits().finally(() => setIsRefreshing(false)) + } const openGrokAccounts = (): void => { openSettingsTarget({ pane: 'accounts', repoId: null, sectionId: 'accounts-grok' }) @@ -57,7 +70,7 @@ export function GrokUsagePane(): React.JSX.Element { grok?.weekly && typeof grok.weekly.usedPercent === 'number' ? Math.round(grok.weekly.usedPercent) : null - const isFetching = grok?.status === 'fetching' + const isFetching = isRefreshing || grok?.status === 'fetching' return (
void refreshGrokRateLimits()} + onClick={handleRefresh} disabled={isFetching} aria-label={translate( 'auto.components.stats.GrokUsagePane.i0j1k2l3m4', From a4cfc82d69a1d5123f5270cd64e2f45a80b25383 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 14 Jul 2026 17:02:06 -0700 Subject: [PATCH 04/15] Surface Grok unified-billing monthly usage instead of a permanent warning (#8769) * Surface Grok unified-billing monthly usage instead of a permanent warning Unified-billing Grok accounts have no weekly credits: the /billing?format=credits view returns a config without creditUsagePercent, so the status bar was stuck on 'Grok billing response did not include credit usage' even though the account has a real quota. The default (format-less) /billing view reports it as an included monthly budget (monthlyLimit/used with the billing period). When the credits view has no weekly credit usage, read the default view and surface monthly usage as the provider's 30-day window (already supported by the tooltip and chip visibility for OpenCode Go). If the fallback read fails, the previous 'unavailable' presentation stands rather than escalating to an error chip. * Review fixes: chip renders monthly-only usage; fallback failures keep stale data - StatusBar ProviderSegment: monthly window is chip-visible when it is the sole window (Grok unified billing); fetching/error no-data guards and the icon-only dot now count monthly, matching tooltip.tsx. OpenCode Go chips are unchanged (monthly stays tooltip-only next to session/weekly). - grok-fetcher: monthly-fallback request failures propagate as 'error' so applyStalePolicy keeps the last good monthly snapshot; 'unavailable' is reserved for a successful response without monthly fields. * Settings: show Grok monthly usage row for unified-billing accounts Why: the Grok accounts section only rendered the weekly-credits row, so unified-billing accounts showed a signed-in state with no usage at all. * Use generated localization keys for Grok monthly copy --- src/main/rate-limits/grok-fetcher.test.ts | 86 ++++++++++- src/main/rate-limits/grok-fetcher.ts | 143 ++++++++++++++---- .../settings/GrokAccountsSection.tsx | 44 ++++-- .../src/components/status-bar/StatusBar.tsx | 18 ++- .../provider-segment-monthly-window.test.tsx | 86 +++++++++++ src/renderer/src/i18n/locales/en.json | 4 +- src/renderer/src/i18n/locales/es.json | 4 +- src/renderer/src/i18n/locales/ja.json | 4 +- src/renderer/src/i18n/locales/ko.json | 4 +- src/renderer/src/i18n/locales/zh.json | 4 +- src/shared/rate-limit-types.ts | 2 +- 11 files changed, 345 insertions(+), 54 deletions(-) create mode 100644 src/renderer/src/components/status-bar/provider-segment-monthly-window.test.tsx diff --git a/src/main/rate-limits/grok-fetcher.test.ts b/src/main/rate-limits/grok-fetcher.test.ts index 07c536f32a0..e387a13e3a1 100644 --- a/src/main/rate-limits/grok-fetcher.test.ts +++ b/src/main/rate-limits/grok-fetcher.test.ts @@ -107,13 +107,95 @@ describe('fetchGrokRateLimits', () => { expect(netFetchMock).not.toHaveBeenCalled() }) - it('returns unavailable when billing has no credit usage', async () => { + it('returns unavailable when neither billing view has usage', async () => { authState.file = freshAuthJson() - netFetchMock.mockResolvedValueOnce(jsonResponse({ config: { subscriptionTier: 'Enterprise' } })) + netFetchMock + .mockResolvedValueOnce(jsonResponse({ config: { subscriptionTier: 'Enterprise' } })) + .mockResolvedValueOnce(jsonResponse({ config: { subscriptionTier: 'Enterprise' } })) const result = await fetchGrokRateLimits() expect(result.status).toBe('unavailable') expect(result.weekly).toBeNull() + expect(result.monthly).toBeUndefined() + }) + + it('maps monthly included usage for unified-billing accounts without weekly credits', async () => { + authState.file = freshAuthJson() + netFetchMock + .mockResolvedValueOnce( + jsonResponse({ + config: { + currentPeriod: { + type: 'USAGE_PERIOD_TYPE_WEEKLY', + start: '2026-07-10T19:38:56.948570+00:00', + end: '2026-07-17T19:38:56.948570+00:00' + }, + isUnifiedBillingUser: true, + subscriptionTier: 'SuperGrok' + } + }) + ) + .mockResolvedValueOnce( + jsonResponse({ + config: { + monthlyLimit: { val: 150000 }, + used: { val: 837 }, + billingPeriodStart: '2026-07-01T00:00:00+00:00', + billingPeriodEnd: '2026-08-01T00:00:00+00:00' + } + }) + ) + + const result = await fetchGrokRateLimits() + expect(result.status).toBe('ok') + expect(result.error).toBeNull() + expect(result.weekly).toBeNull() + expect(result.monthly?.usedPercent).toBeCloseTo((837 / 150000) * 100, 5) + expect(result.monthly?.windowMinutes).toBe(43_200) + expect(result.monthly?.resetsAt).toBe(Date.parse('2026-08-01T00:00:00+00:00')) + expect(result.usageMetadata?.authProvenance).toContain('SuperGrok') + + expect(netFetchMock).toHaveBeenNthCalledWith( + 2, + 'https://cli-chat-proxy.grok.com/v1/billing', + expect.objectContaining({ + headers: expect.objectContaining({ Authorization: 'Bearer access-token' }) + }) + ) + }) + + // Why: 'unavailable' would make applyStalePolicy discard the last good + // monthly snapshot; transient fallback failures must present like transient + // credits-view failures so stale data survives. + it('surfaces an error when the monthly fallback request fails', async () => { + authState.file = freshAuthJson() + netFetchMock + .mockResolvedValueOnce(jsonResponse({ config: { isUnifiedBillingUser: true } })) + .mockResolvedValueOnce(jsonResponse({}, 500)) + + const result = await fetchGrokRateLimits() + expect(result.status).toBe('error') + expect(result.error).toBe('Grok usage request failed (HTTP 500)') + }) + + it('surfaces an error when the monthly fallback request throws', async () => { + authState.file = freshAuthJson() + netFetchMock + .mockResolvedValueOnce(jsonResponse({ config: { isUnifiedBillingUser: true } })) + .mockRejectedValueOnce(new Error('network down')) + + const result = await fetchGrokRateLimits() + expect(result.status).toBe('error') + expect(result.error).toBe('network down') + }) + + it('does not request the default billing view when weekly credits are present', async () => { + authState.file = freshAuthJson() + netFetchMock.mockResolvedValueOnce(jsonResponse(BILLING_RESPONSE)) + + const result = await fetchGrokRateLimits() + expect(result.status).toBe('ok') + expect(netFetchMock).toHaveBeenCalledTimes(1) }) it('returns unavailable when billing response has no config', async () => { diff --git a/src/main/rate-limits/grok-fetcher.ts b/src/main/rate-limits/grok-fetcher.ts index 099298a56c4..eaa140eb2be 100644 --- a/src/main/rate-limits/grok-fetcher.ts +++ b/src/main/rate-limits/grok-fetcher.ts @@ -12,8 +12,12 @@ const GROK_CLI_PROXY_BASE = process.env.GROK_CLI_CHAT_PROXY_BASE_URL?.trim().replace(/\/$/, '') || 'https://cli-chat-proxy.grok.com/v1' const BILLING_CREDITS_URL = `${GROK_CLI_PROXY_BASE}/billing?format=credits` +// Why: unified-billing accounts have no weekly credits; their included monthly +// budget is only present in the default (format-less) billing view. +const BILLING_DEFAULT_URL = `${GROK_CLI_PROXY_BASE}/billing` const API_TIMEOUT_MS = 10_000 const WEEKLY_WINDOW_MINUTES = 10_080 +const MONTHLY_WINDOW_MINUTES = 43_200 const GROK_CLI_AUTH_HEADER = 'xai-grok-cli' @@ -31,6 +35,8 @@ type GrokBillingConfig = { billingPeriodStart?: string billingPeriodEnd?: string subscriptionTier?: string + monthlyLimit?: GrokMoneyVal + used?: GrokMoneyVal onDemandCap?: GrokMoneyVal onDemandUsed?: GrokMoneyVal prepaidBalance?: GrokMoneyVal @@ -81,6 +87,28 @@ function mapWeeklyCredits(config: GrokBillingConfig): RateLimitWindow | null { } } +function parseMoneyVal(value: GrokMoneyVal | undefined): number | null { + const raw = value?.val + const num = typeof raw === 'string' ? Number.parseFloat(raw) : raw + return typeof num === 'number' && Number.isFinite(num) ? num : null +} + +function mapMonthlyUsage(config: GrokBillingConfig): RateLimitWindow | null { + const limit = parseMoneyVal(config.monthlyLimit) + const used = parseMoneyVal(config.used) + if (limit === null || used === null || limit <= 0) { + return null + } + const periodEnd = config.currentPeriod?.end ?? config.billingPeriodEnd + const resetsAt = periodEnd ? Date.parse(periodEnd) : null + return { + usedPercent: Math.min(100, Math.max(0, (used / limit) * 100)), + windowMinutes: MONTHLY_WINDOW_MINUTES, + resetsAt: resetsAt !== null && Number.isFinite(resetsAt) ? resetsAt : null, + resetDescription: parseResetDescription(periodEnd) + } +} + function grokRequestHeaders(session: GrokAuthSession): Record { const headers: Record = { Authorization: `Bearer ${session.accessToken}`, @@ -103,28 +131,22 @@ function resolveBillingConfig(data: GrokBillingResponse): GrokBillingConfig | nu return null } -function mapBillingResponse( - data: GrokBillingResponse, +function billingUsageResult( + windows: { weekly?: RateLimitWindow | null; monthly?: RateLimitWindow | null }, + config: GrokBillingConfig, session: GrokAuthSession ): ProviderRateLimits { - const config = resolveBillingConfig(data) - // Why: a 200 without credit usage means the plan has no weekly credits — - // 'unavailable' hides the bar (like Claude on API-key billing); 'error' - // would paint a permanent alert for a signed-in account that has no quota. - if (!config) { - return result('unavailable', 'Grok billing response did not include config') - } - const weekly = mapWeeklyCredits(config) const tier = config.subscriptionTier?.trim() const authLabel = session.email?.trim() || session.userId || 'Grok account' const provenance = tier ? `${authLabel} (${tier})` : authLabel return { provider: 'grok', session: null, - weekly, + weekly: windows.weekly ?? null, + ...(windows.monthly ? { monthly: windows.monthly } : {}), updatedAt: Date.now(), - error: weekly ? null : 'Grok billing response did not include credit usage', - status: weekly ? 'ok' : 'unavailable', + error: null, + status: 'ok', usageMetadata: { source: 'oauth', authProvenance: provenance @@ -132,6 +154,61 @@ function mapBillingResponse( } } +type GrokBillingFetchOutcome = + | { kind: 'data'; data: GrokBillingResponse } + | { kind: 'result'; result: ProviderRateLimits } + +async function fetchBillingData( + url: string, + session: GrokAuthSession, + signal?: AbortSignal +): Promise { + const requestSignal = signal + ? AbortSignal.any([signal, AbortSignal.timeout(API_TIMEOUT_MS)]) + : AbortSignal.timeout(API_TIMEOUT_MS) + const res = await net.fetch(url, { + headers: grokRequestHeaders(session), + signal: requestSignal + }) + if (res.status === 401 || res.status === 403) { + return { + kind: 'result', + result: result('error', `Grok usage request unauthorized (HTTP ${res.status})`) + } + } + if (!res.ok) { + return { + kind: 'result', + result: result('error', `Grok usage request failed (HTTP ${res.status})`) + } + } + const data: unknown = await res.json() + return { + kind: 'data', + data: typeof data === 'object' && data !== null ? (data as GrokBillingResponse) : {} + } +} + +type GrokMonthlyFallbackOutcome = + | { kind: 'window'; window: RateLimitWindow | null } + | { kind: 'result'; result: ProviderRateLimits } + +// Why: request failures propagate as 'error' (thrown errors reach the caller's +// catch) so the stale policy keeps the last good monthly snapshot — the +// 'unavailable' status would discard it. Only a successful response without +// monthly fields means the account truly has no visible quota. +async function fetchMonthlyUsageFallback( + session: GrokAuthSession, + signal?: AbortSignal +): Promise { + const outcome = await fetchBillingData(BILLING_DEFAULT_URL, session, signal) + if (outcome.kind === 'result') { + return outcome + } + const config = outcome.data.config ?? outcome.data + return { kind: 'window', window: mapMonthlyUsage(config) } +} + // Why: Orca never runs grok login; it only reads the session file the CLI updates. export async function fetchGrokRateLimits( options: { signal?: AbortSignal; authReadResult?: GrokAuthReadResult } = {} @@ -152,24 +229,32 @@ export async function fetchGrokRateLimits( } try { - const signal = options.signal - ? AbortSignal.any([options.signal, AbortSignal.timeout(API_TIMEOUT_MS)]) - : AbortSignal.timeout(API_TIMEOUT_MS) - const res = await net.fetch(BILLING_CREDITS_URL, { - headers: grokRequestHeaders(session), - signal - }) - if (res.status === 401 || res.status === 403) { - return result('error', `Grok usage request unauthorized (HTTP ${res.status})`) + const outcome = await fetchBillingData(BILLING_CREDITS_URL, session, options.signal) + if (outcome.kind === 'result') { + return outcome.result } - if (!res.ok) { - return result('error', `Grok usage request failed (HTTP ${res.status})`) + const config = resolveBillingConfig(outcome.data) + // Why: a 200 without credit usage means the plan has no weekly credits — + // 'unavailable' hides the bar (like Claude on API-key billing); 'error' + // would paint a permanent alert for a signed-in account that has no quota. + if (!config) { + return result('unavailable', 'Grok billing response did not include config') } - const data: unknown = await res.json() - return mapBillingResponse( - typeof data === 'object' && data !== null ? (data as GrokBillingResponse) : {}, - session - ) + const weekly = mapWeeklyCredits(config) + if (weekly) { + return billingUsageResult({ weekly }, config, session) + } + // Why: unified-billing accounts report a monthly included-usage budget + // instead of weekly credits; the credits view omits creditUsagePercent + // for them, so read the default billing view before giving up. + const fallback = await fetchMonthlyUsageFallback(session, options.signal) + if (fallback.kind === 'result') { + return fallback.result + } + if (fallback.window) { + return billingUsageResult({ monthly: fallback.window }, config, session) + } + return result('unavailable', 'Grok billing response did not include credit usage') } catch (err) { return result('error', err instanceof Error ? err.message : 'Grok usage request failed') } diff --git a/src/renderer/src/components/settings/GrokAccountsSection.tsx b/src/renderer/src/components/settings/GrokAccountsSection.tsx index a5eb0ca8d99..474c4c1ee66 100644 --- a/src/renderer/src/components/settings/GrokAccountsSection.tsx +++ b/src/renderer/src/components/settings/GrokAccountsSection.tsx @@ -52,6 +52,10 @@ export function GrokAccountsSection(): React.JSX.Element { const signedIn = status?.signedIn === true const tokenFresh = status?.tokenFresh === true + // Why: unified-billing accounts have no weekly credits; surface their + // monthly included usage instead of hiding the usage row entirely. + const usageIsWeekly = Boolean(grokUsage?.weekly) + const usageWindow = grokUsage?.weekly ?? grokUsage?.monthly ?? null return (
@@ -148,32 +152,46 @@ export function GrokAccountsSection(): React.JSX.Element {
- {grokUsage?.weekly ? ( + {usageWindow ? (
- {Math.round(grokUsage.weekly.usedPercent)}% + {Math.round(usageWindow.usedPercent)}% - {grokUsage.weekly.resetDescription ? ( + {usageWindow.resetDescription ? ( {translate( 'auto.components.settings.GrokAccountsSection.c6d1a8f4e2', 'Resets {{when}}', - { when: grokUsage.weekly.resetDescription } + { when: usageWindow.resetDescription } )} ) : null} - {grokUsage.usageMetadata?.authProvenance ? ( + {grokUsage?.usageMetadata?.authProvenance ? ( {grokUsage.usageMetadata.authProvenance} diff --git a/src/renderer/src/components/status-bar/StatusBar.tsx b/src/renderer/src/components/status-bar/StatusBar.tsx index b3cc8a922c1..2c9a2fb979c 100644 --- a/src/renderer/src/components/status-bar/StatusBar.tsx +++ b/src/renderer/src/components/status-bar/StatusBar.tsx @@ -1114,7 +1114,7 @@ function WindowLabel({ // the rest (Flash Lite, experimental) are secondary and would clutter the bar. const STATUS_BAR_BUCKET_NAMES = new Set(['Flash', 'Pro', '1.5 Pro']) -function ProviderSegment({ +export function ProviderSegment({ p, compact, display @@ -1137,7 +1137,7 @@ function ProviderSegment({ } // Fetching with no prior data - if (p.status === 'fetching' && !p.session && !p.weekly && !p.fableWeekly) { + if (p.status === 'fetching' && !p.session && !p.weekly && !p.fableWeekly && !p.monthly) { return ( @@ -1156,7 +1156,7 @@ function ProviderSegment({ } // Error with no data - if (p.status === 'error' && !p.session && !p.weekly && !p.fableWeekly) { + if (p.status === 'error' && !p.session && !p.weekly && !p.fableWeekly && !p.monthly) { return ( @@ -1215,6 +1215,16 @@ function ProviderSegment({ window: p.fableWeekly, label: translate('auto.components.status.bar.StatusBar.a79c64f87e', 'Fable') } + : null, + // Why: monthly is chip-visible only when it's the sole window (Grok + // unified billing); providers with session/weekly data (OpenCode Go) + // keep monthly tooltip-only so the chip stays uncluttered. + p.monthly && !p.session && !p.weekly + ? { + key: 'monthly', + window: p.monthly, + label: formatWindowLabel(p.monthly.windowMinutes) + } : null ].filter((w): w is { key: string; window: RateLimitWindow; label: string } => w !== null) @@ -1783,7 +1793,7 @@ export function ProviderDetailsMenu({ {iconOnly ? ( {provider.provider === 'claude' diff --git a/src/renderer/src/components/status-bar/provider-segment-monthly-window.test.tsx b/src/renderer/src/components/status-bar/provider-segment-monthly-window.test.tsx new file mode 100644 index 00000000000..866282b8680 --- /dev/null +++ b/src/renderer/src/components/status-bar/provider-segment-monthly-window.test.tsx @@ -0,0 +1,86 @@ +import { renderToStaticMarkup } from 'react-dom/server' +import { describe, expect, it, vi } from 'vitest' +import type { ProviderRateLimits, RateLimitWindow } from '../../../../shared/rate-limit-types' + +vi.mock('@/i18n/i18n', () => ({ + i18n: { language: 'en' }, + translate: (_key: string, fallback: string, values?: Record) => { + let result = fallback + for (const [key, value] of Object.entries(values ?? {})) { + result = result.replace(`{{${key}}}`, value) + } + return result + } +})) + +vi.mock('@/lib/agent-catalog', () => ({ + AgentIcon: () => null +})) + +vi.mock('../../store', () => ({ + useAppStore: (selector: (state: { usagePercentageDisplay: 'used' | 'remaining' }) => unknown) => + selector({ usagePercentageDisplay: 'used' }) +})) + +function windowOf(usedPercent: number, windowMinutes: number): RateLimitWindow { + return { usedPercent, windowMinutes, resetsAt: null, resetDescription: null } +} + +// Grok unified-billing accounts surface a monthly window and nothing else. +function grokMonthlyLimits(status: ProviderRateLimits['status']): ProviderRateLimits { + return { + provider: 'grok', + session: null, + weekly: null, + monthly: windowOf(25, 43200), + updatedAt: Date.now(), + error: null, + status + } +} + +describe('ProviderSegment monthly window', () => { + it('renders a monthly-only snapshot in the chip instead of a bare icon', async () => { + const { ProviderSegment } = await import('./StatusBar') + + const markup = renderToStaticMarkup( + + ) + + expect(markup).toContain('25% used 30d') + }) + + it('shows monthly data while fetching instead of the loading placeholder', async () => { + const { ProviderSegment } = await import('./StatusBar') + + const markup = renderToStaticMarkup( + + ) + + expect(markup).toContain('25% used 30d') + expect(markup).not.toContain('···') + }) + + // Why: providers with session/weekly windows (OpenCode Go) keep monthly + // tooltip-only so the chip stays uncluttered. + it('keeps monthly out of the chip when session and weekly windows exist', async () => { + const { ProviderSegment } = await import('./StatusBar') + + const limits: ProviderRateLimits = { + provider: 'opencode-go', + session: windowOf(10, 300), + weekly: windowOf(20, 10080), + monthly: windowOf(30, 43200), + updatedAt: Date.now(), + error: null, + status: 'ok' + } + const markup = renderToStaticMarkup( + + ) + + expect(markup).toContain('10% used 5h') + expect(markup).toContain('20% used wk') + expect(markup).not.toContain('30d') + }) +}) diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index d3f221f77e8..06e0f50482c 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -9028,7 +9028,9 @@ "3325d996cb": "Refresh usage", "a8f3e2c1b4": "Weekly credits", "b7e2d9f0a3": "Same weekly credit % as the grok /usage screen in the terminal.", - "c6d1a8f4e2": "Resets {{when}}" + "c6d1a8f4e2": "Resets {{when}}", + "e6dadc1e2b": "Monthly usage", + "75e396bf42": "Included monthly usage for Grok unified-billing accounts." }, "AppearanceWindowSidebarSection": { "usagePercentageDisplayUsed": "Used", diff --git a/src/renderer/src/i18n/locales/es.json b/src/renderer/src/i18n/locales/es.json index 903467f9227..2d093ebe8f8 100644 --- a/src/renderer/src/i18n/locales/es.json +++ b/src/renderer/src/i18n/locales/es.json @@ -9028,7 +9028,9 @@ "3325d996cb": "Actualizar uso", "a8f3e2c1b4": "Créditos semanales", "b7e2d9f0a3": "El mismo porcentaje de créditos semanales que la pantalla grok /usage en la terminal.", - "c6d1a8f4e2": "Se restablece {{when}}" + "c6d1a8f4e2": "Se restablece {{when}}", + "e6dadc1e2b": "Monthly usage", + "75e396bf42": "Included monthly usage for Grok unified-billing accounts." }, "AppearanceWindowSidebarSection": { "usagePercentageDisplayUsed": "Usado", diff --git a/src/renderer/src/i18n/locales/ja.json b/src/renderer/src/i18n/locales/ja.json index 123baa012ce..4ad9de324be 100644 --- a/src/renderer/src/i18n/locales/ja.json +++ b/src/renderer/src/i18n/locales/ja.json @@ -9028,7 +9028,9 @@ "3325d996cb": "使用状況を更新", "a8f3e2c1b4": "週次クレジット", "b7e2d9f0a3": "ターミナルの grok /usage 画面と同じ週次クレジット率です。", - "c6d1a8f4e2": "{{when}} にリセット" + "c6d1a8f4e2": "{{when}} にリセット", + "e6dadc1e2b": "Monthly usage", + "75e396bf42": "Included monthly usage for Grok unified-billing accounts." }, "AppearanceWindowSidebarSection": { "usagePercentageDisplayUsed": "使用済み", diff --git a/src/renderer/src/i18n/locales/ko.json b/src/renderer/src/i18n/locales/ko.json index e173a6e1718..dfbefd0700b 100644 --- a/src/renderer/src/i18n/locales/ko.json +++ b/src/renderer/src/i18n/locales/ko.json @@ -9028,7 +9028,9 @@ "3325d996cb": "사용량 새로 고침", "a8f3e2c1b4": "주간 크레딧", "b7e2d9f0a3": "터미널의 grok /usage 화면과 같은 주간 크레딧 비율입니다.", - "c6d1a8f4e2": "{{when}}에 재설정" + "c6d1a8f4e2": "{{when}}에 재설정", + "e6dadc1e2b": "Monthly usage", + "75e396bf42": "Included monthly usage for Grok unified-billing accounts." }, "AppearanceWindowSidebarSection": { "usagePercentageDisplayUsed": "사용", diff --git a/src/renderer/src/i18n/locales/zh.json b/src/renderer/src/i18n/locales/zh.json index a82d1d5fc41..eca7076c294 100644 --- a/src/renderer/src/i18n/locales/zh.json +++ b/src/renderer/src/i18n/locales/zh.json @@ -9028,7 +9028,9 @@ "3325d996cb": "刷新使用量", "a8f3e2c1b4": "每周额度", "b7e2d9f0a3": "与终端中 grok /usage 屏幕显示的每周额度百分比相同。", - "c6d1a8f4e2": "{{when}} 重置" + "c6d1a8f4e2": "{{when}} 重置", + "e6dadc1e2b": "Monthly usage", + "75e396bf42": "Included monthly usage for Grok unified-billing accounts." }, "AppearanceWindowSidebarSection": { "usagePercentageDisplayUsed": "已用", diff --git a/src/shared/rate-limit-types.ts b/src/shared/rate-limit-types.ts index 65c09aaffae..0e77decc76e 100644 --- a/src/shared/rate-limit-types.ts +++ b/src/shared/rate-limit-types.ts @@ -59,7 +59,7 @@ export type ProviderRateLimits = { weekly: RateLimitWindow | null /** Claude Fable 7-day weekly window, null if not available. */ fableWeekly?: RateLimitWindow | null - /** 30-day monthly window (OpenCode Go only), null if not available. */ + /** 30-day monthly window (OpenCode Go, Grok unified billing), null if not available. */ monthly?: RateLimitWindow | null /** Named per-model buckets (Gemini only). */ buckets?: RateLimitBucket[] From 6ee2af357b82c8717ad633f1251b6b310486dd7d Mon Sep 17 00:00:00 2001 From: ishihara takashi Date: Wed, 15 Jul 2026 09:25:10 +0900 Subject: [PATCH 05/15] Add an in-place screenshot markup tool to the browser pane (#6335) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add in-place screenshot markup for the browser pane Adds a Draw button to the browser toolbar (local and remote/SSH panes) that freezes the visible viewport into a still image and overlays a drawing canvas. Users can mark it up with pen, highlighter, arrow, rectangle, ellipse, and text (color, thickness, undo/redo), then copy the composited PNG to the clipboard to paste into their agent — reusing the existing clipboard screenshot path, so it works for local and remote agents alike. The drawing layer is a renderer-side canvas, so the base image is the only environment-specific piece: local panes use webview.capturePage(), remote panes snapshot the already-displayed screencast frame. Drawing model, compositing, and shape rendering are isolated, unit-tested modules. * Fix browser markup clipboard, transparency, text input, and toggle - Copy now always produces a PNG (the clipboard handler accepts PNG only; a JPEG fallback silently produced an empty clipboard) and targets the clipboard size limit so realistic viewports stay full-resolution. - Flatten the captured base image onto opaque white so transparent page backgrounds no longer ghost the live view through the frozen backdrop. - Keep markup text-input keystrokes local (focus the field explicitly and stop propagation) so the browser pane's global handlers can't swallow them. - Make the Draw toolbar button toggle markup mode off on a second click. * Add markup text sizing, object re-editing, and text-input fixes - Add a font-size control (text tool) to the markup toolbar. - Add a Select tool to re-edit committed shapes: click to select, drag to move, change color/width/font-size of the selection, Delete to remove, and double-click text to re-edit its content. Undo/redo now snapshots the whole shape list so every edit (not just adding) is reversible. - Fix the text input: transparent ink-colored field (no dark theme box over the screenshot) and ignore Enter during IME composition so Japanese conversion no longer commits the annotation early. - Split the grown overlay logic into focused modules (editor hook, pointer + keyboard hooks, canvas renderer, document edits, hit-testing) to stay within the file-size budget. * Improve markup text commit and move toolbar to the bottom - Clicking elsewhere while a text box is open now commits the text and switches to the Select tool, instead of discarding it (the input unmounted before its blur fired) and opening a new box at the click. - Move the drawing toolbar to the bottom (stacked above the actions bar) and open the color/size popover upward. * Fix markup text re-editing alignment, doubling, and selection bounds - Hide the text shape being re-edited from the canvas so the live input isn't doubled by its committed render. - Drop the input's padding/border and use the same font as the canvas so the typing preview sits exactly where the text renders (no shift); field-sizing keeps the box hugging the text. - Estimate text width with full-width (CJK) awareness so the selection box and hit bounds track Japanese text instead of clipping it. - Move the font-size control out of the color popover into its own toolbar button showing the current size. * Unify markup text selection and edit framing Draw the selection box on the canvas even while a text shape is being re-edited (the shape stays hidden so it isn't doubled), and strip the box off the text input so it renders only the editable glyphs. The frame is now identical whether selecting or editing, and the text overlays the same position with the same font, so there's no shift between the two. * Align markup text-edit baseline with the committed render Force the text input's line-height to 1 and zero its height/padding so its top edge matches the canvas textBaseline:'top' render — the editing text no longer sits slightly lower than the committed/selected text. * Address review: markup hit-testing, clear reset, and async guards - Hit-test highlights against their rendered (4x) thickness so the select tool doesn't miss large highlight strokes. - Reset transient editor state (pending text, selection, in-progress drag) on Clear all so a pending input blur can't re-add text and no stale selection lingers over the emptied canvas. - Guard markup completion with the capture token so a stale onDeliver can't reset or error a session the user already cancelled or restarted. - Skip the grab-element keyboard shortcut while the markup overlay is open, matching the already-disabled grab toolbar buttons. - Localize the markup error messages and surface them via a toast (the controller's error was otherwise never shown). * Trim browser markup to a draw-only tool and fix two delivery bugs Reshape the screenshot-markup overlay to match its actual use — a throwaway scribble the user copies once into their agent — and drop the re-editable vector-editor machinery that made it an outlier in scope. - Remove the select/move/restyle/re-edit subsystem: hit-testing, per-shape document edits, drag preview, selection frame, double-click text re-edit, Delete-to-remove, and the CJK text-width heuristic / multi-line text plumbing that only fed selection bounds. Keep pen, highlighter, arrow, rect, ellipse, and text with color/thickness/font-size and undo/redo/clear. - Fix silent data loss on HiDPI: the compositor budgeted only bytes, so a large Retina viewport could produce a PNG under the byte limit but over the clipboard handler's pixel ceiling, which the handler drops silently while the UI still reported success. Composite now scales down to fit maxPixels too. - Fix a stranded UI after a capture failure: the controller stayed "active" with no overlay and an inert Escape. It now returns to idle (capture failure) or back to the drawing surface (compose failure) after the error toast. Net ~660 fewer lines and 3 fewer modules; delivery (freeze → composite PNG → existing clipboard-image paste) is unchanged. Unit tests updated. * Add a one-time discovery hint for the browser Draw button Highlight the new screenshot-markup Draw button the first time it's usable so users — including existing ones — notice the tool. A localStorage-gated, show-once tooltip + highlight ring, independent of the browser contextual tour (which is capped at three steps and only shows to users who haven't seen it). - New use-markup-draw-hint hook: opens once per install when the button is enabled and idle, auto-dismisses, and dismisses on click; skips cleanly when storage is unavailable. - MarkupDrawButton forces its tooltip open with the discovery copy and rings the button while the hint shows. - Localize the hint (drawHint) across en/es/ja/ko/zh; normalize the markup key order alphabetically. * Align the markup copy button label with "Copy Screenshot" The browser pane's two image-copy actions read inconsistently in English — "Copy Screenshot" (the grab flow) vs "Copy markup" (the overlay). Match the new markup button to the existing one: "Copy Markup". English only; other locales already use their own sentence-case convention consistently, so they're unchanged. * Drop the markup Draw hint auto-timeout; dismiss on act only The one-time Draw-button hint no longer auto-dismisses on a timer the user might miss. It now stays open until the user clicks Draw or the button stops being usable (grab started, markup open, or a blank tab), which also fixes a case where losing eligibility mid-hint left the forced-open tooltip stuck over a disabled button. * Optimize markup rendering and harden compose delivery Performance: - Rasterize committed shapes once into an offscreen layer; per-frame paint blits that layer and draws only the in-progress shape, so a fast pointermove stream never re-strokes the whole scene. - Encode the composite PNG off the main thread via canvas.toBlob (a large synchronous toDataURL froze the renderer), with a rAF yield so the "composing" UI paints first. Correctness / review fixes: - Don't overwrite the clipboard after cancel: re-check the capture token between the (now async) compose and onDeliver, closing the window where Escape during composing still delivered the markup. - Keep the offscreen layer and visible canvas in lockstep by threading one measured devicePixelRatio through both paints, and re-measure on window resize so a monitor move (dpr change without a CSS-box change) repaints at the new scale. Blit stretched to the canvas box so a transient mismatch never clips. - Drop the stale "or JPEG fallback" note; compose is PNG-only. * Fix test fixtures for markup pixel-budget clamping to use a truly oversi The old 3840×2160×dpr2 fixture (33.2M px) was already under the 32×1024×1024 (~33.55M) ceiling, so the "never over budget" assertions weren't exercising the clamp path. Switch to 4000×3000×dpr2 (48M px), which genuinely exceeds the ceiling. --------- Co-authored-by: Jinjing <6427696+AmethystLiang@users.noreply.github.com> --- .../components/browser-pane/BrowserPane.tsx | 90 +++++- .../browser-pane/markup/MarkupDrawButton.tsx | 63 ++++ .../browser-pane/markup/MarkupOverlay.tsx | 159 ++++++++++ .../browser-pane/markup/MarkupToolbar.tsx | 293 ++++++++++++++++++ .../browser-pane/markup/markup-base-image.ts | 78 +++++ .../markup/markup-canvas-render.ts | 88 ++++++ .../markup/markup-clipboard-delivery.ts | 21 ++ .../markup/markup-drawing-model.test.ts | 166 ++++++++++ .../markup/markup-drawing-model.ts | 213 +++++++++++++ .../markup/markup-screenshot-compose.test.ts | 75 +++++ .../markup/markup-screenshot-compose.ts | 201 ++++++++++++ .../markup/markup-shape-render.test.ts | 153 +++++++++ .../markup/markup-shape-render.ts | 144 +++++++++ .../markup/use-markup-draw-hint.ts | 48 +++ .../browser-pane/markup/useMarkupEditor.ts | 179 +++++++++++ .../markup/useMarkupKeyboardShortcuts.ts | 52 ++++ .../browser-pane/markup/useMarkupMode.ts | 172 ++++++++++ .../markup/useMarkupPointerHandlers.ts | 99 ++++++ src/renderer/src/i18n/locales/en.json | 28 ++ src/renderer/src/i18n/locales/es.json | 28 ++ src/renderer/src/i18n/locales/ja.json | 28 ++ src/renderer/src/i18n/locales/ko.json | 28 ++ src/renderer/src/i18n/locales/zh.json | 28 ++ 23 files changed, 2430 insertions(+), 4 deletions(-) create mode 100644 src/renderer/src/components/browser-pane/markup/MarkupDrawButton.tsx create mode 100644 src/renderer/src/components/browser-pane/markup/MarkupOverlay.tsx create mode 100644 src/renderer/src/components/browser-pane/markup/MarkupToolbar.tsx create mode 100644 src/renderer/src/components/browser-pane/markup/markup-base-image.ts create mode 100644 src/renderer/src/components/browser-pane/markup/markup-canvas-render.ts create mode 100644 src/renderer/src/components/browser-pane/markup/markup-clipboard-delivery.ts create mode 100644 src/renderer/src/components/browser-pane/markup/markup-drawing-model.test.ts create mode 100644 src/renderer/src/components/browser-pane/markup/markup-drawing-model.ts create mode 100644 src/renderer/src/components/browser-pane/markup/markup-screenshot-compose.test.ts create mode 100644 src/renderer/src/components/browser-pane/markup/markup-screenshot-compose.ts create mode 100644 src/renderer/src/components/browser-pane/markup/markup-shape-render.test.ts create mode 100644 src/renderer/src/components/browser-pane/markup/markup-shape-render.ts create mode 100644 src/renderer/src/components/browser-pane/markup/use-markup-draw-hint.ts create mode 100644 src/renderer/src/components/browser-pane/markup/useMarkupEditor.ts create mode 100644 src/renderer/src/components/browser-pane/markup/useMarkupKeyboardShortcuts.ts create mode 100644 src/renderer/src/components/browser-pane/markup/useMarkupMode.ts create mode 100644 src/renderer/src/components/browser-pane/markup/useMarkupPointerHandlers.ts diff --git a/src/renderer/src/components/browser-pane/BrowserPane.tsx b/src/renderer/src/components/browser-pane/BrowserPane.tsx index 9781ea94961..5a0cb9d0615 100644 --- a/src/renderer/src/components/browser-pane/BrowserPane.tsx +++ b/src/renderer/src/components/browser-pane/BrowserPane.tsx @@ -181,6 +181,10 @@ import { shouldPollChromiumErrorPage } from './chromium-error-page-polling' import { useContextualTour } from '@/components/contextual-tours/use-contextual-tour' import { translate } from '@/i18n/i18n' import { isBrowserPagePanePaintable } from './browser-page-paintability' +import { useMarkupMode, type MarkupCaptureContext } from './markup/useMarkupMode' +import { MarkupOverlay } from './markup/MarkupOverlay' +import { MarkupDrawButton } from './markup/MarkupDrawButton' +import { deliverMarkupToClipboard } from './markup/markup-clipboard-delivery' type BrowserTabPageState = Partial< Pick< @@ -2394,6 +2398,30 @@ function RemoteBrowserPagePane({ const remoteFrameStyle = useMemo(() => getRemoteBrowserFrameStyle(frameMetadata), [frameMetadata]) + // Why: markup works on remote panes by snapshotting the already-displayed + // screencast — no in-page injection needed, so it is enabled here even + // though element-grab annotations are not. + const markup = useMarkupMode({ + getCaptureContext: useCallback((): MarkupCaptureContext | null => { + const element = imageRef.current + const container = remoteViewportRef.current + if (!element || !container) { + return null + } + const rect = container.getBoundingClientRect() + if (rect.width <= 0 || rect.height <= 0) { + return null + } + return { + source: { kind: 'image', element }, + cssWidth: rect.width, + cssHeight: rect.height, + outputScale: window.devicePixelRatio || 1 + } + }, []), + onDeliver: deliverMarkupToClipboard + }) + return (
{contextMenu @@ -2600,12 +2628,26 @@ function RemoteBrowserPagePane({ )} + (markup.isActive ? markup.cancel() : void markup.start())} + disabled={!frameUrl} + active={markup.isActive} + className="h-7 w-7" + />
+ {markup.isActive && markup.baseImage ? ( + void markup.complete(input)} + onCancel={markup.cancel} + /> + ) : null} {frameUrl ? ( (null) const [findOpen, setFindOpen] = useState(false) const grab = useGrabMode(browserTab.id) + + const markup = useMarkupMode({ + getCaptureContext: useCallback((): MarkupCaptureContext | null => { + const webview = webviewRef.current + const container = containerRef.current + if (!webview || !container) { + return null + } + const rect = container.getBoundingClientRect() + if (rect.width <= 0 || rect.height <= 0) { + return null + } + return { + source: { kind: 'webview', webview }, + cssWidth: rect.width, + cssHeight: rect.height, + outputScale: window.devicePixelRatio || 1 + } + }, []), + onDeliver: deliverMarkupToClipboard + }) const [grabIntent, setGrabIntent] = useState('copy') const grabIntentRef = useRef(grabIntent) grabIntentRef.current = grabIntent @@ -4149,14 +4212,19 @@ function BrowserPagePane({ if (isEditableKeyboardTarget(e.target)) { return } - if (keybindingMatchesAction('browser.grabElement', e, shortcutPlatform, keybindings)) { + // Why: while the markup overlay is open, don't let the grab shortcut start + // the in-guest picker behind it — matching the disabled grab toolbar buttons. + if ( + !markup.isActive && + keybindingMatchesAction('browser.grabElement', e, shortcutPlatform, keybindings) + ) { e.preventDefault() startGrabIntent('copy') } } window.addEventListener('keydown', handleKeyDown) return () => window.removeEventListener('keydown', handleKeyDown) - }, [isActive, keybindings, startGrabIntent]) + }, [isActive, keybindings, markup.isActive, startGrabIntent]) useEffect(() => { if (!isActive) { @@ -4989,7 +5057,7 @@ function BrowserPagePane({ 'bg-foreground/80 text-background hover:bg-foreground/90' )} onClick={() => startGrabIntent('copy')} - disabled={isBlankTab} + disabled={isBlankTab || markup.isActive} aria-label={translate( 'auto.components.browser.pane.BrowserPane.fdfc7fe0ef', 'Grab page element' @@ -5026,7 +5094,7 @@ function BrowserPagePane({ 'bg-foreground/80 text-background hover:bg-foreground/90' )} onClick={() => startGrabIntent('annotate')} - disabled={isBlankTab} + disabled={isBlankTab || markup.isActive} aria-label={translate( 'auto.components.browser.pane.BrowserPane.fc9be38f6f', 'Annotate page element' @@ -5050,6 +5118,12 @@ function BrowserPagePane({ + (markup.isActive ? markup.cancel() : void markup.start())} + disabled={isBlankTab || grab.state !== 'idle'} + active={markup.isActive} + /> + + + + + {hintOpen + ? translate( + 'auto.components.browser-pane.markup.drawHint', + 'New — draw on the page, then copy it to your agent' + ) + : label} + + + ) +} diff --git a/src/renderer/src/components/browser-pane/markup/MarkupOverlay.tsx b/src/renderer/src/components/browser-pane/markup/MarkupOverlay.tsx new file mode 100644 index 00000000000..28b8c7ef167 --- /dev/null +++ b/src/renderer/src/components/browser-pane/markup/MarkupOverlay.tsx @@ -0,0 +1,159 @@ +import React, { useCallback, useRef, useState } from 'react' +import { Check, X } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { cn } from '@/lib/utils' +import { translate } from '@/i18n/i18n' +import { MarkupToolbar } from './MarkupToolbar' +import type { MarkupBaseImage } from './markup-base-image' +import type { MarkupShape } from './markup-drawing-model' +import { TEXT_FONT_FAMILY } from './markup-shape-render' +import { useMarkupEditor } from './useMarkupEditor' + +export type MarkupOverlayProps = { + baseImage: MarkupBaseImage + busy: boolean + onComplete: (input: { imageElement: HTMLImageElement; shapes: MarkupShape[] }) => void + onCancel: () => void +} + +export function MarkupOverlay({ + baseImage, + busy, + onComplete, + onCancel +}: MarkupOverlayProps): React.JSX.Element { + const baseImgRef = useRef(null) + const [baseLoaded, setBaseLoaded] = useState(false) + const editor = useMarkupEditor(busy, onCancel) + const { pendingText } = editor + + const handleDone = useCallback(() => { + const imageElement = baseImgRef.current + if (!imageElement || !baseLoaded) { + return + } + onComplete({ imageElement, shapes: editor.shapes }) + }, [baseLoaded, editor.shapes, onComplete]) + + return ( +
+ setBaseLoaded(true)} + onError={() => { + // Why: the backdrop is a self-generated data URL, so a decode failure is + // unexpected — but never trap the user with a permanently-disabled Done. + console.error('markup: base screenshot failed to load') + onCancel() + }} + className="pointer-events-none absolute inset-0 block h-full w-full select-none" + style={{ objectFit: 'fill' }} + /> + + + {pendingText ? ( + event.stopPropagation()} + onBlur={(event) => editor.commitPendingText(event.target.value)} + onKeyDown={(event) => { + // Why: keep keystrokes local — without this the browser pane's global + // key handlers can swallow typing before it reaches the input. + event.stopPropagation() + // Why: during IME composition (e.g. Japanese conversion), Enter + // confirms the candidate — it must NOT also commit the annotation. + if (event.key === 'Enter' && !event.nativeEvent.isComposing) { + event.preventDefault() + editor.commitPendingText(event.currentTarget.value) + } else if (event.key === 'Escape') { + event.preventDefault() + editor.cancelPendingText() + } + }} + // Why: the input renders ONLY the editable glyphs (no box of its own) — + // the selection frame is drawn on the canvas so it's identical whether + // selecting or editing. No padding/border + the same font as the canvas + // so the text sits exactly where it commits. field-sizing hugs the text + // so clicks outside it still reach the canvas. + className="absolute z-30 min-w-0 border-0 bg-transparent p-0 leading-none outline-none [field-sizing:content]" + style={{ + left: pendingText.x, + top: pendingText.y, + color: editor.color, + fontSize: editor.fontSize, + fontWeight: 600, + fontFamily: TEXT_FONT_FAMILY, + // Why: force line-height to 1 (the UA default ~1.2 pushes the glyph + // down) and zero the height so the text top aligns with the canvas's + // textBaseline:'top' render — otherwise editing text sits slightly low. + lineHeight: 1, + height: '1em', + boxSizing: 'content-box', + padding: 0, + textShadow: + editor.color.toLowerCase() === '#ffffff' + ? '0 0 3px rgba(0,0,0,0.7)' + : '0 0 3px rgba(255,255,255,0.9), 0 0 2px rgba(255,255,255,0.9)' + }} + /> + ) : null} + +
+
+ +
+
+ + {translate( + 'auto.components.browser-pane.markup.hint', + 'Draw on the page, then copy the markup to paste into your agent.' + )} + + + +
+
+
+ ) +} diff --git a/src/renderer/src/components/browser-pane/markup/MarkupToolbar.tsx b/src/renderer/src/components/browser-pane/markup/MarkupToolbar.tsx new file mode 100644 index 00000000000..ee616b0ce28 --- /dev/null +++ b/src/renderer/src/components/browser-pane/markup/MarkupToolbar.tsx @@ -0,0 +1,293 @@ +import React from 'react' +import { + ArrowUpRight, + Circle, + Highlighter, + Pencil, + Redo2, + Square, + Trash2, + Type, + Undo2 +} from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/components/ui/tooltip' +import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/popover' +import { cn } from '@/lib/utils' +import { translate } from '@/i18n/i18n' +import { + MARKUP_COLORS, + MARKUP_FONT_SIZES, + MARKUP_WIDTHS, + type MarkupTool +} from './markup-drawing-model' + +type ToolItem = { + kind: MarkupTool + icon: React.ComponentType<{ className?: string }> + label: string +} + +function toolItems(): ToolItem[] { + return [ + { + kind: 'pen', + icon: Pencil, + label: translate('auto.components.browser-pane.markup.tool.pen', 'Pen') + }, + { + kind: 'highlight', + icon: Highlighter, + label: translate('auto.components.browser-pane.markup.tool.highlight', 'Highlighter') + }, + { + kind: 'arrow', + icon: ArrowUpRight, + label: translate('auto.components.browser-pane.markup.tool.arrow', 'Arrow') + }, + { + kind: 'rect', + icon: Square, + label: translate('auto.components.browser-pane.markup.tool.rect', 'Rectangle') + }, + { + kind: 'ellipse', + icon: Circle, + label: translate('auto.components.browser-pane.markup.tool.ellipse', 'Ellipse') + }, + { + kind: 'text', + icon: Type, + label: translate('auto.components.browser-pane.markup.tool.text', 'Text') + } + ] +} + +export type MarkupToolbarProps = { + tool: MarkupTool + onToolChange: (tool: MarkupTool) => void + color: string + onColorChange: (color: string) => void + width: number + onWidthChange: (width: number) => void + fontSize: number + onFontSizeChange: (fontSize: number) => void + canUndo: boolean + canRedo: boolean + onUndo: () => void + onRedo: () => void + onClear: () => void +} + +export const MarkupToolbar = React.memo(function MarkupToolbar({ + tool, + onToolChange, + color, + onColorChange, + width, + onWidthChange, + fontSize, + onFontSizeChange, + canUndo, + canRedo, + onUndo, + onRedo, + onClear +}: MarkupToolbarProps) { + return ( + +
+ {toolItems().map((item) => ( + onToolChange(item.kind)} + > + + + ))} + + + + + + + + + + + + {translate('auto.components.browser-pane.markup.style', 'Color and thickness')} + + + +
+ {MARKUP_COLORS.map((swatch) => ( +
+
+ {MARKUP_WIDTHS.map((option) => ( + + ))} +
+ + + + + + + + + + + + {translate('auto.components.browser-pane.markup.fontSize', 'Font size')} + + + +
+ {MARKUP_FONT_SIZES.map((size) => ( + + ))} +
+ + + + + + + + + + + + + + +
+ + ) +}) + +function Divider(): React.JSX.Element { + return +} + +type IconButtonProps = { + label: string + active?: boolean + disabled?: boolean + onClick: () => void + children: React.ReactNode +} + +function IconButton({ + label, + active, + disabled, + onClick, + children +}: IconButtonProps): React.JSX.Element { + return ( + + + + + + {label} + + + ) +} diff --git a/src/renderer/src/components/browser-pane/markup/markup-base-image.ts b/src/renderer/src/components/browser-pane/markup/markup-base-image.ts new file mode 100644 index 00000000000..1841c6ceed8 --- /dev/null +++ b/src/renderer/src/components/browser-pane/markup/markup-base-image.ts @@ -0,0 +1,78 @@ +// Captures the frozen base screenshot the user draws on. Renderer-only and +// environment-aware: local Electron webviews expose capturePage(); remote panes +// already display the streamed frame as an we can snapshot. Keeping this +// out of the main process means the markup feature works identically for local +// and remote/SSH browser panes. + +// A frozen snapshot as a data URL plus its intrinsic pixel size. The overlay +// shows it as an backdrop and reuses that same loaded element as the +// compositor's base layer. +export type MarkupBaseImage = { + dataUrl: string + width: number + height: number +} + +export type MarkupCaptureSource = + | { kind: 'webview'; webview: Electron.WebviewTag } + | { kind: 'image'; element: HTMLImageElement } + +export async function captureMarkupBaseImage( + source: MarkupCaptureSource +): Promise { + if (source.kind === 'webview') { + return captureFromWebview(source.webview) + } + return captureFromImage(source.element) +} + +async function captureFromWebview(webview: Electron.WebviewTag): Promise { + const native = await webview.capturePage() + if (native.isEmpty()) { + throw new Error('markup: webview capturePage returned an empty image') + } + const size = native.getSize() + // Why: capturePage() can include an alpha channel where the page has no opaque + // background, which would let the live webview ghost through the frozen + // backdrop. Flatten onto white (the browser's default canvas color) so the + // base image is fully opaque. + const image = await loadImage(native.toDataURL()) + return { dataUrl: flattenToOpaquePng(image, size.width, size.height), ...size } +} + +function captureFromImage(element: HTMLImageElement): MarkupBaseImage { + const width = element.naturalWidth || element.width + const height = element.naturalHeight || element.height + // Why: drawImage() silently no-ops on an undecoded/detached source, yielding a + // blank PNG. Fail fast instead so the caller surfaces an error. + if (!element.complete || !element.isConnected || width <= 0 || height <= 0) { + throw new Error('markup: remote frame image is not ready') + } + // Why: snapshot the live frame into a detached canvas (opaque white backdrop) + // so later screencast frames can't change the base image and transparent + // regions don't ghost the live stream through. + return { dataUrl: flattenToOpaquePng(element, width, height), width, height } +} + +function flattenToOpaquePng(image: CanvasImageSource, width: number, height: number): string { + const canvas = document.createElement('canvas') + canvas.width = width + canvas.height = height + const ctx = canvas.getContext('2d') + if (!ctx) { + throw new Error('markup: 2d context unavailable for base image') + } + ctx.fillStyle = '#ffffff' + ctx.fillRect(0, 0, width, height) + ctx.drawImage(image, 0, 0, width, height) + return canvas.toDataURL('image/png') +} + +function loadImage(src: string): Promise { + return new Promise((resolve, reject) => { + const image = new Image() + image.onload = () => resolve(image) + image.onerror = () => reject(new Error('markup: failed to decode captured screenshot')) + image.src = src + }) +} diff --git a/src/renderer/src/components/browser-pane/markup/markup-canvas-render.ts b/src/renderer/src/components/browser-pane/markup/markup-canvas-render.ts new file mode 100644 index 00000000000..77d5d4b1255 --- /dev/null +++ b/src/renderer/src/components/browser-pane/markup/markup-canvas-render.ts @@ -0,0 +1,88 @@ +// Renders the live markup scene into the overlay canvas at device resolution. +// Committed shapes are rasterized once into an offscreen layer; the per-frame +// paint blits that layer and draws only the in-progress shape on top, so a fast +// pointermove stream never re-strokes the whole scene. Kept separate from the +// editor hook so the draw composition stays focused. +// +// Why the caller passes dpr: the committed layer and the visible canvas are sized +// in two separate paints. Reading window.devicePixelRatio independently in each +// could size them differently if the ratio changed between them (monitor move), +// misaligning the 1:1 blit. Taking one measured dpr keeps them in lockstep. + +import { clampMarkupScale } from './markup-screenshot-compose' +import { drawShapes } from './markup-shape-render' +import type { MarkupShape } from './markup-drawing-model' + +function sceneDevicePixels( + cssWidth: number, + cssHeight: number, + dpr: number +): { scale: number; width: number; height: number } { + // Why: clamp identically to the compositor so the live preview and the exported + // PNG render shapes at the same scale. + const scale = clampMarkupScale(dpr) + return { + scale, + width: Math.max(1, Math.round(cssWidth * scale)), + height: Math.max(1, Math.round(cssHeight * scale)) + } +} + +// Rasterizes the committed shapes into an offscreen layer. Called only when the +// committed shape list, the size, or the dpr changes — never on the pointermove +// path. +export function renderCommittedLayer( + layer: HTMLCanvasElement, + shapes: readonly MarkupShape[], + cssWidth: number, + cssHeight: number, + dpr: number +): void { + const { scale, width, height } = sceneDevicePixels(cssWidth, cssHeight, dpr) + if (layer.width !== width || layer.height !== height) { + layer.width = width + layer.height = height + } + const ctx = layer.getContext('2d') + if (!ctx) { + return + } + ctx.setTransform(scale, 0, 0, scale, 0, 0) + ctx.clearRect(0, 0, cssWidth, cssHeight) + drawShapes(ctx, shapes) +} + +// Blits the cached committed layer plus the in-progress shape onto the visible +// canvas. Cheap enough for every frame: one image blit + at most one shape. +export function blitMarkupScene( + canvas: HTMLCanvasElement, + committedLayer: HTMLCanvasElement, + inProgress: MarkupShape | null, + cssWidth: number, + cssHeight: number, + dpr: number +): void { + const ctx = canvas.getContext('2d') + if (!ctx) { + return + } + const { scale, width, height } = sceneDevicePixels(cssWidth, cssHeight, dpr) + if (canvas.width !== width || canvas.height !== height) { + canvas.width = width + canvas.height = height + } + // Why: blit the committed layer under the identity transform, stretched to the + // canvas's device box. It's 1:1 when the layer matches (the common case, since + // both are sized from the same dpr) and degrades gracefully — never clips or + // offsets — if a layer render briefly lags a dpr change. + ctx.setTransform(1, 0, 0, 1, 0, 0) + ctx.clearRect(0, 0, width, height) + if (committedLayer.width > 0 && committedLayer.height > 0) { + ctx.drawImage(committedLayer, 0, 0, width, height) + } + // Then switch to the dpr transform to draw the in-progress shape in CSS coords. + ctx.setTransform(scale, 0, 0, scale, 0, 0) + if (inProgress) { + drawShapes(ctx, [inProgress]) + } +} diff --git a/src/renderer/src/components/browser-pane/markup/markup-clipboard-delivery.ts b/src/renderer/src/components/browser-pane/markup/markup-clipboard-delivery.ts new file mode 100644 index 00000000000..f242bbd960b --- /dev/null +++ b/src/renderer/src/components/browser-pane/markup/markup-clipboard-delivery.ts @@ -0,0 +1,21 @@ +import { toast } from 'sonner' +import { translate } from '@/i18n/i18n' +import type { MarkupComposeResult } from './markup-screenshot-compose' + +// Delivery for v1: copy the composited markup PNG to the clipboard, mirroring the +// browser grab "copy" flow. The user pastes it into their agent terminal, where +// Orca's existing clipboard-screenshot paste writes the image to a temp file (on +// the correct host for local or remote/SSH agents) and hands the path to the TUI. +// This reuses proven, environment-agnostic machinery instead of re-plumbing a +// direct send. +export async function deliverMarkupToClipboard(result: MarkupComposeResult): Promise { + await window.api.ui.writeClipboardImage(result.dataUrl) + const isMac = navigator.userAgent.includes('Mac') + toast.success( + translate( + 'auto.components.browser-pane.markup.copiedToast', + 'Markup copied — paste it into your agent ({{value0}})', + { value0: isMac ? '⌘V' : 'Ctrl+V' } + ) + ) +} diff --git a/src/renderer/src/components/browser-pane/markup/markup-drawing-model.test.ts b/src/renderer/src/components/browser-pane/markup/markup-drawing-model.test.ts new file mode 100644 index 00000000000..7fec062ec01 --- /dev/null +++ b/src/renderer/src/components/browser-pane/markup/markup-drawing-model.test.ts @@ -0,0 +1,166 @@ +import { describe, expect, it } from 'vitest' +import { + arrowHeadGeometry, + canRedo, + canUndo, + clearShapes, + commitShape, + createMarkupDocument, + isEmptyDocument, + normalizeRect, + redoShape, + scaleShape, + setShapes, + undoShape, + type ArrowShape, + type MarkupShape, + type PenShape, + type TextShape +} from './markup-drawing-model' + +function pen(id: string): PenShape { + return { id, kind: 'pen', color: '#ef4444', width: 4, points: [{ x: 0, y: 0 }] } +} + +describe('markup document undo/redo', () => { + it('starts empty', () => { + const doc = createMarkupDocument() + expect(doc.shapes).toHaveLength(0) + expect(isEmptyDocument(doc)).toBe(true) + expect(canUndo(doc)).toBe(false) + expect(canRedo(doc)).toBe(false) + }) + + it('commits shapes and clears the redo stack', () => { + let doc = createMarkupDocument() + doc = commitShape(doc, pen('a')) + doc = commitShape(doc, pen('b')) + expect(doc.shapes.map((s) => s.id)).toEqual(['a', 'b']) + expect(canUndo(doc)).toBe(true) + expect(canRedo(doc)).toBe(false) + }) + + it('undoes and redoes in order', () => { + let doc = createMarkupDocument() + doc = commitShape(doc, pen('a')) + doc = commitShape(doc, pen('b')) + doc = undoShape(doc) + expect(doc.shapes.map((s) => s.id)).toEqual(['a']) + expect(canRedo(doc)).toBe(true) + doc = redoShape(doc) + expect(doc.shapes.map((s) => s.id)).toEqual(['a', 'b']) + expect(canRedo(doc)).toBe(false) + }) + + it('forks history: committing after undo discards the redo stack', () => { + let doc = createMarkupDocument() + doc = commitShape(doc, pen('a')) + doc = commitShape(doc, pen('b')) + doc = undoShape(doc) + doc = commitShape(doc, pen('c')) + expect(doc.shapes.map((s) => s.id)).toEqual(['a', 'c']) + expect(canRedo(doc)).toBe(false) + }) + + it('undo/redo on empty document are no-ops returning the same reference', () => { + const doc = createMarkupDocument() + expect(undoShape(doc)).toBe(doc) + expect(redoShape(doc)).toBe(doc) + }) + + it('clears shapes, and is a no-op (same reference) when already empty', () => { + const empty = createMarkupDocument() + expect(clearShapes(empty)).toBe(empty) + let doc = commitShape(empty, pen('a')) + doc = clearShapes(doc) + expect(doc.shapes).toHaveLength(0) + expect(canRedo(doc)).toBe(false) + }) + + it('undoes an edit (not just an add) via whole-list snapshots', () => { + let doc = commitShape(createMarkupDocument(), pen('a')) + doc = setShapes(doc, []) // delete everything + expect(doc.shapes).toHaveLength(0) + doc = undoShape(doc) + expect(doc.shapes.map((s) => s.id)).toEqual(['a']) + }) +}) + +describe('markup geometry', () => { + it('normalizes rectangles from any drag direction', () => { + expect(normalizeRect({ x: 10, y: 20 }, { x: 4, y: 50 })).toEqual({ + x: 4, + y: 20, + width: 6, + height: 30 + }) + }) + + it('scales pen points and width', () => { + const shape: PenShape = { + id: 'a', + kind: 'pen', + color: '#fff', + width: 4, + points: [ + { x: 2, y: 3 }, + { x: 4, y: 5 } + ] + } + const scaled = scaleShape(shape, 2) as PenShape + expect(scaled.width).toBe(8) + expect(scaled.points).toEqual([ + { x: 4, y: 6 }, + { x: 8, y: 10 } + ]) + }) + + it('scales rect/arrow endpoints and text position + font size', () => { + const arrow: ArrowShape = { + id: 'a', + kind: 'arrow', + color: '#fff', + width: 3, + from: { x: 1, y: 1 }, + to: { x: 3, y: 5 } + } + const scaledArrow = scaleShape(arrow, 3) as ArrowShape + expect(scaledArrow.from).toEqual({ x: 3, y: 3 }) + expect(scaledArrow.to).toEqual({ x: 9, y: 15 }) + expect(scaledArrow.width).toBe(9) + + const text: TextShape = { + id: 't', + kind: 'text', + color: '#fff', + at: { x: 10, y: 20 }, + text: 'fix', + fontSize: 18 + } + const scaledText = scaleShape(text, 2) as TextShape + expect(scaledText.at).toEqual({ x: 20, y: 40 }) + expect(scaledText.fontSize).toBe(36) + }) + + it('returns null arrowhead for a zero-length segment', () => { + expect(arrowHeadGeometry({ x: 5, y: 5 }, { x: 5, y: 5 }, 4)).toBeNull() + }) + + it('produces a symmetric arrowhead pointing at the tip', () => { + const head = arrowHeadGeometry({ x: 0, y: 0 }, { x: 10, y: 0 }, 4) + expect(head).not.toBeNull() + if (!head) { + return + } + expect(head.tip).toEqual({ x: 10, y: 0 }) + // Wings sit behind the tip (smaller x) and mirror across the horizontal axis. + expect(head.left.x).toBeLessThan(10) + expect(head.right.x).toBeLessThan(10) + expect(head.left.y).toBeCloseTo(-head.right.y, 6) + }) +}) + +// Why: exhaustiveness guard — if a new MarkupShape kind is added without a +// scaleShape branch, this fails to type-check at the cast site below. +const _exhaustive: MarkupShape['kind'][] = ['pen', 'highlight', 'arrow', 'rect', 'ellipse', 'text'] +void _exhaustive diff --git a/src/renderer/src/components/browser-pane/markup/markup-drawing-model.ts b/src/renderer/src/components/browser-pane/markup/markup-drawing-model.ts new file mode 100644 index 00000000000..790065bf60a --- /dev/null +++ b/src/renderer/src/components/browser-pane/markup/markup-drawing-model.ts @@ -0,0 +1,213 @@ +// Pure drawing model for browser screenshot markup. No DOM access — shapes are +// vector objects in CSS-viewport coordinates so the same data drives both the +// live canvas overlay and the final composited PNG. Keeping this canvas-free +// makes the undo/redo and geometry logic unit-testable. + +export type MarkupToolKind = 'pen' | 'highlight' | 'arrow' | 'rect' | 'ellipse' | 'text' + +// Toolbar selection. Draw-only: markup is a throwaway scribble the user copies +// once, so there is no select/move/restyle cursor. +export type MarkupTool = MarkupToolKind + +export type MarkupPoint = { x: number; y: number } + +type MarkupShapeBase = { id: string; color: string } + +export type PenShape = MarkupShapeBase & { kind: 'pen'; points: MarkupPoint[]; width: number } +export type HighlightShape = MarkupShapeBase & { + kind: 'highlight' + points: MarkupPoint[] + width: number +} +export type ArrowShape = MarkupShapeBase & { + kind: 'arrow' + from: MarkupPoint + to: MarkupPoint + width: number +} +export type RectShape = MarkupShapeBase & { + kind: 'rect' + from: MarkupPoint + to: MarkupPoint + width: number +} +export type EllipseShape = MarkupShapeBase & { + kind: 'ellipse' + from: MarkupPoint + to: MarkupPoint + width: number +} +export type TextShape = MarkupShapeBase & { + kind: 'text' + at: MarkupPoint + text: string + fontSize: number +} + +export type MarkupShape = + | PenShape + | HighlightShape + | ArrowShape + | RectShape + | EllipseShape + | TextShape + +// Concrete ink colors (baked into the exported PNG, so these are content colors, +// not UI-chrome tokens). Ordered for a compact swatch row. +export const MARKUP_COLORS = [ + '#ef4444', + '#f97316', + '#eab308', + '#22c55e', + '#3b82f6', + '#111827', + '#ffffff' +] as const +export const DEFAULT_MARKUP_COLOR: string = MARKUP_COLORS[0] + +export const MARKUP_WIDTHS = [2, 4, 8] as const +export const DEFAULT_MARKUP_WIDTH = 4 +export const MARKUP_FONT_SIZES = [14, 18, 24, 32, 48] as const +export const DEFAULT_MARKUP_FONT_SIZE = 18 + +// Highlight strokes are intentionally fat and translucent. +export const HIGHLIGHT_WIDTH_MULTIPLIER = 4 +export const HIGHLIGHT_ALPHA = 0.35 + +// ─── Document with undo/redo ──────────────────────────────────────────────── + +export type MarkupDocument = { + shapes: MarkupShape[] + /** Prior whole-list states (oldest first) for undo. */ + past: MarkupShape[][] + /** Undone whole-list states (newest first) for redo. */ + future: MarkupShape[][] +} + +export function createMarkupDocument(): MarkupDocument { + return { shapes: [], past: [], future: [] } +} + +// Replace the whole shape list as one undoable step. Snapshot-based so every +// edit — add, move, restyle, delete — is reversible, not just adding a shape. +export function setShapes(doc: MarkupDocument, shapes: MarkupShape[]): MarkupDocument { + return { shapes, past: [...doc.past, doc.shapes], future: [] } +} + +export function commitShape(doc: MarkupDocument, shape: MarkupShape): MarkupDocument { + return setShapes(doc, [...doc.shapes, shape]) +} + +export function undoShape(doc: MarkupDocument): MarkupDocument { + const prev = doc.past.at(-1) + if (!prev) { + return doc + } + return { shapes: prev, past: doc.past.slice(0, -1), future: [doc.shapes, ...doc.future] } +} + +export function redoShape(doc: MarkupDocument): MarkupDocument { + const next = doc.future.at(0) + if (!next) { + return doc + } + return { shapes: next, past: [...doc.past, doc.shapes], future: doc.future.slice(1) } +} + +export function clearShapes(doc: MarkupDocument): MarkupDocument { + return doc.shapes.length === 0 ? doc : setShapes(doc, []) +} + +export function canUndo(doc: MarkupDocument): boolean { + return doc.past.length > 0 +} + +export function canRedo(doc: MarkupDocument): boolean { + return doc.future.length > 0 +} + +export function isEmptyDocument(doc: MarkupDocument): boolean { + return doc.shapes.length === 0 +} + +// ─── Geometry helpers (shared by live overlay + compositor) ───────────────── + +export type NormalizedRect = { x: number; y: number; width: number; height: number } + +export function normalizeRect(from: MarkupPoint, to: MarkupPoint): NormalizedRect { + return { + x: Math.min(from.x, to.x), + y: Math.min(from.y, to.y), + width: Math.abs(to.x - from.x), + height: Math.abs(to.y - from.y) + } +} + +export function scalePoint(point: MarkupPoint, scale: number): MarkupPoint { + return { x: point.x * scale, y: point.y * scale } +} + +// Why: the compositor draws into a canvas sized to the base-image's physical +// pixels, while shapes are authored in CSS-viewport pixels. Scaling every shape +// by (imagePx / cssPx) keeps strokes aligned with the underlying screenshot. +export function scaleShape(shape: MarkupShape, scale: number): MarkupShape { + switch (shape.kind) { + case 'pen': + case 'highlight': + return { + ...shape, + width: shape.width * scale, + points: shape.points.map((point) => scalePoint(point, scale)) + } + case 'arrow': + case 'rect': + case 'ellipse': + return { + ...shape, + width: shape.width * scale, + from: scalePoint(shape.from, scale), + to: scalePoint(shape.to, scale) + } + case 'text': + return { + ...shape, + fontSize: shape.fontSize * scale, + at: scalePoint(shape.at, scale) + } + } +} + +export type ArrowHeadGeometry = { tip: MarkupPoint; left: MarkupPoint; right: MarkupPoint } + +const ARROW_HEAD_ANGLE = 0.45 + +export function arrowHeadLength(width: number): number { + return Math.max(10, width * 3.5) +} + +// Returns the two wing points of an arrowhead at `to`, or null when the segment +// has no direction (from === to) so callers can skip drawing a degenerate head. +export function arrowHeadGeometry( + from: MarkupPoint, + to: MarkupPoint, + width: number +): ArrowHeadGeometry | null { + const dx = to.x - from.x + const dy = to.y - from.y + if (dx === 0 && dy === 0) { + return null + } + const angle = Math.atan2(dy, dx) + const size = arrowHeadLength(width) + const leftAngle = angle + Math.PI - ARROW_HEAD_ANGLE + const rightAngle = angle + Math.PI + ARROW_HEAD_ANGLE + return { + tip: to, + left: { x: to.x + size * Math.cos(leftAngle), y: to.y + size * Math.sin(leftAngle) }, + right: { x: to.x + size * Math.cos(rightAngle), y: to.y + size * Math.sin(rightAngle) } + } +} + +export function highlightWidth(width: number): number { + return width * HIGHLIGHT_WIDTH_MULTIPLIER +} diff --git a/src/renderer/src/components/browser-pane/markup/markup-screenshot-compose.test.ts b/src/renderer/src/components/browser-pane/markup/markup-screenshot-compose.test.ts new file mode 100644 index 00000000000..40f540327b1 --- /dev/null +++ b/src/renderer/src/components/browser-pane/markup/markup-screenshot-compose.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from 'vitest' +import { + clampMarkupScale, + dataUrlByteLength, + effectiveMarkupScale, + markupCanvasSize, + MARKUP_DOWNSCALE_STEPS +} from './markup-screenshot-compose' + +describe('clampMarkupScale', () => { + it('keeps sane scales and clamps to [1, 4]', () => { + expect(clampMarkupScale(2)).toBe(2) + expect(clampMarkupScale(0.5)).toBe(1) + expect(clampMarkupScale(10)).toBe(4) + }) + + it('guards against non-finite / non-positive values', () => { + expect(clampMarkupScale(0)).toBe(1) + expect(clampMarkupScale(-3)).toBe(1) + expect(clampMarkupScale(Number.NaN)).toBe(1) + }) +}) + +describe('markupCanvasSize', () => { + it('sizes the output to the content box times the (clamped) scale', () => { + expect(markupCanvasSize(800, 600, 2)).toEqual({ width: 1600, height: 1200 }) + expect(markupCanvasSize(800, 600, 0.5)).toEqual({ width: 800, height: 600 }) + }) + + it('keeps the composite within the pixel budget (never over)', () => { + // A large HiDPI pane: 4000×3000 CSS × dpr 2 = 48M px > the ~33.55M (32×1024×1024) ceiling. + const maxPixels = 32 * 1024 * 1024 + const size = markupCanvasSize(4000, 3000, 2, maxPixels) + expect(size.width * size.height).toBeLessThanOrEqual(maxPixels) + // Aspect ratio is preserved (one uniform scale). + expect(size.width / size.height).toBeCloseTo(4000 / 3000, 3) + }) +}) + +describe('effectiveMarkupScale', () => { + it('returns the clamped scale when the composite fits the pixel budget', () => { + expect(effectiveMarkupScale(800, 600, 2, 32 * 1024 * 1024)).toBe(2) + }) + + it('reduces the scale so area stays within the pixel budget', () => { + const scale = effectiveMarkupScale(4000, 3000, 2, 32 * 1024 * 1024) + expect(scale).toBeLessThan(2) + expect(4000 * scale * (3000 * scale)).toBeLessThanOrEqual(32 * 1024 * 1024 + 1) + }) + + it('defaults to the clamped scale with no pixel budget', () => { + expect(effectiveMarkupScale(3840, 2160, 2)).toBe(2) + }) +}) + +describe('dataUrlByteLength', () => { + it('decodes the base64 payload size', () => { + // "hi" -> base64 "aGk=" (1 pad char) -> 2 bytes + expect(dataUrlByteLength('data:image/png;base64,aGk=')).toBe(2) + // "man" -> "bWFu" (no padding) -> 3 bytes + expect(dataUrlByteLength('data:image/png;base64,bWFu')).toBe(3) + }) + + it('returns 0 for a malformed data url', () => { + expect(dataUrlByteLength('not-a-data-url')).toBe(0) + }) +}) + +describe('compose budget plans', () => { + it('tries full size first, then shrinks (PNG only, descending)', () => { + expect(MARKUP_DOWNSCALE_STEPS[0]).toBe(1) + expect([...MARKUP_DOWNSCALE_STEPS]).toEqual([...MARKUP_DOWNSCALE_STEPS].sort((a, b) => b - a)) + expect(MARKUP_DOWNSCALE_STEPS.every((s) => s > 0 && s <= 1)).toBe(true) + }) +}) diff --git a/src/renderer/src/components/browser-pane/markup/markup-screenshot-compose.ts b/src/renderer/src/components/browser-pane/markup/markup-screenshot-compose.ts new file mode 100644 index 00000000000..ad2af0969b7 --- /dev/null +++ b/src/renderer/src/components/browser-pane/markup/markup-screenshot-compose.ts @@ -0,0 +1,201 @@ +// Composites the frozen base screenshot + markup shapes into a single PNG under +// the delivery byte and pixel budgets (PNG only — see MarkupComposeResult). +// +// WYSIWYG: the output canvas matches the on-screen content box (its CSS width × +// height) times an output scale for crispness. The base image is stretched to +// fill that box exactly as the user saw it (so a distorted objectFit:fill remote +// frame composites correctly), and shapes — authored in CSS coordinates — are +// scaled by the same single factor. One uniform scale keeps strokes aligned. + +import { scaleShape, type MarkupShape } from './markup-drawing-model' +import { drawShapes } from './markup-shape-render' + +export type MarkupComposeResult = { + dataUrl: string + // Why: PNG only — the clipboard:writeImage handler accepts a PNG data URL and + // silently drops anything else, so a JPEG fallback would "succeed" with an + // empty clipboard. + mimeType: 'image/png' + width: number + height: number + byteLength: number +} + +export type MarkupComposeInput = { + image: CanvasImageSource + /** On-screen size of the content box the canvas overlaid (CSS pixels). */ + displayCssWidth: number + displayCssHeight: number + /** Output resolution multiplier (typically devicePixelRatio). */ + outputScale: number + shapes: readonly MarkupShape[] + /** Hard byte ceiling for the encoded data URL payload. */ + maxBytes: number + /** Hard pixel (width×height) ceiling — the clipboard handler silently drops + * images over its own dimension limit, so the composite must fit it too. */ + maxPixels: number +} + +// ─── Pure helpers (unit-tested) ───────────────────────────────────────────── + +// Keep the output scale sane so a huge DPR or bogus value can't allocate an +// absurd canvas. +export function clampMarkupScale(scale: number): number { + if (!Number.isFinite(scale) || scale <= 0) { + return 1 + } + return Math.min(Math.max(scale, 1), 4) +} + +// The single uniform scale actually applied to the base image and every shape: +// the clamped output scale, reduced further if needed so the composite's pixel +// area stays within `maxPixels`. Reducing one uniform factor keeps strokes aligned. +export function effectiveMarkupScale( + displayCssWidth: number, + displayCssHeight: number, + outputScale: number, + maxPixels: number = Number.POSITIVE_INFINITY +): number { + const scale = clampMarkupScale(outputScale) + const area = displayCssWidth * displayCssHeight + if (area <= 0 || !Number.isFinite(maxPixels) || maxPixels <= 0) { + return scale + } + return Math.min(scale, Math.sqrt(maxPixels / area)) +} + +export function markupCanvasSize( + displayCssWidth: number, + displayCssHeight: number, + outputScale: number, + maxPixels: number = Number.POSITIVE_INFINITY +): { width: number; height: number } { + const scale = effectiveMarkupScale(displayCssWidth, displayCssHeight, outputScale, maxPixels) + // Why: floor (not round) so a rounded-up dimension can't nudge the area back + // over maxPixels at the budget boundary. + return { + width: Math.max(1, Math.floor(displayCssWidth * scale)), + height: Math.max(1, Math.floor(displayCssHeight * scale)) + } +} + +// Bytes represented by a `data:...;base64,` URL's payload. +export function dataUrlByteLength(dataUrl: string): number { + const commaIndex = dataUrl.indexOf(',') + if (commaIndex === -1) { + return 0 + } + const payload = dataUrl.slice(commaIndex + 1) + const padding = payload.endsWith('==') ? 2 : payload.endsWith('=') ? 1 : 0 + return Math.max(0, Math.floor((payload.length * 3) / 4) - padding) +} + +// Progressive raster sizes (full, then shrink) tried to fit the byte budget. +// PNG at every step — never JPEG (see MarkupComposeResult). +export const MARKUP_DOWNSCALE_STEPS = [1, 0.85, 0.7, 0.55, 0.4, 0.3] as const + +// ─── Canvas raster (thin) ─────────────────────────────────────────────────── + +function createCanvas(width: number, height: number): HTMLCanvasElement { + const canvas = document.createElement('canvas') + canvas.width = Math.max(1, Math.round(width)) + canvas.height = Math.max(1, Math.round(height)) + return canvas +} + +function renderComposite(input: MarkupComposeInput): HTMLCanvasElement { + const scale = effectiveMarkupScale( + input.displayCssWidth, + input.displayCssHeight, + input.outputScale, + input.maxPixels + ) + const size = markupCanvasSize( + input.displayCssWidth, + input.displayCssHeight, + input.outputScale, + input.maxPixels + ) + const canvas = createCanvas(size.width, size.height) + const ctx = canvas.getContext('2d') + if (!ctx) { + throw new Error('markup compose: 2d context unavailable') + } + ctx.drawImage(input.image, 0, 0, size.width, size.height) + for (const shape of input.shapes) { + drawShapes(ctx, [scaleShape(shape, scale)]) + } + return canvas +} + +function downscaleCanvas(source: HTMLCanvasElement, factor: number): HTMLCanvasElement { + if (factor >= 1) { + return source + } + const next = createCanvas(source.width * factor, source.height * factor) + const ctx = next.getContext('2d') + if (!ctx) { + return source + } + ctx.drawImage(source, 0, 0, next.width, next.height) + return next +} + +// Encodes a canvas to a PNG data URL. Prefers toBlob because Chromium encodes it +// off the main thread — a multi-megapixel toData('image/png') is synchronous and +// freezes the renderer. Falls back to the sync encode if toBlob yields no blob. +function canvasToPngDataUrl(canvas: HTMLCanvasElement): Promise { + return new Promise((resolve, reject) => { + canvas.toBlob((blob) => { + if (!blob) { + try { + resolve(canvas.toDataURL('image/png')) + } catch (error) { + reject(error instanceof Error ? error : new Error('markup compose: toDataURL failed')) + } + return + } + const reader = new FileReader() + reader.onload = () => resolve(reader.result as string) + reader.onerror = () => reject(reader.error ?? new Error('markup compose: blob read failed')) + reader.readAsDataURL(blob) + }, 'image/png') + }) +} + +// Encodes a PNG within budget: full size, then progressively smaller. Always +// returns a PNG (best effort at the smallest step) so the clipboard handler +// accepts it and delivery never silently drops. Async so the per-step PNG encode +// runs off the main thread instead of blocking the renderer. +export async function composeMarkupDataUrl( + input: MarkupComposeInput +): Promise { + const composite = renderComposite(input) + + let smallest: MarkupComposeResult | null = null + for (const step of MARKUP_DOWNSCALE_STEPS) { + const canvas = downscaleCanvas(composite, step) + const dataUrl = await canvasToPngDataUrl(canvas) + const byteLength = dataUrlByteLength(dataUrl) + const result: MarkupComposeResult = { + dataUrl, + mimeType: 'image/png', + width: canvas.width, + height: canvas.height, + byteLength + } + if (byteLength <= input.maxBytes) { + return result + } + smallest = result + } + + // Why: still over the byte budget at the smallest step — return it anyway. It's + // a PNG within the pixel budget (every step is), so the clipboard handler + // accepts it; a slightly large attachment beats dropping the user's markup. + if (!smallest) { + // Unreachable: MARKUP_DOWNSCALE_STEPS is non-empty, so the loop always sets it. + throw new Error('markup compose: no downscale step produced output') + } + return smallest +} diff --git a/src/renderer/src/components/browser-pane/markup/markup-shape-render.test.ts b/src/renderer/src/components/browser-pane/markup/markup-shape-render.test.ts new file mode 100644 index 00000000000..444f8d01265 --- /dev/null +++ b/src/renderer/src/components/browser-pane/markup/markup-shape-render.test.ts @@ -0,0 +1,153 @@ +import { describe, expect, it } from 'vitest' +import { drawShape } from './markup-shape-render' +import { HIGHLIGHT_ALPHA, type MarkupShape } from './markup-drawing-model' + +// Minimal recording stand-in for CanvasRenderingContext2D — drawShape only uses +// this subset, so we can assert the dispatch without a real canvas. +function makeRecordingCtx() { + const calls: { method: string; args: unknown[] }[] = [] + const record = + (method: string) => + (...args: unknown[]) => { + calls.push({ method, args }) + } + const ctx = { + save: record('save'), + restore: record('restore'), + beginPath: record('beginPath'), + moveTo: record('moveTo'), + lineTo: record('lineTo'), + arc: record('arc'), + ellipse: record('ellipse'), + stroke: record('stroke'), + fill: record('fill'), + strokeRect: record('strokeRect'), + fillText: record('fillText'), + strokeText: record('strokeText'), + lineCap: '', + lineJoin: '', + strokeStyle: '', + fillStyle: '', + lineWidth: 0, + globalAlpha: 1, + font: '', + textBaseline: '' + } + const methods = (name: string) => calls.filter((c) => c.method === name) + return { ctx: ctx as unknown as CanvasRenderingContext2D, calls, methods } +} + +describe('drawShape dispatch', () => { + it('strokes a multi-point pen as a polyline', () => { + const { ctx, methods } = makeRecordingCtx() + const shape: MarkupShape = { + id: 'p', + kind: 'pen', + color: '#ef4444', + width: 4, + points: [ + { x: 0, y: 0 }, + { x: 5, y: 5 }, + { x: 9, y: 1 } + ] + } + drawShape(ctx, shape) + expect(methods('moveTo')).toHaveLength(1) + expect(methods('lineTo')).toHaveLength(2) + expect(methods('stroke')).toHaveLength(1) + }) + + it('draws a dot for a single-point pen', () => { + const { ctx, methods } = makeRecordingCtx() + drawShape(ctx, { id: 'p', kind: 'pen', color: '#fff', width: 4, points: [{ x: 3, y: 3 }] }) + expect(methods('arc')).toHaveLength(1) + expect(methods('fill')).toHaveLength(1) + }) + + it('applies translucent alpha for highlights', () => { + const { ctx } = makeRecordingCtx() + drawShape(ctx, { + id: 'h', + kind: 'highlight', + color: '#eab308', + width: 4, + points: [ + { x: 0, y: 0 }, + { x: 8, y: 0 } + ] + }) + expect(ctx.globalAlpha).toBe(HIGHLIGHT_ALPHA) + }) + + it('draws an arrow as a shaft plus a head', () => { + const { ctx, methods } = makeRecordingCtx() + drawShape(ctx, { + id: 'a', + kind: 'arrow', + color: '#fff', + width: 4, + from: { x: 0, y: 0 }, + to: { x: 20, y: 0 } + }) + // shaft (moveTo+lineTo) + head (moveTo+lineTo+lineTo) + expect(methods('moveTo').length).toBeGreaterThanOrEqual(2) + expect(methods('stroke').length).toBeGreaterThanOrEqual(2) + }) + + it('uses strokeRect for rectangles', () => { + const { ctx, methods } = makeRecordingCtx() + drawShape(ctx, { + id: 'r', + kind: 'rect', + color: '#fff', + width: 2, + from: { x: 10, y: 10 }, + to: { x: 4, y: 30 } + }) + expect(methods('strokeRect')).toHaveLength(1) + expect(methods('strokeRect')[0].args).toEqual([4, 10, 6, 20]) + }) + + it('uses ellipse for ovals', () => { + const { ctx, methods } = makeRecordingCtx() + drawShape(ctx, { + id: 'e', + kind: 'ellipse', + color: '#fff', + width: 2, + from: { x: 0, y: 0 }, + to: { x: 20, y: 10 } + }) + expect(methods('ellipse')).toHaveLength(1) + expect(methods('ellipse')[0].args.slice(0, 4)).toEqual([10, 5, 10, 5]) + }) + + it('renders single-line text with a halo (stroke behind fill)', () => { + const { ctx, methods } = makeRecordingCtx() + drawShape(ctx, { + id: 't', + kind: 'text', + color: '#fff', + at: { x: 5, y: 5 }, + text: 'fix this', + fontSize: 18 + }) + expect(methods('strokeText')).toHaveLength(1) + expect(methods('fillText')).toHaveLength(1) + expect(methods('fillText')[0].args[0]).toBe('fix this') + }) + + it('save/restore brackets every shape so state never leaks', () => { + const { ctx, methods } = makeRecordingCtx() + drawShape(ctx, { + id: 'r', + kind: 'rect', + color: '#fff', + width: 2, + from: { x: 0, y: 0 }, + to: { x: 1, y: 1 } + }) + expect(methods('save')).toHaveLength(1) + expect(methods('restore')).toHaveLength(1) + }) +}) diff --git a/src/renderer/src/components/browser-pane/markup/markup-shape-render.ts b/src/renderer/src/components/browser-pane/markup/markup-shape-render.ts new file mode 100644 index 00000000000..1df81a939f9 --- /dev/null +++ b/src/renderer/src/components/browser-pane/markup/markup-shape-render.ts @@ -0,0 +1,144 @@ +// Canvas rendering for markup shapes. Shared by the live overlay (draws into the +// visible canvas) and the compositor (draws into the offscreen export canvas), so +// on-screen preview and the delivered PNG are pixel-identical. + +import { + arrowHeadGeometry, + highlightWidth, + normalizeRect, + HIGHLIGHT_ALPHA, + type ArrowShape, + type EllipseShape, + type MarkupShape, + type PenShape, + type HighlightShape, + type RectShape, + type TextShape +} from './markup-drawing-model' + +export const TEXT_FONT_FAMILY = + 'ui-sans-serif, system-ui, -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif' + +export function drawShape(ctx: CanvasRenderingContext2D, shape: MarkupShape): void { + ctx.save() + ctx.lineCap = 'round' + ctx.lineJoin = 'round' + ctx.strokeStyle = shape.color + ctx.fillStyle = shape.color + switch (shape.kind) { + case 'pen': + drawStroke(ctx, shape) + break + case 'highlight': + drawHighlight(ctx, shape) + break + case 'arrow': + drawArrow(ctx, shape) + break + case 'rect': + drawRect(ctx, shape) + break + case 'ellipse': + drawEllipse(ctx, shape) + break + case 'text': + drawText(ctx, shape) + break + } + ctx.restore() +} + +export function drawShapes(ctx: CanvasRenderingContext2D, shapes: readonly MarkupShape[]): void { + for (const shape of shapes) { + drawShape(ctx, shape) + } +} + +function drawStroke(ctx: CanvasRenderingContext2D, shape: PenShape): void { + ctx.lineWidth = shape.width + strokePolyline(ctx, shape.points, shape.width) +} + +function drawHighlight(ctx: CanvasRenderingContext2D, shape: HighlightShape): void { + // Why: translucent + fat so it reads as a marker pass over the screenshot + // without hiding the pixels beneath. + ctx.globalAlpha = HIGHLIGHT_ALPHA + const width = highlightWidth(shape.width) + ctx.lineWidth = width + strokePolyline(ctx, shape.points, width) +} + +function strokePolyline( + ctx: CanvasRenderingContext2D, + points: readonly { x: number; y: number }[], + width: number +): void { + if (points.length === 0) { + return + } + if (points.length === 1) { + // Why: a tap (single point) still leaves a visible dot. + const point = points[0] + ctx.beginPath() + ctx.arc(point.x, point.y, Math.max(width / 2, 1), 0, Math.PI * 2) + ctx.fill() + return + } + ctx.beginPath() + ctx.moveTo(points[0].x, points[0].y) + for (let i = 1; i < points.length; i += 1) { + ctx.lineTo(points[i].x, points[i].y) + } + ctx.stroke() +} + +function drawArrow(ctx: CanvasRenderingContext2D, shape: ArrowShape): void { + ctx.lineWidth = shape.width + ctx.beginPath() + ctx.moveTo(shape.from.x, shape.from.y) + ctx.lineTo(shape.to.x, shape.to.y) + ctx.stroke() + const head = arrowHeadGeometry(shape.from, shape.to, shape.width) + if (!head) { + return + } + ctx.beginPath() + ctx.moveTo(head.left.x, head.left.y) + ctx.lineTo(head.tip.x, head.tip.y) + ctx.lineTo(head.right.x, head.right.y) + ctx.stroke() +} + +function drawRect(ctx: CanvasRenderingContext2D, shape: RectShape): void { + ctx.lineWidth = shape.width + const rect = normalizeRect(shape.from, shape.to) + ctx.strokeRect(rect.x, rect.y, rect.width, rect.height) +} + +function drawEllipse(ctx: CanvasRenderingContext2D, shape: EllipseShape): void { + ctx.lineWidth = shape.width + const rect = normalizeRect(shape.from, shape.to) + const cx = rect.x + rect.width / 2 + const cy = rect.y + rect.height / 2 + ctx.beginPath() + ctx.ellipse(cx, cy, rect.width / 2, rect.height / 2, 0, 0, Math.PI * 2) + ctx.stroke() +} + +function drawText(ctx: CanvasRenderingContext2D, shape: TextShape): void { + ctx.font = `600 ${shape.fontSize}px ${TEXT_FONT_FAMILY}` + ctx.textBaseline = 'top' + // Why: a thin contrasting halo keeps text legible over busy screenshots + // regardless of the underlying pixels. + ctx.lineWidth = Math.max(shape.fontSize / 6, 2) + ctx.strokeStyle = haloColor(shape.color) + ctx.lineJoin = 'round' + // Text comes from a single-line input, so there are no newlines to lay out. + ctx.strokeText(shape.text, shape.at.x, shape.at.y) + ctx.fillText(shape.text, shape.at.x, shape.at.y) +} + +// White text gets a dark halo, everything else a light halo. +function haloColor(color: string): string { + return color.toLowerCase() === '#ffffff' ? 'rgba(0,0,0,0.65)' : 'rgba(255,255,255,0.85)' +} diff --git a/src/renderer/src/components/browser-pane/markup/use-markup-draw-hint.ts b/src/renderer/src/components/browser-pane/markup/use-markup-draw-hint.ts new file mode 100644 index 00000000000..f6d1ab2257e --- /dev/null +++ b/src/renderer/src/components/browser-pane/markup/use-markup-draw-hint.ts @@ -0,0 +1,48 @@ +import { useCallback, useEffect, useState } from 'react' +import { useAppStore } from '@/store' + +// One-time discovery highlight for the screenshot-markup Draw button. Shows once +// per install — the first time the button is usable — so users notice the new +// tool. Gated on its own localStorage flag (not a contextual tour), so it fires +// for everyone, including users who already finished the capped browser tour. +// It stays open until the user acts (clicks Draw) or the button stops being +// usable; there is no auto-timeout. +const MARKUP_DRAW_HINT_SEEN_KEY = 'orca.browser.markup-draw-hint-seen' + +// Records the first-ever view and returns whether this call is that first view. +// Returns false when storage is unavailable so a private-mode session never +// risks nagging on every open. +function claimFirstView(): boolean { + try { + if (window.localStorage.getItem(MARKUP_DRAW_HINT_SEEN_KEY) === 'true') { + return false + } + window.localStorage.setItem(MARKUP_DRAW_HINT_SEEN_KEY, 'true') + return true + } catch { + return false + } +} + +export type MarkupDrawHint = { hintOpen: boolean; dismissHint: () => void } + +export function useMarkupDrawHint(eligible: boolean): MarkupDrawHint { + const persistedUIReady = useAppStore((state) => state.persistedUIReady) + const [hintOpen, setHintOpen] = useState(false) + + useEffect(() => { + // Why: only nudge once the app is ready and the button is usable. If it stops + // being usable while showing (grab started, markup open, blank tab), close it + // so the forced-open tooltip can't get stuck over a now-disabled button. + if (!persistedUIReady || !eligible) { + setHintOpen(false) + return + } + if (claimFirstView()) { + setHintOpen(true) + } + }, [eligible, persistedUIReady]) + + const dismissHint = useCallback(() => setHintOpen(false), []) + return { hintOpen, dismissHint } +} diff --git a/src/renderer/src/components/browser-pane/markup/useMarkupEditor.ts b/src/renderer/src/components/browser-pane/markup/useMarkupEditor.ts new file mode 100644 index 00000000000..242d17993cb --- /dev/null +++ b/src/renderer/src/components/browser-pane/markup/useMarkupEditor.ts @@ -0,0 +1,179 @@ +import { useCallback, useEffect, useRef, useState } from 'react' +import { blitMarkupScene, renderCommittedLayer } from './markup-canvas-render' +import { useMarkupKeyboardShortcuts, type PendingText } from './useMarkupKeyboardShortcuts' +import { useMarkupPointerHandlers } from './useMarkupPointerHandlers' +import { + canRedo, + canUndo, + clearShapes, + commitShape, + createMarkupDocument, + DEFAULT_MARKUP_COLOR, + DEFAULT_MARKUP_FONT_SIZE, + DEFAULT_MARKUP_WIDTH, + redoShape, + undoShape, + type MarkupDocument, + type MarkupShape, + type MarkupTool +} from './markup-drawing-model' + +type Size = { width: number; height: number; dpr: number } + +// Owns the markup surface: document, active tool/style, the pending text box, and +// the canvas paint effect. Draw-only — committed shapes are not re-editable. +export function useMarkupEditor(busy: boolean, onCancel: () => void) { + const rootRef = useRef(null) + const canvasRef = useRef(null) + const textInputRef = useRef(null) + // Why: committed shapes are rasterized once into this offscreen layer; the live + // paint blits it instead of re-stroking every committed shape each pointermove. + const committedLayerRef = useRef(null) + if (committedLayerRef.current === null) { + committedLayerRef.current = document.createElement('canvas') + } + + const [size, setSize] = useState({ width: 0, height: 0, dpr: 1 }) + const [doc, setDoc] = useState(() => createMarkupDocument()) + const [inProgress, setInProgress] = useState(null) + const [tool, setTool] = useState('pen') + const [color, setColor] = useState(DEFAULT_MARKUP_COLOR) + const [width, setWidth] = useState(DEFAULT_MARKUP_WIDTH) + const [fontSize, setFontSize] = useState(DEFAULT_MARKUP_FONT_SIZE) + const [pendingText, setPendingText] = useState(null) + + // Track the content-box size so the canvas matches the frozen backdrop exactly. + useEffect(() => { + const root = rootRef.current + if (!root) { + return undefined + } + const measure = () => { + const rect = root.getBoundingClientRect() + const dpr = window.devicePixelRatio || 1 + // Why: bail on an unchanged measurement so an identical ResizeObserver/resize + // tick can't schedule a redundant repaint. + setSize((prev) => + prev.width === rect.width && prev.height === rect.height && prev.dpr === dpr + ? prev + : { width: rect.width, height: rect.height, dpr } + ) + } + measure() + const observer = new ResizeObserver(measure) + observer.observe(root) + // Why: a monitor move changes devicePixelRatio without changing the element's + // CSS box, so ResizeObserver won't fire — window resize (which Chromium emits + // on dpr changes) re-measures the dpr so the canvas repaints at the new scale. + window.addEventListener('resize', measure) + return () => { + observer.disconnect() + window.removeEventListener('resize', measure) + } + }, []) + + // Rasterize committed shapes into the offscreen layer only when they (or the + // size) change — not on every in-progress pointermove. + useEffect(() => { + const layer = committedLayerRef.current + if (!layer) { + return + } + renderCommittedLayer(layer, doc.shapes, size.width, size.height, size.dpr) + }, [doc.shapes, size]) + + // Blit the cached layer + the in-progress shape, coalesced to one paint per + // frame so a burst of pointermove events can't queue redundant full repaints. + useEffect(() => { + const canvas = canvasRef.current + const layer = committedLayerRef.current + if (!canvas || !layer) { + return undefined + } + const handle = requestAnimationFrame(() => { + blitMarkupScene(canvas, layer, inProgress, size.width, size.height, size.dpr) + }) + return () => cancelAnimationFrame(handle) + }, [doc.shapes, inProgress, size]) + + // Why: focus the text input on mount — a placement click can beat autoFocus. + useEffect(() => { + if (!pendingText) { + return undefined + } + const handle = requestAnimationFrame(() => textInputRef.current?.focus()) + return () => cancelAnimationFrame(handle) + }, [pendingText]) + + const undo = useCallback(() => setDoc((current) => undoShape(current)), []) + const redo = useCallback(() => setDoc((current) => redoShape(current)), []) + const clear = useCallback(() => { + // Why: also drop any open text input / in-progress stroke so a clear leaves a + // truly clean slate — otherwise a pending input blur can re-add text. + setPendingText(null) + setInProgress(null) + setDoc((current) => clearShapes(current)) + }, []) + + useMarkupKeyboardShortcuts({ pendingText, setPendingText, undo, redo, onCancel }) + + const pointerHandlers = useMarkupPointerHandlers({ + busy, + tool, + color, + width, + pendingText, + canvasRef, + setInProgress, + setPendingText, + setDoc + }) + + const commitPendingText = useCallback( + (text: string) => { + const at = pendingText + setPendingText(null) + const trimmed = text.trim() + if (!at || trimmed.length === 0) { + return + } + setDoc((document) => + commitShape(document, { + id: crypto.randomUUID(), + kind: 'text', + color, + at, + text: trimmed, + fontSize + }) + ) + }, + [color, fontSize, pendingText] + ) + + const cancelPendingText = useCallback(() => setPendingText(null), []) + + return { + rootRef, + canvasRef, + textInputRef, + tool, + color, + width, + fontSize, + pendingText, + shapes: doc.shapes, + canUndo: canUndo(doc), + canRedo: canRedo(doc), + setTool, + setColor, + setWidth, + setFontSize, + undo, + redo, + clear, + ...pointerHandlers, + commitPendingText, + cancelPendingText + } +} diff --git a/src/renderer/src/components/browser-pane/markup/useMarkupKeyboardShortcuts.ts b/src/renderer/src/components/browser-pane/markup/useMarkupKeyboardShortcuts.ts new file mode 100644 index 00000000000..3fc9ca97f2d --- /dev/null +++ b/src/renderer/src/components/browser-pane/markup/useMarkupKeyboardShortcuts.ts @@ -0,0 +1,52 @@ +import { useEffect } from 'react' + +export type PendingText = { x: number; y: number; initial: string } + +function isTypingTarget(target: EventTarget | null): boolean { + if (!(target instanceof HTMLElement)) { + return false + } + const tag = target.tagName + return tag === 'INPUT' || tag === 'TEXTAREA' || target.isContentEditable +} + +export type MarkupKeyboardParams = { + pendingText: PendingText | null + setPendingText: (value: PendingText | null) => void + undo: () => void + redo: () => void + onCancel: () => void +} + +// Window-level shortcuts for the markup overlay: Escape (close the open text box, +// else exit markup) and platform-correct undo/redo. +export function useMarkupKeyboardShortcuts(params: MarkupKeyboardParams): void { + const { pendingText, setPendingText, undo, redo, onCancel } = params + useEffect(() => { + const isMac = navigator.userAgent.includes('Mac') + const onKeyDown = (event: KeyboardEvent) => { + if (event.key === 'Escape') { + if (pendingText) { + setPendingText(null) + } else { + onCancel() + } + return + } + if (isTypingTarget(event.target)) { + return + } + const mod = isMac ? event.metaKey : event.ctrlKey + if (mod && event.key.toLowerCase() === 'z') { + event.preventDefault() + if (event.shiftKey) { + redo() + } else { + undo() + } + } + } + window.addEventListener('keydown', onKeyDown) + return () => window.removeEventListener('keydown', onKeyDown) + }, [pendingText, setPendingText, undo, redo, onCancel]) +} diff --git a/src/renderer/src/components/browser-pane/markup/useMarkupMode.ts b/src/renderer/src/components/browser-pane/markup/useMarkupMode.ts new file mode 100644 index 00000000000..f7e02df3dbf --- /dev/null +++ b/src/renderer/src/components/browser-pane/markup/useMarkupMode.ts @@ -0,0 +1,172 @@ +import { useCallback, useRef, useState } from 'react' +import { toast } from 'sonner' +import { translate } from '@/i18n/i18n' +import { + CLIPBOARD_IMAGE_MAX_PIXELS, + CLIPBOARD_IMAGE_MAX_SOURCE_BYTES +} from '../../../../../shared/clipboard-image' +import { + captureMarkupBaseImage, + type MarkupBaseImage, + type MarkupCaptureSource +} from './markup-base-image' +import { composeMarkupDataUrl, type MarkupComposeResult } from './markup-screenshot-compose' +import type { MarkupShape } from './markup-drawing-model' + +export type MarkupModeState = 'idle' | 'capturing' | 'drawing' | 'composing' + +// Where to capture the base image from, plus the on-screen geometry used to +// size and scale the composite. Resolved lazily at start() time by the owner +// (BrowserPane) since it depends on which environment the pane is showing. +export type MarkupCaptureContext = { + source: MarkupCaptureSource + cssWidth: number + cssHeight: number + outputScale: number +} + +export type MarkupCompleteInput = { + imageElement: CanvasImageSource + shapes: readonly MarkupShape[] +} + +type UseMarkupModeParams = { + getCaptureContext: () => MarkupCaptureContext | null + onDeliver: (result: MarkupComposeResult) => Promise | void +} + +export type MarkupModeController = { + state: MarkupModeState + isActive: boolean + baseImage: MarkupBaseImage | null + start: () => Promise + cancel: () => void + complete: (input: MarkupCompleteInput) => Promise +} + +export function useMarkupMode({ + getCaptureContext, + onDeliver +}: UseMarkupModeParams): MarkupModeController { + const [state, setState] = useState('idle') + const [baseImage, setBaseImage] = useState(null) + const contextRef = useRef(null) + // Why: a token invalidates an in-flight capture if the user cancels before it + // resolves, so the stale promise can't flip state back to 'drawing'. + const captureTokenRef = useRef(0) + + const reset = useCallback(() => { + captureTokenRef.current += 1 + contextRef.current = null + setBaseImage(null) + setState('idle') + }, []) + + const reportError = useCallback((key: string, fallback: string) => { + toast.error(translate(key, fallback)) + }, []) + + const start = useCallback(async () => { + const context = getCaptureContext() + if (!context) { + reportError( + 'auto.components.browser-pane.markup.errorUnavailable', + 'Screenshot markup is not available on this page.' + ) + return + } + const token = (captureTokenRef.current += 1) + contextRef.current = context + setState('capturing') + try { + const image = await captureMarkupBaseImage(context.source) + if (captureTokenRef.current !== token) { + return + } + setBaseImage(image) + setState('drawing') + } catch { + if (captureTokenRef.current !== token) { + return + } + // Why: a capture failure has no overlay to fall back to, so return to idle + // (not a stuck 'active' state with no surface and an inert Escape). + reportError( + 'auto.components.browser-pane.markup.errorCapture', + 'Could not capture the page to draw on.' + ) + reset() + } + }, [getCaptureContext, reportError, reset]) + + const cancel = useCallback(() => { + reset() + }, [reset]) + + const complete = useCallback( + async ({ imageElement, shapes }: MarkupCompleteInput) => { + const context = contextRef.current + if (!context) { + reset() + return + } + // Why: invalidate this completion if the user cancels or restarts while + // onDeliver is pending, so a stale callback can't reset/error the new session. + const token = captureTokenRef.current + setState('composing') + // Why: yield a frame so the busy 'composing' UI (disabled buttons, progress + // cursor) paints before the synchronous composite raster runs — otherwise + // Copy freezes with no visible feedback. Skip the work if cancelled meanwhile. + await new Promise((resolve) => requestAnimationFrame(() => resolve())) + if (captureTokenRef.current !== token) { + return + } + try { + const result = await composeMarkupDataUrl({ + image: imageElement, + displayCssWidth: context.cssWidth, + displayCssHeight: context.cssHeight, + outputScale: context.outputScale, + shapes, + // Why: target the clipboard handler's own ceilings — the byte limit + // (≈18 MB, well above a normal viewport PNG) and the pixel limit, which + // the handler otherwise enforces by silently dropping oversize images. + maxBytes: CLIPBOARD_IMAGE_MAX_SOURCE_BYTES, + maxPixels: CLIPBOARD_IMAGE_MAX_PIXELS + }) + // Why: re-check before delivering — the async compose is a wide window in + // which the user can Escape/cancel, and onDeliver writes the clipboard + // irreversibly. Without this, a cancelled session still overwrites it. + if (captureTokenRef.current !== token) { + return + } + await onDeliver(result) + if (captureTokenRef.current !== token) { + return + } + reset() + } catch { + if (captureTokenRef.current !== token) { + return + } + // Why: the frozen backdrop is still valid, so return to drawing (not a + // dead-end error state) — the user can retry Copy or Cancel out. + reportError( + 'auto.components.browser-pane.markup.errorAttach', + 'Could not attach the markup screenshot.' + ) + setState('drawing') + } + }, + [onDeliver, reset, reportError] + ) + + return { + state, + isActive: state !== 'idle', + baseImage, + start, + cancel, + complete + } +} diff --git a/src/renderer/src/components/browser-pane/markup/useMarkupPointerHandlers.ts b/src/renderer/src/components/browser-pane/markup/useMarkupPointerHandlers.ts new file mode 100644 index 00000000000..3ddf41fd458 --- /dev/null +++ b/src/renderer/src/components/browser-pane/markup/useMarkupPointerHandlers.ts @@ -0,0 +1,99 @@ +import { useCallback } from 'react' +import type React from 'react' +import type { PendingText } from './useMarkupKeyboardShortcuts' +import { + commitShape, + type MarkupDocument, + type MarkupPoint, + type MarkupShape, + type MarkupTool +} from './markup-drawing-model' + +export type MarkupPointerParams = { + busy: boolean + tool: MarkupTool + color: string + width: number + pendingText: PendingText | null + canvasRef: React.RefObject + setInProgress: React.Dispatch> + setPendingText: (value: PendingText | null) => void + setDoc: React.Dispatch> +} + +// Canvas pointer interactions: draw a new shape, or place text. Split out of +// useMarkupEditor to keep that hook focused. +export function useMarkupPointerHandlers(params: MarkupPointerParams) { + const { busy, tool, color, width, pendingText, canvasRef, setInProgress, setPendingText, setDoc } = + params + + const pointFromEvent = useCallback( + (event: { clientX: number; clientY: number }): MarkupPoint => { + const canvas = canvasRef.current + if (!canvas) { + return { x: 0, y: 0 } + } + const rect = canvas.getBoundingClientRect() + return { x: event.clientX - rect.left, y: event.clientY - rect.top } + }, + [canvasRef] + ) + + const onPointerDown = useCallback( + (event: React.PointerEvent) => { + if (busy || event.button !== 0) { + return + } + const point = pointFromEvent(event) + if (tool === 'text') { + // Why: a box is already open — this click's job is only to commit it (the + // input's blur fires), not to open a second box at the click point. + if (pendingText) { + return + } + // Why: keep focus off the canvas so the mounting text input keeps it. + event.preventDefault() + setPendingText({ x: point.x, y: point.y, initial: '' }) + return + } + event.currentTarget.setPointerCapture(event.pointerId) + const id = crypto.randomUUID() + if (tool === 'pen' || tool === 'highlight') { + setInProgress({ id, kind: tool, color, width, points: [point] }) + } else { + setInProgress({ id, kind: tool, color, width, from: point, to: point }) + } + }, + [busy, color, pendingText, pointFromEvent, setInProgress, setPendingText, tool, width] + ) + + const onPointerMove = useCallback( + (event: React.PointerEvent) => { + setInProgress((current) => { + if (!current) { + return current + } + const point = pointFromEvent(event) + if (current.kind === 'pen' || current.kind === 'highlight') { + return { ...current, points: [...current.points, point] } + } + if (current.kind === 'text') { + return current + } + return { ...current, to: point } + }) + }, + [pointFromEvent, setInProgress] + ) + + const onPointerUp = useCallback(() => { + setInProgress((current) => { + if (current) { + setDoc((document) => commitShape(document, current)) + } + return null + }) + }, [setDoc, setInProgress]) + + return { onPointerDown, onPointerMove, onPointerUp } +} diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index 06e0f50482c..904551378c9 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -13020,6 +13020,34 @@ "searchLabels": "Filter labels…", "searchAssignee": "Filter assignee…", "back": "Back" + }, + "browser-pane": { + "markup": { + "cancel": "Cancel", + "clear": "Clear all", + "copiedToast": "Markup copied — paste it into your agent ({{value0}})", + "copy": "Copy Markup", + "drawButton": "Draw on screenshot", + "drawHint": "New — draw on the page, then copy it to your agent", + "errorAttach": "Could not attach the markup screenshot.", + "errorCapture": "Could not capture the page to draw on.", + "errorUnavailable": "Screenshot markup is not available on this page.", + "fontSize": "Font size", + "hint": "Draw on the page, then copy the markup to paste into your agent.", + "redo": "Redo", + "style": "Color and thickness", + "textInput": "Annotation text", + "tool": { + "arrow": "Arrow", + "ellipse": "Ellipse", + "highlight": "Highlighter", + "pen": "Pen", + "rect": "Rectangle", + "text": "Text" + }, + "undo": "Undo", + "widthOption": "{{value0}} px" + } } }, "i18n": { diff --git a/src/renderer/src/i18n/locales/es.json b/src/renderer/src/i18n/locales/es.json index 2d093ebe8f8..4561dd25d61 100644 --- a/src/renderer/src/i18n/locales/es.json +++ b/src/renderer/src/i18n/locales/es.json @@ -13020,6 +13020,34 @@ "searchLabels": "Filtrar etiquetas…", "searchAssignee": "Filtrar asignatario…", "back": "Atrás" + }, + "browser-pane": { + "markup": { + "cancel": "Cancelar", + "clear": "Borrar todo", + "copiedToast": "Marcado copiado: pégalo en tu agente ({{value0}})", + "copy": "Copiar marcado", + "drawButton": "Dibujar en la captura", + "drawHint": "Nuevo: dibuja en la página y cópialo en tu agente", + "errorAttach": "No se pudo adjuntar la captura de pantalla con marcado.", + "errorCapture": "No se pudo capturar la página para dibujar.", + "errorUnavailable": "El marcado de captura de pantalla no está disponible en esta página.", + "fontSize": "Tamaño de letra", + "hint": "Dibuja en la página y luego copia el marcado para pegarlo en tu agente.", + "redo": "Rehacer", + "style": "Color y grosor", + "textInput": "Texto de anotación", + "tool": { + "arrow": "Flecha", + "ellipse": "Elipse", + "highlight": "Resaltador", + "pen": "Lápiz", + "rect": "Rectángulo", + "text": "Texto" + }, + "undo": "Deshacer", + "widthOption": "{{value0}} px" + } } }, "i18n": { diff --git a/src/renderer/src/i18n/locales/ja.json b/src/renderer/src/i18n/locales/ja.json index 4ad9de324be..f012157d588 100644 --- a/src/renderer/src/i18n/locales/ja.json +++ b/src/renderer/src/i18n/locales/ja.json @@ -13020,6 +13020,34 @@ "searchLabels": "ラベルをフィルター…", "searchAssignee": "担当者をフィルター…", "back": "戻る" + }, + "browser-pane": { + "markup": { + "cancel": "キャンセル", + "clear": "すべて消去", + "copiedToast": "注釈をコピーしました — エージェントに貼り付けてください（{{value0}}）", + "copy": "注釈をコピー", + "drawButton": "スクリーンショットに描き込み", + "drawHint": "新機能 — ページに描き込み、エージェントに貼り付けられます", + "errorAttach": "注釈付きスクリーンショットを添付できませんでした。", + "errorCapture": "描画用にページを取得できませんでした。", + "errorUnavailable": "このページではスクリーンショット注釈を利用できません。", + "fontSize": "文字サイズ", + "hint": "ページに描き込み、注釈をコピーしてエージェントに貼り付けてください。", + "redo": "やり直し", + "style": "色と太さ", + "textInput": "注釈テキスト", + "tool": { + "arrow": "矢印", + "ellipse": "楕円", + "highlight": "ハイライト", + "pen": "ペン", + "rect": "四角形", + "text": "テキスト" + }, + "undo": "元に戻す", + "widthOption": "{{value0}} px" + } } }, "i18n": { diff --git a/src/renderer/src/i18n/locales/ko.json b/src/renderer/src/i18n/locales/ko.json index dfbefd0700b..2fae5839230 100644 --- a/src/renderer/src/i18n/locales/ko.json +++ b/src/renderer/src/i18n/locales/ko.json @@ -13020,6 +13020,34 @@ "searchLabels": "레이블 필터…", "searchAssignee": "담당자 필터…", "back": "뒤로" + }, + "browser-pane": { + "markup": { + "cancel": "취소", + "clear": "모두 지우기", + "copiedToast": "마크업을 복사했습니다 — 에이전트에 붙여넣으세요 ({{value0}})", + "copy": "마크업 복사", + "drawButton": "스크린샷에 그리기", + "drawHint": "새 기능 — 페이지에 그린 다음 에이전트에 붙여넣으세요", + "errorAttach": "마크업 스크린샷을 첨부하지 못했습니다.", + "errorCapture": "그릴 페이지를 캡처하지 못했습니다.", + "errorUnavailable": "이 페이지에서는 스크린샷 마크업을 사용할 수 없습니다.", + "fontSize": "글자 크기", + "hint": "페이지에 그린 다음 마크업을 복사해 에이전트에 붙여넣으세요.", + "redo": "다시 실행", + "style": "색상 및 두께", + "textInput": "주석 텍스트", + "tool": { + "arrow": "화살표", + "ellipse": "타원", + "highlight": "형광펜", + "pen": "펜", + "rect": "사각형", + "text": "텍스트" + }, + "undo": "실행 취소", + "widthOption": "{{value0}} px" + } } }, "i18n": { diff --git a/src/renderer/src/i18n/locales/zh.json b/src/renderer/src/i18n/locales/zh.json index eca7076c294..e23f6853fe3 100644 --- a/src/renderer/src/i18n/locales/zh.json +++ b/src/renderer/src/i18n/locales/zh.json @@ -13020,6 +13020,34 @@ "searchLabels": "筛选标签…", "searchAssignee": "筛选负责人…", "back": "返回" + }, + "browser-pane": { + "markup": { + "cancel": "取消", + "clear": "全部清除", + "copiedToast": "已复制标注 — 粘贴到你的智能体（{{value0}}）", + "copy": "复制标注", + "drawButton": "在截图上绘制", + "drawHint": "新功能 — 在页面上绘制，然后复制到你的智能体", + "errorAttach": "无法附加标注后的截图。", + "errorCapture": "无法捕获页面进行绘制。", + "errorUnavailable": "此页面无法使用截图标注。", + "fontSize": "字号", + "hint": "在页面上绘制，然后复制标注以粘贴到你的智能体。", + "redo": "重做", + "style": "颜色和粗细", + "textInput": "标注文字", + "tool": { + "arrow": "箭头", + "ellipse": "椭圆", + "highlight": "高亮", + "pen": "画笔", + "rect": "矩形", + "text": "文字" + }, + "undo": "撤销", + "widthOption": "{{value0}} px" + } } }, "i18n": { From d44696a74c6b690c86c3cffe000a05e794065d16 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 00:48:41 +0000 Subject: [PATCH 06/15] Update README downloads badge --- docs/assets/readme-downloads.svg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/assets/readme-downloads.svg b/docs/assets/readme-downloads.svg index db2a6fd8806..2c60ec6df8d 100644 --- a/docs/assets/readme-downloads.svg +++ b/docs/assets/readme-downloads.svg @@ -1,5 +1,5 @@ - - downloads: 5.3m + + downloads: 5.4m @@ -15,7 +15,7 @@ downloads downloads - 5.3m - 5.3m + 5.4m + 5.4m From 33a8df93fc0900cee9555c72afb6f35e9a897ccd Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 14 Jul 2026 17:49:13 -0700 Subject: [PATCH 07/15] fix(speech): resume interrupted voice model downloads and surface real errors (#8775) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(speech): resume interrupted voice model downloads and surface real errors Voice model downloads that died late in the transfer (flaky networks, scanning proxies) failed at the UI's 90% progress cap and restarted from byte zero on every retry, so affected users could never complete any model download (#8620 / STA-1775). - Retry transient download failures (net::ERR_* transfer cuts, resets, timeouts, HTTP 5xx/429) with an HTTP Range resume from the bytes already on disk, restarting from the canonical URL each attempt since signed CDN redirect URLs expire; bounded backoff, gives up after repeated zero-progress attempts with a diagnosable error. - Correct catalog sizeBytes to exact upstream asset sizes (parakeet v2/v3 are 482/487 MB, not 170/180; paraformer is 1,047 MB, not 115) — also the progress denominator when content-length is missing. - Show the underlying failure cause in the download-error toast and log it in main; it was previously invisible everywhere. All 7 pinned catalog SHA-256 hashes verified against current upstream assets. Live-verified end to end: a transfer killed mid-body resumes with a 206 and completes through checksum, extraction, and validation. Fixes #8620 * fix(speech): harden resumed model downloads * fix(speech): validate resumed download completion * fix(speech): keep segmented download resumes progressing * fix(speech): don't abandon a download that keeps making progress The retry loop gave up after 8 total failures regardless of progress, so a large model (e.g. the ~1GB Paraformer) over a network that resets every few hundred MB was abandoned mid-download even though every attempt was resuming and advancing — the exact 'stuck then fails' symptom, just later. Replace the all-time failure budget and the separate advancing-segment cap with a single rule: give up only on a genuine stall (repeated attempts with no forward progress) or an absolute request ceiling that bounds a pathological tiny-segment server. Any download that keeps advancing now runs to completion. Verified live under Electron 43: a transfer reset mid-stream on every attempt but advancing each time previously gave up at 67% after 8 attempts; it now completes. All prior resume/cancel/timeout scenarios still pass. --- src/main/speech/model-catalog.ts | 16 +- .../model-manager-download-resume.test.ts | 671 ++++++++++++++++++ src/main/speech/model-manager.ts | 339 ++++++++- .../settings/VoiceSpeechModelSection.tsx | 26 +- 4 files changed, 1020 insertions(+), 32 deletions(-) create mode 100644 src/main/speech/model-manager-download-resume.test.ts diff --git a/src/main/speech/model-catalog.ts b/src/main/speech/model-catalog.ts index 266442eb78d..b77f6c238d6 100644 --- a/src/main/speech/model-catalog.ts +++ b/src/main/speech/model-catalog.ts @@ -1,5 +1,7 @@ import type { SpeechModelManifest } from '../../shared/speech-types' +// Why: sizeBytes must be the exact upstream asset size — it is the UI size +// label and the download-progress denominator when content-length is missing. export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ { id: 'parakeet-tdt-0.6b-v3-int8', @@ -9,7 +11,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ type: 'transducer', provider: 'local', language: 'multilingual', - sizeBytes: 180_000_000, + sizeBytes: 487_170_055, downloadUrl: 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2', archiveSha256: '5793d0fd397c5778d2cf2126994d58e9d56b1be7c04d13c7a15bb1b4eafb16bf', @@ -28,7 +30,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ type: 'transducer', provider: 'local', language: 'en', - sizeBytes: 170_000_000, + sizeBytes: 482_468_385, downloadUrl: 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8.tar.bz2', archiveSha256: '157c157bc51155e03e37d2466522a3a737dd9c72bb25f36eb18912964161e1ad', @@ -45,7 +47,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ type: 'transducer', provider: 'local', language: 'zh-en', - sizeBytes: 130_000_000, + sizeBytes: 511_274_346, downloadUrl: 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2', archiveSha256: '27ffbd9ee24ad186d99acc2f6354d7992b27bcab490812510665fa8f9389c5f8', @@ -68,7 +70,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ type: 'paraformer', provider: 'local', language: 'zh-en', - sizeBytes: 115_000_000, + sizeBytes: 1_047_319_737, downloadUrl: 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2', archiveSha256: '5462a1fce42693deae572af1e8c4687124b12aa85fe61ff4d3168bb5280e205f', @@ -84,7 +86,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ type: 'transducer', provider: 'local', language: 'en', - sizeBytes: 128_000_000, + sizeBytes: 127_887_156, downloadUrl: 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17.tar.bz2', archiveSha256: '9c559283e8498d3fe95913c79ca1cb454bb26281ac2b102b41306c7d752765d9', @@ -106,7 +108,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ type: 'transducer', provider: 'local', language: 'zh', - sizeBytes: 74_000_000, + sizeBytes: 74_004_050, downloadUrl: 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23.tar.bz2', archiveSha256: '2cbd71b640d9c37d3784f29367333a4577b0398b62e9deeed418170b081cba8b', @@ -128,7 +130,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ type: 'whisper', provider: 'local', language: 'multilingual', - sizeBytes: 116_000_000, + sizeBytes: 116_204_861, downloadUrl: 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.tar.bz2', archiveSha256: 'c46116994e539aa165266d96b325252728429c12535eb9d8b6a2b10f129e66b1', diff --git a/src/main/speech/model-manager-download-resume.test.ts b/src/main/speech/model-manager-download-resume.test.ts new file mode 100644 index 00000000000..bbfc88e7487 --- /dev/null +++ b/src/main/speech/model-manager-download-resume.test.ts @@ -0,0 +1,671 @@ +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Readable } from 'node:stream' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { ModelManager } from './model-manager' + +const { netRequestMock } = vi.hoisted(() => ({ + netRequestMock: vi.fn() +})) + +vi.mock('electron', () => ({ + app: { + getPath: () => '/tmp/orca-speech-models-test' + }, + net: { + request: netRequestMock + } +})) + +type ModelManagerInternals = { + downloadArchiveWithRetry: ( + url: string, + archivePath: string, + expectedSize: number, + modelId: string, + isAborted: () => boolean, + signal: AbortSignal + ) => Promise + downloadFile: ( + url: string, + archivePath: string, + expectedSize: number, + modelId: string, + isAborted: () => boolean, + signal?: AbortSignal, + redirectCount?: number, + resumeOffset?: number + ) => Promise +} + +type ScriptedResponse = { + statusCode: number + headers?: Record + chunks?: Buffer[] + failWith?: string +} + +type ScriptedResponseFactory = (sentHeaders: Record) => ScriptedResponse + +// Emulates Electron's ClientRequest/IncomingMessage closely enough for the +// download pipeline: a real Readable body so stream.pipeline semantics +// (including mid-body destroy) match production behavior. +function scriptRequest(factory: ScriptedResponseFactory): { + sentHeaders: Record + abortMock: ReturnType +} { + const sentHeaders: Record = {} + const listeners = new Map void>>() + const addListener = (event: string, cb: (...args: unknown[]) => void): void => { + const set = listeners.get(event) ?? new Set() + set.add(cb) + listeners.set(event, set) + } + const abortMock = vi.fn(() => request) + const request = { + setHeader: vi.fn((name: string, value: string) => { + sentHeaders[name.toLowerCase()] = value + }), + on: vi.fn((event: string, cb: (...args: unknown[]) => void) => { + addListener(event, cb) + return request + }), + off: vi.fn((event: string, cb: (...args: unknown[]) => void) => { + listeners.get(event)?.delete(cb) + return request + }), + abort: abortMock, + end: vi.fn(() => { + queueMicrotask(() => { + const spec = factory(sentHeaders) + const response = Object.assign(new Readable({ read() {} }), { + statusCode: spec.statusCode, + headers: spec.headers ?? {} + }) + for (const cb of listeners.get('response') ?? []) { + cb(response) + } + setTimeout(() => { + for (const chunk of spec.chunks ?? []) { + response.push(chunk) + } + // Why: fail on a later tick so pushed chunks flush to the file + // stream first, mirroring a transfer that dies mid-body. + setTimeout(() => { + if (spec.failWith) { + response.destroy(new Error(spec.failWith)) + } else { + response.push(null) + } + }, 20) + }, 20) + }) + return request + }) + } + netRequestMock.mockImplementationOnce(() => request) + return { sentHeaders, abortMock } +} + +const PAYLOAD = Buffer.from('0123456789abcdefghij') + +describe('ModelManager download resume', () => { + beforeEach(() => { + netRequestMock.mockReset() + }) + + afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() + }) + + it('resumes an interrupted download with a Range request and assembles the full file', async () => { + const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-')) + try { + scriptRequest(() => ({ + statusCode: 200, + headers: { 'content-length': String(PAYLOAD.length) }, + chunks: [PAYLOAD.subarray(0, 10)], + failWith: 'net::ERR_CONTENT_LENGTH_MISMATCH' + })) + const second = scriptRequest((sentHeaders) => { + expect(sentHeaders.range).toBe('bytes=10-') + return { + statusCode: 206, + headers: { + 'content-length': String(PAYLOAD.length - 10), + 'content-range': `bytes 10-${PAYLOAD.length - 1}/${PAYLOAD.length}` + }, + chunks: [PAYLOAD.subarray(10)] + } + }) + const manager = new ModelManager(dir) as unknown as ModelManagerInternals + const archivePath = join(dir, 'model.tar.bz2') + + await manager.downloadArchiveWithRetry( + 'https://example.com/model.tar.bz2', + archivePath, + PAYLOAD.length, + 'm', + () => false, + new AbortController().signal + ) + + expect(netRequestMock).toHaveBeenCalledTimes(2) + expect(second.sentHeaders.range).toBe('bytes=10-') + expect(readFileSync(archivePath)).toEqual(PAYLOAD) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it('uses a complete archive after a late transport failure without requesting past EOF', async () => { + const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-')) + try { + scriptRequest(() => ({ + statusCode: 200, + headers: { 'content-length': String(PAYLOAD.length) }, + chunks: [PAYLOAD], + failWith: 'net::ERR_CONNECTION_RESET' + })) + const manager = new ModelManager(dir) as unknown as ModelManagerInternals + const archivePath = join(dir, 'model.tar.bz2') + + await manager.downloadArchiveWithRetry( + 'https://example.com/model.tar.bz2', + archivePath, + PAYLOAD.length, + 'm', + () => false, + new AbortController().signal + ) + + expect(netRequestMock).toHaveBeenCalledTimes(1) + expect(readFileSync(archivePath)).toEqual(PAYLOAD) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it('requests the remaining bytes when a clean range response ends before the archive total', async () => { + const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-')) + try { + const first = scriptRequest((sentHeaders) => { + expect(sentHeaders.range).toBe('bytes=10-') + return { + statusCode: 206, + headers: { + 'content-length': '5', + 'content-range': `bytes 10-14/${PAYLOAD.length}` + }, + chunks: [PAYLOAD.subarray(10, 15)] + } + }) + const second = scriptRequest((sentHeaders) => { + expect(sentHeaders.range).toBe('bytes=15-') + return { + statusCode: 206, + headers: { + 'content-length': '5', + 'content-range': `bytes 15-19/${PAYLOAD.length}` + }, + chunks: [PAYLOAD.subarray(15)] + } + }) + const manager = new ModelManager(dir) as unknown as ModelManagerInternals + const archivePath = join(dir, 'model.tar.bz2') + writeFileSync(archivePath, PAYLOAD.subarray(0, 10)) + + await manager.downloadArchiveWithRetry( + 'https://example.com/model.tar.bz2', + archivePath, + PAYLOAD.length, + 'm', + () => false, + new AbortController().signal + ) + + expect(netRequestMock).toHaveBeenCalledTimes(2) + expect(first.sentHeaders.range).toBe('bytes=10-') + expect(second.sentHeaders.range).toBe('bytes=15-') + expect(readFileSync(archivePath)).toEqual(PAYLOAD) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it('continues through more than eight advancing range segments', async () => { + const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-')) + try { + for (let offset = 1; offset <= 9; offset += 1) { + scriptRequest((sentHeaders) => { + expect(sentHeaders.range).toBe(`bytes=${offset}-`) + return { + statusCode: 206, + headers: { + 'content-length': '1', + 'content-range': `bytes ${offset}-${offset}/${PAYLOAD.length}` + }, + chunks: [PAYLOAD.subarray(offset, offset + 1)] + } + }) + } + scriptRequest((sentHeaders) => { + expect(sentHeaders.range).toBe('bytes=10-') + return { + statusCode: 206, + headers: { + 'content-length': '10', + 'content-range': `bytes 10-19/${PAYLOAD.length}` + }, + chunks: [PAYLOAD.subarray(10)] + } + }) + const manager = new ModelManager(dir) as unknown as ModelManagerInternals + const archivePath = join(dir, 'model.tar.bz2') + writeFileSync(archivePath, PAYLOAD.subarray(0, 1)) + + await manager.downloadArchiveWithRetry( + 'https://example.com/model.tar.bz2', + archivePath, + PAYLOAD.length, + 'm', + () => false, + new AbortController().signal + ) + + expect(netRequestMock).toHaveBeenCalledTimes(10) + expect(readFileSync(archivePath)).toEqual(PAYLOAD) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it('bounds a server that advances by pathologically tiny segments forever', async () => { + const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-')) + try { + const manager = new ModelManager(dir) as unknown as ModelManagerInternals + const archivePath = join(dir, 'model.tar.bz2') + let bytesWritten = 0 + // Advances one byte per request against a total larger than the request + // ceiling, so it makes forward progress forever without ever completing. + const downloadFileMock = vi.spyOn(manager, 'downloadFile').mockImplementation(() => { + bytesWritten += 1 + writeFileSync(archivePath, Buffer.alloc(bytesWritten)) + return Promise.resolve() + }) + + await expect( + manager.downloadArchiveWithRetry( + 'https://example.com/model.tar.bz2', + archivePath, + 1_000_000, + 'm', + () => false, + new AbortController().signal + ) + ).rejects.toThrow(/too many download requests/) + + // The absolute request ceiling (MAX_TOTAL_DOWNLOAD_REQUESTS) bounds it. + expect(downloadFileMock).toHaveBeenCalledTimes(4_096) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it('keeps resuming a download that advances across many mid-stream drops', async () => { + vi.useFakeTimers() + const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-')) + try { + // Every attempt delivers a small slice then drops mid-stream — the + // classic "dies partway, resumes" pattern. Because each attempt makes + // forward progress, the download must complete no matter how many drops + // it takes (regression guard: a fixed failure budget used to abandon a + // still-advancing large download around attempt 8). + const manager = new ModelManager(dir) as unknown as ModelManagerInternals + const archivePath = join(dir, 'model.tar.bz2') + const SLICE = 2 + let delivered = 0 + const downloadFileMock = vi.spyOn(manager, 'downloadFile').mockImplementation(() => { + delivered = Math.min(delivered + SLICE, PAYLOAD.length) + writeFileSync(archivePath, PAYLOAD.subarray(0, delivered)) + return Promise.reject(new Error('net::ERR_CONNECTION_RESET')) + }) + + const download = manager.downloadArchiveWithRetry( + 'https://example.com/model.tar.bz2', + archivePath, + PAYLOAD.length, + 'm', + () => false, + new AbortController().signal + ) + const outcome = download.then( + () => 'resolved', + (error: unknown) => (error instanceof Error ? error.message : String(error)) + ) + // Drive the per-attempt backoff (1s each, since progress resets the + // stall counter) well past the ten attempts this needs. + await vi.advanceTimersByTimeAsync(30_000) + + await expect(outcome).resolves.toBe('resolved') + expect(downloadFileMock).toHaveBeenCalledTimes(PAYLOAD.length / SLICE) + expect(readFileSync(archivePath)).toEqual(PAYLOAD) + } finally { + vi.useRealTimers() + rmSync(dir, { recursive: true, force: true }) + } + }) + + it('keeps the known archive total when Content-Range omits it', async () => { + const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-')) + try { + scriptRequest((sentHeaders) => { + expect(sentHeaders.range).toBe('bytes=10-') + return { + statusCode: 206, + headers: { + 'content-length': '5', + 'content-range': 'bytes 10-14/*' + }, + chunks: [PAYLOAD.subarray(10, 15)] + } + }) + scriptRequest((sentHeaders) => { + expect(sentHeaders.range).toBe('bytes=15-') + return { + statusCode: 206, + headers: { + 'content-length': '5', + 'content-range': 'bytes 15-19/*' + }, + chunks: [PAYLOAD.subarray(15)] + } + }) + const manager = new ModelManager(dir) as unknown as ModelManagerInternals + const archivePath = join(dir, 'model.tar.bz2') + writeFileSync(archivePath, PAYLOAD.subarray(0, 10)) + + await manager.downloadArchiveWithRetry( + 'https://example.com/model.tar.bz2', + archivePath, + PAYLOAD.length, + 'm', + () => false, + new AbortController().signal + ) + + expect(netRequestMock).toHaveBeenCalledTimes(2) + expect(readFileSync(archivePath)).toEqual(PAYLOAD) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it('rejects and discards a partial when Content-Range does not match the offset', async () => { + const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-')) + try { + const mismatched = scriptRequest(() => ({ + statusCode: 206, + headers: { + 'content-length': '10', + 'content-range': `bytes 0-9/${PAYLOAD.length}` + }, + chunks: [PAYLOAD.subarray(0, 10)] + })) + const manager = new ModelManager(dir) as unknown as ModelManagerInternals + const archivePath = join(dir, 'model.tar.bz2') + writeFileSync(archivePath, PAYLOAD.subarray(0, 10)) + + const error = await manager + .downloadFile( + 'https://example.com/model.tar.bz2', + archivePath, + PAYLOAD.length, + 'm', + () => false, + new AbortController().signal, + 0, + 10 + ) + .catch((cause: unknown) => cause) + + expect(error).toMatchObject({ + message: 'Invalid Content-Range for resume at byte 10', + retryable: true + }) + expect(existsSync(archivePath)).toBe(false) + expect(mismatched.abortMock).toHaveBeenCalledTimes(1) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it('restarts from scratch when the server ignores the Range request', async () => { + const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-')) + try { + scriptRequest(() => ({ + statusCode: 200, + headers: { 'content-length': String(PAYLOAD.length) }, + chunks: [PAYLOAD.subarray(0, 10)], + failWith: 'net::ERR_CONNECTION_RESET' + })) + scriptRequest(() => ({ + statusCode: 200, + headers: { 'content-length': String(PAYLOAD.length) }, + chunks: [PAYLOAD] + })) + const manager = new ModelManager(dir) as unknown as ModelManagerInternals + const archivePath = join(dir, 'model.tar.bz2') + + await manager.downloadArchiveWithRetry( + 'https://example.com/model.tar.bz2', + archivePath, + PAYLOAD.length, + 'm', + () => false, + new AbortController().signal + ) + + expect(netRequestMock).toHaveBeenCalledTimes(2) + expect(readFileSync(archivePath)).toEqual(PAYLOAD) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it('does not retry non-transient failures', async () => { + const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-')) + try { + scriptRequest(() => ({ statusCode: 404 })) + const manager = new ModelManager(dir) as unknown as ModelManagerInternals + + await expect( + manager.downloadArchiveWithRetry( + 'https://example.com/model.tar.bz2', + join(dir, 'model.tar.bz2'), + PAYLOAD.length, + 'm', + () => false, + new AbortController().signal + ) + ).rejects.toThrow('HTTP 404') + + expect(netRequestMock).toHaveBeenCalledTimes(1) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it('aborts an HTTP error response instead of draining it after rejection', async () => { + const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-')) + try { + const scripted = scriptRequest(() => ({ + statusCode: 429, + headers: { 'retry-after': '3' } + })) + const manager = new ModelManager(dir) as unknown as ModelManagerInternals + + const error = await manager + .downloadFile( + 'https://example.com/model.tar.bz2', + join(dir, 'model.tar.bz2'), + PAYLOAD.length, + 'm', + () => false + ) + .catch((cause: unknown) => cause) + + expect(error).toMatchObject({ message: 'HTTP 429', httpStatusCode: 429, retryAfterMs: 3_000 }) + expect(scripted.abortMock).toHaveBeenCalledTimes(1) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it('honors Retry-After before issuing another request', async () => { + vi.useFakeTimers() + const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-')) + try { + const manager = new ModelManager(dir) as unknown as ModelManagerInternals + const archivePath = join(dir, 'model.tar.bz2') + const rateLimitError = Object.assign(new Error('HTTP 429'), { + httpStatusCode: 429, + retryAfterMs: 3_000 + }) + const downloadFileMock = vi + .spyOn(manager, 'downloadFile') + .mockRejectedValueOnce(rateLimitError) + .mockImplementationOnce(() => { + writeFileSync(archivePath, PAYLOAD) + return Promise.resolve() + }) + const download = manager.downloadArchiveWithRetry( + 'https://example.com/model.tar.bz2', + archivePath, + PAYLOAD.length, + 'm', + () => false, + new AbortController().signal + ) + + await vi.advanceTimersByTimeAsync(2_999) + expect(downloadFileMock).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(1) + + await expect(download).resolves.toBeUndefined() + expect(downloadFileMock).toHaveBeenCalledTimes(2) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it('does not retry before an excessively long Retry-After window', async () => { + const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-')) + try { + const manager = new ModelManager(dir) as unknown as ModelManagerInternals + const rateLimitError = Object.assign(new Error('HTTP 429'), { + httpStatusCode: 429, + retryAfterMs: 300_000 + }) + const downloadFileMock = vi.spyOn(manager, 'downloadFile').mockRejectedValue(rateLimitError) + + await expect( + manager.downloadArchiveWithRetry( + 'https://example.com/model.tar.bz2', + join(dir, 'model.tar.bz2'), + PAYLOAD.length, + 'm', + () => false, + new AbortController().signal + ) + ).rejects.toThrow('server requested retry after 300 seconds') + + expect(downloadFileMock).toHaveBeenCalledTimes(1) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it('gives up after repeated zero-progress failures with a diagnosable error', async () => { + vi.useFakeTimers() + const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-')) + try { + const manager = new ModelManager(dir) as unknown as ModelManagerInternals + const downloadFileMock = vi + .spyOn(manager, 'downloadFile') + .mockRejectedValue(new Error('net::ERR_CONNECTION_RESET')) + + const download = manager.downloadArchiveWithRetry( + 'https://example.com/model.tar.bz2', + join(dir, 'model.tar.bz2'), + PAYLOAD.length, + 'm', + () => false, + new AbortController().signal + ) + const outcome = download.then( + () => 'resolved', + (error: unknown) => (error instanceof Error ? error.message : String(error)) + ) + // Let the first rejection schedule its backoff before advancing timers; + // this avoids racing fake time against stream/file I/O under full-suite load. + await vi.advanceTimersByTimeAsync(0) + expect(downloadFileMock).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(999) + expect(downloadFileMock).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(1) + expect(downloadFileMock).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(1_999) + expect(downloadFileMock).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(1) + expect(downloadFileMock).toHaveBeenCalledTimes(3) + await vi.advanceTimersByTimeAsync(3_999) + expect(downloadFileMock).toHaveBeenCalledTimes(3) + await vi.advanceTimersByTimeAsync(1) + + await expect(outcome).resolves.toMatch( + /Model download interrupted at 0% \(0 of 20 bytes\) after 4 attempts: .*net::ERR_CONNECTION_RESET/ + ) + expect(downloadFileMock).toHaveBeenCalledTimes(4) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it('stops retrying once the download is aborted', async () => { + const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-')) + try { + scriptRequest(() => ({ + statusCode: 200, + headers: { 'content-length': String(PAYLOAD.length) }, + failWith: 'net::ERR_CONNECTION_RESET' + })) + const controller = new AbortController() + const manager = new ModelManager(dir) as unknown as ModelManagerInternals + + const download = manager.downloadArchiveWithRetry( + 'https://example.com/model.tar.bz2', + join(dir, 'model.tar.bz2'), + PAYLOAD.length, + 'm', + () => false, + controller.signal + ) + const outcome = download.then( + () => 'resolved', + (error: unknown) => (error instanceof Error ? error.message : String(error)) + ) + // Abort after the transfer failure enters backoff; the next attempt must + // settle as Aborted without issuing another request. + setTimeout(() => controller.abort(), 100) + const message = await outcome + + expect(message).toBe('Aborted') + expect(netRequestMock).toHaveBeenCalledTimes(1) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/src/main/speech/model-manager.ts b/src/main/speech/model-manager.ts index 5671f9f87d3..c8f918f1526 100644 --- a/src/main/speech/model-manager.ts +++ b/src/main/speech/model-manager.ts @@ -1,7 +1,14 @@ /* eslint-disable max-lines -- Why: model download, checksum, extraction, and cleanup share one state machine so progress/error transitions stay coupled. */ import { app, net } from 'electron' import { join, resolve, relative } from 'node:path' -import { existsSync, mkdirSync, createWriteStream, createReadStream, rmSync } from 'node:fs' +import { + existsSync, + mkdirSync, + createWriteStream, + createReadStream, + rmSync, + statSync +} from 'node:fs' import { readdir, rm } from 'node:fs/promises' import { createHash } from 'node:crypto' import { pipeline } from 'node:stream/promises' @@ -28,11 +35,129 @@ type ProgressCallback = (modelId: string, progress: number) => void type DownloadIncomingMessage = Electron.IncomingMessage & NodeJS.ReadableStream & { headers: Record - resume: () => void destroy?: () => void } +type HttpStatusError = Error & { + httpStatusCode?: number + retryAfterMs?: number + retryable?: boolean +} +type DownloadTotals = { totalBytes: number } +type ContentRange = { start: number; end: number; totalBytes?: number } const DOWNLOAD_IDLE_TIMEOUT_MS = 120_000 +// Why: flaky networks and scanning proxies commonly kill long CDN transfers +// near the end (reported as "stuck at 90%, then fails"); resuming with a Range +// request lets those connections finish instead of failing the same way on +// every full restart. +const DOWNLOAD_RETRY_DELAYS_MS = [1_000, 2_000, 4_000] +// Why: abandon a download only after this many CONSECUTIVE attempts make no +// forward progress. A resume that keeps advancing (even across many mid-stream +// drops) is never given up on — that is the whole point of resumable downloads +// for large models over flaky networks, and a fixed all-time failure budget +// would wrongly kill a steadily-progressing large download. +const MAX_NO_PROGRESS_ATTEMPTS = DOWNLOAD_RETRY_DELAYS_MS.length + 1 +// Why: an absolute backstop so a server handing back pathologically tiny (or +// zero-net) segments cannot loop forever; sized to cover the largest (~1GB) +// model even if a proxy caps each range response near ~256KB, far below any +// real CDN chunk. +const MAX_TOTAL_DOWNLOAD_REQUESTS = 4_096 +// Why: a longer server window should be surfaced for a later manual retry, +// rather than leaving the settings download looking stuck for minutes. +const MAX_RETRY_AFTER_MS = 120_000 +const RETRYABLE_NET_ERROR = + /net::ERR_(CONTENT_LENGTH_MISMATCH|INCOMPLETE_CHUNKED_ENCODING|CONNECTION_(RESET|CLOSED|ABORTED|REFUSED|TIMED_OUT)|EMPTY_RESPONSE|NETWORK_CHANGED|TIMED_OUT|INTERNET_DISCONNECTED|ADDRESS_UNREACHABLE|NAME_NOT_RESOLVED|SOCKET_NOT_CONNECTED|HTTP2_PROTOCOL_ERROR|QUIC_PROTOCOL_ERROR)\b/ +const RETRYABLE_HTTP_STATUSES = new Set([408, 416, 425, 429, 500, 502, 503, 504]) + +function isRetryableDownloadError(error: unknown): boolean { + if (!(error instanceof Error)) { + return false + } + const downloadError = error as HttpStatusError + if (downloadError.retryable === true) { + return true + } + const statusCode = downloadError.httpStatusCode + if (statusCode !== undefined) { + return RETRYABLE_HTTP_STATUSES.has(statusCode) + } + return ( + RETRYABLE_NET_ERROR.test(error.message) || error.message.includes('without network activity') + ) +} + +function getHeaderValue(value: string | string[] | undefined): string | undefined { + return Array.isArray(value) ? value[0] : value +} + +function parseContentRange(value: string | string[] | undefined): ContentRange | null { + const match = getHeaderValue(value) + ?.trim() + .match(/^bytes\s+(\d+)-(\d+)\/(\d+|\*)$/i) + if (!match) { + return null + } + const start = Number.parseInt(match[1], 10) + const end = Number.parseInt(match[2], 10) + const totalBytes = match[3] === '*' ? undefined : Number.parseInt(match[3], 10) + if ( + !Number.isSafeInteger(start) || + !Number.isSafeInteger(end) || + end < start || + (totalBytes !== undefined && (!Number.isSafeInteger(totalBytes) || totalBytes <= end)) + ) { + return null + } + return { start, end, totalBytes } +} + +function parseRetryAfterMs(value: string | string[] | undefined): number | undefined { + const header = getHeaderValue(value)?.trim() + if (!header) { + return undefined + } + if (/^\d+$/.test(header)) { + const seconds = Number.parseInt(header, 10) + const delayMs = seconds * 1_000 + return Number.isSafeInteger(delayMs) ? delayMs : undefined + } + const retryAt = Date.parse(header) + return Number.isNaN(retryAt) ? undefined : Math.max(0, retryAt - Date.now()) +} + +function describeInterruptedDownload( + cause: unknown, + receivedBytes: number, + totalBytes: number, + attempts: number +): Error { + const causeMessage = cause instanceof Error ? cause.message : String(cause) + const received = + totalBytes > 0 + ? `${Math.min(99, Math.floor((receivedBytes / totalBytes) * 100))}% (${receivedBytes} of ${totalBytes} bytes)` + : `${receivedBytes} bytes` + return new Error( + `Model download interrupted at ${received} after ${attempts} attempts: ${causeMessage}` + ) +} + +function sleepUnlessAborted(ms: number, signal: AbortSignal): Promise { + return new Promise((resolve) => { + if (signal.aborted) { + resolve() + return + } + const onAbort = (): void => { + clearTimeout(timer) + resolve() + } + const timer = setTimeout(() => { + signal.removeEventListener('abort', onAbort) + resolve() + }, ms) + signal.addEventListener('abort', onAbort, { once: true }) + }) +} export class ModelManager { private modelsDir: string @@ -177,6 +302,15 @@ export class ModelManager { this.updateState(modelId, 'downloading', 0) const archivePath = join(this.modelsDir, `${modelId}.tar.bz2`) + // Why: resume appends to the partial archive from this download run only; + // a leftover file from a crashed run would corrupt the resumed archive. + try { + if (existsSync(archivePath)) { + rmSync(archivePath) + } + } catch { + // best-effort; the first (non-resumed) attempt truncates on write + } let aborted = false const abortController = new AbortController() @@ -191,7 +325,7 @@ export class ModelManager { this.activeDownloads.set(modelId, handle) try { - await this.downloadFile( + await this.downloadArchiveWithRetry( manifest.downloadUrl, archivePath, manifest.sizeBytes, @@ -239,6 +373,7 @@ export class ModelManager { this.updateState(modelId, 'ready') } catch (err) { if (!aborted) { + console.error('[speech] Model download failed:', modelId, err) this.updateState(modelId, 'error', undefined, String(err)) } this.cleanup(modelId, archivePath) @@ -309,6 +444,120 @@ export class ModelManager { } } + private getPartialArchiveBytes(archivePath: string): number { + try { + return statSync(archivePath).size + } catch { + return 0 + } + } + + private async downloadArchiveWithRetry( + url: string, + archivePath: string, + expectedSize: number, + modelId: string, + isAborted: () => boolean, + signal: AbortSignal + ): Promise { + let requestCount = 0 + let noProgressStreak = 0 + const totals: DownloadTotals = { totalBytes: expectedSize } + for (;;) { + requestCount += 1 + const offset = this.getPartialArchiveBytes(archivePath) + // Why: the transport can fail after the final byte reached disk; the + // caller's SHA-256 check is the authoritative completion test. + if (offset === totals.totalBytes) { + return + } + // Why: absolute backstop against a server that never lets the download + // finish; the stall and abort paths handle the common cases first. + if (requestCount > MAX_TOTAL_DOWNLOAD_REQUESTS) { + throw describeInterruptedDownload( + new Error('too many download requests'), + offset, + totals.totalBytes, + requestCount - 1 + ) + } + try { + // Why: each attempt restarts from the canonical URL rather than the + // last redirect target, because CDN redirect URLs are signed and expire. + await this.downloadFile( + url, + archivePath, + expectedSize, + modelId, + isAborted, + signal, + 0, + offset, + totals + ) + const receivedBytes = this.getPartialArchiveBytes(archivePath) + if (receivedBytes === totals.totalBytes) { + return + } + if (receivedBytes > totals.totalBytes) { + throw new Error( + `Model download exceeded its expected size (${receivedBytes} of ${totals.totalBytes} bytes)` + ) + } + const incompleteResponse = new Error( + `Model download response ended at ${receivedBytes} of ${totals.totalBytes} bytes` + ) + if (receivedBytes > offset) { + // Why: some proxies return a valid but bounded range segment; request + // the next segment immediately instead of failing checksum or waiting. + // Forward progress resets the stall counter; the total-request ceiling + // at the loop top bounds a pathological tiny-segment server. + noProgressStreak = 0 + continue + } + const retryableIncompleteResponse = incompleteResponse as HttpStatusError + retryableIncompleteResponse.retryable = true + throw retryableIncompleteResponse + } catch (err) { + if (isAborted() || signal.aborted) { + throw err + } + const receivedBytes = this.getPartialArchiveBytes(archivePath) + if (receivedBytes === totals.totalBytes) { + return + } + noProgressStreak = receivedBytes > offset ? 0 : noProgressStreak + 1 + if (!isRetryableDownloadError(err)) { + throw err + } + // Why: give up only on a genuine stall (repeated no forward progress); + // a download that keeps advancing across drops is allowed to continue. + if (noProgressStreak >= MAX_NO_PROGRESS_ATTEMPTS) { + throw describeInterruptedDownload(err, receivedBytes, totals.totalBytes, requestCount) + } + const retryAfterMs = (err as HttpStatusError).retryAfterMs + if (retryAfterMs !== undefined && retryAfterMs > MAX_RETRY_AFTER_MS) { + const statusCode = (err as HttpStatusError).httpStatusCode + throw new Error( + `HTTP ${statusCode}; server requested retry after ${Math.ceil(retryAfterMs / 1_000)} seconds` + ) + } + console.warn( + `[speech] Model download attempt ${requestCount} failed, retrying:`, + modelId, + err + ) + await sleepUnlessAborted( + retryAfterMs ?? + DOWNLOAD_RETRY_DELAYS_MS[ + Math.min(Math.max(0, noProgressStreak - 1), DOWNLOAD_RETRY_DELAYS_MS.length - 1) + ], + signal + ) + } + } + } + private downloadFile( url: string, dest: string, @@ -316,7 +565,9 @@ export class ModelManager { modelId: string, isAborted: () => boolean, signal?: AbortSignal, - redirectCount = 0 + redirectCount = 0, + resumeOffset = 0, + totals?: DownloadTotals ): Promise { return new Promise((resolve, reject) => { if (signal?.aborted) { @@ -425,28 +676,81 @@ export class ModelManager { modelId, isAborted, signal, - redirectCount + 1 + redirectCount + 1, + resumeOffset, + totals ) .then(resolveOnce) .catch(rejectOnce) } const onResponse = (incoming: Electron.IncomingMessage): void => { const response = incoming as DownloadIncomingMessage - if (response.statusCode !== 200) { - response.resume() - rejectOnce(new Error(`HTTP ${response.statusCode}`)) + const contentLength = response.headers['content-length'] + const headerLength = Number.parseInt(getHeaderValue(contentLength) || '0', 10) + const parsedLength = + Number.isSafeInteger(headerLength) && headerLength > 0 ? headerLength : 0 + const contentRange = parseContentRange(response.headers['content-range']) + const resumed = + resumeOffset > 0 && + response.statusCode === 206 && + contentRange?.start === resumeOffset && + (parsedLength <= 0 || parsedLength === contentRange.end - contentRange.start + 1) + + if (resumeOffset > 0 && response.statusCode === 206 && !resumed) { + // Why: appending an unverified range can silently corrupt the archive; + // discard it so the canonical URL is retried from byte zero. + try { + rmSync(dest) + } catch { + // best-effort + } + const activeRequest = request + const rangeError: HttpStatusError = new Error( + `Invalid Content-Range for resume at byte ${resumeOffset}` + ) + rangeError.retryable = true + rejectOnce(rangeError) + activeRequest?.abort() return } - const contentLength = response.headers['content-length'] - const totalSize = - Number.parseInt( - Array.isArray(contentLength) ? contentLength[0] : contentLength || '0', - 10 - ) || expectedSize + if (response.statusCode !== 200 && !resumed) { + if (response.statusCode === 416) { + // Why: the server rejected our resume offset; drop the partial so + // the retry loop can restart this download from scratch. + try { + rmSync(dest) + } catch { + // best-effort + } + } + const activeRequest = request + const statusError: HttpStatusError = new Error(`HTTP ${response.statusCode}`) + statusError.httpStatusCode = response.statusCode + statusError.retryAfterMs = parseRetryAfterMs(response.headers['retry-after']) + rejectOnce(statusError) + // Why: retrying must not leave a prior error body draining without + // cancellation or idle-timeout ownership. + activeRequest?.abort() + return + } + + // Why: a 200 despite our Range request means the server restarted the + // transfer from byte zero, so the partial file must be overwritten. + const progressBase = resumed ? resumeOffset : 0 + // Why: Content-Length on a 206 describes only this segment; when + // Content-Range uses '*', retain the known complete archive size. + const totalSize = resumed + ? (contentRange?.totalBytes ?? totals?.totalBytes ?? expectedSize) + : parsedLength > 0 + ? parsedLength + : expectedSize + if (totals) { + totals.totalBytes = totalSize + } let downloaded = 0 - const fileStream = createWriteStream(dest) + const fileStream = createWriteStream(dest, { flags: resumed ? 'a' : 'w' }) const cleanupResponseProgressListener = (): void => { response.off('data', onResponseData) @@ -460,7 +764,7 @@ export class ModelManager { return } downloaded += chunk.length - const progress = Math.min(0.9, downloaded / totalSize) + const progress = Math.min(0.9, (progressBase + downloaded) / totalSize) this.updateState(modelId, 'downloading', progress) } @@ -481,6 +785,9 @@ export class ModelManager { } request = net.request({ method: 'GET', url: parsedUrl.toString() }) + if (resumeOffset > 0) { + request.setHeader('Range', `bytes=${resumeOffset}-`) + } // Why: Electron's net stack honors app proxy settings, unlike Node's // https client, but it does not expose request.setTimeout(). diff --git a/src/renderer/src/components/settings/VoiceSpeechModelSection.tsx b/src/renderer/src/components/settings/VoiceSpeechModelSection.tsx index ea96c68fd39..acd8409e74d 100644 --- a/src/renderer/src/components/settings/VoiceSpeechModelSection.tsx +++ b/src/renderer/src/components/settings/VoiceSpeechModelSection.tsx @@ -13,6 +13,13 @@ import { import { Cloud, Download, Trash2, Loader2, ChevronDown, Check } from 'lucide-react' import { translate } from '@/i18n/i18n' +function describeSpeechModelDownloadError(error: unknown): string { + const message = error instanceof Error ? error.message : String(error) + // Why: ipcRenderer.invoke wraps main-process rejections; strip the transport + // prefix so the toast shows only the underlying download failure. + return message.replace(/^Error invoking remote method '[^']+': (?:Error: )?/, '') +} + type VoiceSpeechModelSectionProps = { voiceSettings: VoiceSettings catalog: SpeechModelManifest[] @@ -90,16 +97,17 @@ export function VoiceSpeechModelSection({ } else if (!isDownloading) { // Why: download progress appears in this menu, so starting one should not dismiss it. event.preventDefault() - void window.api.speech - .downloadModel(manifest.id) - .catch(() => - toast.error( - translate( - 'auto.components.settings.VoicePane.cfde55c7b0', - 'Failed to download model.' - ) - ) + void window.api.speech.downloadModel(manifest.id).catch((error: unknown) => + toast.error( + translate( + 'auto.components.settings.VoicePane.cfde55c7b0', + 'Failed to download model.' + ), + // Why: the raw cause (e.g. net::ERR_CONTENT_LENGTH_MISMATCH) + // is the only diagnosable signal users can report back. + { description: describeSpeechModelDownloadError(error) } ) + ) } }} className={`group flex items-center gap-2.5 py-2.5 ${ From a03377fe0fefc698831731f636e237e85e635917 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 00:50:08 +0000 Subject: [PATCH 08/15] release: v1.4.142-rc.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index dfe8c1c51b2..58fbc7aabef 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "orca", - "version": "1.4.142-rc.1", + "version": "1.4.142-rc.2", "description": "Next-gen IDE for parallel agentic development", "homepage": "https://github.com/stablyai/orca", "author": "stablyai", From 53c8a5583314c01c15e3769885f501aaf3293f29 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 14 Jul 2026 19:15:50 -0700 Subject: [PATCH 09/15] Add mobile notification opt-in onboarding (#8780) * feat(mobile): add notification opt-in onboarding * fix(mobile): deliver alerts despite desktop focus --- mobile/app/_layout.tsx | 4 + mobile/app/index.tsx | 19 +- mobile/app/notification-opt-in.tsx | 219 ++++++++++++++++++ mobile/app/notifications.tsx | 4 +- mobile/app/pair-confirm.tsx | 11 +- mobile/app/pair-scan.tsx | 11 +- .../notification-opt-in-gate.test.ts | 76 ++++++ .../notifications/notification-opt-in-gate.ts | 33 +++ mobile/src/storage/preferences.test.ts | 43 ++++ mobile/src/storage/preferences.ts | 28 ++- src/main/ipc/notifications.test.ts | 24 +- src/main/ipc/notifications.ts | 62 ++--- 12 files changed, 481 insertions(+), 53 deletions(-) create mode 100644 mobile/app/notification-opt-in.tsx create mode 100644 mobile/src/notifications/notification-opt-in-gate.test.ts create mode 100644 mobile/src/notifications/notification-opt-in-gate.ts diff --git a/mobile/app/_layout.tsx b/mobile/app/_layout.tsx index b3bd5c85b14..1059c95502a 100644 --- a/mobile/app/_layout.tsx +++ b/mobile/app/_layout.tsx @@ -178,6 +178,10 @@ export default function RootLayout() { + diff --git a/mobile/app/index.tsx b/mobile/app/index.tsx index 45821de712b..e26893afd84 100644 --- a/mobile/app/index.tsx +++ b/mobile/app/index.tsx @@ -36,6 +36,7 @@ import { } from '../src/transport/client-context' import { classifyConnection } from '../src/transport/connection-health' import { subscribeToDesktopNotifications } from '../src/notifications/mobile-notifications' +import { shouldPresentNotificationOptIn } from '../src/notifications/notification-opt-in-gate' import type { ConnectionState, HostProfile } from '../src/transport/types' import { triggerMediumImpact } from '../src/platform/haptics' import { OrcaLogo } from '../src/components/OrcaLogo' @@ -317,6 +318,7 @@ export default function HomeScreen() { const [lastVisited, setLastVisited] = useState<{ hostId: string; worktreeId: string } | null>( null ) + const notificationOptInCheckedRef = useRef(false) // Why: read shared clients from the per-host store. Replaces the prior // pattern of opening N independent WebSockets here. See @@ -395,9 +397,18 @@ export default function HomeScreen() { useFocusEffect( useCallback(() => { let stale = false - void loadHosts().then((h) => { - if (!stale) { - setHosts(h) + void loadHosts().then(async (h) => { + if (stale) { + return + } + setHosts(h) + if (h.length === 0 || notificationOptInCheckedRef.current) { + return + } + notificationOptInCheckedRef.current = true + const showNotificationOptIn = await shouldPresentNotificationOptIn() + if (!stale && showNotificationOptIn) { + router.replace('/notification-opt-in') } }) void AsyncStorage.getItem('orca:last-visited-worktree').then((raw) => { @@ -419,7 +430,7 @@ export default function HomeScreen() { return () => { stale = true } - }, []) + }, [router]) ) const sortedHosts = useMemo( diff --git a/mobile/app/notification-opt-in.tsx b/mobile/app/notification-opt-in.tsx new file mode 100644 index 00000000000..41f2c2168e2 --- /dev/null +++ b/mobile/app/notification-opt-in.tsx @@ -0,0 +1,219 @@ +import { useCallback, useState } from 'react' +import { ActivityIndicator, BackHandler, Pressable, StyleSheet, Text, View } from 'react-native' +import { useFocusEffect, useLocalSearchParams, useRouter } from 'expo-router' +import { SafeAreaView } from 'react-native-safe-area-context' +import { BellRing } from 'lucide-react-native' +import { OrcaLogo } from '../src/components/OrcaLogo' +import { ensureNotificationPermissions } from '../src/notifications/mobile-notifications' +import { savePushNotificationsEnabled } from '../src/storage/preferences' +import { colors, radii, spacing, typography } from '../src/theme/mobile-theme' + +export default function NotificationOptInScreen() { + const router = useRouter() + const params = useLocalSearchParams<{ hostId?: string | string[] }>() + const hostId = Array.isArray(params.hostId) ? params.hostId[0] : params.hostId + const [busyChoice, setBusyChoice] = useState<'enable' | 'skip' | null>(null) + const [error, setError] = useState(null) + + // Why: this one-time screen requires an explicit Enable or Not now choice; + // disabling back gestures alone would still leave Android hardware back open. + useFocusEffect( + useCallback(() => { + const subscription = BackHandler.addEventListener('hardwareBackPress', () => true) + return () => subscription.remove() + }, []) + ) + + const continueToApp = useCallback(() => { + router.replace(hostId ? `/h/${hostId}` : '/') + }, [hostId, router]) + + const choose = useCallback( + async (choice: 'enable' | 'skip') => { + if (busyChoice) { + return + } + setBusyChoice(choice) + setError(null) + try { + const enabled = choice === 'enable' ? await ensureNotificationPermissions() : false + await savePushNotificationsEnabled(enabled) + continueToApp() + } catch { + setError('Notification settings could not be updated. Try again.') + setBusyChoice(null) + } + }, + [busyChoice, continueToApp] + ) + + return ( + + + + Orca + + + + + + + Notifications + Stay updated while away + + Get notified on this device when an agent needs your input or finishes a task. + + + + + {error ? ( + + {error} + + ) : null} + [ + styles.primaryButton, + pressed && styles.buttonPressed, + busyChoice !== null && styles.buttonDisabled + ]} + onPress={() => void choose('enable')} + > + {busyChoice === 'enable' ? ( + + ) : ( + Enable notifications + )} + + [ + styles.secondaryButton, + pressed && styles.buttonPressed, + busyChoice !== null && styles.buttonDisabled + ]} + onPress={() => void choose('skip')} + > + {busyChoice === 'skip' ? ( + + ) : ( + Not now + )} + + You can change this any time in Settings. + + + ) +} + +const styles = StyleSheet.create({ + container: { + flex: 1, + backgroundColor: colors.bgBase, + paddingHorizontal: spacing.xl + }, + brandRow: { + minHeight: 52, + flexDirection: 'row', + alignItems: 'center', + gap: spacing.sm + }, + brandName: { + color: colors.textPrimary, + fontSize: 17, + fontWeight: '700' + }, + content: { + flex: 1, + alignItems: 'center', + justifyContent: 'center', + paddingBottom: spacing.xl + }, + iconSurface: { + width: 64, + height: 64, + borderRadius: radii.card, + alignItems: 'center', + justifyContent: 'center', + backgroundColor: colors.bgRaised, + marginBottom: spacing.xl + }, + eyebrow: { + color: colors.textMuted, + fontSize: 11, + fontWeight: '600', + letterSpacing: 0.55, + textTransform: 'uppercase', + marginBottom: spacing.sm + }, + title: { + maxWidth: 420, + color: colors.textPrimary, + fontSize: 26, + fontWeight: '700', + letterSpacing: -0.3, + textAlign: 'center' + }, + body: { + maxWidth: 420, + color: colors.textSecondary, + fontSize: typography.bodySize, + lineHeight: 21, + textAlign: 'center', + marginTop: spacing.md + }, + footer: { + width: '100%', + maxWidth: 420, + alignSelf: 'center', + paddingBottom: spacing.lg + }, + primaryButton: { + height: 44, + alignItems: 'center', + justifyContent: 'center', + borderRadius: radii.button, + backgroundColor: colors.surfaceBright + }, + primaryButtonText: { + color: colors.bgBase, + fontSize: typography.bodySize, + fontWeight: '600' + }, + secondaryButton: { + height: 44, + alignItems: 'center', + justifyContent: 'center', + borderRadius: radii.button, + marginTop: spacing.xs + }, + secondaryButtonText: { + color: colors.textSecondary, + fontSize: typography.bodySize, + fontWeight: '500' + }, + buttonPressed: { + opacity: 0.72 + }, + buttonDisabled: { + opacity: 0.58 + }, + footerNote: { + color: colors.textMuted, + fontSize: typography.metaSize, + lineHeight: 18, + textAlign: 'center', + marginTop: spacing.sm + }, + error: { + color: colors.statusRed, + fontSize: typography.metaSize, + lineHeight: 18, + textAlign: 'center', + marginBottom: spacing.sm + } +}) diff --git a/mobile/app/notifications.tsx b/mobile/app/notifications.tsx index f7922074d6e..4adacfc016f 100644 --- a/mobile/app/notifications.tsx +++ b/mobile/app/notifications.tsx @@ -69,7 +69,7 @@ export default function NotificationsScreen() { const notificationsBlocked = permissionState.status === 'denied' const hint = notificationsBlocked ? 'Notifications are disabled in system settings.' - : 'Receive notifications when an agent task completes on your desktop.' + : 'Get notified on this device when an agent needs your input or finishes a task.' return ( @@ -82,7 +82,7 @@ export default function NotificationsScreen() { - Push Notifications + Agent notifications ({ + readPushNotificationsPreference: vi.fn(), + savePushNotificationsEnabled: vi.fn() +})) + +vi.mock('./mobile-notifications', () => ({ + getNotificationPermissionState: vi.fn() +})) + +describe('notification opt-in gate', () => { + beforeEach(() => { + vi.mocked(readPushNotificationsPreference).mockReset() + vi.mocked(savePushNotificationsEnabled).mockReset() + vi.mocked(getNotificationPermissionState).mockReset() + }) + + it('presents only when the local preference and system decision are both unset', async () => { + vi.mocked(readPushNotificationsPreference).mockResolvedValue({ value: null, loaded: true }) + vi.mocked(getNotificationPermissionState).mockResolvedValue({ + granted: false, + status: 'undetermined', + canAskAgain: true + }) + + await expect(shouldPresentNotificationOptIn()).resolves.toBe(true) + expect(savePushNotificationsEnabled).not.toHaveBeenCalled() + }) + + it.each([true, false])('preserves an existing %s mobile preference', async (value) => { + vi.mocked(readPushNotificationsPreference).mockResolvedValue({ value, loaded: true }) + + await expect(shouldPresentNotificationOptIn()).resolves.toBe(false) + expect(getNotificationPermissionState).not.toHaveBeenCalled() + }) + + it('adopts existing system authorization without prompting', async () => { + vi.mocked(readPushNotificationsPreference).mockResolvedValue({ value: null, loaded: true }) + vi.mocked(getNotificationPermissionState).mockResolvedValue({ + granted: true, + status: 'granted', + canAskAgain: true + }) + + await expect(shouldPresentNotificationOptIn()).resolves.toBe(false) + expect(savePushNotificationsEnabled).toHaveBeenCalledWith(true) + }) + + it('skips the gate when iOS has already denied permission', async () => { + vi.mocked(readPushNotificationsPreference).mockResolvedValue({ value: null, loaded: true }) + vi.mocked(getNotificationPermissionState).mockResolvedValue({ + granted: false, + status: 'denied', + canAskAgain: false + }) + + await expect(shouldPresentNotificationOptIn()).resolves.toBe(false) + expect(savePushNotificationsEnabled).toHaveBeenCalledWith(false) + }) + + it('does not block startup when storage or permission checks fail', async () => { + vi.mocked(readPushNotificationsPreference).mockResolvedValue({ value: null, loaded: false }) + await expect(shouldPresentNotificationOptIn()).resolves.toBe(false) + + vi.mocked(readPushNotificationsPreference).mockResolvedValue({ value: null, loaded: true }) + vi.mocked(getNotificationPermissionState).mockRejectedValue(new Error('unavailable')) + await expect(shouldPresentNotificationOptIn()).resolves.toBe(false) + }) +}) diff --git a/mobile/src/notifications/notification-opt-in-gate.ts b/mobile/src/notifications/notification-opt-in-gate.ts new file mode 100644 index 00000000000..9d7b4eed6cc --- /dev/null +++ b/mobile/src/notifications/notification-opt-in-gate.ts @@ -0,0 +1,33 @@ +import { + readPushNotificationsPreference, + savePushNotificationsEnabled +} from '../storage/preferences' +import { getNotificationPermissionState } from './mobile-notifications' + +export async function shouldPresentNotificationOptIn(): Promise { + const preference = await readPushNotificationsPreference() + if (!preference.loaded || preference.value !== null) { + return false + } + + try { + const permission = await getNotificationPermissionState() + if (permission.granted) { + // Why: an already-authorized device should inherit the useful default + // without seeing an onboarding decision it has effectively made. + await savePushNotificationsEnabled(true) + return false + } + if (permission.status === 'denied' || !permission.canAskAgain) { + // Why: iOS cannot show its authorization prompt again, so a blocking + // onboarding screen would be a dead end; Settings remains the recovery. + await savePushNotificationsEnabled(false) + return false + } + return permission.status === 'undetermined' + } catch { + // Why: permission or persistence failures must not trap startup behind a + // decision screen whose result cannot be applied reliably. + return false + } +} diff --git a/mobile/src/storage/preferences.test.ts b/mobile/src/storage/preferences.test.ts index 1e5aee95de1..f8e07fb4d98 100644 --- a/mobile/src/storage/preferences.test.ts +++ b/mobile/src/storage/preferences.test.ts @@ -10,11 +10,14 @@ import { clampHostSidebarWidth, loadDisabledTerminalLiveInputHandles, loadHostSidebarWidth, + loadPushNotificationsEnabled, loadTerminalAutocompleteEnabled, loadTerminalLinkOpenMode, + readPushNotificationsPreference, readDisabledTerminalLiveInputHandlesPreference, saveDisabledTerminalLiveInputHandles, saveHostSidebarWidth, + savePushNotificationsEnabled, saveTerminalAutocompleteEnabled, saveTerminalLinkOpenMode } from './preferences' @@ -26,6 +29,46 @@ vi.mock('@react-native-async-storage/async-storage', () => ({ } })) +describe('push notification preference', () => { + beforeEach(() => { + vi.mocked(AsyncStorage.getItem).mockReset() + vi.mocked(AsyncStorage.setItem).mockReset() + }) + + it('distinguishes an unset preference from an explicit disabled choice', async () => { + vi.mocked(AsyncStorage.getItem).mockResolvedValue(null) + await expect(readPushNotificationsPreference()).resolves.toEqual({ + value: null, + loaded: true + }) + await expect(loadPushNotificationsEnabled()).resolves.toBe(false) + + vi.mocked(AsyncStorage.getItem).mockResolvedValue('false') + await expect(readPushNotificationsPreference()).resolves.toEqual({ + value: false, + loaded: true + }) + }) + + it('reports storage failures without enabling notifications', async () => { + vi.mocked(AsyncStorage.getItem).mockRejectedValue(new Error('storage unavailable')) + + await expect(readPushNotificationsPreference()).resolves.toEqual({ + value: null, + loaded: false + }) + await expect(loadPushNotificationsEnabled()).resolves.toBe(false) + }) + + it('persists the onboarding decision in the existing mobile toggle', async () => { + await savePushNotificationsEnabled(true) + expect(AsyncStorage.setItem).toHaveBeenCalledWith('orca:pushNotificationsEnabled', 'true') + + await savePushNotificationsEnabled(false) + expect(AsyncStorage.setItem).toHaveBeenCalledWith('orca:pushNotificationsEnabled', 'false') + }) +}) + describe('terminal autocomplete preference', () => { beforeEach(() => { vi.mocked(AsyncStorage.getItem).mockReset() diff --git a/mobile/src/storage/preferences.ts b/mobile/src/storage/preferences.ts index d5b0db3a6e7..5173ac5bc8a 100644 --- a/mobile/src/storage/preferences.ts +++ b/mobile/src/storage/preferences.ts @@ -3,23 +3,29 @@ import AsyncStorage from '@react-native-async-storage/async-storage' const PINS_PREFIX = 'orca:pins:' const NOTIF_KEY = 'orca:pushNotificationsEnabled' -// Why: default-off so the iOS notification permission prompt never -// fires until the user explicitly opts in via Settings → Notifications. -// Apple's review guideline 4.5.4 and HIG both prefer user-initiated -// permission prompts; default-on would fire the prompt the moment the -// desktop sent its first notification, which can read as unsolicited. -export async function loadPushNotificationsEnabled(): Promise { +export type PushNotificationsPreference = { + readonly value: boolean | null + readonly loaded: boolean +} + +// Why: null distinguishes people who have never made the one-time onboarding +// decision from people who explicitly chose Not now or disabled notifications. +export async function readPushNotificationsPreference(): Promise { try { const raw = await AsyncStorage.getItem(NOTIF_KEY) - if (raw === null) { - return false - } - return raw === 'true' + return { value: raw === null ? null : raw === 'true', loaded: true } } catch { - return false + return { value: null, loaded: false } } } +// Why: default-off prevents background notification events from opening the +// system prompt; only the onboarding CTA or Settings switch requests permission. +export async function loadPushNotificationsEnabled(): Promise { + const preference = await readPushNotificationsPreference() + return preference.value ?? false +} + export async function savePushNotificationsEnabled(enabled: boolean): Promise { await AsyncStorage.setItem(NOTIF_KEY, String(enabled)) } diff --git a/src/main/ipc/notifications.test.ts b/src/main/ipc/notifications.test.ts index 8fb5566a895..d7930066ad7 100644 --- a/src/main/ipc/notifications.test.ts +++ b/src/main/ipc/notifications.test.ts @@ -899,7 +899,7 @@ describe('registerNotificationHandlers', () => { expect(dispatchMobileNotification).not.toHaveBeenCalled() }) - it('does not dispatch mobile notifications for focused active-worktree notifications', async () => { + it('dispatches one mobile notification when the active worktree is focused on desktop', async () => { getAllWindowsMock.mockReturnValue([ { isDestroyed: () => false, @@ -922,17 +922,25 @@ describe('registerNotificationHandlers', () => { ) const handler = getDispatchHandler() - expect( - await handler( - {}, - { source: 'agent-task-complete', worktreeId: 'repo::wt1', isActiveWorktree: true } - ) - ).toEqual({ + const focusedNotification = { + source: 'agent-task-complete' as const, + worktreeId: 'repo::wt1', + isActiveWorktree: true + } + expect(await handler({}, focusedNotification)).toEqual({ + delivered: false, + reason: 'suppressed-focus' + }) + expect(await handler({}, focusedNotification)).toEqual({ delivered: false, reason: 'suppressed-focus' }) - expect(dispatchMobileNotification).not.toHaveBeenCalled() + expect(dispatchMobileNotification).toHaveBeenCalledTimes(1) + expect(dispatchMobileNotification).toHaveBeenCalledWith( + expect.objectContaining({ source: 'agent-task-complete', worktreeId: 'repo::wt1' }) + ) + expect(notificationCtorMock).not.toHaveBeenCalled() }) it('does not dispatch mobile notifications for cooldown-suppressed bursts', async () => { diff --git a/src/main/ipc/notifications.ts b/src/main/ipc/notifications.ts index 47baa7e7725..4eb8938f2b4 100644 --- a/src/main/ipc/notifications.ts +++ b/src/main/ipc/notifications.ts @@ -312,8 +312,24 @@ function pruneRecentNotifications(recentNotifications: Map, now: } } +function reserveNotificationCooldown( + recentNotifications: Map, + dedupeKey: string, + now: number +): boolean { + const lastSentAt = recentNotifications.get(dedupeKey) ?? 0 + if (now - lastSentAt < NOTIFICATION_COOLDOWN_MS) { + return false + } + recentNotifications.delete(dedupeKey) + recentNotifications.set(dedupeKey, now) + pruneRecentNotifications(recentNotifications, now) + return true +} + export function registerNotificationHandlers(store: Store, runtime?: OrcaRuntimeService): void { - const recentNotifications = new Map() + const recentDesktopNotifications = new Map() + const recentMobileNotifications = new Map() // Why: handler registration marks a fresh session — permission evidence // from a previous registration must not leak into the new one. lastObservedDeliveryOutcome = null @@ -441,6 +457,24 @@ export function registerNotificationHandlers(store: Store, runtime?: OrcaRuntime return { delivered: false, reason: 'source-disabled' } } + const notificationOptions = buildNotificationOptions(args) + + // Why: desktop focus only means this computer has the worktree visible; + // the paired phone may be locked or elsewhere and still needs the alert. + if (runtime && args.source !== 'test') { + const dedupeKey = args.worktreeId ?? args.worktreeLabel ?? 'global' + if (reserveNotificationCooldown(recentMobileNotifications, dedupeKey, Date.now())) { + runtime.dispatchMobileNotification({ + type: 'notification', + source: args.source, + title: notificationOptions.title, + body: notificationOptions.body, + worktreeId: args.worktreeId, + ...(args.notificationId ? { notificationId: args.notificationId } : {}) + }) + } + } + const browserWindow = BrowserWindow.getAllWindows().find((window) => !window.isDestroyed()) ?? null if ( @@ -458,33 +492,9 @@ export function registerNotificationHandlers(store: Store, runtime?: OrcaRuntime // Dedupe by worktree, not by source — an agent finishing and a terminal bell // often fire within the same data chunk so only the first one should surface. const dedupeKey = args.worktreeId ?? args.worktreeLabel ?? 'global' - const now = Date.now() - const lastSentAt = recentNotifications.get(dedupeKey) ?? 0 - if (now - lastSentAt < NOTIFICATION_COOLDOWN_MS) { + if (!reserveNotificationCooldown(recentDesktopNotifications, dedupeKey, Date.now())) { return { delivered: false, reason: 'cooldown' } } - recentNotifications.delete(dedupeKey) - recentNotifications.set(dedupeKey, now) - - // Why: a storm across many worktrees should not make every - // notification dispatch scan an ever-growing cooldown table. - pruneRecentNotifications(recentNotifications, now) - } - - const notificationOptions = buildNotificationOptions(args) - - // Why: paired mobile clients should follow the same user-facing - // notification gates as desktop delivery, while still working on hosts - // where Electron native notifications are unavailable. - if (runtime && args.source !== 'test') { - runtime.dispatchMobileNotification({ - type: 'notification', - source: args.source, - title: notificationOptions.title, - body: notificationOptions.body, - worktreeId: args.worktreeId, - ...(args.notificationId ? { notificationId: args.notificationId } : {}) - }) } if (!Notification.isSupported()) { From 70e16cb9003603c6d27f71ae0a22079979033edb Mon Sep 17 00:00:00 2001 From: iridite Date: Wed, 15 Jul 2026 10:19:43 +0800 Subject: [PATCH 10/15] fix: stop renderer heap leak from unhandled terminal teardown rejections (#8279) Two renderer paths let Promise rejections escape unhandled, where the crash-breadcrumb system retains them and grows the heap until the renderer white-screens (#8260): - The terminal file-link provider fires a `void Promise.all(...)` whose async mappers probe path existence over SSH. On disconnect these reject with "Remote connection dropped/reconnecting"; add a `.catch` that resolves the callback with no links so the rejection is handled. - "ResizeObserver loop completed" is a benign Chromium quirk; suppress it in crash-diagnostics so it stops filling the breadcrumb buffer and inflating the renderer error count. Scope note: the original fix for #8260 also proposed Windows pty:kill signal handling (killPtyProcess / isPtyAlreadyGoneError) and a ResizeObserver double-rAF. Those landed on main independently (Windows kill handling by v1.4.135; the pane-title-overlay rect-equality guard from #2756 already breaks the loop at the source), so this change carries only the two guards still missing upstream. Co-authored-by: Yixian HU Co-authored-by: Claude --- .../terminal-link-handlers.test.ts | 15 +++++++ .../terminal-pane/terminal-link-handlers.ts | 40 +++++++++++-------- .../src/lib/crash-diagnostics.test.ts | 14 +++++++ src/renderer/src/lib/crash-diagnostics.ts | 7 ++++ 4 files changed, 60 insertions(+), 16 deletions(-) diff --git a/src/renderer/src/components/terminal-pane/terminal-link-handlers.test.ts b/src/renderer/src/components/terminal-pane/terminal-link-handlers.test.ts index 3840c7c962a..dbd2057e309 100644 --- a/src/renderer/src/components/terminal-pane/terminal-link-handlers.test.ts +++ b/src/renderer/src/components/terminal-pane/terminal-link-handlers.test.ts @@ -1217,6 +1217,21 @@ describe('createFilePathLinkProvider range bounds', () => { ) }) + it('recovers with no links when a path-existence probe rejects (SSH teardown)', async () => { + // Regression: a rejected probe used to escape the void Promise.all as an + // unhandled rejection the crash-breadcrumb buffer retained, leaking heap (#8260). + const shellPathExists = vi.mocked(window.api.shell.pathExists) + shellPathExists.mockRejectedValueOnce(new Error('Remote connection dropped/reconnecting')) + const { provider } = createProviderSetup([makeBufferLine('CLAUDE.md')], new Map()) + + const links = await new Promise((resolve) => { + provider.provideLinks(1, (provided) => resolve(provided ?? [])) + }) + + expect(links).toEqual([]) + expect(shellPathExists).toHaveBeenCalled() + }) + it('shows switch and external-open hint for known worktree root hover', async () => { setPlatform('Macintosh') storeState.worktreesByRepo = { diff --git a/src/renderer/src/components/terminal-pane/terminal-link-handlers.ts b/src/renderer/src/components/terminal-pane/terminal-link-handlers.ts index 462dbdeeedc..8ca9fe0bef1 100644 --- a/src/renderer/src/components/terminal-pane/terminal-link-handlers.ts +++ b/src/renderer/src/components/terminal-pane/terminal-link-handlers.ts @@ -214,23 +214,31 @@ export function createFilePathLinkProvider( } ) ) - ).then((resolvedLinks) => { - const latestFingerprints = new Set( - buildCandidateLogicalLinesForBufferPosition(buffer, bufferLineNumber).map( - (logicalLine) => logicalLine.fingerprint + ) + .then((resolvedLinks) => { + const latestFingerprints = new Set( + buildCandidateLogicalLinesForBufferPosition(buffer, bufferLineNumber).map( + (logicalLine) => logicalLine.fingerprint + ) ) - ) - const providedLinks = resolvedLinks.filter( - (link): link is ProvidedFileLink => link !== null - ) - const links = preferLongestNonOverlappingLinks(providedLinks) - .filter(({ logicalLine }) => latestFingerprints.has(logicalLine.fingerprint)) - .map(({ link }) => link) - if (providedLinks.length > 0 && links.length === 0) { - return - } - callback(links.length > 0 ? links : undefined) - }) + const providedLinks = resolvedLinks.filter( + (link): link is ProvidedFileLink => link !== null + ) + const links = preferLongestNonOverlappingLinks(providedLinks) + .filter(({ logicalLine }) => latestFingerprints.has(logicalLine.fingerprint)) + .map(({ link }) => link) + if (providedLinks.length > 0 && links.length === 0) { + return + } + callback(links.length > 0 ? links : undefined) + }) + .catch(() => { + // Why: remote path-existence probes reject with "Remote connection + // dropped/reconnecting" during SSH teardown. Without a catch the + // rejected Promise.all is unhandled and the crash-breadcrumb buffer + // retains it, growing the renderer heap until it crashes (#8260). + callback(undefined) + }) } } } diff --git a/src/renderer/src/lib/crash-diagnostics.test.ts b/src/renderer/src/lib/crash-diagnostics.test.ts index 8abe9aaf9ea..d093c8faf61 100644 --- a/src/renderer/src/lib/crash-diagnostics.test.ts +++ b/src/renderer/src/lib/crash-diagnostics.test.ts @@ -116,6 +116,20 @@ describe('renderer crash diagnostics', () => { }) }) + it('suppresses benign ResizeObserver loop errors instead of recording them', () => { + diagnostics.installRendererCrashDiagnostics() + recordBreadcrumbMock.mockClear() + + const preventDefault = vi.fn() + listeners.get('error')?.[0]?.({ + message: 'ResizeObserver loop completed with undelivered notifications.', + preventDefault + }) + + expect(preventDefault).toHaveBeenCalledTimes(1) + expect(recordBreadcrumbMock).not.toHaveBeenCalled() + }) + it('disposes global listeners and the memory interval', () => { diagnostics.installRendererCrashDiagnostics() diff --git a/src/renderer/src/lib/crash-diagnostics.ts b/src/renderer/src/lib/crash-diagnostics.ts index bb0875f1433..b718917d9ed 100644 --- a/src/renderer/src/lib/crash-diagnostics.ts +++ b/src/renderer/src/lib/crash-diagnostics.ts @@ -64,6 +64,13 @@ if (typeof import.meta !== 'undefined' && import.meta.hot) { } function recordRendererError(event: ErrorEvent): void { + // Why: "ResizeObserver loop completed" is a benign, self-resolving Chromium + // quirk. Recording it fills the breadcrumb buffer and inflates the error + // count without diagnostic value, contributing to renderer heap growth (#8260). + if (/ResizeObserver loop/i.test(event.message)) { + event.preventDefault() + return + } recordRendererCrashBreadcrumb( 'renderer_error', compactBreadcrumbData({ From d9f7fd08f71646d4d6d363b3a07f0634746c5c0d Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 14 Jul 2026 19:20:36 -0700 Subject: [PATCH 11/15] fix(mobile): harden notification opt-in onboarding (#8792) --- mobile/app/notification-opt-in.tsx | 139 ++++++++++-------- mobile/app/notifications.tsx | 3 +- .../mobile-notifications.test.ts | 30 +++- .../src/notifications/mobile-notifications.ts | 7 +- .../notification-opt-in-gate.test.ts | 22 ++- .../notifications/notification-opt-in-gate.ts | 3 + 6 files changed, 137 insertions(+), 67 deletions(-) diff --git a/mobile/app/notification-opt-in.tsx b/mobile/app/notification-opt-in.tsx index 41f2c2168e2..a4476687e09 100644 --- a/mobile/app/notification-opt-in.tsx +++ b/mobile/app/notification-opt-in.tsx @@ -1,5 +1,13 @@ import { useCallback, useState } from 'react' -import { ActivityIndicator, BackHandler, Pressable, StyleSheet, Text, View } from 'react-native' +import { + ActivityIndicator, + BackHandler, + Pressable, + ScrollView, + StyleSheet, + Text, + View +} from 'react-native' import { useFocusEffect, useLocalSearchParams, useRouter } from 'expo-router' import { SafeAreaView } from 'react-native-safe-area-context' import { BellRing } from 'lucide-react-native' @@ -49,63 +57,65 @@ export default function NotificationOptInScreen() { return ( - - - Orca - - - - - + + + + Orca - Notifications - Stay updated while away - - Get notified on this device when an agent needs your input or finishes a task. - - - - {error ? ( - - {error} + + + + + Notifications + Stay updated while away + + Get notified on this device when an agent needs your input or finishes a task. - ) : null} - [ - styles.primaryButton, - pressed && styles.buttonPressed, - busyChoice !== null && styles.buttonDisabled - ]} - onPress={() => void choose('enable')} - > - {busyChoice === 'enable' ? ( - - ) : ( - Enable notifications - )} - - [ - styles.secondaryButton, - pressed && styles.buttonPressed, - busyChoice !== null && styles.buttonDisabled - ]} - onPress={() => void choose('skip')} - > - {busyChoice === 'skip' ? ( - - ) : ( - Not now - )} - - You can change this any time in Settings. - + + + + {error ? ( + + {error} + + ) : null} + [ + styles.primaryButton, + pressed && styles.buttonPressed, + busyChoice !== null && styles.buttonDisabled + ]} + onPress={() => void choose('enable')} + > + {busyChoice === 'enable' ? ( + + ) : ( + Enable notifications + )} + + [ + styles.secondaryButton, + pressed && styles.buttonPressed, + busyChoice !== null && styles.buttonDisabled + ]} + onPress={() => void choose('skip')} + > + {busyChoice === 'skip' ? ( + + ) : ( + Not now + )} + + You can change this any time in Settings. + + ) } @@ -116,6 +126,11 @@ const styles = StyleSheet.create({ backgroundColor: colors.bgBase, paddingHorizontal: spacing.xl }, + // Why: this decision screen cannot be dismissed with Back, so every action + // must remain reachable in landscape and with accessibility text scaling. + scrollContent: { + flexGrow: 1 + }, brandRow: { minHeight: 52, flexDirection: 'row', @@ -128,10 +143,10 @@ const styles = StyleSheet.create({ fontWeight: '700' }, content: { - flex: 1, + flexGrow: 1, alignItems: 'center', justifyContent: 'center', - paddingBottom: spacing.xl + paddingVertical: spacing.xl }, iconSurface: { width: 64, @@ -173,11 +188,12 @@ const styles = StyleSheet.create({ paddingBottom: spacing.lg }, primaryButton: { - height: 44, + minHeight: 44, alignItems: 'center', justifyContent: 'center', borderRadius: radii.button, - backgroundColor: colors.surfaceBright + backgroundColor: colors.surfaceBright, + paddingVertical: spacing.sm }, primaryButtonText: { color: colors.bgBase, @@ -185,11 +201,12 @@ const styles = StyleSheet.create({ fontWeight: '600' }, secondaryButton: { - height: 44, + minHeight: 44, alignItems: 'center', justifyContent: 'center', borderRadius: radii.button, - marginTop: spacing.xs + marginTop: spacing.xs, + paddingVertical: spacing.sm }, secondaryButtonText: { color: colors.textSecondary, diff --git a/mobile/app/notifications.tsx b/mobile/app/notifications.tsx index 4adacfc016f..d9696251a94 100644 --- a/mobile/app/notifications.tsx +++ b/mobile/app/notifications.tsx @@ -17,7 +17,8 @@ import { const DEFAULT_PERMISSION_STATE: NotificationPermissionState = { granted: false, status: 'undetermined', - canAskAgain: true + canAskAgain: true, + authorizationReflectsUserChoice: false } export default function NotificationsScreen() { diff --git a/mobile/src/notifications/mobile-notifications.test.ts b/mobile/src/notifications/mobile-notifications.test.ts index 2d6cf868461..b1f7c89578e 100644 --- a/mobile/src/notifications/mobile-notifications.test.ts +++ b/mobile/src/notifications/mobile-notifications.test.ts @@ -1,6 +1,8 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import * as Notifications from 'expo-notifications' +import { Platform } from 'react-native' import { + getNotificationPermissionState, setScheduledNotificationsMaxForTests, subscribeToDesktopNotifications } from './mobile-notifications' @@ -18,7 +20,7 @@ vi.mock('expo-notifications', () => ({ })) vi.mock('react-native', () => ({ - Platform: { OS: 'ios' } + Platform: { OS: 'ios', Version: 18 } })) // Why: mobile-notifications now persists the catch-up watermark to @@ -35,6 +37,32 @@ vi.mock('../storage/preferences', () => ({ loadPushNotificationsEnabled: vi.fn() })) +beforeEach(() => { + Object.assign(Platform, { OS: 'ios', Version: 18 }) +}) + +describe('getNotificationPermissionState', () => { + it.each([ + { os: 'android', version: 32, expected: false }, + { os: 'android', version: 33, expected: true }, + { os: 'ios', version: 18, expected: true } + ])( + 'reports whether a granted $os $version authorization reflects user choice', + async ({ os, version, expected }) => { + Object.assign(Platform, { OS: os, Version: version }) + vi.mocked(Notifications.getPermissionsAsync).mockResolvedValue({ + status: 'granted', + canAskAgain: true + } as never) + + await expect(getNotificationPermissionState()).resolves.toMatchObject({ + granted: true, + authorizationReflectsUserChoice: expected + }) + } + ) +}) + describe('subscribeToDesktopNotifications', () => { beforeEach(() => { vi.clearAllMocks() diff --git a/mobile/src/notifications/mobile-notifications.ts b/mobile/src/notifications/mobile-notifications.ts index f91e5981366..d3866393116 100644 --- a/mobile/src/notifications/mobile-notifications.ts +++ b/mobile/src/notifications/mobile-notifications.ts @@ -82,6 +82,7 @@ export type NotificationPermissionState = { granted: boolean status: string canAskAgain: boolean + authorizationReflectsUserChoice: boolean } export async function getNotificationPermissionState(): Promise { @@ -89,7 +90,11 @@ export async function getNotificationPermissionState(): Promise= 33) } } diff --git a/mobile/src/notifications/notification-opt-in-gate.test.ts b/mobile/src/notifications/notification-opt-in-gate.test.ts index fb5a95ac642..ded3d7eca39 100644 --- a/mobile/src/notifications/notification-opt-in-gate.test.ts +++ b/mobile/src/notifications/notification-opt-in-gate.test.ts @@ -27,7 +27,8 @@ describe('notification opt-in gate', () => { vi.mocked(getNotificationPermissionState).mockResolvedValue({ granted: false, status: 'undetermined', - canAskAgain: true + canAskAgain: true, + authorizationReflectsUserChoice: false }) await expect(shouldPresentNotificationOptIn()).resolves.toBe(true) @@ -46,19 +47,34 @@ describe('notification opt-in gate', () => { vi.mocked(getNotificationPermissionState).mockResolvedValue({ granted: true, status: 'granted', - canAskAgain: true + canAskAgain: true, + authorizationReflectsUserChoice: true }) await expect(shouldPresentNotificationOptIn()).resolves.toBe(false) expect(savePushNotificationsEnabled).toHaveBeenCalledWith(true) }) + it('still presents when a pre-Android 13 default grant is not an opt-in decision', async () => { + vi.mocked(readPushNotificationsPreference).mockResolvedValue({ value: null, loaded: true }) + vi.mocked(getNotificationPermissionState).mockResolvedValue({ + granted: true, + status: 'granted', + canAskAgain: true, + authorizationReflectsUserChoice: false + }) + + await expect(shouldPresentNotificationOptIn()).resolves.toBe(true) + expect(savePushNotificationsEnabled).not.toHaveBeenCalled() + }) + it('skips the gate when iOS has already denied permission', async () => { vi.mocked(readPushNotificationsPreference).mockResolvedValue({ value: null, loaded: true }) vi.mocked(getNotificationPermissionState).mockResolvedValue({ granted: false, status: 'denied', - canAskAgain: false + canAskAgain: false, + authorizationReflectsUserChoice: false }) await expect(shouldPresentNotificationOptIn()).resolves.toBe(false) diff --git a/mobile/src/notifications/notification-opt-in-gate.ts b/mobile/src/notifications/notification-opt-in-gate.ts index 9d7b4eed6cc..a909fad16d7 100644 --- a/mobile/src/notifications/notification-opt-in-gate.ts +++ b/mobile/src/notifications/notification-opt-in-gate.ts @@ -13,6 +13,9 @@ export async function shouldPresentNotificationOptIn(): Promise { try { const permission = await getNotificationPermissionState() if (permission.granted) { + if (!permission.authorizationReflectsUserChoice) { + return true + } // Why: an already-authorized device should inherit the useful default // without seeing an onboarding decision it has effectively made. await savePushNotificationsEnabled(true) From 42ee45f392e431380e4bdadb8de2128791ec2b59 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Tue, 14 Jul 2026 19:36:47 -0700 Subject: [PATCH 12/15] Fix restored mobile terminals and workspace visibility parity (#8789) * Fix mobile cutover activation and usage refresh loops Co-authored-by: Orca * Fix restored mobile terminal state parity Co-authored-by: Orca * Fix migrated PTY workspace attribution Co-authored-by: Orca * Fix overlapping mobile terminal surface swaps Co-authored-by: Orca --------- Co-authored-by: Orca --- .../app/h/[hostId]/session/[worktreeId].tsx | 147 +++++---- .../session/mobile-session-route-helpers.ts | 28 ++ .../mobile-session-startup-source.test.ts | 17 +- .../mobile-session-tab-activation.test.ts | 75 +++++ .../session/mobile-session-tab-activation.ts | 90 ++++++ .../mobile-terminal-diagnostics.test.ts | 46 +++ .../session/mobile-terminal-diagnostics.ts | 291 ++++++++++++++++++ .../session/session-tab-snapshot-gate.test.ts | 13 + .../src/session/session-tab-snapshot-gate.ts | 6 + mobile/src/terminal/terminal-webview-html.ts | 31 +- .../terminal-webview-init-surface.test.ts | 132 ++++++++ .../terminal-webview-surface-swap-injected.ts | 49 +++ src/main/rate-limits/service.test.ts | 51 +++ src/main/rate-limits/service.ts | 11 + src/main/runtime/orca-runtime.test.ts | 182 +++++++++++ src/main/runtime/orca-runtime.ts | 96 +++++- ...ime-request-connection.integration.test.ts | 5 + src/main/runtime/rpc/methods/accounts.test.ts | 58 ++++ src/main/runtime/rpc/methods/accounts.ts | 8 +- 19 files changed, 1240 insertions(+), 96 deletions(-) create mode 100644 mobile/src/session/mobile-session-tab-activation.test.ts create mode 100644 mobile/src/session/mobile-session-tab-activation.ts create mode 100644 mobile/src/session/mobile-terminal-diagnostics.test.ts create mode 100644 mobile/src/session/mobile-terminal-diagnostics.ts create mode 100644 mobile/src/terminal/terminal-webview-init-surface.test.ts create mode 100644 mobile/src/terminal/terminal-webview-surface-swap-injected.ts create mode 100644 src/main/runtime/rpc/methods/accounts.test.ts diff --git a/mobile/app/h/[hostId]/session/[worktreeId].tsx b/mobile/app/h/[hostId]/session/[worktreeId].tsx index b4e7207d6a5..d4d1b35896f 100644 --- a/mobile/app/h/[hostId]/session/[worktreeId].tsx +++ b/mobile/app/h/[hostId]/session/[worktreeId].tsx @@ -184,6 +184,7 @@ import { useLiveWorktreeName } from '../../../../src/session/use-live-worktree-n import { acceptSessionSnapshot, applyClosedTabTombstones, + confirmsMirroredTabSelection, type AppliedSnapshotMarker } from '../../../../src/session/session-tab-snapshot-gate' import { @@ -197,8 +198,14 @@ import { isDictationSetupRequiredError } from '../../../../src/dictation/mobile-dictation-setup' import { TerminalPaneView } from '../../../../src/session/TerminalPaneView' +import { + activateMobileSessionTab, + focusMobileTerminal +} from '../../../../src/session/mobile-session-tab-activation' +import { MobileTerminalDiagnostics } from '../../../../src/session/mobile-terminal-diagnostics' import { getRepoIdFromMobileWorktreeId, + getActiveTabIdForHandle, isFileExistsErrorMessage, isGestureMouseTrackingMode, MOBILE_SESSION_STATUS_LABELS, @@ -206,7 +213,8 @@ import { TERMINAL_GESTURE_INPUT_FLUSH_DELAY_MS, TERMINAL_GESTURE_INPUT_MAX_PENDING_SEQUENCES, TERMINAL_GESTURE_INPUT_MAX_QUEUE_AGE_MS, - TERMINAL_GESTURE_INPUT_REFILL_PER_SECOND + TERMINAL_GESTURE_INPUT_REFILL_PER_SECOND, + updateTerminalCwdFromStreamEvent } from '../../../../src/session/mobile-session-route-helpers' import { resolveMarkdownFloatingActionsBottom } from '../../../../src/session/markdown-floating-actions-layout' import { resolveTabStripScrollOffset } from '../../../../src/session/tab-strip-scroll' @@ -245,21 +253,6 @@ type TerminalLiveAccessoryInput = ReturnType => - tab.type === 'terminal' && tab.terminal === terminalHandle - )?.id ?? terminalHandle - ) -} - function MarkdownReader({ documentId, doc, @@ -800,21 +793,6 @@ function FileReader({ return renderSourceText(doc.content) } -function updateTerminalCwdFromStreamEvent( - handle: string, - data: Record, - terminalCwd: Map -): void { - if (!('cwd' in data)) { - return - } - if (typeof data.cwd === 'string' && data.cwd.trim().length > 0) { - terminalCwd.set(handle, data.cwd) - return - } - terminalCwd.delete(handle) -} - export default function SessionScreen() { const { hostId, @@ -1031,6 +1009,7 @@ export default function SessionScreen() { const terminalUnsubsRef = useRef void>>(new Map()) const subscribingHandlesRef = useRef>(new Set()) const initializedHandlesRef = useRef>(new Set()) + const terminalDiagnosticsRef = useRef(new MobileTerminalDiagnostics()) // Why: WebViews load xterm.js from CDN asynchronously. Hidden WebViews // (opacity:0) may have delayed JS execution on iOS. We must not subscribe // until the WebView has fired web-ready, otherwise init() messages queue @@ -1315,6 +1294,7 @@ export default function SessionScreen() { terminalUnsubsRef.current.get(handle)?.() terminalUnsubsRef.current.delete(handle) subscribingHandlesRef.current.delete(handle) + terminalDiagnosticsRef.current.terminalUnsubscribed(handle) subscribeSeqRef.current.set(handle, (subscribeSeqRef.current.get(handle) ?? 0) + 1) // Why: a fresh subscription will land on a new server-side state machine // run (or the same one with a higher seq); reset the high-water mark so @@ -1329,6 +1309,7 @@ export default function SessionScreen() { terminalUnsubsRef.current.clear() subscribingHandlesRef.current.clear() initializedHandlesRef.current.clear() + terminalDiagnosticsRef.current.clearTerminalCache() webReadyHandlesRef.current.clear() subscribeSeqRef.current.clear() layoutSeqRef.current.clear() @@ -1350,6 +1331,7 @@ export default function SessionScreen() { const dims = await getTerminalRef(handle)?.measureFitDimensions( terminalFrameHeightRef.current || undefined ) + terminalDiagnosticsRef.current.viewportMeasured(handle, dims, terminalFrameHeightRef.current) if (dims) { viewportRef.current = dims viewportMeasuredRef.current = true @@ -1360,25 +1342,34 @@ export default function SessionScreen() { const subscribeToTerminal = useCallback( (handle: string) => { + const diagnostics = terminalDiagnosticsRef.current + const logSkippedGate = (reason: string) => + diagnostics.streamSkipped(handle, reason, handle === activeHandleRef.current) if (!client) { + logSkippedGate('no-client') return } if (terminalUnsubsRef.current.has(handle)) { + logSkippedGate('already-subscribed') return } if (subscribingHandlesRef.current.has(handle)) { + logSkippedGate('subscribe-in-flight') return } if (!getTerminalRef(handle)) { + logSkippedGate('no-webview-ref') return } if (!webReadyHandlesRef.current.has(handle)) { + logSkippedGate('webview-not-ready') return } subscribingHandlesRef.current.add(handle) const seq = (subscribeSeqRef.current.get(handle) ?? 0) + 1 subscribeSeqRef.current.set(handle, seq) + diagnostics.streamArmed(handle, seq, viewportRef.current) // Why: server handles auto-fit on subscribe — no terminal.focus call needed. // The viewport is embedded in the subscribe params so the server resizes @@ -1397,6 +1388,7 @@ export default function SessionScreen() { return } const data = result as Record + diagnostics.firstStreamEvent(handle, seq, data.type) // Why: stale-event filter. Server-side state machine bumps a // monotonic seq on every applyLayout. Drop `resized` events // whose seq is strictly older than what we've already observed @@ -1429,6 +1421,7 @@ export default function SessionScreen() { return } if (data.type === 'scrollback') { + diagnostics.streamScrollback(handle, seq, eventSeq, data) if (initializedHandlesRef.current.has(handle)) { return } @@ -1520,6 +1513,7 @@ export default function SessionScreen() { // server still has a null viewport for THIS subscriber // record — we MUST resubscribe so the server stores it. if (dims) { + diagnostics.streamResubscribing(handle, seq, dims) viewportRef.current = dims viewportMeasuredRef.current = true unsubscribeTerminal(handle) @@ -1564,6 +1558,7 @@ export default function SessionScreen() { const cols = (data.cols as number) || 80 const rows = (data.rows as number) || 24 const serialized = typeof data.serialized === 'string' ? data.serialized : null + diagnostics.streamResized(handle, seq, eventSeq, data, getTerminalRef(handle) != null) const oscLinks = isTerminalOscLinkRanges(data.oscLinks) ? data.oscLinks : undefined if (serialized != null) { getTerminalRef(handle)?.init(cols, rows, serialized, true, oscLinks) @@ -1733,6 +1728,7 @@ export default function SessionScreen() { const applySessionTabs = useCallback( (result: SessionTabsResult) => { + const diagnostics = terminalDiagnosticsRef.current // Reject out-of-order snapshots, then suppress just-closed tabs until the // publisher confirms their absence. See session-tab-snapshot-gate. if (!acceptSessionSnapshot(result, appliedSnapshotMarkerRef.current)) { @@ -1789,15 +1785,21 @@ export default function SessionScreen() { const pendingActiveSessionTabId = pendingActiveSessionTabIdRef.current const pendingActiveTerminalHandle = pendingActiveTerminalHandleRef.current let active = snapshotActive + let selectionSource = 'snapshot' if (pendingActiveSessionTabId) { if (snapshotActive?.id === pendingActiveSessionTabId) { - pendingActiveSessionTabIdRef.current = null + if (confirmsMirroredTabSelection(result.publicationEpoch)) { + pendingActiveSessionTabIdRef.current = null + } else { + selectionSource = 'pending-tab-local-ack' + } } else { const pendingTab = nextTabs.find((tab) => tab.id === pendingActiveSessionTabId) if (pendingTab) { // Why: desktop tab snapshots can lag a mobile tap while activate RPC // is in flight. Keep the locally selected tab to avoid snapping back. active = pendingTab + selectionSource = 'pending-tab' } else { pendingActiveSessionTabIdRef.current = null } @@ -1815,12 +1817,17 @@ export default function SessionScreen() { snapshotActive?.type === 'terminal' && snapshotActive.terminal === pendingActiveTerminalHandle ) { - pendingActiveTerminalHandleRef.current = null + if (confirmsMirroredTabSelection(result.publicationEpoch)) { + pendingActiveTerminalHandleRef.current = null + } else { + selectionSource = 'pending-handle-local-ack' + } } else if (pendingTerminalTab) { // Why: desktop active flags can lag a mobile terminal tap. Key by // terminal handle too, because fallback PTY tabs may not yet have a // stable session tab id during new-worktree startup. active = pendingTerminalTab + selectionSource = 'pending-handle-tab' } else if (pendingTerminalExists) { const nextActiveTabId = getActiveTabIdForHandle(nextTabs, pendingActiveTerminalHandle) activeSessionTabIdRef.current = nextActiveTabId @@ -1833,6 +1840,7 @@ export default function SessionScreen() { pendingActiveTerminalHandleRef.current = null } } + diagnostics.tabsApplied(result, nextTabs, active, selectionSource) activeSessionTabTypeRef.current = active?.type ?? null activeSessionTabIdRef.current = active?.id ?? null setActiveSessionTabId(active?.id ?? null) @@ -2377,20 +2385,25 @@ export default function SessionScreen() { const fetchSessionTabs = useCallback(async () => { if (!client) { + terminalDiagnosticsRef.current.tabsFetchSkipped('no-client') return } if (fetchSessionTabsInFlightRef.current) { + terminalDiagnosticsRef.current.tabsFetchSkipped('already-in-flight') return } fetchSessionTabsInFlightRef.current = true + terminalDiagnosticsRef.current.tabsFetchStarted(worktreeId) try { const response = await client.sendRequest('session.tabs.list', { worktree: `id:${worktreeId}` }) if (!response.ok) { + terminalDiagnosticsRef.current.tabsFetchFailed((response as RpcFailure).error.code) return } const result = (response as RpcSuccess).result as SessionTabsResult + terminalDiagnosticsRef.current.tabsFetchSucceeded(result) applySessionTabs(result) // Focus a just-opened browser tab once it appears in the snapshot, via the // normal activate path so it sticks and the user can still switch away. @@ -2404,7 +2417,8 @@ export default function SessionScreen() { switchSessionTabRef.current?.(browserTab) } } - } catch { + } catch (error) { + terminalDiagnosticsRef.current.tabsFetchErrored(error) // Keep the last tab snapshot visible during reconnect/backoff. } finally { fetchSessionTabsInFlightRef.current = false @@ -2689,6 +2703,7 @@ export default function SessionScreen() { pendingBrowserFocusPageIdRef.current = null pendingTerminalActivationAttemptRef.current = null initialEmptySessionAutoCreateRef.current = null + terminalDiagnosticsRef.current.resetRoute() appliedSnapshotMarkerRef.current = { epoch: null, version: -1 } closedTabTombstonesRef.current.clear() for (const queued of terminalGestureInputQueuesRef.current.values()) { @@ -2912,6 +2927,7 @@ export default function SessionScreen() { (tab): tab is Extract => tab.type === 'terminal' && tab.terminal === handle ) + terminalDiagnosticsRef.current.tabSwitch('terminal', matchingTab?.id ?? '', false, handle) pendingActiveSessionTabIdRef.current = matchingTab?.id ?? null pendingActiveTerminalHandleRef.current = handle activeSessionTabTypeRef.current = 'terminal' @@ -2931,15 +2947,15 @@ export default function SessionScreen() { } subscribeToTerminal(handle) if (client) { - void client.sendRequest('terminal.focus', { terminal: handle }).catch(() => {}) + void focusMobileTerminal(client, handle).catch(() => {}) if (matchingTab) { - void client - .sendRequest('session.tabs.activate', { - worktree: `id:${worktreeId}`, - tabId: matchingTab.id, - notifyClients: false - }) - .catch(() => {}) + // Why: persist selection for headless hosts; the snapshot gate keeps + // this phone-local acknowledgement from impersonating desktop focus. + void activateMobileSessionTab(client, { + worktree: `id:${worktreeId}`, + tabId: matchingTab.id, + notifyClients: false + }).catch(() => {}) } } }, @@ -2960,6 +2976,7 @@ export default function SessionScreen() { switchTab(tab.terminal) return } + terminalDiagnosticsRef.current.tabSwitch('terminal', tab.id, true) triggerSelection() pendingActiveSessionTabIdRef.current = tab.id pendingActiveTerminalHandleRef.current = null @@ -2973,18 +2990,17 @@ export default function SessionScreen() { activeHandleRef.current = null setActiveHandle(null) if (client) { - void client - .sendRequest('session.tabs.activate', { - worktree: `id:${worktreeId}`, - tabId: tab.id, - notifyClients: false - }) - .catch(() => {}) + void activateMobileSessionTab(client, { + worktree: `id:${worktreeId}`, + tabId: tab.id, + notifyClients: false + }).catch(() => {}) } return } triggerSelection() + terminalDiagnosticsRef.current.tabSwitch(tab.type, tab.id, false) pendingActiveSessionTabIdRef.current = tab.id pendingActiveTerminalHandleRef.current = null activeSessionTabTypeRef.current = tab.type @@ -2997,13 +3013,11 @@ export default function SessionScreen() { activeHandleRef.current = null setActiveHandle(null) if (client) { - void client - .sendRequest('session.tabs.activate', { - worktree: `id:${worktreeId}`, - tabId: tab.id, - notifyClients: false - }) - .catch(() => {}) + void activateMobileSessionTab(client, { + worktree: `id:${worktreeId}`, + tabId: tab.id, + notifyClients: false + }).catch(() => {}) } if (tab.type === 'browser') { return @@ -3031,6 +3045,7 @@ export default function SessionScreen() { // init messages. This prevents the blank terminal race where init() was // queued before the WebView loaded. const setTerminalWebViewRef = useCallback((handle: string, ref: TerminalWebViewHandle | null) => { + terminalDiagnosticsRef.current.webViewRef(handle, ref != null) if (ref) { terminalRefs.current.set(handle, ref) } else { @@ -3049,6 +3064,11 @@ export default function SessionScreen() { (handle: string) => { const wasAlreadyReady = webReadyHandlesRef.current.has(handle) webReadyHandlesRef.current.add(handle) + terminalDiagnosticsRef.current.webViewReady( + handle, + wasAlreadyReady, + handle === activeHandleRef.current + ) if (wasAlreadyReady && initializedHandlesRef.current.has(handle)) { // Why: the native WebView reloaded (Metro hot reload or Android // process churn). The old xterm buffer is gone, so force a fresh @@ -3085,7 +3105,7 @@ export default function SessionScreen() { })() } }, - [measureViewportOnce, subscribeToTerminal, unsubscribeTerminal] + [getTerminalRef, measureViewportOnce, subscribeToTerminal, unsubscribeTerminal] ) useEffect(() => { @@ -4302,13 +4322,12 @@ export default function SessionScreen() { // Why: a hydrated headless/server-owned tab can already be active but still // pending; activation is the RPC that materializes or focuses its PTY handle. pendingTerminalActivationAttemptRef.current = activationKey - void client - .sendRequest('session.tabs.activate', { - worktree: `id:${worktreeId}`, - tabId: activePendingTerminalTab.id, - leafId: activePendingTerminalTab.leafId, - notifyClients: false - }) + void activateMobileSessionTab(client, { + worktree: `id:${worktreeId}`, + tabId: activePendingTerminalTab.id, + leafId: activePendingTerminalTab.leafId, + notifyClients: false + }) .then((response) => { if (!response.ok) { if (pendingTerminalActivationAttemptRef.current === activationKey) { diff --git a/mobile/src/session/mobile-session-route-helpers.ts b/mobile/src/session/mobile-session-route-helpers.ts index 18434e21f5a..b5a5049dd06 100644 --- a/mobile/src/session/mobile-session-route-helpers.ts +++ b/mobile/src/session/mobile-session-route-helpers.ts @@ -33,3 +33,31 @@ export function isGestureMouseTrackingMode( ): boolean { return mode === 'x10' || mode === 'vt200' || mode === 'drag' || mode === 'any' } + +export function getActiveTabIdForHandle( + tabs: ReadonlyArray<{ id: string; type: string; terminal?: string | null }>, + terminalHandle: string | null +): string | null { + if (!terminalHandle) { + return null + } + return ( + tabs.find((tab) => tab.type === 'terminal' && tab.terminal === terminalHandle)?.id ?? + terminalHandle + ) +} + +export function updateTerminalCwdFromStreamEvent( + handle: string, + data: Readonly>, + terminalCwd: Map +): void { + if (!('cwd' in data)) { + return + } + if (typeof data.cwd === 'string' && data.cwd.trim().length > 0) { + terminalCwd.set(handle, data.cwd) + return + } + terminalCwd.delete(handle) +} diff --git a/mobile/src/session/mobile-session-startup-source.test.ts b/mobile/src/session/mobile-session-startup-source.test.ts index aca17cfb509..c4083b1a545 100644 --- a/mobile/src/session/mobile-session-startup-source.test.ts +++ b/mobile/src/session/mobile-session-startup-source.test.ts @@ -58,7 +58,7 @@ describe('mobile session startup', () => { expect(pendingActivationEffect).toContain( 'pendingTerminalActivationAttemptRef.current === activationKey' ) - expect(pendingActivationEffect).toContain("sendRequest('session.tabs.activate'") + expect(pendingActivationEffect).toContain('activateMobileSessionTab(client,') expect(pendingActivationEffect).toContain('tabId: activePendingTerminalTab.id') expect(pendingActivationEffect).toContain('leafId: activePendingTerminalTab.leafId') expect(pendingActivationEffect).toContain('notifyClients: false') @@ -68,8 +68,19 @@ describe('mobile session startup', () => { expect(pendingActivationEffect).toContain('scheduleDelayedAction(() => void fetchSessionTabs()') }) - it('keeps mobile session tab activation local to the phone', () => { - const activationRequests = source.split("sendRequest('session.tabs.activate'").slice(1) + it('mirrors ready terminal taps while persisting them for headless hosts', () => { + const readyTerminalSwitch = sliceBetween( + 'const switchTab = useCallback(', + 'const switchSessionTab = useCallback(' + ) + + expect(readyTerminalSwitch).toContain('focusMobileTerminal(client, handle)') + expect(readyTerminalSwitch).toContain('activateMobileSessionTab(client,') + expect(readyTerminalSwitch).toContain('notifyClients: false') + }) + + it('keeps background and pending session-tab activation local to the phone', () => { + const activationRequests = source.split('activateMobileSessionTab(client,').slice(1) expect(activationRequests).toHaveLength(4) for (const request of activationRequests) { diff --git a/mobile/src/session/mobile-session-tab-activation.test.ts b/mobile/src/session/mobile-session-tab-activation.test.ts new file mode 100644 index 00000000000..d448d7e2f53 --- /dev/null +++ b/mobile/src/session/mobile-session-tab-activation.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it, vi } from 'vitest' +import { LogicalClientCutoverError } from '../transport/stable-logical-rpc-client' +import type { RpcClient } from '../transport/rpc-client' +import type { RpcResponse } from '../transport/types' +import { activateMobileSessionTab, focusMobileTerminal } from './mobile-session-tab-activation' + +function success(): RpcResponse { + return { id: 'rpc-1', ok: true, result: {}, _meta: { runtimeId: 'runtime-1' } } +} + +function clientWith(sendRequest: RpcClient['sendRequest']): Pick { + return { sendRequest } +} + +describe('mobile session tab activation', () => { + it('retries terminal focus once on the authenticated replacement after cutover', async () => { + const sendRequest = vi + .fn() + .mockRejectedValueOnce(new LogicalClientCutoverError()) + .mockResolvedValueOnce(success()) + + await expect(focusMobileTerminal(clientWith(sendRequest), 'terminal-1')).resolves.toMatchObject( + { + ok: true + } + ) + expect(sendRequest).toHaveBeenCalledTimes(2) + expect(sendRequest).toHaveBeenNthCalledWith(1, 'terminal.focus', { terminal: 'terminal-1' }) + expect(sendRequest).toHaveBeenNthCalledWith(2, 'terminal.focus', { terminal: 'terminal-1' }) + }) + + it('retries session-tab activation with the same target after cutover', async () => { + const sendRequest = vi + .fn() + .mockRejectedValueOnce(new LogicalClientCutoverError()) + .mockResolvedValueOnce(success()) + const params = { + worktree: 'id:worktree-1', + tabId: 'tab-1', + leafId: 'leaf-1', + notifyClients: false as const + } + + await expect(activateMobileSessionTab(clientWith(sendRequest), params)).resolves.toMatchObject({ + ok: true + }) + expect(sendRequest).toHaveBeenCalledTimes(2) + expect(sendRequest).toHaveBeenNthCalledWith(1, 'session.tabs.activate', params) + expect(sendRequest).toHaveBeenNthCalledWith(2, 'session.tabs.activate', params) + }) + + it('does not retry unrelated transport failures', async () => { + const sendRequest = vi.fn().mockRejectedValue(new Error('offline')) + + await expect(focusMobileTerminal(clientWith(sendRequest), 'terminal-1')).rejects.toThrow( + 'offline' + ) + expect(sendRequest).toHaveBeenCalledOnce() + }) + + it('retries at most once when consecutive cutovers interrupt activation', async () => { + const sendRequest = vi + .fn() + .mockRejectedValue(new LogicalClientCutoverError()) + + await expect( + activateMobileSessionTab(clientWith(sendRequest), { + worktree: 'id:worktree-1', + tabId: 'tab-1', + notifyClients: false + }) + ).rejects.toBeInstanceOf(LogicalClientCutoverError) + expect(sendRequest).toHaveBeenCalledTimes(2) + }) +}) diff --git a/mobile/src/session/mobile-session-tab-activation.ts b/mobile/src/session/mobile-session-tab-activation.ts new file mode 100644 index 00000000000..4b07bff1eee --- /dev/null +++ b/mobile/src/session/mobile-session-tab-activation.ts @@ -0,0 +1,90 @@ +import type { RpcClient } from '../transport/rpc-client' +import { LogicalClientCutoverError } from '../transport/stable-logical-rpc-client' +import type { RpcResponse } from '../transport/types' +import { + getMobileTerminalDiagnosticErrorName, + logMobileTerminalDiagnostic, + shortenMobileTerminalDiagnosticId +} from './mobile-terminal-diagnostics' + +type ActivationClient = Pick + +type MobileSessionTabActivationParams = { + worktree: string + tabId: string + leafId?: string + notifyClients: false +} + +async function retryIdempotentActivationAfterCutover( + request: () => Promise, + operation: 'terminal.focus' | 'session.tabs.activate', + target: string +): Promise { + const diagnosticTarget = shortenMobileTerminalDiagnosticId(target) + logMobileTerminalDiagnostic('activation-request', { operation, target: diagnosticTarget }) + try { + const response = await request() + logMobileTerminalDiagnostic('activation-result', { + operation, + target: diagnosticTarget, + ok: response.ok, + rpcCode: response.ok ? null : response.error.code + }) + return response + } catch (error) { + if (!(error instanceof LogicalClientCutoverError)) { + logMobileTerminalDiagnostic('activation-error', { + operation, + target: diagnosticTarget, + errorName: getMobileTerminalDiagnosticErrorName(error) + }) + throw error + } + logMobileTerminalDiagnostic('activation-cutover-retry', { + operation, + target: diagnosticTarget + }) + // Why: cutover rejects ambiguous in-flight work after the replacement is + // active; these state-setting requests are idempotent and safe to repeat once. + try { + const response = await request() + logMobileTerminalDiagnostic('activation-result', { + operation, + target: diagnosticTarget, + ok: response.ok, + rpcCode: response.ok ? null : response.error.code + }) + return response + } catch (retryError) { + logMobileTerminalDiagnostic('activation-error', { + operation, + target: diagnosticTarget, + errorName: getMobileTerminalDiagnosticErrorName(retryError) + }) + throw retryError + } + } +} + +export function focusMobileTerminal( + client: ActivationClient, + terminal: string +): Promise { + return retryIdempotentActivationAfterCutover( + () => client.sendRequest('terminal.focus', { terminal }), + 'terminal.focus', + terminal + ) +} + +export function activateMobileSessionTab( + client: ActivationClient, + params: MobileSessionTabActivationParams +): Promise { + return retryIdempotentActivationAfterCutover( + () => client.sendRequest('session.tabs.activate', params), + 'session.tabs.activate', + params.tabId + ) +} diff --git a/mobile/src/session/mobile-terminal-diagnostics.test.ts b/mobile/src/session/mobile-terminal-diagnostics.test.ts new file mode 100644 index 00000000000..831ad4bfccc --- /dev/null +++ b/mobile/src/session/mobile-terminal-diagnostics.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it, vi } from 'vitest' +import { + getMobileTerminalDiagnosticErrorName, + logMobileTerminalDiagnostic, + MobileTerminalDiagnostics, + shortenMobileTerminalDiagnosticId +} from './mobile-terminal-diagnostics' + +describe('mobile terminal diagnostics', () => { + it('keeps only the correlatable suffix of identifiers', () => { + expect(shortenMobileTerminalDiagnosticId('terminal-secret-prefix-12345678')).toBe('12345678') + expect(shortenMobileTerminalDiagnosticId('short')).toBe('short') + expect(shortenMobileTerminalDiagnosticId(null)).toBeNull() + }) + + it('reports thrown error types without copying potentially sensitive messages', () => { + expect(getMobileTerminalDiagnosticErrorName(new TypeError('/private/worktree failed'))).toBe( + 'TypeError' + ) + expect(getMobileTerminalDiagnosticErrorName('raw failure')).toBe('string') + }) + + it('uses one filterable structured log tag', () => { + const log = vi.spyOn(console, 'log').mockImplementation(() => {}) + + logMobileTerminalDiagnostic('stream-armed', { handle: '12345678', seq: 2 }) + + expect(log).toHaveBeenCalledWith('[terminal-diagnostic]', 'stream-armed', { + handle: '12345678', + seq: 2 + }) + log.mockRestore() + }) + + it('forgets first-event state when a terminal unsubscribes', () => { + const log = vi.spyOn(console, 'log').mockImplementation(() => {}) + const diagnostics = new MobileTerminalDiagnostics() + + diagnostics.firstStreamEvent('terminal-1', 1, 'subscribed') + diagnostics.terminalUnsubscribed('terminal-1') + diagnostics.firstStreamEvent('terminal-1', 1, 'subscribed') + + expect(log).toHaveBeenCalledTimes(2) + log.mockRestore() + }) +}) diff --git a/mobile/src/session/mobile-terminal-diagnostics.ts b/mobile/src/session/mobile-terminal-diagnostics.ts new file mode 100644 index 00000000000..8695928ed03 --- /dev/null +++ b/mobile/src/session/mobile-terminal-diagnostics.ts @@ -0,0 +1,291 @@ +const MOBILE_TERMINAL_DIAGNOSTIC_TAG = '[terminal-diagnostic]' + +type MobileTerminalDiagnosticValue = string | number | boolean | null | undefined + +export type MobileTerminalDiagnosticDetails = Readonly< + Record +> + +type DiagnosticTab = { + readonly id: string + readonly type: string + readonly isActive: boolean + readonly terminal?: string | null +} + +type DiagnosticTabsSnapshot = { + readonly publicationEpoch?: string + readonly snapshotVersion: number + readonly tabs: readonly DiagnosticTab[] +} + +type DiagnosticDimensions = { readonly cols: number; readonly rows: number } | null | undefined + +// Why: full runtime identifiers make shared logs unnecessarily sensitive; the +// suffix is enough to correlate lifecycle events within one reproduction. +export function shortenMobileTerminalDiagnosticId(value: string | null | undefined): string | null { + if (!value) { + return null + } + return value.slice(-8) +} + +export function getMobileTerminalDiagnosticErrorName(error: unknown): string { + if (error instanceof Error && error.name) { + return error.name + } + return typeof error +} + +export function logMobileTerminalDiagnostic( + event: string, + details: MobileTerminalDiagnosticDetails = {} +): void { + // Why: lifecycle diagnostics are intentionally available for HMR repros, + // but high-frequency WebView events must not add production log overhead. + if (typeof __DEV__ !== 'undefined' && !__DEV__) { + return + } + // Keep this structured and content-free so users can safely share a filtered log. + console.log(MOBILE_TERMINAL_DIAGNOSTIC_TAG, event, details) +} + +export class MobileTerminalDiagnostics { + private readonly streamGateByHandle = new Map() + private readonly firstStreamEventSeqByHandle = new Map() + private lastFetchedTabsSignature: string | null = null + private lastAppliedTabsSignature: string | null = null + private lastTabsFetchStartAt = 0 + private tabsFetchSkipLogged = false + + clearTerminalCache(): void { + this.streamGateByHandle.clear() + this.firstStreamEventSeqByHandle.clear() + } + + resetRoute(): void { + this.clearTerminalCache() + this.lastFetchedTabsSignature = null + this.lastAppliedTabsSignature = null + this.lastTabsFetchStartAt = 0 + this.tabsFetchSkipLogged = false + } + + terminalUnsubscribed(handle: string): void { + this.streamGateByHandle.delete(handle) + this.firstStreamEventSeqByHandle.delete(handle) + } + + viewportMeasured(handle: string, dims: DiagnosticDimensions, frameHeight: number): void { + logMobileTerminalDiagnostic('viewport-measure', { + handle: shortenMobileTerminalDiagnosticId(handle), + ok: dims != null, + cols: dims?.cols, + rows: dims?.rows, + frameHeight: Math.round(frameHeight) + }) + } + + streamSkipped(handle: string, reason: string, isActive: boolean): void { + if (this.streamGateByHandle.get(handle) === reason) { + return + } + this.streamGateByHandle.set(handle, reason) + logMobileTerminalDiagnostic('stream-skipped', { + handle: shortenMobileTerminalDiagnosticId(handle), + reason, + isActive + }) + } + + streamArmed(handle: string, seq: number, viewport: DiagnosticDimensions): void { + this.streamGateByHandle.delete(handle) + logMobileTerminalDiagnostic('stream-armed', { + handle: shortenMobileTerminalDiagnosticId(handle), + seq, + hasViewport: viewport != null, + viewportCols: viewport?.cols, + viewportRows: viewport?.rows + }) + } + + firstStreamEvent(handle: string, seq: number, type: unknown): void { + if (this.firstStreamEventSeqByHandle.get(handle) === seq) { + return + } + this.firstStreamEventSeqByHandle.set(handle, seq) + logMobileTerminalDiagnostic('stream-first-event', { + handle: shortenMobileTerminalDiagnosticId(handle), + seq, + type: typeof type === 'string' ? type : 'unknown' + }) + } + + streamScrollback( + handle: string, + seq: number, + eventSeq: number | null, + data: Readonly> + ): void { + logMobileTerminalDiagnostic('stream-scrollback', { + handle: shortenMobileTerminalDiagnosticId(handle), + seq, + eventSeq, + cols: typeof data.cols === 'number' ? data.cols : null, + rows: typeof data.rows === 'number' ? data.rows : null, + serializedLength: typeof data.serialized === 'string' ? data.serialized.length : 0, + displayMode: typeof data.displayMode === 'string' ? data.displayMode : null, + source: typeof data.source === 'string' ? data.source : null, + scrollbackRows: typeof data.scrollbackRows === 'number' ? data.scrollbackRows : null, + truncated: data.truncated === true || data.truncatedByByteBudget === true + }) + } + + streamResubscribing(handle: string, seq: number, dims: { cols: number; rows: number }): void { + logMobileTerminalDiagnostic('stream-resubscribe-for-viewport', { + handle: shortenMobileTerminalDiagnosticId(handle), + seq, + cols: dims.cols, + rows: dims.rows + }) + } + + streamResized( + handle: string, + seq: number, + eventSeq: number | null, + data: Readonly>, + hasRef: boolean + ): void { + logMobileTerminalDiagnostic('stream-resized', { + handle: shortenMobileTerminalDiagnosticId(handle), + seq, + eventSeq, + cols: typeof data.cols === 'number' ? data.cols : null, + rows: typeof data.rows === 'number' ? data.rows : null, + serializedLength: typeof data.serialized === 'string' ? data.serialized.length : 0, + hasRef + }) + } + + tabsApplied( + snapshot: DiagnosticTabsSnapshot, + tabs: readonly DiagnosticTab[], + activeTab: DiagnosticTab | null, + selectionSource: string + ): void { + const activeHandle = + activeTab?.type === 'terminal' && typeof activeTab.terminal === 'string' + ? activeTab.terminal + : null + const appliedSnapshot = { ...snapshot, tabs } + const signature = [ + appliedSnapshot.publicationEpoch ?? '', + appliedSnapshot.snapshotVersion, + activeTab?.id ?? '', + activeHandle ?? '', + selectionSource + ].join(':') + if (this.lastAppliedTabsSignature === signature) { + return + } + this.lastAppliedTabsSignature = signature + this.logTabs('tabs-applied', appliedSnapshot, activeTab, activeHandle, { selectionSource }) + } + + tabsFetchSkipped(reason: string): void { + if (reason === 'already-in-flight' && this.tabsFetchSkipLogged) { + return + } + this.tabsFetchSkipLogged = reason === 'already-in-flight' + logMobileTerminalDiagnostic('tabs-fetch-skipped', { reason }) + } + + tabsFetchStarted(worktreeId: string): void { + this.tabsFetchSkipLogged = false + const now = Date.now() + if (this.lastFetchedTabsSignature != null && now - this.lastTabsFetchStartAt < 10_000) { + return + } + this.lastTabsFetchStartAt = now + logMobileTerminalDiagnostic('tabs-fetch-start', { + worktree: shortenMobileTerminalDiagnosticId(worktreeId) + }) + } + + tabsFetchFailed(rpcCode: string): void { + logMobileTerminalDiagnostic('tabs-fetch-rpc-failure', { rpcCode }) + } + + tabsFetchErrored(error: unknown): void { + logMobileTerminalDiagnostic('tabs-fetch-error', { + errorName: getMobileTerminalDiagnosticErrorName(error) + }) + } + + tabsFetchSucceeded(snapshot: DiagnosticTabsSnapshot): void { + const activeTab = snapshot.tabs.find((tab) => tab.isActive) ?? null + const signature = [ + snapshot.publicationEpoch ?? '', + snapshot.snapshotVersion, + snapshot.tabs.length, + activeTab?.id ?? '', + activeTab?.type ?? '' + ].join(':') + if (this.lastFetchedTabsSignature === signature) { + return + } + this.lastFetchedTabsSignature = signature + const activeHandle = + activeTab?.type === 'terminal' && typeof activeTab.terminal === 'string' + ? activeTab.terminal + : null + this.logTabs('tabs-fetch-success', snapshot, activeTab, activeHandle) + } + + tabSwitch(tabType: string, tabId: string, pending: boolean, handle?: string): void { + logMobileTerminalDiagnostic('tab-switch', { + tabType, + tab: shortenMobileTerminalDiagnosticId(tabId), + handle: shortenMobileTerminalDiagnosticId(handle), + pending + }) + } + + webViewRef(handle: string, attached: boolean): void { + logMobileTerminalDiagnostic('webview-ref', { + handle: shortenMobileTerminalDiagnosticId(handle), + attached + }) + } + + webViewReady(handle: string, reload: boolean, isActive: boolean): void { + logMobileTerminalDiagnostic('webview-ready', { + handle: shortenMobileTerminalDiagnosticId(handle), + reload, + isActive + }) + } + + private logTabs( + event: 'tabs-applied' | 'tabs-fetch-success', + snapshot: DiagnosticTabsSnapshot, + activeTab: DiagnosticTab | null, + activeHandle: string | null, + extra: MobileTerminalDiagnosticDetails = {} + ): void { + logMobileTerminalDiagnostic(event, { + publication: shortenMobileTerminalDiagnosticId(snapshot.publicationEpoch), + snapshotVersion: snapshot.snapshotVersion, + tabCount: snapshot.tabs.length, + terminalTabCount: snapshot.tabs.filter((tab) => tab.type === 'terminal').length, + pendingTerminalCount: snapshot.tabs.filter( + (tab) => tab.type === 'terminal' && typeof tab.terminal !== 'string' + ).length, + activeTab: shortenMobileTerminalDiagnosticId(activeTab?.id), + activeType: activeTab?.type ?? null, + activeHandle: shortenMobileTerminalDiagnosticId(activeHandle), + ...extra + }) + } +} diff --git a/mobile/src/session/session-tab-snapshot-gate.test.ts b/mobile/src/session/session-tab-snapshot-gate.test.ts index 865c297fdb2..9cb742629ec 100644 --- a/mobile/src/session/session-tab-snapshot-gate.test.ts +++ b/mobile/src/session/session-tab-snapshot-gate.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest' import { acceptSessionSnapshot, applyClosedTabTombstones, + confirmsMirroredTabSelection, type AppliedSnapshotMarker } from './session-tab-snapshot-gate' @@ -50,6 +51,18 @@ describe('acceptSessionSnapshot', () => { }) }) +describe('confirmsMirroredTabSelection', () => { + it('does not treat phone-local persistence as desktop focus confirmation', () => { + expect(confirmsMirroredTabSelection('mobile-local:abc')).toBe(false) + }) + + it('accepts renderer, headless, and legacy publications as confirmation', () => { + expect(confirmsMirroredTabSelection('renderer:abc')).toBe(true) + expect(confirmsMirroredTabSelection('headless:abc')).toBe(true) + expect(confirmsMirroredTabSelection()).toBe(true) + }) +}) + describe('applyClosedTabTombstones', () => { const tab = (id: string): { id: string } => ({ id }) diff --git a/mobile/src/session/session-tab-snapshot-gate.ts b/mobile/src/session/session-tab-snapshot-gate.ts index f3d7601ef4a..1bcf9cadb99 100644 --- a/mobile/src/session/session-tab-snapshot-gate.ts +++ b/mobile/src/session/session-tab-snapshot-gate.ts @@ -33,6 +33,12 @@ export function acceptSessionSnapshot( return true } +export function confirmsMirroredTabSelection(publicationEpoch?: string): boolean { + // Why: phone-local persistence acknowledges the mutation, not host focus; + // keep the pending guard until any host publication confirms the selection. + return !publicationEpoch?.startsWith('mobile-local:') +} + /** * Drops tabs the user just closed locally (tombstoned) until the publisher's * snapshot also drops them or the tombstone expires. Mutates `tombstones`, diff --git a/mobile/src/terminal/terminal-webview-html.ts b/mobile/src/terminal/terminal-webview-html.ts index 69b2e6f6b51..e9d188b74ef 100644 --- a/mobile/src/terminal/terminal-webview-html.ts +++ b/mobile/src/terminal/terminal-webview-html.ts @@ -6,6 +6,7 @@ import { TERMINAL_TEXT_SCALES } from '../storage/preferences' import { TERMINAL_PATH_TAP_JS } from './terminal-path-tap-injected' import { XTERM_ENGINE_CSS, XTERM_ENGINE_JS } from './terminal-webview-engine.generated' import { TERMINAL_REFLOW_JS } from './terminal-webview-reflow-injected' +import { TERMINAL_SURFACE_SWAP_JS } from './terminal-webview-surface-swap-injected' import { TERMINAL_TAP_DISPATCH_JS } from './terminal-webview-tap-dispatch-injected' import { TERMINAL_WEBVIEW_THEME_JS } from './terminal-webview-theme-injected' import { TERMINAL_QUERY_REPLY_JS } from './terminal-webview-query-reply-injected' @@ -217,6 +218,7 @@ window.onerror = function(msg) { var statusDotPendingSelector = false; var PRIVATE_MODE_SCAN_TAIL_LIMIT = 4096; var term = null; ${TERMINAL_QUERY_REPLY_JS} + ${TERMINAL_SURFACE_SWAP_JS} var scrollIndicator = document.getElementById('scroll-indicator'); var scrollThumb = document.getElementById('scroll-thumb'); var scrollIndicatorHideTimer = null; @@ -713,22 +715,8 @@ ${TERMINAL_WEBGL_RECOVERY_JS} initialOscLinks = Array.isArray(nextOscLinks) ? nextOscLinks : []; initialOscLinkRowOffset = 0; initialOscLinkEvictionReady = false; - var oldTerm = term; - var oldSurface = surface; - var nextSurface = null; - disposeTermObservers(); - if (oldTerm) { - nextSurface = document.createElement('div'); - nextSurface.id = 'terminal-surface'; - nextSurface.style.visibility = 'hidden'; - nextSurface.style.position = 'absolute'; - nextSurface.style.left = '0'; - nextSurface.style.top = '0'; - document.getElementById('terminal-container').appendChild(nextSurface); - surface = nextSurface; - attachSurfaceEventHandlers(surface); - oldSurface.removeAttribute('id'); - } + var surfaceSwap = beginTerminalSurfaceSwap(); + var nextSurface = surfaceSwap.nextSurface; applyTerminalTheme(nextTheme); term = new Terminal({ @@ -749,6 +737,8 @@ ${TERMINAL_WEBGL_RECOVERY_JS} convertEol: false, allowProposedApi: true }); + var nextTerm = term; + pendingTerm = nextTerm; term.open(surface); attachWebglAddon(true); if (window.Unicode11Addon && window.Unicode11Addon.Unicode11Addon) try { term.loadAddon(new window.Unicode11Addon.Unicode11Addon()); term.unicode.activeVersion = '11'; } catch (e) {} @@ -768,14 +758,7 @@ ${TERMINAL_WEBGL_RECOVERY_JS} everReady = true; afterWritesDrained(function() { if (gen !== terminalGeneration) return; - if (nextSurface && oldSurface) { - nextSurface.style.visibility = 'visible'; - nextSurface.style.position = ''; - nextSurface.style.left = ''; - nextSurface.style.top = ''; - oldSurface.remove(); - if (oldTerm) oldTerm.dispose(); - } + commitTerminalSurfaceSwap(surfaceSwap, nextTerm); // Why: restore the reader's place after the rewrapped buffer replays. // Replay lands at bottom, so only act when they were scrolled up (rows>0). if (scrollAnchorRows > 0 && term && term.buffer && term.buffer.active) { diff --git a/mobile/src/terminal/terminal-webview-init-surface.test.ts b/mobile/src/terminal/terminal-webview-init-surface.test.ts new file mode 100644 index 00000000000..6c7c4e5f173 --- /dev/null +++ b/mobile/src/terminal/terminal-webview-init-surface.test.ts @@ -0,0 +1,132 @@ +// @vitest-environment happy-dom +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { XTERM_HTML } from './terminal-webview-html' + +function iifeSource(): string { + const start = XTERM_HTML.indexOf('(function() {') + const end = XTERM_HTML.lastIndexOf('})();') + return XTERM_HTML.slice(start, end + '})();'.length) +} + +function bodyMarkup(): string { + const start = XTERM_HTML.indexOf('') + ''.length + const end = XTERM_HTML.indexOf('