From 4e42cc2a3f8992c8aa9a09d5091dc00ceac3660a Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:49:37 +0000 Subject: [PATCH 01/59] fix(speech): keep a slow GPU model load on the GPU while it still makes progress (#40472) The GPU stall watch gave the whole model load one 25 s budget, so a cold disk or busy machine restarted a healthy load on the CPU and could miss the 60 s start limit. The speech process now reports each load step, and the watch restarts on it; a step that hangs for 25 s still moves to the CPU. --- src/main/speech/stt-recognizer.ts | 2 ++ .../speech/stt-transcribe-cpp-recognizer.ts | 5 ++++ .../speech/stt-worker-cpu-fallback.test.ts | 27 ++++++++++++++++++- src/main/speech/stt-worker-cpu-fallback.ts | 9 +++++-- 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/main/speech/stt-recognizer.ts b/src/main/speech/stt-recognizer.ts index 2b91dd8d43f..1998151130c 100644 --- a/src/main/speech/stt-recognizer.ts +++ b/src/main/speech/stt-recognizer.ts @@ -15,6 +15,8 @@ export type SttRecognizerEvent = | { type: 'chunk'; streamId: string; text: string; samples: number } /** Pause-cut streams only: speech was heard that no chunk has committed yet. */ | { type: 'speech-pending'; streamId: string; pending: boolean } + /** A load step finished, so a slow load is told apart from a hung GPU. */ + | { type: 'loading' } export type SttRecognizerPost = (event: SttRecognizerEvent) => void diff --git a/src/main/speech/stt-transcribe-cpp-recognizer.ts b/src/main/speech/stt-transcribe-cpp-recognizer.ts index e8c62efda14..9d5b51f0852 100644 --- a/src/main/speech/stt-transcribe-cpp-recognizer.ts +++ b/src/main/speech/stt-transcribe-cpp-recognizer.ts @@ -91,7 +91,12 @@ export async function createTranscribeCppRecognizer( if (!gguf) { throw new Error(`No .gguf model file in: ${init.files.join(', ')}`) } + // Device setup (Metal shader library, Vulkan drivers) and reading the weights each report done, + // so a cold disk or a busy machine isn't taken for a GPU that hung. + await transcribe.initialize() + post({ type: 'loading' }) const model = await transcribe.TranscribeModel.load(join(init.modelDir, gguf)) + post({ type: 'loading' }) // The child runs one call at a time, so every stream can share one session. const session = model.createSession({ nThreads: performanceCoreThreads() }) const { languages } = model.capabilities diff --git a/src/main/speech/stt-worker-cpu-fallback.test.ts b/src/main/speech/stt-worker-cpu-fallback.test.ts index 582acc144d1..ffc7a6cb42c 100644 --- a/src/main/speech/stt-worker-cpu-fallback.test.ts +++ b/src/main/speech/stt-worker-cpu-fallback.test.ts @@ -1,5 +1,5 @@ import { EventEmitter } from 'node:events' -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { spawnSttWorkerWithCpuFallback } from './stt-worker-cpu-fallback' class FakeChild extends EventEmitter { @@ -115,6 +115,31 @@ describe('GPU speech process falling back to the CPU', () => { expect(marks()).toBe(0) }) + it('keeps a slow GPU load that still finishes its steps, and moves a silent one to the CPU', () => { + vi.useFakeTimers() + try { + const { children, events } = setup() + const [gpu] = children + vi.advanceTimersByTime(20_000) + gpu.emit('message', { type: 'loading' }) + vi.advanceTimersByTime(20_000) + gpu.emit('message', { type: 'loading' }) + vi.advanceTimersByTime(20_000) + gpu.emit('message', { type: 'ready' }) + expect(children).toHaveLength(1) + expect(events).toEqual([{ type: 'ready' }]) + + const silent = setup() + vi.advanceTimersByTime(20_000) + silent.children[0].emit('message', { type: 'loading' }) + vi.advanceTimersByTime(25_000) + expect(silent.children).toHaveLength(2) + expect(silent.children[1].cpuOnly).toBe(true) + } finally { + vi.useRealTimers() + } + }) + it('fails the worker when the CPU replacement cannot load the model mid-dictation', () => { const { worker, children } = setup() const exits: number[] = [] diff --git a/src/main/speech/stt-worker-cpu-fallback.ts b/src/main/speech/stt-worker-cpu-fallback.ts index 63bdeba05a4..d86c3d1b15b 100644 --- a/src/main/speech/stt-worker-cpu-fallback.ts +++ b/src/main/speech/stt-worker-cpu-fallback.ts @@ -17,7 +17,7 @@ type ChildMessage = | { type: 'chunk'; streamId: string; text: string; samples: number } | { type: 'speech-pending'; streamId: string; pending: boolean } | { - type: 'ready' | 'stopped' | 'error' | 'partial' | 'final' + type: 'ready' | 'loading' | 'stopped' | 'error' | 'partial' | 'final' streamId?: string text?: string error?: string @@ -33,7 +33,8 @@ type StreamBacklog = { pending: 'stop' | 'discard' | null } -// Well under the session's 60 s start and stop timeouts, so a CPU replay still fits; covers first-run shader compiles. +// Per load step or decode. Well under the session's 60 s start and stop timeouts, so a CPU replay still fits; +// covers first-run shader compiles. const GPU_STALL_TIMEOUT_MS = 25_000 /** @@ -132,6 +133,10 @@ class SttCpuFallbackWorker extends EventEmitter implements SttWorkerProcess { this.emit('exit', 1) return } + if (message.type === 'loading') { + this.watchForStall(true) + return + } if (message.type === 'chunk') { const { streamId, text, samples } = message const stream = this.streams.get(streamId) From ab9f29c1da8e6e08ac145231d504d266bfa97442 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:57:55 -0800 Subject: [PATCH 02/59] Fork Codex replies after rewinding a chat (#40140) * Fork Codex replies from the current history after rewind * Strengthen Codex fork preparation failure coverage * Preserve Codex fork compatibility and original chat guidance * Explain when Codex cannot start for a fork --- ...le-structured-agent-session-launch.test.ts | 28 ++ .../mobile-structured-agent-session-launch.ts | 6 +- .../codex/codex-conversation-copy.test.ts | 344 +++++++++++-- src/main/codex/codex-conversation-copy.ts | 459 +++++++++--------- src/main/codex/codex-fork-history.ts | 100 ++++ .../agent-conversation-copy-error.ts | 2 + .../structured-agent-session-fork-source.ts | 24 +- .../structured-agent-session-fork.ts | 2 +- ...ed-agent-session-precommit-refusal.test.ts | 61 +++ 9 files changed, 754 insertions(+), 272 deletions(-) create mode 100644 src/main/codex/codex-fork-history.ts diff --git a/mobile/src/session/mobile-structured-agent-session-launch.test.ts b/mobile/src/session/mobile-structured-agent-session-launch.test.ts index b5b62c474c0..457d3fb3c2e 100644 --- a/mobile/src/session/mobile-structured-agent-session-launch.test.ts +++ b/mobile/src/session/mobile-structured-agent-session-launch.test.ts @@ -57,3 +57,31 @@ it('settles a proven missing-original fork refusal but keeps a lost earlier atte (await createMobileStructuredAgentSession(lost, 'workspace-1', 'opencode', forkFrom)).kind ).toBe('unknown') }) + +it('settles a proven fork read refusal without settling a prior lost copy', async () => { + const refusal = { + ok: true, + result: { + ok: false, + refusal: { + code: 'agent_session_identity_required', + details: { reason: 'transcriptUnreadable' }, + createAttemptRefusedBeforeCommit: true, + message: 'Codex could not read the saved chat.' + } + } + } + const client = clientReturning({ ok: true, result: { supported: true } }, refusal) + const forkFrom = { sessionId: 'source-chat', throughItemId: 'codex:source:selected:1' } + expect( + (await createMobileStructuredAgentSession(client, 'workspace-1', 'codex', forkFrom)).kind + ).toBe('failed') + const lost = clientReturning({ ok: true, result: { supported: true } }, refusal) + lost.sendRequest + .mockImplementationOnce(async () => ({ ok: true, result: { supported: true } })) + .mockRejectedValueOnce(markRpcDeliveryUnknown(new Error('copy reply lost'))) + .mockResolvedValueOnce(refusal) + expect( + (await createMobileStructuredAgentSession(lost, 'workspace-1', 'codex', forkFrom)).kind + ).toBe('unknown') +}) diff --git a/mobile/src/session/mobile-structured-agent-session-launch.ts b/mobile/src/session/mobile-structured-agent-session-launch.ts index 55dbc1ebef0..d7cb85e2a31 100644 --- a/mobile/src/session/mobile-structured-agent-session-launch.ts +++ b/mobile/src/session/mobile-structured-agent-session-launch.ts @@ -7,7 +7,8 @@ import type { } from '../../../src/shared/agent-session/agent-session-wire' import { isDefinitiveAgentSessionCreateRefusal, - isAgentSessionForkSourceAdmissionRefusal + isAgentSessionForkSourceAdmissionRefusal, + isAgentSessionCreateAdmissionRefusal } from '../../../src/shared/agent-session/failures/agent-session-definitive-refusal' import { createStructuredAgentSessionId, @@ -187,7 +188,8 @@ export async function createMobileStructuredAgentSession( forkFrom && !priorCreateOutcomeUnknown && result.refusal.createAttemptRefusedBeforeCommit === true && - isAgentSessionForkSourceAdmissionRefusal(result.refusal) + (isAgentSessionForkSourceAdmissionRefusal(result.refusal) || + isAgentSessionCreateAdmissionRefusal(result.refusal)) ) { return { kind: 'failed', diff --git a/src/main/codex/codex-conversation-copy.test.ts b/src/main/codex/codex-conversation-copy.test.ts index bafac8ca255..17938242104 100644 --- a/src/main/codex/codex-conversation-copy.test.ts +++ b/src/main/codex/codex-conversation-copy.test.ts @@ -10,15 +10,24 @@ import { join } from 'node:path' import { afterEach, beforeEach, expect, it, vi } from 'vitest' import { agentJournalItemKey } from '../../shared/agent-session/journal/agent-session-journal-item-key' import { agentSessionRecordFixture } from '../native-chat/agent-session-record-test-fixture' -import { copyCodexConversation, codexForkHistory } from './codex-conversation-copy' +import { copyCodexConversation } from './codex-conversation-copy' +import { codexForkHistory } from './codex-fork-history' +import { CodexAppServerTimeoutError } from './app-server/codex-app-server-session' import type { CodexAppServerRpc } from './app-server/codex-app-server-session' -const state = vi.hoisted(() => ({ - home: '', - paths: new Map(), - request: vi.fn(), - initialize: vi.fn() -})) +const state = vi.hoisted(() => { + const session: { + run?: (body: (rpc: CodexAppServerRpc) => Promise) => Promise + transport?: 'native' | 'unsupported-read' + } = {} + return { + home: '', + paths: new Map(), + request: vi.fn(), + initialize: vi.fn(), + session + } +}) vi.mock('../native-chat/codex-local-history', async (importOriginal) => { const original = await importOriginal() return { ...original, readCodexLocalHistory: vi.fn(original.readCodexLocalHistory) } @@ -30,19 +39,58 @@ vi.mock('../native-chat/codex-rollout-history-stream', async (importOriginal) => vi.mock('../native-chat/session-file-resolver', () => ({ resolveSessionFilePath: async (_agent: string, id: string) => state.paths.get(id) ?? null })) -vi.mock('./app-server/codex-app-server-session', async (importOriginal) => ({ - ...(await importOriginal()), - runCodexAppServerSession: async ( - launch: unknown, - body: (rpc: CodexAppServerRpc) => Promise - ) => { - state.initialize(launch) - return body({ request: state.request, notify: () => {} }) +vi.mock('./app-server/codex-app-server-session', async (importOriginal) => { + const original = await importOriginal() + return { + ...original, + runCodexAppServerSession: async ( + launch: AppServerSession.CodexAppServerInvocation, + body: (rpc: CodexAppServerRpc) => Promise + ) => { + state.initialize(launch) + if (state.session.transport) { + const server = String.raw` + let experimental = false + require('node:readline').createInterface({ input: process.stdin }).on('line', line => { + const message = JSON.parse(line) + if (typeof message.id !== 'number') return + if (message.method === 'initialize') { + experimental = message.params.capabilities?.experimentalApi === true + } + const error = message.method === 'thread/read' && process.argv[1] === 'unsupported-read' + ? { code: -32601, message: 'thread/read is not supported yet' } + : message.method === 'thread/turns/list' && !experimental + ? { code: -32600, message: 'thread/turns/list requires experimentalApi capability' } + : null + process.stdout.write(JSON.stringify({ id: message.id, ...(error ? { error } : { result: {} }) }) + '\n') + }) + ` + return original.runCodexAppServerSession( + { + ...launch, + command: process.execPath, + args: ['-e', server, state.session.transport], + cliPath: null + }, + (rpc) => + body({ + ...rpc, + request: async (method, params) => { + await rpc.request(method, params) + return state.request(method, params) + } + }) + ) + } + return state.session.run + ? state.session.run(body) + : body({ request: state.request, notify: () => {} }) + } } -})) +}) -const SOURCE = '00000000-0000-4000-8000-000000000001' -const CHILD = '00000000-0000-4000-8000-000000000002' +const SOURCE = '01a1bdd2-6d33-7a21-9a1d-c14f0117bc8c' +const CHILD = '01a1bde0-5ded-7f03-a1bf-3dae1613d8c7' let root: string beforeEach(async () => { @@ -50,6 +98,8 @@ beforeEach(async () => { state.home = root state.paths.clear() state.request.mockReset() + state.session.run = undefined + state.session.transport = undefined state.initialize.mockReset() vi.mocked(readCodexLocalHistory).mockClear() vi.mocked(readCodexRolloutBytes).mockClear() @@ -85,7 +135,12 @@ async function history(id: string, records: object[]) { state.paths.set(id, path) return path } -function copy(selected = `${SOURCE}:a1`, settled = true, turnId?: string) { +function copy( + selected = `${SOURCE}:a1`, + settled = true, + turnId?: string, + options: { signal?: AbortSignal; selectedItemId?: string } = {} +) { return copyCodexConversation( { record: agentSessionRecordFixture({ @@ -102,15 +157,18 @@ function copy(selected = `${SOURCE}:a1`, settled = true, turnId?: string) { providerHandle: null }, providerSessionId: SOURCE, + signal: options.signal, title: 'Source fork', accountHome: { variable: 'CODEX_HOME', path: root }, checkpoint: { - selectedItemId: agentJournalItemKey({ - provider: 'legacy', - agent: 'codex', - sessionId: SOURCE, - recordId: selected - }), + selectedItemId: + options.selectedItemId ?? + agentJournalItemKey({ + provider: 'legacy', + agent: 'codex', + sessionId: SOURCE, + recordId: selected + }), settled, ...(turnId ? { turnId } : {}) } @@ -150,10 +208,12 @@ async function childPrefix(lines?: number) { return path } -it('uses a persisted terminal canonical boundary for an idle imported latest reply with no live owner', async () => { +it('keeps omitted-mode compatibility for a persisted terminal imported reply', async () => { await history(SOURCE, turn('canonical', 1)) await childPrefix() - state.request.mockResolvedValue({ thread: { id: CHILD } }) + state.request.mockImplementation(async (method) => ({ + thread: { id: method === 'thread/read' ? SOURCE : CHILD } + })) await expect(copy(undefined, false)).resolves.toEqual({ sessionId: CHILD, coverage: { unit: 'codex-turn', ids: ['canonical'] } @@ -168,9 +228,11 @@ it('uses beforeTurnId only for the immediate next effective root and opts into t await history(SOURCE, [...turn(undefined, 1), ...turn('canonical-next', 2)]) await childPrefix(3) state.request.mockImplementation(async (method) => - method === 'thread/fork' - ? { thread: { id: CHILD } } - : { data: [{ id: 'legacy-first' }, { id: 'canonical-next' }], nextCursor: null } + method === 'thread/read' + ? { thread: { id: SOURCE, historyMode: 'paginated' } } + : method === 'thread/fork' + ? { thread: { id: CHILD } } + : { data: [{ id: 'legacy-first' }, { id: 'canonical-next' }], nextCursor: null } ) await expect(copy()).resolves.toEqual({ sessionId: CHILD, @@ -196,6 +258,9 @@ it('forks and reverts only the child for legacy, legacy, canonical, counting aga state.paths.set(CHILD, path) let reverted = false state.request.mockImplementation(async (method) => { + if (method === 'thread/read') { + return { thread: { id: SOURCE, historyMode: 'paginated' } } + } if (method === 'thread/fork') { return { thread: { id: CHILD, historyMode: 'paginated' } } } @@ -249,7 +314,8 @@ it('uses exact physical identities across repeated prompts rather than a retaine expect(result.selected.id).toBe('second') }) -it('freezes all-unrecorded root IDs on the source, excluding child replacements and later source growth', async () => { +it('keeps method-not-found compatibility with frozen unrecorded IDs despite source growth and child replacements', async () => { + state.session.transport = 'unsupported-read' await history(SOURCE, [...turn(undefined, 1), ...turn(undefined, 2), ...turn(undefined, 3)]) let reverted = false state.request.mockImplementation(async (method, params) => { @@ -290,7 +356,7 @@ it('freezes all-unrecorded root IDs on the source, excluding child replacements }) }) -it('certifies recorded turns in a standalone copied rollout with no physical history base', async () => { +it('keeps legacy-mode compatibility for recorded turns in a standalone copied rollout', async () => { const rows = await recordedCodexRollout('c') const source = rows.find((row) => row.type === 'session_meta')! const started = rows.find( @@ -312,9 +378,221 @@ it('certifies recorded turns in a standalone copied rollout with no physical his .join('\n') await writeFile(path, `${serialized}\n`) } - state.request.mockResolvedValue({ thread: { id: CHILD } }) + state.request.mockImplementation(async (method) => ({ + thread: { id: method === 'thread/read' ? SOURCE : CHILD, historyMode: 'legacy' } + })) await expect(copy(undefined, true, String(started.payload.turn_id))).resolves.toEqual({ sessionId: CHILD, coverage: { unit: 'codex-turn', ids: ['01a1b0d7-ac1d-7e00-85b8-806a4d8d494b'] } }) }) + +it('forks the current paged native prefix after rewind without source or child rollout authority', async () => { + state.session.transport = 'native' + await history(SOURCE, [ + ...turn('01a1bdd2-6d5e-7960-8fbb-30c962a5eeaa', 1), + ...turn('01a1bdd3-1293-7bc2-b750-017ca47738f9', 2) + ]) + const selected = '01a1bdd4-2977-7f81-8495-eb160b46419a' + state.request.mockImplementation(async (method, params) => { + if (method === 'thread/read') { + return { thread: { id: SOURCE, historyMode: 'paginated', status: { type: 'notLoaded' } } } + } + if (method === 'thread/fork') { + return { thread: { id: CHILD, historyMode: 'paginated' } } + } + if (!params?.cursor) { + return { + data: [{ id: '01a1bdd2-6d5e-7960-8fbb-30c962a5eeaa', status: 'completed' }], + nextCursor: 'page-2' + } + } + return { + data: [ + { id: '01a1bdd2-b7a1-7c63-8b19-cc361c2baefb', status: 'completed' }, + { id: selected, status: 'completed' }, + ...(params.threadId === CHILD + ? [{ id: '01a1bde0-5e1e-7ad0-ab18-f9ed07fd00e7', status: 'completed' }] + : []) + ], + nextCursor: null + } + }) + await expect( + copy(undefined, false, selected, { selectedItemId: `codex:${SOURCE}:${selected}:1` }) + ).resolves.toEqual({ + sessionId: '01a1bde0-5ded-7f03-a1bf-3dae1613d8c7', + coverage: { + unit: 'codex-turn', + ids: [ + '01a1bdd2-6d5e-7960-8fbb-30c962a5eeaa', + '01a1bdd2-b7a1-7c63-8b19-cc361c2baefb', + '01a1bdd4-2977-7f81-8495-eb160b46419a' + ] + } + }) + expect(state.request).toHaveBeenCalledWith('thread/fork', { + threadId: SOURCE, + lastTurnId: '01a1bdd4-2977-7f81-8495-eb160b46419a', + excludeTurns: true, + cwd: root + }) +}) + +it('bridges a saved physical reply with an absent command candidate without restoring a removed native turn', async () => { + await history(SOURCE, [...turn('kept', 1), ...turn('removed', 2)]) + state.request.mockImplementation(async (method) => { + if (method === 'thread/read') { + return { thread: { id: SOURCE, historyMode: 'paginated' } } + } + if (method === 'thread/fork') { + return { thread: { id: CHILD } } + } + return { data: [{ id: 'kept', status: 'completed' }], nextCursor: null } + }) + await expect(copy(undefined, true, 'orca-command')).resolves.toEqual({ + sessionId: CHILD, + coverage: { unit: 'codex-turn', ids: ['kept'] } + }) + await expect(copy(`${SOURCE}:a2`, true, 'orca-command')).rejects.toMatchObject({ + refusal: { + code: 'agent_session_operation_invalid', + details: { reason: 'savedConversationMissing' }, + createAttemptRefusedBeforeCommit: true + } + }) + await expect(copy(undefined, true)).resolves.toEqual({ + sessionId: CHILD, + coverage: { unit: 'codex-turn', ids: ['kept'] } + }) + await expect( + copy(undefined, true, 'removed', { selectedItemId: `codex:${SOURCE}:removed:1` }) + ).rejects.toMatchObject({ + refusal: { + details: { reason: 'savedConversationMissing' }, + createAttemptRefusedBeforeCommit: true + } + }) +}) + +it('keeps read faults in preparation and a contradictory copied prefix uncertain', async () => { + const selected = '01a1bdd4-2977-7f81-8495-eb160b46419a' + await history(SOURCE, turn(selected, 1)) + await childPrefix() + const prepare = () => copy(undefined, true, selected) + state.request.mockRejectedValue(new Error('read failed')) + await expect(prepare()).rejects.toMatchObject({ + refusal: { + code: 'agent_session_identity_required', + details: { reason: 'transcriptUnreadable' }, + createAttemptRefusedBeforeCommit: true + } + }) + state.request.mockResolvedValue({ thread: { id: CHILD, historyMode: 'paginated' } }) + await expect(prepare()).rejects.toMatchObject({ + refusal: { + code: 'agent_session_identity_required', + details: { reason: 'transcriptUnreadable' }, + createAttemptRefusedBeforeCommit: true + } + }) + state.request.mockImplementation(async (method, params) => { + if (method === 'thread/read') { + return { thread: { id: SOURCE, historyMode: 'paginated' } } + } + if (method === 'thread/fork') { + return { thread: { id: CHILD } } + } + return { + data: (params?.threadId === SOURCE + ? ['01a1bdd2-6d5e-7960-8fbb-30c962a5eeaa', selected] + : [selected, '01a1bdd2-6d5e-7960-8fbb-30c962a5eeaa'] + ).map((id) => ({ + id + })), + nextCursor: null + } + }) + await expect(prepare()).rejects.not.toHaveProperty('refusal') +}) + +it('refuses invalid and repeated source cursors before copying', async () => { + state.request.mockImplementation(async (method) => + method === 'thread/read' + ? { thread: { id: SOURCE, historyMode: 'paginated' } } + : { data: [{ id: 'first' }], nextCursor: 'same-page' } + ) + await expect(copy(undefined, true, 'missing')).rejects.toMatchObject({ + refusal: { details: { reason: 'transcriptUnreadable' }, createAttemptRefusedBeforeCommit: true } + }) + state.request.mockImplementation(async (method) => + method === 'thread/read' + ? { thread: { id: SOURCE, historyMode: 'paginated' } } + : { data: [], nextCursor: 17 } + ) + await expect(copy(undefined, true, 'missing')).rejects.toMatchObject({ + refusal: { details: { reason: 'transcriptUnreadable' }, createAttemptRefusedBeforeCommit: true } + }) +}) + +it('cannot dispatch an abandoned preparation after its lifetime ends, but keeps an invoked fork uncertain', async () => { + const listed = Promise.withResolvers() + const release = Promise.withResolvers() + const expired = Promise.withResolvers() + const lifetime = new AbortController() + let abandoned: Promise | undefined + let childCreated = false + state.request.mockImplementation(async (method) => { + if (method === 'thread/read') { + return { thread: { id: SOURCE, historyMode: 'paginated' } } + } + if (method === 'thread/fork') { + childCreated = true + return { thread: { id: CHILD } } + } + listed.resolve() + return release.promise + }) + state.session.run = async (body) => { + abandoned = body({ request: state.request, notify: () => {}, signal: lifetime.signal }) + return Promise.race([abandoned, expired.promise]) + } + const preparation = copy(undefined, true, 'selected') + await listed.promise + lifetime.abort(new CodexAppServerTimeoutError('session ended')) + expired.reject(lifetime.signal.reason) + await expect(preparation).rejects.toMatchObject({ + refusal: { createAttemptRefusedBeforeCommit: true } + }) + release.resolve({ data: [{ id: 'selected' }], nextCursor: null }) + await expect(abandoned).rejects.toBe(lifetime.signal.reason) + expect(childCreated).toBe(false) + const forked = Promise.withResolvers() + const lost = Promise.withResolvers() + const pending = Promise.withResolvers() + state.session.run = async (body) => + Promise.race([body({ request: state.request, notify: () => {} }), lost.promise]) + state.request.mockImplementation(async (method) => { + if (method === 'thread/read') { + return { thread: { id: SOURCE, historyMode: 'paginated' } } + } + if (method === 'thread/fork') { + forked.resolve() + return pending.promise + } + return { data: [{ id: 'selected' }], nextCursor: null } + }) + const invoked = copy(undefined, true, 'selected') + await forked.promise + const timeout = new CodexAppServerTimeoutError('reply lost') + lost.reject(timeout) + await expect(invoked).rejects.not.toHaveProperty('refusal') + pending.reject(timeout) + state.session.run = undefined + state.session.transport = undefined + const caller = new AbortController() + caller.abort(new Error('caller cancelled')) + await expect(copy(undefined, true, 'selected', { signal: caller.signal })).rejects.toBe( + caller.signal.reason + ) +}) diff --git a/src/main/codex/codex-conversation-copy.ts b/src/main/codex/codex-conversation-copy.ts index f0bc1e43c49..cb3b3a9adee 100644 --- a/src/main/codex/codex-conversation-copy.ts +++ b/src/main/codex/codex-conversation-copy.ts @@ -1,15 +1,19 @@ -import { AgentConversationCopyError } from '../native-chat/agent-conversation-copy-error' +import { TEXT_GENERATION_FAILURE_COPY } from '../../shared/text-generation-failure-copy' +import { codexForkHistory } from './codex-fork-history' +import { + AgentConversationCopyError, + AgentConversationCopyStartError, + agentConversationCopyRefusal +} from '../native-chat/agent-conversation-copy-error' +import { isAgentSessionRefusalError } from '../../shared/agent-session/failures/agent-session-wire-refusals' import { join } from 'node:path' import { requireLegacyAgentSessionAccountHome } from '../../shared/agent-session/lifecycle/agent-session-account-home' import { parseAgentJournalItemKey } from '../../shared/agent-session/journal/agent-session-journal-item-key' -import { asRecord, parseJsonObject } from '../ai-vault/session-scanner-values' +import { asRecord } from '../ai-vault/session-scanner-values' import type { StructuredAgentSessionAdapter } from '../native-chat/agent-session-wire/providers/structured-agent-session-adapter' -import { createLegacyIdentityTracker } from '../native-chat/agent-session-journal/history-import/journal-legacy-identity' -import { readCodexLocalHistory } from '../native-chat/codex-local-history' import { resolveSessionFilePath } from '../native-chat/session-file-resolver' -import { decodeCodexTranscriptLine } from '../native-chat/transcript-line-decoders-codex' import { - CodexAppServerTimeoutError, + isCodexAppServerUnsupportedError, type CodexAppServerRpc } from './app-server/codex-app-server-session' import { @@ -18,117 +22,28 @@ import { } from './structured/session-lifecycle/codex-provider-session-rename' type CopyInput = Parameters>[0] -type Root = { id?: string; terminal: boolean; prompt: boolean } function unavailable(): never { - throw new AgentConversationCopyError( - 'Codex could not locate an exact saved boundary in this older chat. Fork a response in a newer chat.' - ) -} - -/** Use the importer's physical lineage and identity accounting, including discarded window rows. */ -export async function codexForkHistory(input: { - path: string - roots: string[] - selectedItemId: string - turnId?: string - sessionId: string - signal?: AbortSignal - inheritedOnly?: boolean -}) { - const identity = parseAgentJournalItemKey(input.selectedItemId) - const tracker = createLegacyIdentityTracker({ - transcriptAgent: 'codex', - agent: 'codex', - sessionId: input.sessionId - }) - const roots: Root[] = [] - let selected: Root | undefined - let lineIndex = 0 - let referenceBacked = false - await readCodexLocalHistory({ - filePath: input.path, - roots: input.roots, - visitRecord: ({ record, selected }) => { - if (selected && record?.type === 'session_meta') { - referenceBacked = asRecord(record.payload)?.history_base != null - } - }, - visit: (line, fallbackId, selectedSegment, recordId) => { - if (input.inheritedOnly && referenceBacked && selectedSegment) { - return - } - input.signal?.throwIfAborted() - const tracked = tracker.identify(line, lineIndex++) - const raw = parseJsonObject(line) - const payload = asRecord(raw?.payload) - if ( - raw?.type === 'event_msg' && - payload?.type === 'thread_rolled_back' && - typeof payload.num_turns === 'number' - ) { - roots.splice(Math.max(0, roots.length - payload.num_turns)) - return - } - if (raw?.type === 'event_msg' && payload?.type === 'task_started') { - roots.push({ - id: typeof payload.turn_id === 'string' ? payload.turn_id : undefined, - terminal: false, - prompt: false - }) - } - const message = decodeCodexTranscriptLine(line, fallbackId, { historyImport: true }) - if (message?.role === 'user') { - const previous = roots.at(-1) - if (!previous || (previous.prompt && (!previous.id || previous.terminal))) { - roots.push({ terminal: false, prompt: true }) - } else { - previous.prompt = true - } - } - const root = roots.at(-1) - if ( - root && - raw?.type === 'event_msg' && - (payload?.type === 'task_complete' || payload?.type === 'turn_aborted') && - (!payload.turn_id || payload.turn_id === root.id) - ) { - root.terminal = true - } - const importedMatch = - identity?.provider === 'legacy' && - identity.recordId === - (recordId ?? (tracked.provider === 'legacy' ? tracked.recordId : undefined)) - if ( - root && - ((importedMatch && message?.role === 'assistant') || - (input.turnId && root.id === input.turnId)) - ) { - if (selected && selected !== root) { - return unavailable() - } - selected = root - } - } - }) - const index = selected ? roots.indexOf(selected) : -1 - if (index < 0) { - return unavailable() - } - return { roots, index, selected: roots[index] } + throw new AgentConversationCopyError('Codex could not verify the copied chat.') } async function turns( rpc: CodexAppServerRpc, threadId: string, - count: number -): Promise<{ id: string }[]> { - const all: { id: string }[] = [] + through: number | string +): Promise<{ id: string; status?: unknown }[]> { + const all: { id: string; status?: unknown }[] = [] let cursor: string | null = null const seen = new Set() do { const page = asRecord( - await rpc.request('thread/turns/list', { threadId, cursor, sortDirection: 'asc', limit: 100 }) + await rpc.request('thread/turns/list', { + threadId, + cursor, + sortDirection: 'asc', + limit: 100, + itemsView: 'summary' + }) ) if (!Array.isArray(page?.data)) { return unavailable() @@ -138,7 +53,13 @@ async function turns( if (typeof turn?.id !== 'string') { return unavailable() } - all.push({ id: turn.id }) + all.push({ id: turn.id, status: turn.status }) + if (turn.id === through) { + return all + } + } + if (page.nextCursor != null && (typeof page.nextCursor !== 'string' || !page.nextCursor)) { + return unavailable() } cursor = typeof page.nextCursor === 'string' ? page.nextCursor : null if (cursor && seen.has(cursor)) { @@ -147,7 +68,7 @@ async function turns( if (cursor) { seen.add(cursor) } - } while (cursor && all.length < count) + } while (cursor && (typeof through === 'string' || all.length < through)) return all } @@ -155,140 +76,226 @@ export async function copyCodexConversation( input: CopyInput, invocation: CodexProviderInvocationDeps | undefined ) { - if (!input.record || !input.checkpoint || !invocation) { - return unavailable() - } - const { record, checkpoint } = input - const signal = AbortSignal.any([ - AbortSignal.timeout(60_000), - ...(input.signal ? [input.signal] : []) - ]) - const home = requireLegacyAgentSessionAccountHome(record.accountHome).path - const stores = [join(home, 'sessions'), join(home, 'archived_sessions')] - const find = async (id: string) => - (await resolveSessionFilePath( - 'codex', - id, - { codexSessionsDirs: stores, strictErrors: true }, - signal - )) ?? unavailable() - const history = await codexForkHistory({ - path: await find(input.providerSessionId), - roots: stores, - selectedItemId: checkpoint.selectedItemId, - turnId: checkpoint.turnId, - sessionId: input.providerSessionId, - signal - }) - const beforeTurnId = history.roots[history.index + 1]?.id - const lastTurnId = history.selected.id - if (!checkpoint.settled && !history.selected.terminal && !beforeTurnId) { - throw new AgentConversationCopyError( - "Codex hasn't finished this response yet. Wait for it to finish, then try again." - ) - } - return runCodexProviderAppServer( - record, - { ...invocation, resolveLaunchArgs: () => record.launchArgs ?? invocation.resolveLaunchArgs() }, - { timeoutMs: 60_000, signal, experimentalApi: !lastTurnId && Boolean(beforeTurnId) }, - async (rpc, cwd) => { - const prefix = history.roots.slice(0, history.index + 1) - const sourceTurns = prefix.every((root) => root.id) - ? prefix.map((root) => ({ id: root.id! })) - : await turns(rpc, input.providerSessionId, history.roots.length) - if ( - sourceTurns.length < prefix.length || - prefix.some((root, index) => root.id && root.id !== sourceTurns[index]?.id) - ) { - return unavailable() - } - const coverage = { - unit: 'codex-turn' as const, - ids: sourceTurns.slice(0, prefix.length).map((turn) => turn.id) - } - const fork = asRecord( - await rpc.request('thread/fork', { - threadId: input.providerSessionId, - ...(lastTurnId ? { lastTurnId } : beforeTurnId ? { beforeTurnId } : {}), - excludeTurns: true, - cwd - }) - ) - const thread = asRecord(fork?.thread) - const id = thread?.id - if (typeof id !== 'string' || !id.trim() || id === input.providerSessionId) { - return unavailable() - } - const captured = await codexForkHistory({ - path: await find(id), + let starting = false + let dispatched = false + try { + if (!input.record || !input.checkpoint || !invocation) { + return unavailable() + } + const { record, checkpoint } = input + const signal = AbortSignal.any([ + AbortSignal.timeout(60_000), + ...(input.signal ? [input.signal] : []) + ]) + const home = requireLegacyAgentSessionAccountHome(record.accountHome).path + const stores = [join(home, 'sessions'), join(home, 'archived_sessions')] + const legacy = parseAgentJournalItemKey(checkpoint.selectedItemId)?.provider === 'legacy' + const find = async (id: string) => + (await resolveSessionFilePath( + 'codex', + id, + { codexSessionsDirs: stores, strictErrors: true }, + signal + )) ?? unavailable() + const physicalHistory = async (turnId: string | undefined) => { + const path = await find(input.providerSessionId) + return codexForkHistory({ + path, roots: stores, selectedItemId: checkpoint.selectedItemId, - turnId: checkpoint.turnId, - sessionId: id, - inheritedOnly: true, + turnId, + sessionId: input.providerSessionId, signal }) - if ( - captured.index !== history.index || - captured.roots.length < prefix.length || - captured.roots - .slice(0, prefix.length) - .some((root, index) => root.id && root.id !== coverage.ids[index]) - ) { - return unavailable() - } - if (lastTurnId || beforeTurnId) { - if (captured.roots.length !== prefix.length) { - return unavailable() - } - } else { - // Trim the actual capture; coverage identities were frozen on the source before fork. - if (thread?.historyMode !== 'paginated') { - return unavailable() - } - const childTurns = await turns(rpc, id, captured.roots.length + 1) - if (childTurns.length !== captured.roots.length) { - return unavailable() - } - for (const [index, root] of captured.roots.entries()) { - if (root.id && root.id !== childTurns[index]?.id) { - return unavailable() + } + const missing = (): never => { + throw agentConversationCopyRefusal( + 'agent_session_operation_invalid', + { reason: 'savedConversationMissing' }, + 'Codex could not find this response in the saved chat. Open the chat again, then fork a response still in its history.' + ) + } + let history = legacy && !checkpoint.turnId ? await physicalHistory(undefined) : undefined + starting = true + return await runCodexProviderAppServer( + record, + { + ...invocation, + resolveLaunchArgs: () => record.launchArgs ?? invocation.resolveLaunchArgs() + }, + { timeoutMs: 60_000, signal, experimentalApi: true }, + async (rpc, cwd) => { + starting = false + let metadata + try { + metadata = asRecord( + asRecord( + await rpc.request('thread/read', { + threadId: input.providerSessionId, + includeTurns: false + }) + )?.thread + ) + if ( + metadata?.id !== input.providerSessionId || + (metadata.historyMode != null && + metadata.historyMode !== 'legacy' && + metadata.historyMode !== 'paginated') + ) { + throw new Error('Invalid source metadata') + } + } catch (error) { + if (!isCodexAppServerUnsupportedError(error)) { + throw error } } - const firstRemoved = childTurns[captured.index + 1] - if (firstRemoved) { - await rpc.request('thread/revert', { threadId: id, beforeTurnId: firstRemoved.id }) - const kept = await turns(rpc, id, captured.index + 2) + let native = metadata?.historyMode === 'paginated' + let sourceTurns = + native && checkpoint.turnId + ? await turns(rpc, input.providerSessionId, checkpoint.turnId) + : undefined + if (!native || !sourceTurns || sourceTurns.at(-1)?.id !== checkpoint.turnId) { + if (native && !legacy) { + return missing() + } + history ??= await physicalHistory(native && legacy ? undefined : checkpoint.turnId) + if (native && history.selected.id) { + sourceTurns = await turns(rpc, input.providerSessionId, history.selected.id) + if (sourceTurns.at(-1)?.id !== history.selected.id) { + return missing() + } + } else { + native = false + } + } + const lastTurnId = native ? sourceTurns!.at(-1)!.id : history!.selected.id + const beforeTurnId = native ? undefined : history!.roots[history!.index + 1]?.id + const terminal = native + ? ['completed', 'failed', 'interrupted'].includes(String(sourceTurns!.at(-1)?.status)) + : history!.selected.terminal + if (!checkpoint.settled && !terminal && !beforeTurnId) { + throw agentConversationCopyRefusal( + 'agent_session_operation_invalid', + { reason: 'turnActive' }, + "Codex hasn't finished this response yet. Wait for it to finish, then try again." + ) + } + const prefix = native ? sourceTurns! : history!.roots.slice(0, history!.index + 1) + sourceTurns = native + ? sourceTurns! + : prefix.every((root) => root.id) + ? prefix.map((root) => ({ id: root.id! })) + : await turns(rpc, input.providerSessionId, history!.roots.length) + if ( + sourceTurns.length < prefix.length || + prefix.some((root, index) => root.id && root.id !== sourceTurns![index]?.id) + ) { + return missing() + } + const coverage = { + unit: 'codex-turn' as const, + ids: sourceTurns.slice(0, prefix.length).map((turn) => turn.id) + } + rpc.signal?.throwIfAborted() + signal.throwIfAborted() + dispatched = true + const fork = asRecord( + await rpc.request('thread/fork', { + threadId: input.providerSessionId, + ...(lastTurnId ? { lastTurnId } : beforeTurnId ? { beforeTurnId } : {}), + excludeTurns: true, + cwd + }) + ) + const thread = asRecord(fork?.thread) + const id = thread?.id + if (typeof id !== 'string' || !id.trim() || id === input.providerSessionId) { + return unavailable() + } + if (native) { + const captured = await turns(rpc, id, lastTurnId!) if ( - kept.length !== captured.index + 1 || - kept.at(-1)?.id !== childTurns[captured.index]?.id + captured.length !== coverage.ids.length || + captured.some((turn, index) => turn.id !== coverage.ids[index]) ) { return unavailable() } + return { sessionId: id, coverage } } + const captured = await codexForkHistory({ + path: await find(id), + roots: stores, + selectedItemId: checkpoint.selectedItemId, + turnId: checkpoint.turnId, + sessionId: id, + inheritedOnly: true, + signal + }) + if ( + captured.index !== history!.index || + captured.roots.length < prefix.length || + captured.roots + .slice(0, prefix.length) + .some((root, index) => root.id && root.id !== coverage.ids[index]) + ) { + return unavailable() + } + if (lastTurnId || beforeTurnId) { + if (captured.roots.length !== prefix.length) { + return unavailable() + } + } else { + // Trim the actual capture; coverage identities were frozen on the source before fork. + if (thread?.historyMode !== 'paginated') { + return unavailable() + } + const childTurns = await turns(rpc, id, captured.roots.length + 1) + if (childTurns.length !== captured.roots.length) { + return unavailable() + } + for (const [index, root] of captured.roots.entries()) { + if (root.id && root.id !== childTurns[index]?.id) { + return unavailable() + } + } + const firstRemoved = childTurns[captured.index + 1] + if (firstRemoved) { + await rpc.request('thread/revert', { threadId: id, beforeTurnId: firstRemoved.id }) + const kept = await turns(rpc, id, captured.index + 2) + if ( + kept.length !== captured.index + 1 || + kept.at(-1)?.id !== childTurns[captured.index]?.id + ) { + return unavailable() + } + } + } + return { sessionId: id, coverage } } - return { sessionId: id, coverage } + ) + } catch (error) { + input.signal?.throwIfAborted() + if (!dispatched) { + const refusal = isAgentSessionRefusalError(error) + ? error + : agentConversationCopyRefusal( + 'agent_session_identity_required', + { reason: 'transcriptUnreadable' }, + starting + ? TEXT_GENERATION_FAILURE_COPY.agentCouldNotStart.replace('{{label}}', 'Codex') + : 'Codex could not read the saved chat. Open it again, then try forking the response.' + ) + if (starting) { + refusal.cause = new AgentConversationCopyStartError(refusal.refusal.message) + } + refusal.refusal.createAttemptRefusedBeforeCommit = true + throw refusal } - ).catch((error: unknown) => { if (error instanceof AgentConversationCopyError) { throw error } - if (error instanceof CodexAppServerTimeoutError || signal.aborted) { - throw new AgentConversationCopyError('Codex took too long to copy this chat. Try again.', { - cause: error - }) - } - if ( - error instanceof Error && - /(?:in[- ]progress|active turn|turn.*running)/i.test(error.message) - ) { - throw new AgentConversationCopyError( - 'Codex could not copy this chat while a response was running. Wait for it to finish, then try again.', - { cause: error } - ) - } - throw new AgentConversationCopyError('Codex could not copy this chat. Try again.', { + throw new AgentConversationCopyError('Codex could not verify the copied chat.', { cause: error }) - }) + } } diff --git a/src/main/codex/codex-fork-history.ts b/src/main/codex/codex-fork-history.ts new file mode 100644 index 00000000000..26d345e2693 --- /dev/null +++ b/src/main/codex/codex-fork-history.ts @@ -0,0 +1,100 @@ +import { AgentConversationCopyError } from '../native-chat/agent-conversation-copy-error' +import { parseAgentJournalItemKey } from '../../shared/agent-session/journal/agent-session-journal-item-key' +import { asRecord, parseJsonObject } from '../ai-vault/session-scanner-values' +import { createLegacyIdentityTracker } from '../native-chat/agent-session-journal/history-import/journal-legacy-identity' +import { readCodexLocalHistory } from '../native-chat/codex-local-history' +import { decodeCodexTranscriptLine } from '../native-chat/transcript-line-decoders-codex' + +type Root = { id?: string; terminal: boolean; prompt: boolean } + +/** Use the importer's physical lineage and identity accounting, including discarded window rows. */ +export async function codexForkHistory(input: { + path: string + roots: string[] + selectedItemId: string + turnId?: string + sessionId: string + signal?: AbortSignal + inheritedOnly?: boolean +}) { + const identity = parseAgentJournalItemKey(input.selectedItemId) + const tracker = createLegacyIdentityTracker({ + transcriptAgent: 'codex', + agent: 'codex', + sessionId: input.sessionId + }) + const roots: Root[] = [] + let selected: Root | undefined + let lineIndex = 0 + let referenceBacked = false + await readCodexLocalHistory({ + filePath: input.path, + roots: input.roots, + visitRecord: ({ record, selected }) => { + if (selected && record?.type === 'session_meta') { + referenceBacked = asRecord(record.payload)?.history_base != null + } + }, + visit: (line, fallbackId, selectedSegment, recordId) => { + if (input.inheritedOnly && referenceBacked && selectedSegment) { + return + } + input.signal?.throwIfAborted() + const tracked = tracker.identify(line, lineIndex++) + const raw = parseJsonObject(line) + const payload = asRecord(raw?.payload) + if ( + raw?.type === 'event_msg' && + payload?.type === 'thread_rolled_back' && + typeof payload.num_turns === 'number' + ) { + roots.splice(Math.max(0, roots.length - payload.num_turns)) + return + } + if (raw?.type === 'event_msg' && payload?.type === 'task_started') { + roots.push({ + id: typeof payload.turn_id === 'string' ? payload.turn_id : undefined, + terminal: false, + prompt: false + }) + } + const message = decodeCodexTranscriptLine(line, fallbackId, { historyImport: true }) + if (message?.role === 'user') { + const previous = roots.at(-1) + if (!previous || (previous.prompt && (!previous.id || previous.terminal))) { + roots.push({ terminal: false, prompt: true }) + } else { + previous.prompt = true + } + } + const root = roots.at(-1) + if ( + root && + raw?.type === 'event_msg' && + (payload?.type === 'task_complete' || payload?.type === 'turn_aborted') && + (!payload.turn_id || payload.turn_id === root.id) + ) { + root.terminal = true + } + const importedMatch = + identity?.provider === 'legacy' && + identity.recordId === + (recordId ?? (tracked.provider === 'legacy' ? tracked.recordId : undefined)) + if ( + root && + ((importedMatch && message?.role === 'assistant') || + (input.turnId && root.id === input.turnId)) + ) { + if (selected && selected !== root) { + throw new AgentConversationCopyError('Codex could not verify the copied chat.') + } + selected = root + } + } + }) + const index = selected ? roots.indexOf(selected) : -1 + if (index < 0) { + throw new AgentConversationCopyError('Codex could not verify the copied chat.') + } + return { roots, index, selected: roots[index] } +} diff --git a/src/main/native-chat/agent-conversation-copy-error.ts b/src/main/native-chat/agent-conversation-copy-error.ts index 8924223a2a2..64fb08a76f8 100644 --- a/src/main/native-chat/agent-conversation-copy-error.ts +++ b/src/main/native-chat/agent-conversation-copy-error.ts @@ -8,6 +8,8 @@ import type { AgentSessionRefusalDetailsByCode } from '../../shared/agent-sessio /** Only copy errors authored here may pass through the host's generic provider failure. */ export class AgentConversationCopyError extends Error {} +export class AgentConversationCopyStartError extends AgentConversationCopyError {} + export function agentConversationCopyRefusal( code: C, details: NoInfer, diff --git a/src/main/native-chat/agent-session-wire/acquisition/structured-agent-session-fork-source.ts b/src/main/native-chat/agent-session-wire/acquisition/structured-agent-session-fork-source.ts index 5c60bb1c6bc..dba1bd926a7 100644 --- a/src/main/native-chat/agent-session-wire/acquisition/structured-agent-session-fork-source.ts +++ b/src/main/native-chat/agent-session-wire/acquisition/structured-agent-session-fork-source.ts @@ -9,7 +9,10 @@ import { type AgentSessionExecutionLocation } from '../../../../shared/agent-session/lifecycle/agent-session-record' import type { StructuredAgentId } from '../../../../shared/agent-session/lifecycle/agent-session-provider-handle' -import { isAgentSessionRefusalError } from '../../../../shared/agent-session/failures/agent-session-wire-refusals' +import { + isAgentSessionRefusalError, + type AgentSessionRefusalError +} from '../../../../shared/agent-session/failures/agent-session-wire-refusals' import { isAgentSessionForkSourceAdmissionRefusal } from '../../../../shared/agent-session/failures/agent-session-definitive-refusal' import { AGENT_SESSION_FAILURE_COPY, @@ -17,6 +20,7 @@ import { } from '../../../../shared/agent-session/failures/agent-session-failure-copy' import { AgentConversationCopyError, + AgentConversationCopyStartError, agentConversationCopyRefusal } from '../../agent-conversation-copy-error' import { acpAgentName } from '../../../acp/session/acp-structured-agent-definitions' @@ -70,7 +74,7 @@ async function resolveForkSource(input: StructuredAgentSessionForkInput) { const capability = host.deps.agents.capabilities(input.agent)?.fork const { head, pendingClear } = activeProviderContext(record) const fail = ( - message: string, + message: string | AgentSessionRefusalError, reason: | 'conversationCleared' | 'rewindUnconfirmed' @@ -79,16 +83,16 @@ async function resolveForkSource(input: StructuredAgentSessionForkInput) { | 'providerRejected' | 'launchFolderMissing' = 'savedConversationMissing' ): never => { - const refusal = agentConversationCopyRefusal( - 'agent_session_operation_invalid', - { reason }, - message - ) - if (original) { + const refusal = + typeof message === 'string' + ? agentConversationCopyRefusal('agent_session_operation_invalid', { reason }, message) + : message + const failureReason = refusal.refusal.details?.reason + if (original && !(refusal.cause instanceof AgentConversationCopyStartError)) { const piece = - reason === 'savedConversationMissing' + failureReason === 'savedConversationMissing' ? 'forkOriginalMissing' - : reason === 'turnActive' + : failureReason === 'turnActive' ? 'forkOriginalLatestOnly' : 'forkOriginalUnavailable' const title = record.conversationName?.trim() diff --git a/src/main/native-chat/agent-session-wire/acquisition/structured-agent-session-fork.ts b/src/main/native-chat/agent-session-wire/acquisition/structured-agent-session-fork.ts index ae112b21465..184a31f819a 100644 --- a/src/main/native-chat/agent-session-wire/acquisition/structured-agent-session-fork.ts +++ b/src/main/native-chat/agent-session-wire/acquisition/structured-agent-session-fork.ts @@ -51,7 +51,7 @@ export async function prepareStructuredAgentSessionFork( accountHome: record.accountHome }).catch((error: unknown) => { if (isAgentSessionRefusalError(error)) { - throw error + return fail(error) } host.deps.logger.warn('native conversation copy failed', { scope: 'chat-fork', diff --git a/src/main/runtime/rpc/methods/structured-sessions/creation/structured-agent-session-precommit-refusal.test.ts b/src/main/runtime/rpc/methods/structured-sessions/creation/structured-agent-session-precommit-refusal.test.ts index fdab266aaf1..441bff0acff 100644 --- a/src/main/runtime/rpc/methods/structured-sessions/creation/structured-agent-session-precommit-refusal.test.ts +++ b/src/main/runtime/rpc/methods/structured-sessions/creation/structured-agent-session-precommit-refusal.test.ts @@ -1,3 +1,6 @@ +import { copyCodexConversation } from '../../../../../codex/codex-conversation-copy' +import type * as AppServerSession from '../../../../../codex/app-server/codex-app-server-session' +import { tmpdir } from 'node:os' import { createRestTestRig } from '../../../../../native-chat/agent-session-wire/host/testing/structured-agent-session-rest-test-rig' import { forkTestSource } from '../../../../../native-chat/agent-session-wire/acquisition/testing/structured-agent-session-fork.test-fixture' import { agentSessionRecordFixture } from '../../../../../native-chat/agent-session-record-test-fixture' @@ -21,6 +24,15 @@ import { RpcDispatcher } from '../../../dispatcher' import { STRUCTURED_AGENT_SESSION_METHODS } from '../structured-agent-session' import './structured-agent-session-create' +const codex = vi.hoisted(() => ({ request: vi.fn() })) +vi.mock('../../../../../codex/app-server/codex-app-server-session', async (importOriginal) => ({ + ...(await importOriginal()), + runCodexAppServerSession: async ( + _launch: unknown, + body: (rpc: AppServerSession.CodexAppServerRpc) => Promise + ) => body({ request: codex.request, notify: () => {} }) +})) + const SESSION = 'session-alpha' const OPERATION = '1800000000000-00000000000000000000000000000001' const WORKTREE = 'id:workspace-1' @@ -310,3 +322,52 @@ it('does not prove no copy when source revalidation fails after the native copy' await rig.dispose() } }) + +it('settles a redirected Codex preparation refusal but not a dispatched child-read failure', async () => { + const { rig, test, createFork } = await forkHost() + test.record.conversationName = ' Release notes ' + try { + test.copy.mockImplementation((input) => + copyCodexConversation(input, { + resolveLaunchArgs: () => [], + resolveWorkspacePath: async () => tmpdir(), + resolveCommand: () => ({ command: 'codex', prefixArgs: [] }) + }) + ) + codex.request.mockRejectedValue(new Error('source read failed')) + expect(refusalOf(await createFork('codex_source_read_failed'))).toMatchObject({ + code: 'agent_session_identity_required', + details: { reason: 'transcriptUnreadable' }, + createAttemptRefusedBeforeCommit: true + }) + expect(rig.store.getRecord('codex_source_read_failed')).toBeNull() + codex.request.mockImplementation(async (method: string) => { + if (method === 'thread/read') { + return { thread: { id: '019e6b26-18e2-7103-b7a0-7ba6bb071123', historyMode: 'paginated' } } + } + if (method === 'thread/fork') { + return { thread: { id: '01a1bde0-5ded-7f03-a1bf-3dae1613d8c7' } } + } + throw new Error('child read failed') + }) + // Source listing succeeds before the child read fails. + codex.request + .mockImplementationOnce(async () => ({ + thread: { id: '019e6b26-18e2-7103-b7a0-7ba6bb071123', historyMode: 'paginated' } + })) + .mockImplementationOnce(async () => ({ + data: [{ id: 'completed-turn', status: 'completed' }], + nextCursor: null + })) + const refusal = refusalOf(await createFork('codex_child_read_failed')) + expect(refusal).toMatchObject({ + code: 'agent_session_operation_invalid', + details: { reason: 'providerRejected' } + }) + expect(refusal).not.toHaveProperty('createAttemptRefusedBeforeCommit') + expect(rig.store.getRecord('codex_child_read_failed')).toBeNull() + } finally { + setStructuredAgentSessionHost(null) + await rig.dispose() + } +}) From c55bee048576a43d529b413924678721dc4d30b8 Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:06:44 +0000 Subject: [PATCH 03/59] refactor(claude): move the kill-delivered receipt check out of the session adapter (#40481) The adapter crossed the 300-line cap after #40069 and main's lint fails. The same receipt check ran twice in it; one shared predicate brings the file back under the cap with no behaviour change. --- .../claude-kill-delivered-exit-receipt.ts | 10 ++++++++++ .../claude-structured-session-adapter.ts | 16 +++------------- 2 files changed, 13 insertions(+), 13 deletions(-) create mode 100644 src/main/claude/structured/claude-kill-delivered-exit-receipt.ts diff --git a/src/main/claude/structured/claude-kill-delivered-exit-receipt.ts b/src/main/claude/structured/claude-kill-delivered-exit-receipt.ts new file mode 100644 index 00000000000..4fa1bbb75ef --- /dev/null +++ b/src/main/claude/structured/claude-kill-delivered-exit-receipt.ts @@ -0,0 +1,10 @@ +import { AgentSessionAcquisitionRootExitObservedError } from '../../native-chat/agent-session-wire/acquisition/structured-agent-session-acquisition-errors' + +export function isKillDeliveredExitReceipt( + receipt: Error | undefined +): receipt is AgentSessionAcquisitionRootExitObservedError { + return ( + receipt instanceof AgentSessionAcquisitionRootExitObservedError && + receipt.deathEvidenceKind === 'kill-delivered' + ) +} diff --git a/src/main/claude/structured/claude-structured-session-adapter.ts b/src/main/claude/structured/claude-structured-session-adapter.ts index 45ea207c93a..f2b69b423d1 100644 --- a/src/main/claude/structured/claude-structured-session-adapter.ts +++ b/src/main/claude/structured/claude-structured-session-adapter.ts @@ -1,5 +1,5 @@ import { dispatchClaudeCommand } from './claude-structured-command-dispatch' -import { AgentSessionAcquisitionRootExitObservedError } from '../../native-chat/agent-session-wire/acquisition/structured-agent-session-acquisition-errors' +import { isKillDeliveredExitReceipt } from './claude-kill-delivered-exit-receipt' import { claudeAcquireFailure } from '../claude-startup-failure' import { claudeStartPublication, openClaudeStart } from './claude-structured-start-ending' import { AgentSessionStartEndings } from '../../native-chat/agent-session-wire/startup/structured-agent-session-start-failure' @@ -259,11 +259,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda try { const closed = await close() const receipt = this.settledExitErrors.get(sessionId) - if ( - closed && - receipt instanceof AgentSessionAcquisitionRootExitObservedError && - receipt.deathEvidenceKind === 'kill-delivered' - ) { + if (closed && isKillDeliveredExitReceipt(receipt)) { throw receipt } return closed @@ -290,13 +286,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda options ) => this.releaseAcquisition({ sessionId, closeOptions: options }).finally(() => { - const receipt = this.settledExitErrors.get(sessionId) - if ( - !( - receipt instanceof AgentSessionAcquisitionRootExitObservedError && - receipt.deathEvidenceKind === 'kill-delivered' - ) - ) { + if (!isKillDeliveredExitReceipt(this.settledExitErrors.get(sessionId))) { this.settledExitErrors.delete(sessionId) } }) From 313fb8733e37d4c6e10f725e720177ea7c022a78 Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:18:06 +0000 Subject: [PATCH 04/59] fix(speech): reload the warm recognizer once the VAD download lands (#40477) A recognizer that loaded before the Silero VAD file existed kept cutting text only at stop until its next restart. When the VAD lands, the speech service now unloads that recognizer once no dictation uses it and warms it again, so pause cuts start on first use. A running dictation is never interrupted. --- src/main/speech/speech-vad-model.ts | 25 ++++++++++--- src/main/speech/stt-service.test.ts | 54 ++++++++++++++++++++++++---- src/main/speech/stt-service.ts | 22 ++++++++++++ src/main/speech/stt-session-start.ts | 3 +- src/main/speech/stt-session-state.ts | 2 ++ 5 files changed, 93 insertions(+), 13 deletions(-) diff --git a/src/main/speech/speech-vad-model.ts b/src/main/speech/speech-vad-model.ts index 5aa954a01b0..12403954e06 100644 --- a/src/main/speech/speech-vad-model.ts +++ b/src/main/speech/speech-vad-model.ts @@ -19,10 +19,7 @@ export function downloadSileroVadInBackground(models: Pick -): string | undefined { +function placedSileroVadPath(models: Pick): string | undefined { const [file] = SILERO_VAD_MODEL.downloadFiles const path = join(models.getModelDir(SILERO_VAD_MODEL.id), file.name) if (path === verifiedVadPath) { @@ -36,6 +33,24 @@ export function sileroVadModelPath( } catch { // Not downloaded yet, e.g. the model came before VAD support. } - downloadSileroVadInBackground(models) return undefined } + +/** Whether a model-state change for `modelId` means the Silero VAD file is now in place. */ +export function sileroVadLanded( + models: Pick, + modelId: string +): boolean { + return modelId === SILERO_VAD_MODEL.id && placedSileroVadPath(models) !== undefined +} + +/** The downloaded Silero VAD file; when it's missing, starts fetching it and returns undefined. */ +export function sileroVadModelPath( + models: Pick +): string | undefined { + const path = placedSileroVadPath(models) + if (!path) { + downloadSileroVadInBackground(models) + } + return path +} diff --git a/src/main/speech/stt-service.test.ts b/src/main/speech/stt-service.test.ts index cdd27198850..f397c55b819 100644 --- a/src/main/speech/stt-service.test.ts +++ b/src/main/speech/stt-service.test.ts @@ -1,6 +1,9 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import type { ModelManager } from './model-manager' import type { SpeechModelState } from '../../shared/speech-types' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { beforeEach, describe, expect, it, onTestFinished, vi } from 'vitest' const { MockWorker, getCreatedWorkerCount, getLastWorker, resetWorkers } = vi.hoisted(() => { class HoistedMockWorker extends EventTarget { @@ -104,9 +107,16 @@ const stoppedReply = (worker: InstanceType) => ({ streamId: worker.messages.findLast((message) => message.type === 'stop')?.streamId }) -const testModels = (models: Pick): ModelManager => - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SttService reads getModelState and getModelDir, and fetches the VAD with downloadModel. - ({ ...models, downloadModel: vi.fn().mockResolvedValue(undefined) }) as never +const testModels = ( + models: Pick, + onModelStateChange: ModelManager['setProgressCallback'] = () => () => {} +): ModelManager => + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SttService reads getModelState and getModelDir, fetches the VAD with downloadModel and watches it land. + ({ + ...models, + downloadModel: vi.fn().mockResolvedValue(undefined), + setProgressCallback: onModelStateChange + }) as never const readyModels = (): ModelManager => testModels({ @@ -122,6 +132,7 @@ async function flushMicrotasks(): Promise { } vi.mock('./model-catalog', () => ({ + isLocalSpeechModel: (manifest: { provider: string }) => manifest.provider === 'local', getCatalogModel: (id: string) => id === 'openai-model' ? { @@ -174,6 +185,31 @@ describe('SttService', () => { resetWorkers() }) + it('reloads a recognizer loaded before the VAD landed once the dictation using it ends', async () => { + const modelsDir = mkdtempSync(join(tmpdir(), 'stt-vad-')) + onTestFinished(() => rmSync(modelsDir, { recursive: true, force: true })) + const watchModels = vi.fn(() => () => {}) + const models = testModels( + { getModelState: readyModels().getModelState, getModelDir: (id) => join(modelsDir, id) }, + watchModels + ) + const service = new SttService(models, spawnMockWorker) + await service.startDictation('model-a', vi.fn(), undefined, 'desktop') + const initOf = (worker = getLastWorker()) => worker?.messages.find((m) => m.type === 'init') + expect(initOf()).toMatchObject({ vadModelPath: undefined }) + + const vadPath = join(modelsDir, 'silero-vad-v6.2', 'silero-vad-v6.2-F32.gguf') + mkdirSync(join(modelsDir, 'silero-vad-v6.2')) + writeFileSync(vadPath, Buffer.alloc(1_241_056)) + watchModels.mock.calls[0]?.[0]('silero-vad-v6.2', -1) + await flushMicrotasks() + expect(getCreatedWorkerCount()).toBe(1) + + await service.stopDictation('desktop') + await vi.waitFor(() => expect(getCreatedWorkerCount()).toBe(2)) + expect(initOf()).toMatchObject({ vadModelPath: vadPath }) + }) + it('gives each device dictating at once only its own text from one recognizer', async () => { const service = new SttService(readyModels(), spawnMockWorker) const desktop = vi.fn() @@ -739,15 +775,19 @@ describe('SttService', () => { async (event) => { vi.useFakeTimers() try { - const models: Pick = { + const models: Pick< + ModelManager, + 'getModelState' | 'getModelDir' | 'downloadModel' | 'setProgressCallback' + > = { getModelState: vi.fn().mockResolvedValue({ id: 'model-a', status: 'ready' }), getModelDir: vi.fn().mockReturnValue('/tmp/model-a'), - downloadModel: vi.fn().mockResolvedValue(undefined) + downloadModel: vi.fn().mockResolvedValue(undefined), + setProgressCallback: () => () => {} } - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: This lifecycle reads getModelState/getModelDir and downloadModel (VAD) on the model manager. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: This lifecycle reads getModelState/getModelDir, downloadModel (VAD) and setProgressCallback on the model manager. const service = new SttService(models as ModelManager, spawnMockWorker) const sink = vi.fn() await service.startDictation('model-a', sink, undefined, 'desktop') diff --git a/src/main/speech/stt-service.ts b/src/main/speech/stt-service.ts index 048c4872135..2333dfae0ed 100644 --- a/src/main/speech/stt-service.ts +++ b/src/main/speech/stt-service.ts @@ -6,6 +6,7 @@ import { } from '../../shared/dictation-text-pipeline' import { getCatalogModel } from './model-catalog' import type { ModelManager } from './model-manager' +import { sileroVadLanded } from './speech-vad-model' import type { SpawnSttWorkerProcess } from './stt-worker-process' import { prewarmSttModel } from './stt-session-prewarm' import { startSttDictation } from './stt-session-start' @@ -45,6 +46,27 @@ export class SttService { constructor(modelManager: ModelManager, spawnWorker: SpawnSttWorkerProcess) { this.state = createSttSessionState(modelManager, spawnWorker) + modelManager.setProgressCallback((modelId) => { + if (sileroVadLanded(modelManager, modelId)) { + void this.reloadForVad() + } + }) + } + + /** A recognizer loaded before the VAD landed cuts text only at stop; reload it once no dictation uses it. */ + private async reloadForVad(): Promise { + const engine = this.state.engine + if (!engine || engine.retired || engine.vadModelPath) { + return + } + const { modelId, language } = engine + const unloaded = this.unloadModelWhenIdle() + const generation = this.state.prewarmGeneration + await unloaded + // A newer pre-warm or a dictation took over; loading is theirs now. + if (this.state.prewarmGeneration === generation && !this.state.engine) { + await this.prewarm(modelId, language) + } } /** `terms`: the dictionary for where this dictation goes, repo terms first (mergeDictationTerms). */ diff --git a/src/main/speech/stt-session-start.ts b/src/main/speech/stt-session-start.ts index cd815fbc920..ee7bd5b1833 100644 --- a/src/main/speech/stt-session-start.ts +++ b/src/main/speech/stt-session-start.ts @@ -187,7 +187,7 @@ export function spawnSttEngine( if (!speechModulesDir) { throw new Error(LOCAL_SPEECH_UNAVAILABLE_ERROR) } - // A VAD fetched now serves the next engine start. + // A VAD fetched now reloads this engine once it is idle (SttService). const vadModelPath = sileroVadModelPath(state.modelManager) clearSttIdleTeardownTimer(state) const worker = state.spawnWorker() @@ -210,6 +210,7 @@ export function spawnSttEngine( worker, detach, retired: false, + vadModelPath, onUnloaded: [], ready: readyPromise.catch((error: unknown) => { failSttEngine(state, engine, STT_WORKER_EXITED_ERROR) diff --git a/src/main/speech/stt-session-state.ts b/src/main/speech/stt-session-state.ts index 21c081dacf9..c54d4dd5eed 100644 --- a/src/main/speech/stt-session-state.ts +++ b/src/main/speech/stt-session-state.ts @@ -19,6 +19,8 @@ export type SttEngine = SttRecognizerConfig & { detach: () => void /** Torn down on purpose, so a load it cut short is not a failure of the model. */ retired: boolean + /** Absent when the VAD wasn't downloaded yet at load, so this process cuts text only at stop. */ + vadModelPath: string | undefined /** Called once this process is gone; set when it must not outlive its last dictation. */ onUnloaded: (() => void)[] } From 18996530407d9744890f196813f3d5cc8061cd4b Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:18:14 +0000 Subject: [PATCH 05/59] feat(settings): manage dictation vocabulary in Voice settings, per user and per repo (#40471) * feat(settings): add a Vocabulary group to Voice settings for personal and repo terms * feat(speech): use the repo name as a dictation hint without recasing one-word names * fix(settings): use Input and Button styling as the design system defines it * fix(settings): chain quick repo-term edits, report a host that drops them, keep the hint off phone replies * fix(settings): key a pending repo-term write by host too * refactor(settings): catch a dropped repo-term write in updateRepo and keep the name hint out of stored terms * fix(settings): one vocabulary chip per term with its aliases, and no repo suffix on the scope tab * fix(settings): skip a bare vocabulary term that already has an entry --- src/main/ipc/speech.ts | 7 +- .../runtime/speech/runtime-dictation-terms.ts | 6 +- .../speech/runtime-mobile-speech-catalog.ts | 4 +- src/main/speech/stt-service.ts | 2 +- .../components/dictation/dictation-context.ts | 6 +- .../components/settings/voice/VoicePane.tsx | 8 + .../settings/voice/VoiceVocabularySetting.tsx | 269 ++++++++++++++++++ .../settings/voice/voice-pane-search.ts | 18 ++ src/renderer/src/i18n/locales/de.json | 23 +- src/renderer/src/i18n/locales/en.json | 23 +- src/renderer/src/i18n/locales/es.json | 23 +- src/renderer/src/i18n/locales/fr.json | 23 +- src/renderer/src/i18n/locales/ja.json | 23 +- src/renderer/src/i18n/locales/ko.json | 23 +- src/renderer/src/i18n/locales/pt.json | 23 +- src/renderer/src/i18n/locales/ru.json | 23 +- src/renderer/src/i18n/locales/tr.json | 23 +- src/renderer/src/i18n/locales/zh-Hant.json | 23 +- src/renderer/src/i18n/locales/zh.json | 23 +- src/renderer/src/store/repos/repo-update.ts | 8 + src/shared/dictation-dictionary.test.ts | 21 +- src/shared/dictation-dictionary.ts | 47 ++- src/shared/speech-types.ts | 3 + 23 files changed, 619 insertions(+), 33 deletions(-) create mode 100644 src/renderer/src/components/settings/voice/VoiceVocabularySetting.tsx diff --git a/src/main/ipc/speech.ts b/src/main/ipc/speech.ts index 2451c5acff7..98f7f52f9b0 100644 --- a/src/main/ipc/speech.ts +++ b/src/main/ipc/speech.ts @@ -6,7 +6,7 @@ import { type MessagePortMain } from 'electron' import { abortWhenRendererGone } from './renderer-lifetime-abort' -import { mergeDictationTerms } from '../../shared/dictation-dictionary' +import { dictationTermsFor } from '../../shared/dictation-dictionary' import { SPEECH_MODEL_CATALOG } from '../speech/model-catalog' import { deleteLocalSpeechModel } from '../speech/speech-model-deletion' import { getSpeechModelManager, getSpeechSttService } from '../speech/speech-runtime-service' @@ -220,7 +220,10 @@ export function registerSpeechHandlers(store: Store): void { break } }, - mergeDictationTerms(context?.repoTerms, store.getSettings().voice?.dictationTerms), + dictationTermsFor(store.getSettings().voice, { + dictationTerms: context?.repoTerms, + displayName: context?.repoName + }), owner, store.getSettings().voice ) diff --git a/src/main/runtime/speech/runtime-dictation-terms.ts b/src/main/runtime/speech/runtime-dictation-terms.ts index 13785a2cc2d..63838db9179 100644 --- a/src/main/runtime/speech/runtime-dictation-terms.ts +++ b/src/main/runtime/speech/runtime-dictation-terms.ts @@ -1,12 +1,12 @@ -import { mergeDictationTerms, type DictationTerm } from '../../../shared/dictation-dictionary' +import { dictationTermsFor, type DictationTerm } from '../../../shared/dictation-dictionary' import { getRepoIdFromWorktreeId } from '../../../shared/worktree/id' import type { RuntimeStore } from '../construction/runtime-store-contract' -/** This host's dictionary for a worktree: its repo's terms, then the personal ones. A folder workspace has no repo row, so it gets the personal ones. */ +/** This host's dictionary for a worktree; a folder workspace has no repo row, so it gets the personal terms. */ export function hostDictationTerms( store: RuntimeStore, worktreeId: string | undefined ): DictationTerm[] { const repo = worktreeId ? store.getRepo(getRepoIdFromWorktreeId(worktreeId)) : undefined - return mergeDictationTerms(repo?.dictationTerms, store.getSettings().voice?.dictationTerms) + return dictationTermsFor(store.getSettings().voice, repo) } diff --git a/src/main/runtime/speech/runtime-mobile-speech-catalog.ts b/src/main/runtime/speech/runtime-mobile-speech-catalog.ts index fa042e5be68..6619e4f7e0b 100644 --- a/src/main/runtime/speech/runtime-mobile-speech-catalog.ts +++ b/src/main/runtime/speech/runtime-mobile-speech-catalog.ts @@ -69,7 +69,9 @@ export class RuntimeMobileSpeechCatalog { dictationMode: voice.dictationMode === 'hold' ? 'hold' : 'toggle', dictationHoldToRecord: voice.dictationHoldToRecord !== false, removeFillerWords: voice.removeFillerWords !== false, - dictationTerms: hostDictationTerms(store, worktreeId), + // The phone has no Whisper prompt to bias, and its text pass would only join the name; keeping the + // hint out also spares a phone page from before hints, which would treat the name as a full term. + dictationTerms: hostDictationTerms(store, worktreeId).filter((entry) => !entry.hint), models } } diff --git a/src/main/speech/stt-service.ts b/src/main/speech/stt-service.ts index 2333dfae0ed..bacc4b753ed 100644 --- a/src/main/speech/stt-service.ts +++ b/src/main/speech/stt-service.ts @@ -69,7 +69,7 @@ export class SttService { } } - /** `terms`: the dictionary for where this dictation goes, repo terms first (mergeDictationTerms). */ + /** `terms`: the dictionary for where this dictation goes, repo terms first (dictationTermsFor). */ startDictation( modelId: string, sink: SttEventSink, diff --git a/src/renderer/src/components/dictation/dictation-context.ts b/src/renderer/src/components/dictation/dictation-context.ts index f2cbe4088fe..08f9db5290a 100644 --- a/src/renderer/src/components/dictation/dictation-context.ts +++ b/src/renderer/src/components/dictation/dictation-context.ts @@ -11,5 +11,9 @@ export function captureDictationContext(): DictationContext { return {} } const repo = selectRepoByIdForActiveWorkspace(state, getRepoIdFromWorktreeId(worktreeId)) - return { worktreeId, ...(repo?.dictationTerms ? { repoTerms: repo.dictationTerms } : {}) } + return { + worktreeId, + ...(repo?.dictationTerms ? { repoTerms: repo.dictationTerms } : {}), + ...(repo ? { repoName: repo.displayName } : {}) + } } diff --git a/src/renderer/src/components/settings/voice/VoicePane.tsx b/src/renderer/src/components/settings/voice/VoicePane.tsx index b29ea9ff4a7..e95683cda94 100644 --- a/src/renderer/src/components/settings/voice/VoicePane.tsx +++ b/src/renderer/src/components/settings/voice/VoicePane.tsx @@ -12,6 +12,7 @@ import { OpenAiCompatibleProviderDialog } from './OpenAiCompatibleProviderDialog import { OpenAiTranscriptionSettingsRow } from './OpenAiTranscriptionSettingsRow' import { handleVoiceDictationToggle } from './voice-dictation-toggle' import { VoiceDictationLanguageSetting } from './VoiceDictationLanguageSetting' +import { VoiceVocabularySetting } from './VoiceVocabularySetting' import { VOICE_MICROPHONE_ROW_ID, VoiceInputSettings } from './VoiceInputSettings' import { VoiceSetupCard } from './VoiceSetupCard' import { cancelSpeechModelDownload, startSpeechModelDownload } from './speech-model-download' @@ -331,6 +332,13 @@ export function VoicePane({ settings, updateSettings }: VoicePaneProps): React.J + + + +
) => void +} + +const NO_TERMS: readonly DictationTerm[] = [] + +export function VoiceVocabularySetting({ + voiceSettings, + onUpdateVoiceSettings +}: VoiceVocabularySettingProps): React.JSX.Element { + const t = useTranslate() + // The repo you're working in; a folder workspace has none, so only personal terms show. + const repo = useAppStore((s) => selectRepoByIdForActiveWorkspace(s, s.activeRepoId)) + const updateRepo = useAppStore((s) => s.updateRepo) + const [chosenScope, setScope] = useState('personal') + // A repo write is a round trip (RPC for a remote host); the next edit builds on it, not on the stored list. + const [pending, setPending] = useState<{ + repoId: string + hostId: ExecutionHostId + terms: DictationTerm[] + } | null>(null) + const scope = repo ? chosenScope : 'personal' + const personalTerms = voiceSettings.dictationTerms ?? NO_TERMS + const repoTerms = + pending && repo && pending.repoId === repo.id && pending.hostId === getRepoExecutionHostId(repo) + ? pending.terms + : (repo?.dictationTerms ?? NO_TERMS) + const terms = scope === 'repo' ? repoTerms : personalTerms + + const saveRepoTerms = async ( + repoId: string, + hostId: ExecutionHostId, + next: DictationTerm[] + ): Promise => { + const write = { repoId, hostId, terms: next } + setPending(write) + if (!(await updateRepo(repoId, { dictationTerms: next }, { hostId }))) { + toast.error( + t( + 'components.settings.voice.vocabulary.repoSaveFailed', + "Couldn't save the repo's terms. The host running this repo may need an Orca update." + ) + ) + } + setPending((current) => (current === write ? null : current)) + } + + const saveTerms = (next: DictationTerm[]): void => { + if (scope === 'repo' && repo) { + void saveRepoTerms(repo.id, getRepoExecutionHostId(repo), next) + } else { + onUpdateVoiceSettings({ dictationTerms: next }) + } + } + + return ( +
+ +
+

+ {scope === 'repo' && repo + ? t( + 'components.settings.voice.vocabulary.repoScope', + 'Only used while you dictate in {{repo}}.', + { repo: repo.displayName } + ) + : t('components.settings.voice.vocabulary.personalScope', 'Used in every repo.')} +

+ {repo ? ( + + ) : null} +
+ {terms.length > 0 ? ( +
    + {chipsBySpelling(terms).map((chip) => ( + saveTerms(terms.filter((entry) => entry.term !== chip.term))} + /> + ))} +
+ ) : null} + = MAX_DICTATION_TERMS} + onAdd={(entry) => { + // A term already keys its own spelling, so a bare one needs no second entry. + const exists = terms.some( + (other) => + other.term === entry.term && (!entry.heardAs || other.heardAs === entry.heardAs) + ) + if (!exists) { + saveTerms([...terms, entry]) + } + }} + /> + + onUpdateVoiceSettings({ + dictationRepoNameHint: voiceSettings.dictationRepoNameHint === false + }) + } + /> +
+ ) +} + +type VocabularyChipModel = { term: string; aliases: string[] } + +// One chip per spelling: the stored list keeps one entry per "heard as". +function chipsBySpelling(terms: readonly DictationTerm[]): VocabularyChipModel[] { + const chips = new Map() + for (const { term, heardAs } of terms) { + const chip = chips.get(term) ?? { term, aliases: [] } + if (heardAs) { + chip.aliases.push(heardAs) + } + chips.set(term, chip) + } + return [...chips.values()] +} + +function VocabularyChip({ + chip, + onRemove +}: { + chip: VocabularyChipModel + onRemove: () => void +}): React.JSX.Element { + const t = useTranslate() + return ( +
  • + {chip.term} + {chip.aliases.length > 0 ? ( + + {t('components.settings.voice.vocabulary.heardAsChip', '← {{heardAs}}', { + heardAs: chip.aliases.join(', ') + })} + + ) : null} + +
  • + ) +} + +function VocabularyAddRow({ + full, + onAdd +}: { + full: boolean + onAdd: (entry: DictationTerm) => void +}): React.JSX.Element { + const t = useTranslate() + const [term, setTerm] = useState('') + const [heardAs, setHeardAs] = useState('') + const canAdd = !full && term.trim() !== '' + + const add = (): void => { + if (!canAdd) { + return + } + onAdd({ term: term.trim(), ...(heardAs.trim() ? { heardAs: heardAs.trim() } : {}) }) + setTerm('') + setHeardAs('') + } + const addOnEnter = (event: React.KeyboardEvent): void => { + if (event.key === 'Enter') { + event.preventDefault() + add() + } + } + + return ( +
    + setTerm(event.target.value)} + onKeyDown={addOnEnter} + /> + setHeardAs(event.target.value)} + onKeyDown={addOnEnter} + /> + +
    + ) +} diff --git a/src/renderer/src/components/settings/voice/voice-pane-search.ts b/src/renderer/src/components/settings/voice/voice-pane-search.ts index ff52e8b254c..55abb31d426 100644 --- a/src/renderer/src/components/settings/voice/voice-pane-search.ts +++ b/src/renderer/src/components/settings/voice/voice-pane-search.ts @@ -120,6 +120,24 @@ export const getVoicePaneSearchEntries = createLocalizedCatalog(() => [ ...translateSearchKeyword('auto.components.settings.voice.pane.search.10d45a9fce', 'stt') ] }, + { + title: translate('auto.components.settings.voice.pane.search.vocabularyTitle', 'Vocabulary'), + description: translate( + 'auto.components.settings.voice.pane.search.vocabularyDescription', + 'Names and code terms dictation spells your way, for you or for one repo.' + ), + keywords: [ + ...translateSearchKeyword('auto.components.settings.voice.pane.search.7640ed9848', 'voice'), + ...translateSearchKeyword( + 'auto.components.settings.voice.pane.search.089d31a45b', + 'dictation' + ), + ...translateSearchKeyword( + 'auto.components.settings.voice.pane.search.vocabularyKeyword', + 'dictionary' + ) + ] + }, { title: translate( 'auto.components.settings.voice.pane.search.fillerWordsTitle', diff --git a/src/renderer/src/i18n/locales/de.json b/src/renderer/src/i18n/locales/de.json index 454f252e06d..47e9692fe35 100644 --- a/src/renderer/src/i18n/locales/de.json +++ b/src/renderer/src/i18n/locales/de.json @@ -11299,7 +11299,10 @@ "modelMemoryKeyword": "speicher", "fillerWordsTitle": "Füllwörter entfernen", "fillerWordsDescription": "Entfernt „äh“, „ähm“ und ähnliche Laute aus diktiertem Text.", - "fillerWordsKeyword": "füllwörter" + "fillerWordsKeyword": "füllwörter", + "vocabularyTitle": "Wortschatz", + "vocabularyDescription": "Namen und Code-Begriffe, die das Diktat auf deine Weise schreibt, für dich oder ein Repo.", + "vocabularyKeyword": "wörterbuch" } } }, @@ -20746,6 +20749,24 @@ "local": "Auf diesem Computer", "cloudDescription": "Cloud-Modelle senden deine Audiodaten an den Anbieter. Modelle auf diesem Computer behalten sie hier.", "sendsToCompatible": "Sendet deine Audiodaten an den Dienst, den du eingerichtet hast." + }, + "vocabulary": { + "title": "Wortschatz", + "description": "Namen und Code-Begriffe, die du auf deine Weise geschrieben haben willst. Gib an, wie das Modell sie falsch hört, wenn es sie immer wieder verwechselt.", + "repoScope": "Nur beim Diktieren in {{repo}}.", + "personalScope": "In jedem Repo.", + "scope": "Begriffe für", + "personal": "Persönlich · {{total}}", + "repo": "{{repo}} · {{total}}", + "heardAsChip": "← {{heardAs}}", + "remove": "{{term}} entfernen", + "term": "Wort oder Wendung", + "heardAs": "Gehört als (optional)", + "heardAsLabel": "Gehört als", + "add": "Hinzufügen", + "repoNameHint": "Repo-Namen als Hinweis verwenden", + "repoNameHintDescription": "Hilft, den Namen des Repos zu erkennen.", + "repoSaveFailed": "Die Begriffe des Repos konnten nicht gespeichert werden. Auf dem Host dieses Repos ist eventuell ein Orca-Update nötig." } }, "repositoryHooks": { diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index fd3e66b2e85..c0bd1dd1335 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -11635,7 +11635,10 @@ "modelMemoryKeyword": "memory", "fillerWordsTitle": "Remove Filler Words", "fillerWordsDescription": "Drop “um”, “uh” and similar sounds from dictated text.", - "fillerWordsKeyword": "filler words" + "fillerWordsKeyword": "filler words", + "vocabularyTitle": "Vocabulary", + "vocabularyDescription": "Names and code terms dictation spells your way, for you or for one repo.", + "vocabularyKeyword": "dictionary" } } }, @@ -21445,6 +21448,24 @@ "where": "Where models run", "local": "On this computer", "cloudDescription": "Cloud models send your audio to the provider. Models on this computer keep it here." + }, + "vocabulary": { + "title": "Vocabulary", + "description": "Names and code terms to spell your way. Add what the model mishears it as, if it keeps getting it wrong.", + "repoScope": "Only used while you dictate in {{repo}}.", + "personalScope": "Used in every repo.", + "scope": "Terms for", + "personal": "Personal · {{total}}", + "repo": "{{repo}} · {{total}}", + "heardAsChip": "← {{heardAs}}", + "remove": "Remove {{term}}", + "term": "Word or phrase", + "heardAs": "Heard as (optional)", + "heardAsLabel": "Heard as", + "add": "Add", + "repoNameHint": "Use the repo name as a hint", + "repoNameHintDescription": "Helps recognize the repo's name.", + "repoSaveFailed": "Couldn't save the repo's terms. The host running this repo may need an Orca update." } }, "repositoryHooks": { diff --git a/src/renderer/src/i18n/locales/es.json b/src/renderer/src/i18n/locales/es.json index 76d24032fdf..fdbe57e1fcb 100644 --- a/src/renderer/src/i18n/locales/es.json +++ b/src/renderer/src/i18n/locales/es.json @@ -10882,7 +10882,10 @@ "modelMemoryKeyword": "memoria", "fillerWordsTitle": "Quitar muletillas", "fillerWordsDescription": "Quita «ehm», «em» y sonidos parecidos del texto dictado.", - "fillerWordsKeyword": "muletillas" + "fillerWordsKeyword": "muletillas", + "vocabularyTitle": "Vocabulario", + "vocabularyDescription": "Nombres y términos de código que el dictado escribe a tu manera, para ti o para un repo.", + "vocabularyKeyword": "diccionario" } } }, @@ -20534,6 +20537,24 @@ "local": "En este equipo", "cloudDescription": "Los modelos en la nube envían tu audio al proveedor. Los de este equipo lo mantienen aquí.", "sendsToCompatible": "Envía tu audio al servicio que configuraste." + }, + "vocabulary": { + "title": "Vocabulario", + "description": "Nombres y términos de código que se escriben a tu manera. Añade cómo lo oye mal el modelo si se equivoca una y otra vez.", + "repoScope": "Solo se usa al dictar en {{repo}}.", + "personalScope": "Se usa en todos los repos.", + "scope": "Términos para", + "personal": "Personal · {{total}}", + "repo": "{{repo}} · {{total}}", + "heardAsChip": "← {{heardAs}}", + "remove": "Quitar {{term}}", + "term": "Palabra o frase", + "heardAs": "Se oye como (opcional)", + "heardAsLabel": "Se oye como", + "add": "Añadir", + "repoNameHint": "Usar el nombre del repo como pista", + "repoNameHintDescription": "Ayuda a reconocer el nombre del repo.", + "repoSaveFailed": "No se pudieron guardar los términos del repo. Puede que el host de este repo necesite actualizar Orca." } }, "repositoryHooks": { diff --git a/src/renderer/src/i18n/locales/fr.json b/src/renderer/src/i18n/locales/fr.json index 5e54997ebbb..28f2e380f9a 100644 --- a/src/renderer/src/i18n/locales/fr.json +++ b/src/renderer/src/i18n/locales/fr.json @@ -11167,7 +11167,10 @@ "modelMemoryKeyword": "mémoire", "fillerWordsTitle": "Supprimer les hésitations", "fillerWordsDescription": "Supprime « euh », « hum » et les sons similaires du texte dicté.", - "fillerWordsKeyword": "hésitations" + "fillerWordsKeyword": "hésitations", + "vocabularyTitle": "Vocabulaire", + "vocabularyDescription": "Noms et termes de code que la dictée écrit à votre façon, pour vous ou pour un dépôt.", + "vocabularyKeyword": "dictionnaire" } } }, @@ -20851,6 +20854,24 @@ "local": "Sur cet ordinateur", "cloudDescription": "Les modèles cloud envoient votre audio au fournisseur. Ceux de cet ordinateur le gardent ici.", "sendsToCompatible": "Envoie votre audio au service que vous avez configuré." + }, + "vocabulary": { + "title": "Vocabulaire", + "description": "Noms et termes de code à écrire à votre façon. Ajoutez ce que le modèle entend à la place s’il se trompe toujours.", + "repoScope": "Utilisé seulement quand vous dictez dans {{repo}}.", + "personalScope": "Utilisé dans tous les dépôts.", + "scope": "Termes pour", + "personal": "Personnel · {{total}}", + "repo": "{{repo}} · {{total}}", + "heardAsChip": "← {{heardAs}}", + "remove": "Retirer {{term}}", + "term": "Mot ou expression", + "heardAs": "Entendu comme (facultatif)", + "heardAsLabel": "Entendu comme", + "add": "Ajouter", + "repoNameHint": "Utiliser le nom du dépôt comme indice", + "repoNameHintDescription": "Aide à reconnaître le nom du dépôt.", + "repoSaveFailed": "Impossible d’enregistrer les termes du dépôt. L’hôte de ce dépôt doit peut-être mettre Orca à jour." } }, "repositoryHooks": { diff --git a/src/renderer/src/i18n/locales/ja.json b/src/renderer/src/i18n/locales/ja.json index 6f309bdfab3..4d90f59f72d 100644 --- a/src/renderer/src/i18n/locales/ja.json +++ b/src/renderer/src/i18n/locales/ja.json @@ -10626,7 +10626,10 @@ "modelMemoryKeyword": "メモリ", "fillerWordsTitle": "フィラーを削除", "fillerWordsDescription": "「えーと」「うーん」などのつなぎ言葉を音声入力のテキストから取り除きます。", - "fillerWordsKeyword": "フィラー" + "fillerWordsKeyword": "フィラー", + "vocabularyTitle": "語彙", + "vocabularyDescription": "音声入力が自分の書き方で表記する名前やコード用語。個人用またはリポジトリ用。", + "vocabularyKeyword": "辞書" } } }, @@ -20027,6 +20030,24 @@ "local": "このコンピューター", "cloudDescription": "クラウドモデルは音声をプロバイダーに送信します。このコンピューター上のモデルなら音声は外に出ません。", "sendsToCompatible": "音声を設定したサービスに送信します。" + }, + "vocabulary": { + "title": "語彙", + "description": "名前やコード用語を自分の書き方で入力します。モデルが繰り返し聞き間違える場合は、聞き取られる言い方も追加してください。", + "repoScope": "{{repo}} で音声入力するときだけ使います。", + "personalScope": "すべてのリポジトリで使います。", + "scope": "用語の対象", + "personal": "個人 · {{total}}", + "repo": "{{repo}} · {{total}}", + "heardAsChip": "← {{heardAs}}", + "remove": "{{term}} を削除", + "term": "単語またはフレーズ", + "heardAs": "聞き取られ方(任意)", + "heardAsLabel": "聞き取られ方", + "add": "追加", + "repoNameHint": "リポジトリ名をヒントとして使う", + "repoNameHintDescription": "リポジトリ名を認識しやすくします。", + "repoSaveFailed": "リポジトリの用語を保存できませんでした。このリポジトリのホストで Orca の更新が必要かもしれません。" } }, "repositoryHooks": { diff --git a/src/renderer/src/i18n/locales/ko.json b/src/renderer/src/i18n/locales/ko.json index 068db9b1fbd..db599a834a2 100644 --- a/src/renderer/src/i18n/locales/ko.json +++ b/src/renderer/src/i18n/locales/ko.json @@ -10589,7 +10589,10 @@ "modelMemoryKeyword": "메모리", "fillerWordsTitle": "군말 제거", "fillerWordsDescription": "받아쓴 텍스트에서 “음”, “흠” 같은 소리를 지웁니다.", - "fillerWordsKeyword": "군말" + "fillerWordsKeyword": "군말", + "vocabularyTitle": "어휘", + "vocabularyDescription": "받아쓰기가 원하는 대로 표기하는 이름과 코드 용어, 개인용 또는 저장소용.", + "vocabularyKeyword": "사전" } } }, @@ -20027,6 +20030,24 @@ "local": "이 컴퓨터", "cloudDescription": "클라우드 모델은 오디오를 제공업체로 보냅니다. 이 컴퓨터의 모델은 오디오를 밖으로 보내지 않습니다.", "sendsToCompatible": "오디오를 설정한 서비스로 보냅니다." + }, + "vocabulary": { + "title": "어휘", + "description": "원하는 대로 표기할 이름과 코드 용어입니다. 모델이 계속 잘못 들으면 잘못 들리는 말도 추가하세요.", + "repoScope": "{{repo}}에서 받아쓸 때만 사용합니다.", + "personalScope": "모든 저장소에서 사용합니다.", + "scope": "용어 대상", + "personal": "개인 · {{total}}", + "repo": "{{repo}} · {{total}}", + "heardAsChip": "← {{heardAs}}", + "remove": "{{term}} 삭제", + "term": "단어 또는 구문", + "heardAs": "들리는 말(선택)", + "heardAsLabel": "들리는 말", + "add": "추가", + "repoNameHint": "저장소 이름을 힌트로 사용", + "repoNameHintDescription": "저장소 이름을 더 잘 인식하도록 돕습니다.", + "repoSaveFailed": "저장소 용어를 저장하지 못했습니다. 이 저장소의 호스트에서 Orca 업데이트가 필요할 수 있습니다." } }, "repositoryHooks": { diff --git a/src/renderer/src/i18n/locales/pt.json b/src/renderer/src/i18n/locales/pt.json index 12065876a1f..1aad807a1d8 100644 --- a/src/renderer/src/i18n/locales/pt.json +++ b/src/renderer/src/i18n/locales/pt.json @@ -6597,7 +6597,10 @@ "modelMemoryKeyword": "memória", "fillerWordsTitle": "Remover hesitações", "fillerWordsDescription": "Remove “hã”, “hum” e sons parecidos do texto ditado.", - "fillerWordsKeyword": "hesitações" + "fillerWordsKeyword": "hesitações", + "vocabularyTitle": "Vocabulário", + "vocabularyDescription": "Nomes e termos de código que o ditado escreve do seu jeito, para você ou para um repositório.", + "vocabularyKeyword": "dicionário" } } }, @@ -21716,6 +21719,24 @@ "local": "Neste computador", "cloudDescription": "Modelos na nuvem enviam seu áudio ao provedor. Os deste computador o mantêm aqui.", "sendsToCompatible": "Envia seu áudio para o serviço que você configurou." + }, + "vocabulary": { + "title": "Vocabulário", + "description": "Nomes e termos de código escritos do seu jeito. Adicione como o modelo os ouve errado, se ele continuar errando.", + "repoScope": "Usado só ao ditar em {{repo}}.", + "personalScope": "Usado em todos os repositórios.", + "scope": "Termos para", + "personal": "Pessoal · {{total}}", + "repo": "{{repo}} · {{total}}", + "heardAsChip": "← {{heardAs}}", + "remove": "Remover {{term}}", + "term": "Palavra ou frase", + "heardAs": "Ouvido como (opcional)", + "heardAsLabel": "Ouvido como", + "add": "Adicionar", + "repoNameHint": "Usar o nome do repositório como dica", + "repoNameHintDescription": "Ajuda a reconhecer o nome do repositório.", + "repoSaveFailed": "Não foi possível salvar os termos do repositório. O host deste repositório pode precisar de uma atualização do Orca." } }, "repositoryHooks": { diff --git a/src/renderer/src/i18n/locales/ru.json b/src/renderer/src/i18n/locales/ru.json index 6cc23c88138..58e12c44554 100644 --- a/src/renderer/src/i18n/locales/ru.json +++ b/src/renderer/src/i18n/locales/ru.json @@ -11651,7 +11651,10 @@ "modelMemoryKeyword": "память", "fillerWordsTitle": "Удалять слова-паразиты", "fillerWordsDescription": "Убирает «э», «эм» и похожие звуки из надиктованного текста.", - "fillerWordsKeyword": "паразиты" + "fillerWordsKeyword": "паразиты", + "vocabularyTitle": "Словарь", + "vocabularyDescription": "Имена и термины из кода, которые диктовка пишет по-вашему, для вас или для репозитория.", + "vocabularyKeyword": "словарь" } } }, @@ -21433,6 +21436,24 @@ "local": "На этом компьютере", "cloudDescription": "Облачные модели отправляют звук провайдеру. Модели на этом компьютере оставляют его здесь.", "sendsToCompatible": "Отправляет звук в настроенный вами сервис." + }, + "vocabulary": { + "title": "Словарь", + "description": "Имена и термины из кода, которые нужно писать по-вашему. Если модель всё время ошибается, добавьте, как она их слышит.", + "repoScope": "Используется только при диктовке в {{repo}}.", + "personalScope": "Используется во всех репозиториях.", + "scope": "Термины для", + "personal": "Личные · {{total}}", + "repo": "{{repo}} · {{total}}", + "heardAsChip": "← {{heardAs}}", + "remove": "Удалить {{term}}", + "term": "Слово или фраза", + "heardAs": "Слышится как (необязательно)", + "heardAsLabel": "Слышится как", + "add": "Добавить", + "repoNameHint": "Использовать имя репозитория как подсказку", + "repoNameHintDescription": "Помогает распознавать имя репозитория.", + "repoSaveFailed": "Не удалось сохранить термины репозитория. Возможно, на хосте этого репозитория нужно обновить Orca." } }, "repositoryHooks": { diff --git a/src/renderer/src/i18n/locales/tr.json b/src/renderer/src/i18n/locales/tr.json index 002982551d7..0893ffd23e7 100644 --- a/src/renderer/src/i18n/locales/tr.json +++ b/src/renderer/src/i18n/locales/tr.json @@ -11283,7 +11283,10 @@ "modelMemoryKeyword": "bellek", "fillerWordsTitle": "Dolgu seslerini kaldır", "fillerWordsDescription": "Dikte edilen metinden “hmm” gibi duraksama seslerini çıkarır.", - "fillerWordsKeyword": "dolgu" + "fillerWordsKeyword": "dolgu", + "vocabularyTitle": "Sözcük dağarcığı", + "vocabularyDescription": "Diktenin senin yazımınla yazdığı adlar ve kod terimleri, senin için veya bir depo için.", + "vocabularyKeyword": "sözlük" } } }, @@ -20728,6 +20731,24 @@ "local": "Bu bilgisayarda", "cloudDescription": "Bulut modelleri sesinizi sağlayıcıya gönderir. Bu bilgisayardaki modeller sesi burada tutar.", "sendsToCompatible": "Sesinizi kurduğunuz hizmete gönderir." + }, + "vocabulary": { + "title": "Sözcük dağarcığı", + "description": "Kendi yazımınla yazılacak adlar ve kod terimleri. Model sürekli yanlış duyuyorsa nasıl duyduğunu da ekle.", + "repoScope": "Yalnızca {{repo}} içinde dikte ederken kullanılır.", + "personalScope": "Her depoda kullanılır.", + "scope": "Terimler", + "personal": "Kişisel · {{total}}", + "repo": "{{repo}} · {{total}}", + "heardAsChip": "← {{heardAs}}", + "remove": "{{term}} kaldır", + "term": "Sözcük veya ifade", + "heardAs": "Duyulan biçim (isteğe bağlı)", + "heardAsLabel": "Duyulan biçim", + "add": "Ekle", + "repoNameHint": "Depo adını ipucu olarak kullan", + "repoNameHintDescription": "Depo adının tanınmasına yardımcı olur.", + "repoSaveFailed": "Deponun terimleri kaydedilemedi. Bu deponun ana makinesinde Orca güncellemesi gerekebilir." } }, "repositoryHooks": { diff --git a/src/renderer/src/i18n/locales/zh-Hant.json b/src/renderer/src/i18n/locales/zh-Hant.json index 11bd936a4f6..b04ad95e54d 100644 --- a/src/renderer/src/i18n/locales/zh-Hant.json +++ b/src/renderer/src/i18n/locales/zh-Hant.json @@ -11120,7 +11120,10 @@ "modelMemoryKeyword": "記憶體", "fillerWordsTitle": "移除語氣詞", "fillerWordsDescription": "從聽寫文字中移除「嗯」「呃」之類的語氣詞。", - "fillerWordsKeyword": "語氣詞" + "fillerWordsKeyword": "語氣詞", + "vocabularyTitle": "詞彙", + "vocabularyDescription": "聽寫依你的寫法拼寫的名稱和程式碼術語,可用於個人或某個儲存庫。", + "vocabularyKeyword": "詞典" } } }, @@ -20429,6 +20432,24 @@ "local": "這台電腦", "cloudDescription": "雲端模型會把你的音訊傳送給服務商。本機模型會把音訊留在這台電腦上。", "sendsToCompatible": "會把你的音訊傳送到你設定的服務。" + }, + "vocabulary": { + "title": "詞彙", + "description": "依你的寫法拼寫的名稱和程式碼術語。如果模型總是聽錯,可以加上它聽成的說法。", + "repoScope": "僅在 {{repo}} 中聽寫時使用。", + "personalScope": "在所有儲存庫中使用。", + "scope": "術語範圍", + "personal": "個人 · {{total}}", + "repo": "{{repo}} · {{total}}", + "heardAsChip": "← {{heardAs}}", + "remove": "移除 {{term}}", + "term": "詞語或片語", + "heardAs": "聽成(選填)", + "heardAsLabel": "聽成", + "add": "新增", + "repoNameHint": "將儲存庫名稱用作提示", + "repoNameHintDescription": "幫助辨識儲存庫名稱。", + "repoSaveFailed": "無法儲存儲存庫的術語。執行此儲存庫的主機可能需要更新 Orca。" } }, "repositoryHooks": { diff --git a/src/renderer/src/i18n/locales/zh.json b/src/renderer/src/i18n/locales/zh.json index 31c5adba076..8985678ae5f 100644 --- a/src/renderer/src/i18n/locales/zh.json +++ b/src/renderer/src/i18n/locales/zh.json @@ -10589,7 +10589,10 @@ "modelMemoryKeyword": "内存", "fillerWordsTitle": "移除语气词", "fillerWordsDescription": "从听写文本中去掉“嗯”“呃”之类的语气词。", - "fillerWordsKeyword": "语气词" + "fillerWordsKeyword": "语气词", + "vocabularyTitle": "词汇", + "vocabularyDescription": "听写按你的写法拼写的名称和代码术语,可用于个人或某个仓库。", + "vocabularyKeyword": "词典" } } }, @@ -20026,6 +20029,24 @@ "local": "这台电脑", "cloudDescription": "云端模型会把你的音频发送给服务商。本机模型会把音频留在这台电脑上。", "sendsToCompatible": "会把你的音频发送到你设置的服务。" + }, + "vocabulary": { + "title": "词汇", + "description": "按你的写法拼写的名称和代码术语。如果模型总是听错,可以加上它听成的说法。", + "repoScope": "仅在 {{repo}} 中听写时使用。", + "personalScope": "在所有仓库中使用。", + "scope": "术语范围", + "personal": "个人 · {{total}}", + "repo": "{{repo}} · {{total}}", + "heardAsChip": "← {{heardAs}}", + "remove": "移除 {{term}}", + "term": "词语或短语", + "heardAs": "听成(可选)", + "heardAsLabel": "听成", + "add": "添加", + "repoNameHint": "将仓库名用作提示", + "repoNameHintDescription": "帮助识别仓库名称。", + "repoSaveFailed": "无法保存仓库的术语。运行此仓库的主机可能需要更新 Orca。" } }, "repositoryHooks": { diff --git a/src/renderer/src/store/repos/repo-update.ts b/src/renderer/src/store/repos/repo-update.ts index 3a7604b3831..995331a18a7 100644 --- a/src/renderer/src/store/repos/repo-update.ts +++ b/src/renderer/src/store/repos/repo-update.ts @@ -158,6 +158,14 @@ export function createRepoUpdateActions( ) { return false } + // A host from before dictionaries drops the field and still answers with its row. + if ( + sanitizedUpdates.dictationTerms !== undefined && + JSON.stringify(updatedRepo?.dictationTerms ?? []) !== + JSON.stringify(sanitizedUpdates.dictationTerms) + ) { + return false + } set((s) => { const nextRepos = s.repos.map((r) => { const matchesOwner = ownerHasExplicitHost diff --git a/src/shared/dictation-dictionary.test.ts b/src/shared/dictation-dictionary.test.ts index 1e81f897337..f7b159cd011 100644 --- a/src/shared/dictation-dictionary.test.ts +++ b/src/shared/dictation-dictionary.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from 'vitest' -import { createDictationDictionary, mergeDictationTerms } from './dictation-dictionary' +import { + createDictationDictionary, + dictationTermsFor, + dictationVocabulary +} from './dictation-dictionary' import { createDictationTextPipeline } from './dictation-text-pipeline' const spell = (terms: Parameters[0], text: string): string => @@ -55,8 +59,21 @@ describe('createDictationDictionary', () => { ) }) + it('lets the repo name join its spoken words but never recase a single word', () => { + const oneWord = dictationTermsFor({}, { displayName: 'orca' }) + const words = dictationTermsFor({}, { displayName: 'dd-scratch-repo' }) + + expect(createDictationDictionary(oneWord)).toBeNull() + expect(dictationVocabulary(oneWord)).toEqual(['orca']) + expect(spell(words, 'Open dd scratch repo now.')).toBe('Open dd-scratch-repo now.') + expect(dictationTermsFor({ dictationRepoNameHint: false }, { displayName: 'orca' })).toEqual([]) + }) + it("prefers a repo's spelling over a personal one", () => { - const terms = mergeDictationTerms([{ term: 'RPC' }], [{ term: 'rpc' }]) + const terms = dictationTermsFor( + { dictationTerms: [{ term: 'rpc' }] }, + { dictationTerms: [{ term: 'RPC' }] } + ) expect(spell(terms, 'Call the rpc.')).toBe('Call the RPC.') }) }) diff --git a/src/shared/dictation-dictionary.ts b/src/shared/dictation-dictionary.ts index a66d3f5785b..ad448055809 100644 --- a/src/shared/dictation-dictionary.ts +++ b/src/shared/dictation-dictionary.ts @@ -1,5 +1,8 @@ -/** A word or phrase dictation should spell `term`'s way, optionally also replacing what the model hears instead. */ -export type DictationTerm = { term: string; heardAs?: string } +/** + * A word or phrase dictation should spell `term`'s way, optionally also replacing what the model hears instead. + * `hint` marks the repo name: it biases recognition and joins its words, but never recases a single word. + */ +export type DictationTerm = { term: string; heardAs?: string; hint?: true } export const MAX_DICTATION_TERMS = 500 export const MAX_DICTATION_TERM_LENGTH = 100 @@ -20,32 +23,49 @@ function wordKey(text: string): string { return (text.normalize('NFKC').toLowerCase().match(WORD_RE) ?? []).join('') } -/** Well-formed entries only, trimmed and capped; undefined when the value isn't a list. */ +/** Well-formed, distinct entries only, trimmed and capped; undefined when the value isn't a list. */ export function normalizeDictationTerms(value: unknown): DictationTerm[] | undefined { if (!Array.isArray(value)) { return undefined } const terms: DictationTerm[] = [] + const seen = new Set() for (const entry of value) { const term: unknown = entry?.term const heardAs: unknown = entry?.heardAs if (typeof term !== 'string' || !term.trim()) { continue } - const alias = typeof heardAs === 'string' ? heardAs.trim() : '' + const spelling = term.trim().slice(0, MAX_DICTATION_TERM_LENGTH) + const alias = + typeof heardAs === 'string' ? heardAs.trim().slice(0, MAX_DICTATION_TERM_LENGTH) : '' + // The same spelling and alias twice is one stored entry. + const id = `${spelling}\u0000${alias}` + if (seen.has(id)) { + continue + } + seen.add(id) terms.push({ - term: term.trim().slice(0, MAX_DICTATION_TERM_LENGTH), - ...(alias ? { heardAs: alias.slice(0, MAX_DICTATION_TERM_LENGTH) } : {}) + term: spelling, + ...(alias ? { heardAs: alias } : {}) }) } return terms.slice(0, MAX_DICTATION_TERMS) } -/** Repo terms first, so a repo's spelling wins over a personal one for the same words. */ -export function mergeDictationTerms(repoTerms: unknown, personalTerms: unknown): DictationTerm[] { +/** + * The dictionary for a dictation into `repo` (none for a folder workspace): its terms first, so a repo's + * spelling wins over a personal one, then the personal terms, then its name as a hint unless that's off. + */ +export function dictationTermsFor( + voice: { dictationTerms?: unknown; dictationRepoNameHint?: boolean } | undefined, + repo: { dictationTerms?: unknown; displayName?: string } | undefined +): DictationTerm[] { + const name = repo?.displayName?.trim().slice(0, MAX_DICTATION_TERM_LENGTH) return [ - ...(normalizeDictationTerms(repoTerms) ?? []), - ...(normalizeDictationTerms(personalTerms) ?? []) + ...(normalizeDictationTerms(repo?.dictationTerms) ?? []), + ...(normalizeDictationTerms(voice?.dictationTerms) ?? []), + ...(name && voice?.dictationRepoNameHint !== false ? [{ term: name, hint: true as const }] : []) ] } @@ -75,8 +95,11 @@ export function createDictationDictionary( add(wordKey(heardAs), term) } } - for (const { term } of terms ?? []) { - add(wordKey(term), term) + for (const { term, hint } of terms ?? []) { + // A one-word repo name like "orca" would recase every "Orca"; only a name of several words joins. + if (!hint || (term.match(WORD_RE)?.length ?? 0) > 1) { + add(wordKey(term), term) + } } if (byKey.size === 0 && unspacedAliases.length === 0) { return null diff --git a/src/shared/speech-types.ts b/src/shared/speech-types.ts index bedd641f99f..dafcdfeba76 100644 --- a/src/shared/speech-types.ts +++ b/src/shared/speech-types.ts @@ -132,10 +132,13 @@ export type VoiceSettings = { speechGpu?: SpeechGpuMode /** Personal dictionary, used in every repo; each repo adds its own Repo.dictationTerms. */ dictationTerms?: DictationTerm[] + /** Adds the current repo's name as a hint (DictationTerm.hint). Absent = on. */ + dictationRepoNameHint?: boolean } /** Where a dictation goes: an engine in the desktop's own process takes repoTerms, a host resolves worktreeId itself. */ export type DictationContext = { worktreeId?: string repoTerms?: DictationTerm[] + repoName?: string } From 854928969e2d5a7251cfba06bea708d401350a75 Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:19:11 +0000 Subject: [PATCH 06/59] feat(mobile): add Shift, Ctrl and Alt latches and opt-in agent keys to the terminal key row (#38776) * feat(mobile): smart rail keys for the terminal agent's state, with Shift/Ctrl/Alt latches * fix(mobile): cancel key presses only when the key row really moves * fix(mobile): keep Smart rail on through Reset Defaults and finish two-write keys * fix(mobile): only a finger scrolling the key row cancels its presses * fix(mobile): hold the smart group until the tap's press lands, and drop clicks on keys that moved * refactor(mobile): simplify the smart rail touch hook and layout read * chore(mobile): allowlist the Shift, Ctrl and Alt key names * feat(mobile): Codex async question keys on the smart rail * fix(mobile): drop the Codex Always key, whose key differs by approval Codex binds "don't ask again" to p for a command-prefix rule and a for session and file approvals (0.136 through 0.166). The permission hook doesn't say which one is on screen. --- ...obile-localization-coverage-allowlist.json | 21 ++ .../src/components/CustomKeyModal.styles.ts | 3 + mobile/src/components/CustomKeyModal.tsx | 10 +- mobile/src/components/SmartRailRow.tsx | 24 +++ .../components/TerminalShortcutSettings.tsx | 22 +- mobile/src/i18n/locales/en.json | 23 ++ .../src/session/MobileSessionCommandDock.tsx | 10 +- mobile/src/session/MobileSessionSheets.tsx | 3 + .../session/use-mobile-session-lifecycle.ts | 21 +- .../use-mobile-session-screen-state.ts | 3 + .../session/MobileSessionAccessoryKeys.tsx | 163 +++++++++++--- .../session/MobileSessionLatchKeys.tsx | 58 +++++ .../session/MobileSessionSmartKeys.tsx | 112 ++++++++++ .../session/mobile-terminal-tab-agent.ts | 21 ++ .../terminal/session/terminal-dock-styles.ts | 54 +++++ .../session/use-terminal-dock-keys.ts | 6 +- .../session/use-terminal-key-row-touch.ts | 141 ++++++++++++ .../src/terminal/terminal-accessory-layout.ts | 69 ++++-- .../terminal/input/agent-smart-keys.test.ts | 71 +++++++ src/shared/terminal/input/agent-smart-keys.ts | 200 ++++++++++++++++++ 20 files changed, 967 insertions(+), 68 deletions(-) create mode 100644 mobile/src/components/SmartRailRow.tsx create mode 100644 mobile/src/terminal/session/MobileSessionLatchKeys.tsx create mode 100644 mobile/src/terminal/session/MobileSessionSmartKeys.tsx create mode 100644 mobile/src/terminal/session/use-terminal-key-row-touch.ts create mode 100644 src/shared/terminal/input/agent-smart-keys.test.ts create mode 100644 src/shared/terminal/input/agent-smart-keys.ts diff --git a/config/mobile-localization-coverage-allowlist.json b/config/mobile-localization-coverage-allowlist.json index 50d8243f0f7..265dd13cf28 100644 --- a/config/mobile-localization-coverage-allowlist.json +++ b/config/mobile-localization-coverage-allowlist.json @@ -307,6 +307,27 @@ "dynamic": false, "count": 1 }, + { + "filePath": "mobile/src/terminal/session/MobileSessionLatchKeys.tsx", + "kind": "object-property:label", + "text": "Shift", + "dynamic": false, + "count": 1 + }, + { + "filePath": "mobile/src/terminal/session/MobileSessionLatchKeys.tsx", + "kind": "object-property:label", + "text": "Ctrl", + "dynamic": false, + "count": 1 + }, + { + "filePath": "mobile/src/terminal/session/MobileSessionLatchKeys.tsx", + "kind": "object-property:label", + "text": "Alt", + "dynamic": false, + "count": 1 + }, { "filePath": "mobile/src/tasks/mobile-smart-source-modes.ts", "kind": "object-property:label", diff --git a/mobile/src/components/CustomKeyModal.styles.ts b/mobile/src/components/CustomKeyModal.styles.ts index a95b5abe3be..2921025e632 100644 --- a/mobile/src/components/CustomKeyModal.styles.ts +++ b/mobile/src/components/CustomKeyModal.styles.ts @@ -41,6 +41,9 @@ export const customKeyModalStyles = themedStyleSheet(() => ({ borderRadius: radii.surface, overflow: 'hidden' }, + smartRailGroup: { + marginTop: spacing.md + }, shortcutForm: { paddingTop: spacing.sm }, diff --git a/mobile/src/components/CustomKeyModal.tsx b/mobile/src/components/CustomKeyModal.tsx index b6dfac3cdc6..42afc17fd3c 100644 --- a/mobile/src/components/CustomKeyModal.tsx +++ b/mobile/src/components/CustomKeyModal.tsx @@ -19,6 +19,7 @@ import { useMobileTranslation } from '../i18n/use-mobile-translation' import { MobileSwitch } from './MobileSwitch' import { ListGroup, ListRow } from './list/ListGroup' import { FieldTextInput } from './composer-input/FieldTextInput' +import { SmartRailRow } from './SmartRailRow' type Step = 'choose-type' | 'shortcut-combo' | 'special-keys' | 'text-macro' @@ -62,9 +63,11 @@ type Props = { onClose: () => void onKeysChanged: (keys: CustomKey[]) => void onManageShortcuts?: () => void + smartRail?: { enabled: boolean; onChange: (enabled: boolean) => void } } -export function CustomKeyModal({ visible, onClose, onKeysChanged, onManageShortcuts }: Props) { +export function CustomKeyModal(props: Props) { + const { visible, onClose, onKeysChanged, onManageShortcuts, smartRail } = props useMobileColorScheme() const t = useMobileTranslation() const [step, setStep] = useState('choose-type') @@ -235,6 +238,11 @@ export function CustomKeyModal({ visible, onClose, onKeysChanged, onManageShortc ) : null} )} + {step === 'choose-type' && smartRail ? ( + + + + ) : null} {step === 'shortcut-combo' && ( diff --git a/mobile/src/components/SmartRailRow.tsx b/mobile/src/components/SmartRailRow.tsx new file mode 100644 index 00000000000..8789fcd0f42 --- /dev/null +++ b/mobile/src/components/SmartRailRow.tsx @@ -0,0 +1,24 @@ +import { ListRow } from './list/ListGroup' +import { MobileSwitch } from './MobileSwitch' +import { useMobileTranslation } from '../i18n/use-mobile-translation' + +/** The Smart rail setting: keys for what the terminal's agent is doing, at the front of the key row. */ +export function SmartRailRow({ + enabled, + onChange +}: { + enabled: boolean + onChange: (enabled: boolean) => void +}): React.JSX.Element { + const t = useMobileTranslation() + const title = t('components.smartRailRow.title', 'Smart rail') + return ( + + } + /> + ) +} diff --git a/mobile/src/components/TerminalShortcutSettings.tsx b/mobile/src/components/TerminalShortcutSettings.tsx index db5bfe5b64e..37e14d0620f 100644 --- a/mobile/src/components/TerminalShortcutSettings.tsx +++ b/mobile/src/components/TerminalShortcutSettings.tsx @@ -27,6 +27,7 @@ import { setTerminalAccessoryBuiltInVisible } from '../terminal/terminal-accessory-layout' import { MobileSwitch } from './MobileSwitch' +import { SmartRailRow } from './SmartRailRow' // Absolutely positioned DragReorderList rows require one fixed height per section. const REORDER_ROW_HEIGHT = 56 @@ -148,10 +149,13 @@ export function TerminalShortcutSettings({ [persistLayout] ) + // Resetting the keys leaves the Smart rail setting alone. const resetBuiltInKeys = useCallback(() => { - const next = defaultLayout() - setShortcutLayout(next) - persistLayout(next) + setShortcutLayout((current) => { + const next = { ...defaultLayout(), smartRail: current.smartRail } + persistLayout(next) + return next + }) }, [defaultLayout, persistLayout]) const reorderCustomKeys = useCallback( @@ -182,6 +186,18 @@ export function TerminalShortcutSettings({ return ( <> + + + setShortcutLayout((current) => { + const next = { ...current, smartRail: enabled } + persistLayout(next) + return next + }) + } + /> + {hasPad ? ( { - // A ctrl/alt/shift held on the Duo rail turns the next typed letter into a combo, while the rail shows it. - if ( - !keysInRail || - !dockKeys.interceptTypedCharacter(text, event.nativeEvent.text, liveInputRef) - ) { + // A held Shift, Ctrl or Alt on the key row or the Duo rail turns the next typed letter into a combo. + if (!dockKeys.interceptTypedCharacter(text, event.nativeEvent.text, liveInputRef)) { handleLiveInputChange(event) } }} diff --git a/mobile/src/session/MobileSessionSheets.tsx b/mobile/src/session/MobileSessionSheets.tsx index 027af33cf82..fe6f590a3d2 100644 --- a/mobile/src/session/MobileSessionSheets.tsx +++ b/mobile/src/session/MobileSessionSheets.tsx @@ -77,6 +77,8 @@ export function MobileSessionSheets({ controller }: { controller: MobileSessionC confirmDiscardMarkdown, handleDeleteCustomKey, handleManageShortcuts, + smartRailEnabled, + handleSmartRailChange, handleClearTerminal, handleCreateTerminal, launchQuickCommand, @@ -363,6 +365,7 @@ export function MobileSessionSheets({ controller }: { controller: MobileSessionC onClose={() => setShowCustomKeyModal(false)} onKeysChanged={setCustomKeys} onManageShortcuts={handleManageShortcuts} + smartRail={{ enabled: smartRailEnabled, onChange: handleSmartRailChange }} /> { + // A read that started before the toggle must not flip it back. + layoutReadRevisionRef.current += 1 + setSmartRailEnabled(enabled) + saveTerminalSmartRail(enabled, padAvailable).catch((error: unknown) => { + console.warn('[terminal-keys] the store would not take the Smart rail setting', error) + setSmartRailEnabled(!enabled) + }) + }, + [padAvailable] + ) + useFocusEffect( useCallback(() => { const releaseKeys = refreshCustomKeys() @@ -195,6 +213,7 @@ export function useMobileSessionLifecycle(scope: MobileSessionTabReconciliationM }, [connState, scheduleDelayedAction, subscribeToTerminal, terminalResume, unsubscribeTerminal]) return { accessoryPreferencesReady: customKeysLoaded && accessoryLayoutLoaded, + handleSmartRailChange, pendingForegroundRecoveryRef } } diff --git a/mobile/src/session/use-mobile-session-screen-state.ts b/mobile/src/session/use-mobile-session-screen-state.ts index eb0f7d28b74..bde653c4045 100644 --- a/mobile/src/session/use-mobile-session-screen-state.ts +++ b/mobile/src/session/use-mobile-session-screen-state.ts @@ -82,6 +82,7 @@ export function useMobileSessionScreenState(scope: MobileSessionFoundationModel) const [renameTarget, setRenameTarget] = useState(null) const [customKeys, setCustomKeys] = useState([]) const [directionPadEnabled, setDirectionPadEnabled] = useState(true) + const [smartRailEnabled, setSmartRailEnabled] = useState(false) const [visibleBuiltInIds, setVisibleBuiltInIds] = useState( getDefaultTerminalAccessoryBuiltInIds ) @@ -182,6 +183,8 @@ export function useMobileSessionScreenState(scope: MobileSessionFoundationModel) setCustomKeys, directionPadEnabled, setDirectionPadEnabled, + smartRailEnabled, + setSmartRailEnabled, visibleBuiltInIds, setVisibleBuiltInIds, showCustomKeyModal, diff --git a/mobile/src/terminal/session/MobileSessionAccessoryKeys.tsx b/mobile/src/terminal/session/MobileSessionAccessoryKeys.tsx index f88d4d336c7..737327fab48 100644 --- a/mobile/src/terminal/session/MobileSessionAccessoryKeys.tsx +++ b/mobile/src/terminal/session/MobileSessionAccessoryKeys.tsx @@ -21,6 +21,17 @@ import { styles as sessionStyles } from '../../session/mobile-session-styles' import { useKeyboardPersistingTaps } from '../../platform/keyboard-persisting-taps' import type { MobileSessionController } from '../../session/use-mobile-session-controller' import { useMobileSessionKeyboardLift } from './use-mobile-session-keyboard-lift' +import { + AGENT_SMART_KEY_STEP_MS, + type AgentSmartKey +} from '../../../../src/shared/terminal/input/agent-smart-keys' +import type { TerminalAccessoryKey } from '../terminal-accessory-keys' +import { resolveMobileTerminalTabSmartKeys } from './mobile-terminal-tab-agent' +import { MobileSessionSmartKeys } from './MobileSessionSmartKeys' +import { MobileSessionLatchKeys } from './MobileSessionLatchKeys' +import { isAccessibilityClick, useTerminalKeyRowTouch } from './use-terminal-key-row-touch' + +type PreparedPress = ReturnType // The terminal's key row above the live input; every key goes straight to the terminal. export function MobileSessionAccessoryKeys({ @@ -46,10 +57,26 @@ export function MobileSessionAccessoryKeys({ stopAccessoryRepeat, cancelAccessoryRepeat, prepareAccessoryPress, - handlePaste + handlePaste, + dockKeys, + activeSessionTab, + terminals, + smartRailEnabled } = controller const keyboardLift = useMobileSessionKeyboardLift(controller.dockBottomInset) - const accessoryPresses = useRef(new Map>()) + const accessoryPresses = useRef(new Map()) + const scrollRef = useRef(null) + const latestSmartKeys = useMemo( + () => + smartRailEnabled ? resolveMobileTerminalTabSmartKeys(activeSessionTab, terminals) : null, + [smartRailEnabled, activeSessionTab, terminals] + ) + const touch = useTerminalKeyRowTouch({ + scrollRef, + latest: latestSmartKeys, + cancelAccessoryRepeat + }) + const smartKeys = touch.shown const keepsKeyboard = useKeyboardPersistingTaps('always') // An unmounting Pressable never fires onPressOut, so a held key would outlive its row. useEffect(() => cancelAccessoryRepeat, [cancelAccessoryRepeat]) @@ -70,8 +97,57 @@ export function MobileSessionAccessoryKeys({ ] const sendKeyLabel = (label: string) => t('session.accessoryKeys.sendKey', 'Send {{label}}', { label }) + // Held Shift, Ctrl or Alt turn a key into its combo and are used up when it sends; keys that take none send as they are. + const prepareBuiltInPress = (key: TerminalAccessoryKey): PreparedPress => { + const chord = dockKeys.chordInput(key.id) + const press = prepareAccessoryPress(chord ?? createTerminalLiveAccessoryInput(key)) + const takeChord = () => { + if (chord) { + dockKeys.takeModifiers() + } + } + return { + tap: () => { + takeChord() + return press.tap() + }, + hold: (firstRepeatDelayMs) => { + takeChord() + press.hold(firstRepeatDelayMs) + } + } + } + // Later writes belong to the same press, so a scroll after release mustn't drop them; a tab switch does. + const sendSmartKey = (key: AgentSmartKey) => { + const handle = controller.activeHandleRef.current + void (async () => { + for (const [index, bytes] of key.writes.entries()) { + if (index > 0) { + await new Promise((resolve) => setTimeout(resolve, AGENT_SMART_KEY_STEP_MS)) + } + if (controller.activeHandleRef.current !== handle) { + return + } + if (!(await prepareAccessoryPress({ bytes }).tap())) { + return + } + } + })() + } + const latchIndex = builtInKeys.findIndex((key) => key.id === 'tab') + 1 + const latchKeys = ( + touch.arm('latch')} + onPress={(modifier, event) => + touch.fire('latch', event, () => dockKeys.tapModifier(modifier)) + } + /> + ) return ( - + {/* Why: fixed keyboard escape hatch; outside the ScrollView so it can't scroll away (#5106). */} {keyboardLift > 0 && ( + {smartKeys ? ( + touch.arm(`smart:${key.id}`)} + onPress={(key, event) => touch.fire(`smart:${key.id}`, event, () => sendSmartKey(key))} + /> + ) : null} { - if (activeHandle) { - void toggleDisplayMode(activeHandle) - } - }} + onPressIn={() => touch.arm('displayMode')} + onPress={(event) => + touch.fire('displayMode', event, () => { + if (activeHandle) { + void toggleDisplayMode(activeHandle) + } + }) + } accessibilityRole="button" accessibilityLabel={ phoneDisplay @@ -133,7 +222,8 @@ export function MobileSessionAccessoryKeys({ style={keyStyle} disabled={!canSend} hitSlop={TERMINAL_KEY_HIT_SLOP} - onPress={() => void handlePaste()} + onPressIn={() => touch.arm('paste')} + onPress={(event) => touch.fire('paste', event, () => void handlePaste())} accessibilityRole="button" accessibilityLabel={t( 'session.accessoryKeys.pasteFromClipboard', @@ -143,20 +233,22 @@ export function MobileSessionAccessoryKeys({ {t('session.accessoryKeys.paste', 'Paste')} )} - {builtInKeys.map((key) => ( + {latchIndex === 0 ? latchKeys : null} + {builtInKeys.flatMap((key, index) => [ { + touch.arm(key.id) if (!key.repeatable) { return } - const press = prepareAccessoryPress(createTerminalLiveAccessoryInput(key)) - accessoryPresses.current.set(key.id, press) + const prepared = prepareBuiltInPress(key) + accessoryPresses.current.set(key.id, prepared) if (event.type === 'keydown') { - press.hold(TERMINAL_ACCESSORY_REPEAT_DELAY_MS) + prepared.hold(TERMINAL_ACCESSORY_REPEAT_DELAY_MS) } }} onPressOut={() => { @@ -165,23 +257,23 @@ export function MobileSessionAccessoryKeys({ } }} onPress={(event) => { - if (key.repeatable) { - // Accessibility clicks have no touch-down to prepare a sender. - const accessibilityClick = - event.type === 'click' && - 'detail' in event.nativeEvent && - event.nativeEvent.detail === 0 - const press = - (!accessibilityClick && accessoryPresses.current.get(key.id)) || - prepareAccessoryPress(createTerminalLiveAccessoryInput(key)) - accessoryPresses.current.delete(key.id) - void press.tap() - return - } - void handleAccessoryKey(createTerminalLiveAccessoryInput(key)) + // Accessibility clicks have no touch-down to prepare a sender. + const prepared = + (key.repeatable && + !isAccessibilityClick(event) && + accessoryPresses.current.get(key.id)) || + prepareBuiltInPress(key) + accessoryPresses.current.delete(key.id) + touch.fire(key.id, event, () => void prepared.tap()) }} onLongPress={ - key.repeatable ? () => accessoryPresses.current.get(key.id)?.hold() : undefined + key.repeatable + ? () => { + if (touch.stillArmed(key.id)) { + accessoryPresses.current.get(key.id)?.hold() + } + } + : undefined } delayLongPress={TERMINAL_ACCESSORY_REPEAT_DELAY_MS} // A web