diff --git a/src/main/automations/retained-run-reconciliation.test.ts b/src/main/automations/retained-run-reconciliation.test.ts index 57afd5ba963..a99aab6ccb6 100644 --- a/src/main/automations/retained-run-reconciliation.test.ts +++ b/src/main/automations/retained-run-reconciliation.test.ts @@ -232,6 +232,37 @@ describe('reconciling retained runs against a graph that has not published yet', service.stop() }) + it('keeps a shell run visibly uncertain after the host is lost, then reattaches without dispatching again', async () => { + const store = await createStore() + const automation = createAutomation(store) + const shellAutomation = store.updateAutomation(automation.id, { agentId: null }) + const retained = retainDispatchedRun(store, shellAutomation) + const surface = createPaneSurface() + const dispatch = vi.fn(async () => ({ ...LAUNCH_TARGET })) + const service = new AutomationService(store, { + terminalObserver: surface.observer, + headlessDispatcher: dispatch + }) + + service.start() + await vi.advanceTimersByTimeAsync(AFTER_SETTLE_MS) + + expect(readRun(store, automation.id, retained.id)).toMatchObject({ + status: 'dispatched', + completionCondition: 'exit', + error: 'Orca is waiting for the execution host to confirm this command’s completion.' + }) + expect(dispatch).not.toHaveBeenCalled() + surface.mountPane() + await vi.advanceTimersByTimeAsync(30_000) + expect(surface.observedHandles).toEqual(['handle-1']) + surface.settleObservation({ status: 'completed', error: null }) + await vi.advanceTimersByTimeAsync(0) + expect(readRun(store, automation.id, retained.id).status).toBe('completed') + expect(dispatch).not.toHaveBeenCalled() + service.stop() + }) + it('still closes out a run the ready surface cannot find after the settle window', async () => { const store = await createStore() const automation = createAutomation(store) diff --git a/src/main/automations/retained-run-reconciliation.ts b/src/main/automations/retained-run-reconciliation.ts index eeeb6d39b79..b243cd040b5 100644 --- a/src/main/automations/retained-run-reconciliation.ts +++ b/src/main/automations/retained-run-reconciliation.ts @@ -12,6 +12,7 @@ export type RetainedRunReconcilerDeps = { attach: (run: AutomationRun) => boolean /** False once the run reached a terminal status by some other path. */ stillRetained: (run: AutomationRun) => boolean + canStrand?: (run: AutomationRun) => boolean strand: (run: AutomationRun) => void } @@ -27,7 +28,7 @@ export type RetainedRunReconcilerDeps = { export class RetainedRunReconciler { private readonly deps: RetainedRunReconcilerDeps private readonly pending = new Map() - private timer: ReturnType | null = null + private timer: ReturnType | null = null private surfaceReadyAt: number | null = null private disposed = false @@ -35,16 +36,20 @@ export class RetainedRunReconciler { this.deps = deps } - reconcile(runs: readonly AutomationRun[]): void { + reconcile(runs: readonly AutomationRun[], defer = false): void { if (this.disposed) { return } for (const run of runs) { - if (!this.deps.attach(run)) { + if (defer || !this.deps.attach(run)) { this.pending.set(run.id, run) } } - this.sweep() + if (defer) { + this.arm() + } else { + this.sweep() + } } /** The authority's terminal surface can now answer pane lookups. */ @@ -71,7 +76,7 @@ export class RetainedRunReconciler { this.pending.delete(runId) continue } - if (strandAt !== null && Date.now() >= strandAt) { + if (strandAt !== null && Date.now() >= strandAt && this.deps.canStrand?.(run) !== false) { this.pending.delete(runId) this.deps.strand(run) } @@ -87,7 +92,15 @@ export class RetainedRunReconciler { if (this.timer || this.disposed) { return } - const timer = setInterval(() => this.sweep(), RETRY_INTERVAL_MS) + const settled = + this.surfaceReadyAt !== null && Date.now() >= this.surfaceReadyAt + SURFACE_SETTLE_MS + const timer = setTimeout( + () => { + this.timer = null + this.sweep() + }, + settled ? 30_000 : RETRY_INTERVAL_MS + ) // Why: a pending retry must never be the reason a process stays alive. ;(timer as { unref?: () => void }).unref?.() this.timer = timer @@ -97,7 +110,7 @@ export class RetainedRunReconciler { if (!this.timer) { return } - clearInterval(this.timer) + clearTimeout(this.timer) this.timer = null } } diff --git a/src/main/automations/run-completion-watcher.test.ts b/src/main/automations/run-completion-watcher.test.ts index 09216f0aeb1..a5afba00697 100644 --- a/src/main/automations/run-completion-watcher.test.ts +++ b/src/main/automations/run-completion-watcher.test.ts @@ -120,7 +120,7 @@ describe('authority-owned automation run completion', () => { }) it.each(['renderer', 'headless', 'retained'] as const)( - 'keeps %s shell runs out of the agent-idle completion watcher', + 'routes %s shell runs through host completion observation', async (mode) => { const store = await createStore() const automation = createAutomation(store, null) @@ -145,14 +145,77 @@ describe('authority-owned automation run completion', () => { service.start() } } - expect(resolveTerminal).not.toHaveBeenCalled() - expect(readRun(store, automation.id, run.id).status).toBe('dispatched') + expect(resolveTerminal).toHaveBeenCalledWith( + expect.objectContaining({ completionCondition: 'exit' }) + ) + await vi.waitFor(() => + expect(readRun(store, automation.id, run.id).status).toBe('completed') + ) } finally { service.stop() } } ) + it('binds a reserved shell run before execution and refuses replacement identities or late acknowledgements', async () => { + const store = await createStore() + const automation = createAutomation(store, null) + const observer = createObserver(async () => ({ + status: 'completed', + outputSnapshot: { + format: 'plain_text', + content: 'host output', + capturedAt: Date.now(), + truncated: false + } + })) + const observing = vi.spyOn(observer, 'observeCompletion') + const service = new AutomationService(store, { terminalObserver: observer }) + const run = store.createAutomationRun(automation, Date.now(), 'manual') + await service.markDispatchResult({ + runId: run.id, + status: 'dispatching', + workspaceId: 'wt1', + terminalPaneKey: LAUNCH_TARGET.terminalPaneKey + }) + const historyReads = vi.spyOn(store, 'listAutomationRuns') + service.bindShellRunTerminal({ + workspaceId: 'other', + paneKey: LAUNCH_TARGET.terminalPaneKey, + ptyId: 'pty-1', + incarnationId: 'original' + }) + expect(observing).not.toHaveBeenCalled() + expect(historyReads).not.toHaveBeenCalled() + service.bindShellRunTerminal({ + workspaceId: 'wt1', + paneKey: LAUNCH_TARGET.terminalPaneKey, + ptyId: 'pty-1', + incarnationId: 'original' + }) + await vi.waitFor(() => expect(readRun(store, automation.id, run.id).status).toBe('completed')) + await service.markDispatchResult({ runId: run.id, status: 'dispatched', ...LAUNCH_TARGET }) + await service.markDispatchResult({ + runId: run.id, + status: 'dispatch_failed', + error: 'late renderer failure' + }) + service.bindShellRunTerminal({ + workspaceId: 'wt1', + paneKey: LAUNCH_TARGET.terminalPaneKey, + ptyId: 'pty-1', + incarnationId: 'replacement' + }) + expect(readRun(store, automation.id, run.id)).toMatchObject({ + status: 'completed', + terminalIncarnationId: 'original', + outputSnapshot: { content: 'host output' }, + error: null + }) + expect(observing).toHaveBeenCalledOnce() + service.stop() + }) + it('leaves a headless dispatched run alone when the authority cannot observe it', async () => { const store = await createStore() const automation = createAutomation(store) diff --git a/src/main/automations/run-completion-watcher.ts b/src/main/automations/run-completion-watcher.ts index 4ac125082fb..fae8e164744 100644 --- a/src/main/automations/run-completion-watcher.ts +++ b/src/main/automations/run-completion-watcher.ts @@ -18,7 +18,12 @@ export type AutomationRunTerminalObserver = { resolveRunTerminal: (run: AutomationRun) => string | null observeCompletion: ( handle: string, - options: { signal: AbortSignal } + options: { + signal: AbortSignal + run?: AutomationRun + onCommandExit?: (exitCode: number) => void + onUnverifiable?: (outputSnapshot: AutomationRunOutputSnapshot | null) => void + } ) => Promise } @@ -60,6 +65,7 @@ export class AutomationRunCompletionWatcher { const current = this.readRun(run.automationId, run.id) return Boolean(current && !isFinalAutomationRunStatus(current.status)) }, + canStrand: (run) => run.completionCondition !== 'exit', strand: (run) => { void this.finalize(run, { status: 'dispatch_failed', @@ -83,7 +89,8 @@ export class AutomationRunCompletionWatcher { } } - private attachRetainedRun(run: AutomationRun): boolean { + private attachRetainedRun(retained: AutomationRun): boolean { + const run = this.readRun(retained.automationId, retained.id) ?? retained if (this.disposed) { return false } @@ -114,12 +121,61 @@ export class AutomationRunCompletionWatcher { controller: AbortController ): Promise { let observation: AutomationRunCompletionObservation + const recordUnverifiable = (outputSnapshot: AutomationRunOutputSnapshot | null): void => { + const current = this.readRun(run.automationId, run.id) + const error = 'Orca cannot verify this command’s completion. Waiting for its execution host.' + if (!current || isFinalAutomationRunStatus(current.status)) { + return + } + if ( + current.error === error && + current.outputSnapshot?.content === outputSnapshot?.content && + current.outputSnapshot?.truncated === outputSnapshot?.truncated + ) { + return + } + void this.markDispatchResult({ + runId: run.id, + status: current.status, + outputSnapshot, + error + }).catch((error) => + console.error('[automations] Failed to save pending shell evidence:', error) + ) + } try { - observation = await this.observer.observeCompletion(handle, { signal: controller.signal }) + observation = await this.observer.observeCompletion(handle, { + signal: controller.signal, + run, + onCommandExit: (exitCode) => { + const current = this.readRun(run.automationId, run.id) + if ( + !current || + isFinalAutomationRunStatus(current.status) || + current.terminalCommandExitCode === exitCode + ) { + return + } + void this.markDispatchResult({ + runId: run.id, + status: current.status, + terminalCommandExitCode: exitCode + }).catch((error) => + console.error('[automations] Failed to save command exit receipt:', error) + ) + }, + onUnverifiable: recordUnverifiable + }) } catch (error) { if (controller.signal.aborted) { return } + if (run.completionCondition === 'exit') { + recordUnverifiable(this.readRun(run.automationId, run.id)?.outputSnapshot ?? null) + this.watching.delete(run.id) + this.reconciler.reconcile([run], true) + return + } observation = { status: 'dispatch_failed', error: describeObservationError(error) } } try { diff --git a/src/main/automations/runtime-terminal-run-observer.ts b/src/main/automations/runtime-terminal-run-observer.ts index 0dc6750e8de..b68a965317e 100644 --- a/src/main/automations/runtime-terminal-run-observer.ts +++ b/src/main/automations/runtime-terminal-run-observer.ts @@ -3,6 +3,8 @@ import type { AutomationRunCompletionObservation, AutomationRunTerminalObserver } from './run-completion-watcher' +import type { TerminalExitCause } from '../../shared/terminal-exit-cause' +import { observeShellRunCompletion } from './shell-run-completion' import type { AutomationRunOutputSnapshot } from '../../shared/automations-types' const TERMINAL_SNAPSHOT_LIMIT = 2_000 @@ -27,11 +29,26 @@ const OBSERVE_DEADLINE_MS = 6 * 60 * 60 * 1000 /** The runtime surface an authority uses to observe its own terminals. */ export type AutomationRunTerminalHost = { getTerminalHandleForPaneKey(paneKey: string): string | null + resolveTerminalPane?( + paneKey: string, + workspaceId?: string + ): { handle: string; ptyId: string | null; incarnationId?: string | null; connected?: boolean } + subscribeToTerminalData?(ptyId: string, listener: (data: string) => void): () => void + subscribeToPtyExit?(ptyId: string, listener: () => void): () => void waitForTerminal( handle: string, - options?: { condition?: 'tui-idle'; timeoutMs?: number; signal?: AbortSignal } - ): Promise<{ satisfied: boolean; blockedReason?: string }> - readTerminal(handle: string, opts?: { limit?: number }): Promise<{ tail: string[] }> + options?: { condition?: 'tui-idle' | 'exit'; timeoutMs?: number; signal?: AbortSignal } + ): Promise<{ + satisfied: boolean + blockedReason?: string + exitCode?: number | null + exitCause?: TerminalExitCause | null + }> + readTerminal( + handle: string, + opts?: { limit?: number }, + snapshot?: { streamOnly?: boolean } + ): Promise<{ tail: string[]; truncated?: boolean; limited?: boolean }> } function isTerminalWaitTimeout(error: unknown): boolean { @@ -149,9 +166,40 @@ export function createRuntimeAutomationRunTerminalObserver( runtime: AutomationRunTerminalHost ): AutomationRunTerminalObserver { return { - resolveRunTerminal: (run) => - run.terminalPaneKey ? runtime.getTerminalHandleForPaneKey(run.terminalPaneKey) : null, - observeCompletion: async (handle, { signal }) => { + resolveRunTerminal: (run) => { + if (!run.terminalPaneKey) { + return null + } + if (run.completionCondition !== 'exit') { + return runtime.getTerminalHandleForPaneKey(run.terminalPaneKey) + } + if (!run.terminalPtyId || !run.terminalIncarnationId) { + return null + } + try { + const terminal = runtime.resolveTerminalPane?.( + run.terminalPaneKey, + run.workspaceId ?? undefined + ) + return terminal?.ptyId === run.terminalPtyId && + terminal.incarnationId === run.terminalIncarnationId + ? terminal.handle + : null + } catch { + return null + } + }, + observeCompletion: async (handle, { signal, run, onUnverifiable, onCommandExit }) => { + if (run?.completionCondition === 'exit') { + return await observeShellRunCompletion( + runtime, + handle, + run, + signal, + onUnverifiable, + onCommandExit + ) + } const startedAt = Date.now() // Why: tui-idle is level-triggered, so a reused pane still idle from the // PREVIOUS run satisfies it before this run's agent has typed a character. diff --git a/src/main/automations/service.ts b/src/main/automations/service.ts index a2d5a6ccd59..e0791af14b4 100644 --- a/src/main/automations/service.ts +++ b/src/main/automations/service.ts @@ -1,3 +1,7 @@ +import { + ShellRunTerminalBindings, + type ShellRunTerminalBinding +} from './shell-run-terminal-binding' import type { WebContents } from 'electron' /** All the service asks of the renderer: is it still there, and take this message. Narrower @@ -54,6 +58,7 @@ export class AutomationService { private readonly allowRemoteHostScheduling: boolean private readonly headlessDispatcher: HeadlessAutomationDispatcher | null private readonly publish: PublishAutomationsChanged | null + private readonly shellRunBindings = new ShellRunTerminalBindings() private readonly runs: AutomationRunWriter private readonly completionWatcher: AutomationRunCompletionWatcher | null /** Installed by desktop IPC registration, where external probes live; null on @@ -117,9 +122,9 @@ export class AutomationService { this.timer = setInterval(() => { void this.evaluateDueRuns() }, this.tickMs) - this.completionWatcher?.reconcileRetainedRuns( - this.store.listAutomationRuns().filter((run) => this.isAgentRun(run)) - ) + const retained = this.store.listAutomationRuns() + this.shellRunBindings.restore(retained, this.runs) + this.completionWatcher?.reconcileRetainedRuns(retained) reportAutomationScheduleDrift(this.store.listAutomations()) // Why: headless serve never gets a renderer-ready IPC, but due runs still // need the same startup catch-up pass desktop gets after renderer attach. @@ -189,9 +194,10 @@ export class AutomationService { async markDispatchResult(result: AutomationDispatchResult): Promise { const run = this.runs.updateRun(result) + this.shellRunBindings.remember(run) clearAutomationDispatchTokens(run.automationId, run.id) if (!isFinalAutomationRunStatus(run.status)) { - if (run.status === 'dispatched' && this.completionWatcher && this.isAgentRun(run)) { + if (run.status === 'dispatched' && this.completionWatcher) { this.completionWatcher.watch(run) } return run @@ -213,11 +219,11 @@ export class AutomationService { }) } - private isAgentRun(run: AutomationRun): boolean { - return ( - this.store.listAutomations().find((automation) => automation.id === run.automationId) - ?.agentId !== null - ) + bindShellRunTerminal(binding: ShellRunTerminalBinding): void { + const run = this.shellRunBindings.bind(this.store, this.runs, binding) + if (run) { + this.completionWatcher?.watch(run) + } } private async evaluateDueRuns(): Promise { @@ -310,7 +316,7 @@ export class AutomationService { runPrecheck: () => this.runPrecheck(automation.id, run.id), markDispatchResult: (result) => this.markDispatchResult(result), watchRun: (dispatched) => { - if (this.completionWatcher && this.isAgentRun(dispatched)) { + if (this.completionWatcher) { this.completionWatcher.watch(dispatched) } } diff --git a/src/main/automations/shell-run-completion.test.ts b/src/main/automations/shell-run-completion.test.ts new file mode 100644 index 00000000000..5fff92f774d --- /dev/null +++ b/src/main/automations/shell-run-completion.test.ts @@ -0,0 +1,511 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { AutomationRun } from '../../shared/automations-types' +import type { AutomationRunTerminalHost } from './runtime-terminal-run-observer' +import { observeShellRunCompletion } from './shell-run-completion' + +const run: AutomationRun = { + id: 'shell-run', + automationId: 'shell-automation', + title: 'Shell command', + scheduledFor: 1, + status: 'dispatched', + trigger: 'manual', + workspaceId: 'folder-workspace', + sessionKind: 'terminal', + completionCondition: 'exit', + terminalIncarnationId: 'incarnation-1', + terminalPtyId: 'pty-1', + terminalPaneKey: 'tab-1:leaf-1', + terminalSessionId: 'tab-1', + chatSessionId: null, + outputSnapshot: null, + precheckResult: null, + usage: null, + error: null, + startedAt: 1, + dispatchedAt: 1, + createdAt: 1 +} + +type WaitResult = Awaited> + +function exited(exitCode: number): WaitResult { + return { satisfied: true, exitCode, exitCause: { kind: 'exited', exitCode } } +} + +function receipt(exitCode: number, runId = run.id): string { + return `\u001b]133;D;${exitCode};orca-automation:${runId}\u0007` +} + +function createHost(initialTail: string[] = []) { + const listeners = new Set<(data: string) => void>() + const unsubscribe = vi.fn(() => listeners.clear()) + const readTerminal = vi + .fn() + .mockResolvedValue({ tail: initialTail }) + const waitForTerminal = vi + .fn() + .mockResolvedValue(exited(0)) + const subscribeToTerminalData = vi.fn((_ptyId: string, listener: (data: string) => void) => { + listeners.add(listener) + return unsubscribe + }) + const runtime: AutomationRunTerminalHost = { + getTerminalHandleForPaneKey: () => 'terminal-1', + resolveTerminalPane: () => ({ + handle: 'terminal-1', + ptyId: 'pty-1', + incarnationId: 'incarnation-1' + }), + readTerminal, + waitForTerminal, + subscribeToTerminalData + } + return { + runtime, + readTerminal, + waitForTerminal, + subscribeToTerminalData, + unsubscribe, + emit: (chunk: string) => listeners.forEach((listener) => listener(chunk)) + } +} + +function createConnectedHost() { + const host = createHost() + const exitListeners = new Set<() => void>() + const unsubscribeExit = vi.fn(() => exitListeners.clear()) + const terminal = { + handle: 'terminal-1', + ptyId: 'pty-1', + incarnationId: 'incarnation-1', + connected: true + } + host.runtime.resolveTerminalPane = () => terminal + const subscribeToPtyExit = vi.fn((_ptyId: string, listener: () => void) => { + exitListeners.add(listener) + return unsubscribeExit + }) + host.runtime.subscribeToPtyExit = subscribeToPtyExit + return { + ...host, + terminal, + subscribeToPtyExit, + unsubscribeExit, + exit: () => { + terminal.connected = false + exitListeners.forEach((listener) => listener()) + } + } +} + +afterEach(() => vi.useRealTimers()) + +function observe(host: ReturnType, retainedRun = run) { + return observeShellRunCompletion( + host.runtime, + 'terminal-1', + retainedRun, + new AbortController().signal + ) +} + +describe('host-owned shell run completion', () => { + it('retains streamed output when the terminal tail is removed at exit', async () => { + const host = createHost(['command started']) + host.waitForTerminal.mockImplementation(async () => { + host.emit('\u001b[32mcommand finished\u001b[0m\r\n') + host.emit(receipt(0)) + host.readTerminal.mockRejectedValue(new Error('terminal_not_found')) + return exited(0) + }) + expect(await observe(host)).toMatchObject({ + status: 'completed', + error: null, + outputSnapshot: { + format: 'plain_text', + content: 'command started\ncommand finished', + truncated: false + } + }) + expect(host.readTerminal).toHaveBeenCalledOnce() + expect(host.subscribeToTerminalData).toHaveBeenCalledWith('pty-1', expect.any(Function)) + expect(host.waitForTerminal).toHaveBeenCalledWith( + 'terminal-1', + expect.objectContaining({ condition: 'exit' }) + ) + expect(host.unsubscribe).toHaveBeenCalledOnce() + }) + + it('reports a nonzero process exit and preserves its diagnostic output', async () => { + const host = createHost() + host.waitForTerminal.mockImplementation(async () => { + host.emit('permission denied\n') + host.emit(receipt(7)) + return exited(7) + }) + expect(await observe(host)).toMatchObject({ + status: 'dispatch_failed', + error: 'Automation process exited with code 7.', + outputSnapshot: { content: 'permission denied' } + }) + expect(host.unsubscribe).toHaveBeenCalledOnce() + }) + + it.each([ + { satisfied: true }, + { satisfied: true, exitCode: null }, + { satisfied: true, exitCode: -1 }, + { satisfied: true, exitCode: -1, exitCause: { kind: 'operator_close' } }, + { satisfied: true, exitCode: -1, exitCause: { kind: 'signaled', signal: 9 } }, + { satisfied: true, exitCode: 0 }, + { satisfied: false, exitCode: 0, exitCause: { kind: 'exited', exitCode: 0 } }, + { satisfied: true, exitCode: 0, exitCause: { kind: 'unknown', reason: 'cause_unreported' } }, + { + satisfied: true, + exitCode: 0, + exitCause: { kind: 'unknown', reason: 'host_status_unavailable' } + } + ])('hands unverifiable evidence back to the reconciler: %j', async (wait) => { + vi.useFakeTimers() + const host = createHost(['retained progress']) + const unknown = vi.fn() + host.waitForTerminal.mockResolvedValue(wait) + await expect( + observeShellRunCompletion( + host.runtime, + 'terminal-1', + run, + new AbortController().signal, + unknown + ) + ).rejects.toThrow('automation_exit_unverifiable') + expect(unknown).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ content: 'retained progress' }) + ) + expect(host.waitForTerminal).toHaveBeenCalledOnce() + expect(host.unsubscribe).toHaveBeenCalledOnce() + expect(vi.getTimerCount()).toBe(0) + }) + + it.each([ + { satisfied: true, exitCode: 0, exitCause: { kind: 'signaled', signal: 9 } }, + { satisfied: true, exitCode: 0, exitCause: { kind: 'operator_close' } } + ])('never reports interrupted execution as success: %j', async (wait) => { + const host = createHost(['interrupted output']) + host.waitForTerminal.mockResolvedValue(wait) + const result = await observe(host) + expect(result.status).toBe('dispatch_failed') + expect(result.error).toMatch(/signal 9|closed before the command completed/) + expect(result.outputSnapshot?.content).toBe('interrupted output') + expect(host.unsubscribe).toHaveBeenCalledOnce() + }) + + it('preserves transport errors and output without inventing an execution failure', async () => { + const host = createHost(['before disconnect']) + const unknown = vi.fn() + const error = new Error('connection_lost') + host.waitForTerminal.mockRejectedValue(error) + await expect( + observeShellRunCompletion( + host.runtime, + 'terminal-1', + run, + new AbortController().signal, + unknown + ) + ).rejects.toBe(error) + expect(unknown).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ content: 'before disconnect' }) + ) + expect(host.unsubscribe).toHaveBeenCalledOnce() + }) + + it('can complete from proven exit evidence when the initial tail cannot be read', async () => { + const host = createHost() + host.readTerminal.mockRejectedValue(new Error('terminal_not_found')) + expect(await observe(host, { ...run, terminalCommandExitCode: 0 })).toMatchObject({ + status: 'completed', + outputSnapshot: null, + error: null + }) + expect(host.waitForTerminal).toHaveBeenCalledOnce() + expect(host.unsubscribe).toHaveBeenCalledOnce() + }) + + it('unsubscribes when an active process wait is aborted', async () => { + const host = createHost() + const controller = new AbortController() + const unknown = vi.fn() + host.waitForTerminal.mockImplementation( + (_handle, options) => + new Promise((_resolve, reject) => { + options?.signal?.addEventListener('abort', () => reject(new Error('request_aborted')), { + once: true + }) + }) + ) + const observation = observeShellRunCompletion( + host.runtime, + 'terminal-1', + run, + controller.signal, + unknown + ) + const aborted = expect(observation).rejects.toThrow('request_aborted') + await Promise.resolve() + await Promise.resolve() + expect(host.waitForTerminal).toHaveBeenCalledOnce() + controller.abort() + await aborted + expect(host.unsubscribe).toHaveBeenCalledOnce() + expect(unknown).not.toHaveBeenCalled() + }) + + it('caps retained output at 256 Ki characters and marks older output as truncated', async () => { + const host = createHost(['old output']) + host.waitForTerminal.mockImplementation(async () => { + host.emit(receipt(0)) + host.emit('x'.repeat(256 * 1024)) + host.emit('last line') + return exited(0) + }) + const result = await observe(host) + expect(result.outputSnapshot?.content).toHaveLength(256 * 1024) + expect(result.outputSnapshot?.content.endsWith('last line')).toBe(true) + expect(result.outputSnapshot?.content).not.toContain('old output') + expect(result.outputSnapshot?.truncated).toBe(true) + }) + + it('rejects a replacement incarnation before reading or subscribing', async () => { + const host = createHost(['replacement output']) + host.runtime.resolveTerminalPane = () => ({ + handle: 'terminal-1', + ptyId: 'pty-1', + incarnationId: 'replacement' + }) + await expect(observe(host)).rejects.toThrow('terminal_incarnation_changed') + expect(host.readTerminal).not.toHaveBeenCalled() + expect(host.subscribeToTerminalData).not.toHaveBeenCalled() + expect(host.waitForTerminal).not.toHaveBeenCalled() + }) + + it('preserves the truncation flag when recovering output from the saved run', async () => { + const host = createHost() + const retainedRun: AutomationRun = { + ...run, + terminalCommandExitCode: 0, + outputSnapshot: { + format: 'plain_text', + content: 'retained tail', + capturedAt: 1, + truncated: true + } + } + expect(await observe(host, retainedRun)).toMatchObject({ + status: 'completed', + outputSnapshot: { content: 'retained tail', truncated: true } + }) + }) + + it('recovers the same incarnation through a new attempt with saved interim output', async () => { + const host = createHost() + const retainedRun: AutomationRun = { ...run } + const unknown = vi.fn((snapshot: AutomationRun['outputSnapshot']) => { + retainedRun.outputSnapshot = snapshot + }) + host.waitForTerminal + .mockImplementationOnce(async () => { + host.emit('before disconnect\n') + return { satisfied: true, exitCode: -1 } + }) + .mockImplementationOnce(async () => { + host.emit('after reconnect\n') + host.emit(receipt(0)) + return exited(0) + }) + await expect( + observeShellRunCompletion( + host.runtime, + 'terminal-1', + run, + new AbortController().signal, + unknown + ) + ).rejects.toThrow('automation_exit_unverifiable') + expect(unknown).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ content: 'before disconnect' }) + ) + expect(host.unsubscribe).toHaveBeenCalledOnce() + host.runtime.resolveTerminalPane = () => ({ + handle: 'terminal-2', + ptyId: 'pty-1', + incarnationId: 'incarnation-1' + }) + expect(await observe(host, retainedRun)).toMatchObject({ + status: 'completed', + outputSnapshot: { content: 'before disconnect\nafter reconnect', truncated: false } + }) + expect(host.waitForTerminal).toHaveBeenNthCalledWith( + 2, + 'terminal-2', + expect.objectContaining({ condition: 'exit' }) + ) + expect(host.unsubscribe).toHaveBeenCalledTimes(2) + }) + it.each(['', receipt(0, 'another-run')])( + 'rejects missing or unrelated command receipts: %j', + async (token) => { + const host = createHost() + host.waitForTerminal.mockImplementation(async () => { + host.emit(token) + return exited(0) + }) + const onCommandExit = vi.fn() + await expect( + observeShellRunCompletion( + host.runtime, + 'terminal-1', + run, + new AbortController().signal, + undefined, + onCommandExit + ) + ).rejects.toThrow('automation_exit_unverifiable') + expect(onCommandExit).not.toHaveBeenCalled() + expect(host.unsubscribe).toHaveBeenCalledOnce() + } + ) + + it('persists a matching command receipt even before host exit can be confirmed', async () => { + const host = createHost() + host.waitForTerminal.mockImplementation(async () => { + host.emit(receipt(7)) + return { satisfied: true, exitCode: -1 } + }) + const onCommandExit = vi.fn() + await expect( + observeShellRunCompletion( + host.runtime, + 'terminal-1', + run, + new AbortController().signal, + undefined, + onCommandExit + ) + ).rejects.toThrow('automation_exit_unverifiable') + expect(onCommandExit).toHaveBeenCalledExactlyOnceWith(7) + expect(host.unsubscribe).toHaveBeenCalledOnce() + }) + + it('uses the saved command exit status rather than a successful wrapper exit after recovery', async () => { + const host = createHost() + expect(await observe(host, { ...run, terminalCommandExitCode: 7 })).toMatchObject({ + status: 'dispatch_failed', + error: 'Automation process exited with code 7.' + }) + }) + + it('recovers a matching command receipt from retained terminal output', async () => { + const host = createHost(['retained command output', receipt(0)]) + expect(await observe(host)).toMatchObject({ + status: 'completed', + outputSnapshot: { content: 'retained command output' } + }) + }) + + it.each(['terminal-1', 'renewed-terminal'])( + 'uses the host exit event across graph handle changes: %s', + async (handle) => { + const host = createConnectedHost() + const onCommandExit = vi.fn() + const observation = observeShellRunCompletion( + host.runtime, + 'terminal-1', + run, + new AbortController().signal, + undefined, + onCommandExit + ) + expect(host.subscribeToPtyExit).toHaveBeenCalledWith('pty-1', expect.any(Function)) + expect(host.subscribeToPtyExit.mock.invocationCallOrder[0]).toBeLessThan( + host.subscribeToTerminalData.mock.invocationCallOrder[0] ?? Infinity + ) + host.emit('before reload\n') + await Promise.resolve() + await Promise.resolve() + expect(host.waitForTerminal).not.toHaveBeenCalled() + host.terminal.handle = handle + host.emit(`after reload\n${receipt(0)}`) + expect(onCommandExit).toHaveBeenCalledExactlyOnceWith(0) + expect(host.waitForTerminal).not.toHaveBeenCalled() + expect(host.unsubscribe).not.toHaveBeenCalled() + host.exit() + expect(await observation).toMatchObject({ + status: 'completed', + outputSnapshot: { content: 'before reload\nafter reload' } + }) + expect(host.waitForTerminal).toHaveBeenCalledExactlyOnceWith( + handle, + expect.objectContaining({ condition: 'exit' }) + ) + expect(host.subscribeToTerminalData).toHaveBeenCalledOnce() + expect(host.unsubscribe).toHaveBeenCalledOnce() + expect(host.unsubscribeExit).toHaveBeenCalledOnce() + } + ) + + it('disposes output and exit subscriptions when observation is aborted during execution', async () => { + const host = createConnectedHost() + const controller = new AbortController() + const observation = observeShellRunCompletion( + host.runtime, + 'terminal-1', + run, + controller.signal + ) + const aborted = expect(observation).rejects.toThrow('request_aborted') + await Promise.resolve() + await Promise.resolve() + expect(host.waitForTerminal).not.toHaveBeenCalled() + controller.abort() + await aborted + expect(host.waitForTerminal).not.toHaveBeenCalled() + expect(host.unsubscribe).toHaveBeenCalledOnce() + expect(host.unsubscribeExit).toHaveBeenCalledOnce() + }) + + it('refuses a verdict for a replacement incarnation after the exit event', async () => { + const host = createConnectedHost() + const onCommandExit = vi.fn() + const observation = observeShellRunCompletion( + host.runtime, + 'terminal-1', + run, + new AbortController().signal, + undefined, + onCommandExit + ) + const rejected = expect(observation).rejects.toThrow('terminal_incarnation_changed') + await Promise.resolve() + await Promise.resolve() + host.terminal.incarnationId = 'replacement-incarnation' + host.emit(receipt(0)) + host.exit() + await rejected + expect(onCommandExit).not.toHaveBeenCalled() + expect(host.waitForTerminal).not.toHaveBeenCalled() + expect(host.unsubscribe).toHaveBeenCalledOnce() + expect(host.unsubscribeExit).toHaveBeenCalledOnce() + }) + + it('reads an already-ended process verdict without waiting for a new exit event', async () => { + const host = createConnectedHost() + host.terminal.connected = false + expect(await observe(host, { ...run, terminalCommandExitCode: 0 })).toMatchObject({ + status: 'completed' + }) + expect(host.waitForTerminal).toHaveBeenCalledOnce() + expect(host.unsubscribeExit).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/automations/shell-run-completion.ts b/src/main/automations/shell-run-completion.ts new file mode 100644 index 00000000000..814235a58b9 --- /dev/null +++ b/src/main/automations/shell-run-completion.ts @@ -0,0 +1,143 @@ +import type { AutomationRun, AutomationRunOutputSnapshot } from '../../shared/automations-types' +import { + stripAnsiEscapeSequences, + TERMINAL_CONTROL_CHARACTER_PATTERN +} from '../../shared/ansi-escape-sequences' +import { isProvenProcessExit } from '../../shared/terminal-exit-cause' +import { createAutomationShellReceiptScanner } from '../../shared/automation-shell-exit-receipt' +import { createHeadlessAutomationOutputSnapshotBuffer } from './headless-dispatch' +import type { AutomationRunTerminalHost } from './runtime-terminal-run-observer' +import type { AutomationRunCompletionObservation } from './run-completion-watcher' + +function resolveRunHandle( + runtime: AutomationRunTerminalHost, + run: AutomationRun, + fallback: string +): string { + if (!runtime.resolveTerminalPane || !run.terminalPaneKey) { + return fallback + } + const terminal = runtime.resolveTerminalPane(run.terminalPaneKey, run.workspaceId ?? undefined) + if ( + terminal.ptyId !== run.terminalPtyId || + terminal.incarnationId !== run.terminalIncarnationId + ) { + throw new Error('terminal_incarnation_changed') + } + return terminal.handle +} + +export async function observeShellRunCompletion( + runtime: AutomationRunTerminalHost, + handle: string, + run: AutomationRun, + signal: AbortSignal, + onUnverifiable?: (snapshot: AutomationRunOutputSnapshot | null) => void, + onCommandExit?: (exitCode: number) => void +): Promise { + if (!run.terminalPtyId || !runtime.subscribeToTerminalData) { + throw new Error('terminal_not_found') + } + const boundHandle = resolveRunHandle(runtime, run, handle) + let wakeExit = (): void => {} + const hostExit = new Promise((resolve) => { + wakeExit = resolve + }) + const unsubscribeExit = runtime.subscribeToPtyExit?.(run.terminalPtyId, () => wakeExit()) + signal.addEventListener('abort', wakeExit, { once: true }) + // Capture the existing host tail before subscribing, without an async renderer read between them. + const initial = runtime + .readTerminal(boundHandle, { limit: 2_000 }, { streamOnly: true }) + .catch(() => ({ tail: [], truncated: false, limited: false })) + const live = createHeadlessAutomationOutputSnapshotBuffer() + let commandExitCode: number | null = run.terminalCommandExitCode ?? null + const receipt = createAutomationShellReceiptScanner(run.id, (code) => { + commandExitCode = code + onCommandExit?.(code) + }) + const unsubscribe = runtime.subscribeToTerminalData(run.terminalPtyId, (data) => { + try { + resolveRunHandle(runtime, run, boundHandle) + } catch { + return + } + receipt.scan(data) + live.append(data) + }) + try { + const seed = await initial + receipt.scan(seed.tail.join('\n')) + const snapshot = (): AutomationRunOutputSnapshot | null => { + const buffer = createHeadlessAutomationOutputSnapshotBuffer() + buffer.append(seed.tail.join('\n') || run.outputSnapshot?.content || '') + const captured = live.snapshot() + if ((seed.tail.length || run.outputSnapshot?.content) && captured) { + buffer.append('\n') + } + buffer.append(captured?.content ?? '') + const output = buffer.snapshot() + if (output) { + output.content = stripAnsiEscapeSequences(output.content) + .replace(/\r\n/g, '\n') + .replace(/\r/g, '\n') + .replace(TERMINAL_CONTROL_CHARACTER_PATTERN, '') + .trim() + output.truncated ||= + (captured?.truncated ?? false) || seed.truncated === true || seed.limited === true + if (!seed.tail.length) { + output.truncated ||= run.outputSnapshot?.truncated ?? false + } + } + return output + } + try { + const running = + run.terminalPaneKey && + runtime.resolveTerminalPane?.(run.terminalPaneKey, run.workspaceId ?? undefined).connected + if (unsubscribeExit && running && !signal.aborted) { + await hostExit + } + if (signal.aborted) { + throw new Error('request_aborted') + } + const wait = await runtime.waitForTerminal(resolveRunHandle(runtime, run, boundHandle), { + condition: 'exit', + signal + }) + const hostObservedExit = + typeof wait.exitCode === 'number' && isProvenProcessExit(wait.exitCode) + const exitCode = hostObservedExit ? commandExitCode : null + if (wait.satisfied && exitCode !== null) { + return { + status: exitCode === 0 ? 'completed' : 'dispatch_failed', + outputSnapshot: snapshot(), + error: exitCode === 0 ? null : `Automation process exited with code ${exitCode}.` + } + } + if ( + wait.satisfied && + hostObservedExit && + (wait.exitCause?.kind === 'signaled' || wait.exitCause?.kind === 'operator_close') + ) { + return { + status: 'dispatch_failed', + outputSnapshot: snapshot(), + error: + wait.exitCause.kind === 'signaled' + ? `Automation process was killed by signal ${wait.exitCause.signal}.` + : 'The automation terminal was closed before the command completed.' + } + } + throw new Error('automation_exit_unverifiable') + } catch (error) { + if (!signal.aborted) { + onUnverifiable?.(snapshot()) + } + throw error + } + } finally { + signal.removeEventListener('abort', wakeExit) + unsubscribeExit?.() + unsubscribe() + } +} diff --git a/src/main/automations/shell-run-runtime-integration.test.ts b/src/main/automations/shell-run-runtime-integration.test.ts new file mode 100644 index 00000000000..e13c0e47d7c --- /dev/null +++ b/src/main/automations/shell-run-runtime-integration.test.ts @@ -0,0 +1,174 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { installFakeAppEnvironment } from '../../../config/scripts/vitest-host-ports-setup' +import { HEADLESS_RUNTIME_WINDOW_ID } from '../../shared/runtime-types' +import { AutomationService } from './service' +import { createRuntimeAutomationRunTerminalObserver } from './runtime-terminal-run-observer' + +const testState = { dir: '' } + +vi.mock('electron', () => ({ + app: { getPath: () => testState.dir }, + safeStorage: { isEncryptionAvailable: () => false } +})) +vi.mock('../telemetry/client', () => ({ track: vi.fn() })) +vi.mock('../telemetry/cohort-classifier', () => ({ getCohortAtEmit: vi.fn() })) + +const TAB_ID = 'shell-run-tab' +const LEAF_ID = '11111111-2222-4333-8444-555555555555' +const PANE_KEY = `${TAB_ID}:${LEAF_ID}` +const PTY_ID = 'shell-run-pty' +const INCARNATION_ID = 'shell-run-incarnation' + +describe('shell run completion through the execution runtime', () => { + beforeEach(() => { + testState.dir = mkdtempSync(join(tmpdir(), 'orca-shell-run-runtime-')) + }) + + afterEach(() => { + rmSync(testState.dir, { recursive: true, force: true }) + }) + + it.each([false, true])( + 'retains immediate output and completion across pruning and late acknowledgement (renderer reload: %s)', + async (reloadRenderer) => { + vi.resetModules() + installFakeAppEnvironment({ getPath: () => testState.dir }) + const { Store, initDataPath } = await import('../persistence') + const { OrcaRuntimeService } = await import('../runtime/orca-runtime') + initDataPath() + const store = new Store() + store.addRepo({ + id: 'shell-run-repo', + path: testState.dir, + displayName: 'Shell run test', + badgeColor: 'blue', + addedAt: 1, + kind: 'git' + }) + const workspaceId = `shell-run-repo::${testState.dir}` + const automation = store.createAutomation({ + name: 'Shell receipt test', + prompt: 'printf done', + agentId: null, + projectId: 'shell-run-repo', + workspaceMode: 'existing', + workspaceId, + timezone: 'UTC', + rrule: 'FREQ=DAILY', + dtstart: Date.now() + }) + const run = store.createAutomationRun(automation, Date.now(), 'manual') + const runtime = new OrcaRuntimeService(store) + if (reloadRenderer) { + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { + tabs: [ + { + tabId: TAB_ID, + worktreeId: workspaceId, + title: 'Terminal', + activeLeafId: LEAF_ID, + layout: null + } + ], + leaves: [ + { + tabId: TAB_ID, + worktreeId: workspaceId, + leafId: LEAF_ID, + paneRuntimeId: 1, + ptyId: PTY_ID + } + ] + }) + } else { + runtime.syncWindowGraph(HEADLESS_RUNTIME_WINDOW_ID, { tabs: [], leaves: [] }) + } + const service = new AutomationService(store, { + terminalObserver: createRuntimeAutomationRunTerminalObserver(runtime) + }) + runtime.setAutomationService(service) + + const waitStarted = vi.spyOn(runtime, 'waitForTerminal') + const subscribeExit = vi.spyOn(runtime, 'subscribeToPtyExit') + try { + await service.markDispatchResult({ + runId: run.id, + status: 'dispatching', + workspaceId, + terminalSessionId: TAB_ID, + terminalPaneKey: PANE_KEY, + terminalPtyId: PTY_ID + }) + runtime.registerPty(PTY_ID, workspaceId, null, { + tabId: TAB_ID, + leafId: LEAF_ID, + incarnationId: INCARNATION_ID + }) + expect(store.listAutomationRuns(automation.id)[0]).toMatchObject({ + status: 'dispatched', + terminalPtyId: PTY_ID, + terminalIncarnationId: INCARNATION_ID + }) + const terminal = runtime.resolveTerminalPane(PANE_KEY, workspaceId) + + if (reloadRenderer) { + await vi.waitFor(() => + expect(subscribeExit).toHaveBeenCalledWith(PTY_ID, expect.any(Function)) + ) + expect(waitStarted).not.toHaveBeenCalled() + expect(runtime.markRendererReloading(1)).not.toBeNull() + runtime.syncWindowGraph(1, { tabs: [], leaves: [] }) + } + + // A fast command can finish before the dispatch acknowledgement or any microtask. + runtime.onPtyData(PTY_ID, 'finished before renderer acknowledgement\r\n', Date.now()) + runtime.onPtyData(PTY_ID, `\u001b]133;D;0;orca-automation:${run.id}\u0007`, Date.now()) + runtime.onPtyExit(PTY_ID, 0, INCARNATION_ID, { + hostExitConfirmed: true, + cause: { kind: 'exited', exitCode: 0 } + }) + + await vi.waitFor(() => { + expect(store.listAutomationRuns(automation.id)[0]).toMatchObject({ + status: 'completed', + terminalCommandExitCode: 0, + outputSnapshot: { + content: 'finished before renderer acknowledgement', + truncated: false + } + }) + }) + expect(waitStarted).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ condition: 'exit' }) + ) + if (!reloadRenderer) { + expect((await runtime.readTerminal(terminal.handle)).tail).toEqual([]) + } + + const lateAcknowledgement = await service.markDispatchResult({ + runId: run.id, + status: 'dispatched', + terminalSessionId: TAB_ID, + terminalPaneKey: PANE_KEY, + terminalPtyId: PTY_ID + }) + expect(lateAcknowledgement.status).toBe('completed') + + const reloaded = new Store() + expect(reloaded.listAutomationRuns(automation.id)[0]).toMatchObject({ + status: 'completed', + terminalCommandExitCode: 0, + terminalIncarnationId: INCARNATION_ID, + outputSnapshot: { content: 'finished before renderer acknowledgement' } + }) + } finally { + service.stop() + } + } + ) +}) diff --git a/src/main/automations/shell-run-terminal-binding.ts b/src/main/automations/shell-run-terminal-binding.ts new file mode 100644 index 00000000000..1302650dec7 --- /dev/null +++ b/src/main/automations/shell-run-terminal-binding.ts @@ -0,0 +1,72 @@ +import { isFinalAutomationRunStatus, type AutomationRun } from '../../shared/automations-types' +import type { Store } from '../persistence' +import type { AutomationRunWriter } from './automation-run-writer' + +export type ShellRunTerminalBinding = { + workspaceId: string + paneKey: string + ptyId: string + incarnationId: string +} + +export class ShellRunTerminalBindings { + private readonly pending = new Map() + + restore(retained: AutomationRun[], runs: AutomationRunWriter): void { + for (const run of retained) { + this.remember(run) + if (run.completionCondition === 'exit' && !isFinalAutomationRunStatus(run.status)) { + runs.updateRun({ + runId: run.id, + status: run.status, + error: 'Orca is waiting for the execution host to confirm this command’s completion.' + }) + } + } + } + + remember(run: AutomationRun): void { + if (run.completionCondition !== 'exit' || !run.workspaceId || !run.terminalPaneKey) { + return + } + const key = JSON.stringify([run.workspaceId, run.terminalPaneKey]) + if (isFinalAutomationRunStatus(run.status) || run.terminalIncarnationId) { + this.pending.delete(key) + } else { + this.pending.set(key, run) + } + } + + bind( + store: Pick, + runs: AutomationRunWriter, + binding: ShellRunTerminalBinding + ): AutomationRun | null { + const key = JSON.stringify([binding.workspaceId, binding.paneKey]) + const reserved = this.pending.get(key) + if (!reserved) { + return null + } + const run = store + .listAutomationRuns(reserved.automationId) + .find((run) => run.id === reserved.id) + if (!run || isFinalAutomationRunStatus(run.status)) { + this.pending.delete(key) + return null + } + if ( + (run.terminalPtyId && run.terminalPtyId !== binding.ptyId) || + (run.terminalIncarnationId && run.terminalIncarnationId !== binding.incarnationId) + ) { + return null + } + const bound = runs.updateRun({ + runId: run.id, + status: 'dispatched', + terminalPtyId: binding.ptyId, + terminalIncarnationId: binding.incarnationId + }) + this.pending.delete(key) + return bound + } +} diff --git a/src/main/persistence/scheduling-automations/automation-run-operations.ts b/src/main/persistence/scheduling-automations/automation-run-operations.ts index 0dc9e61731a..a56aa9d1377 100644 --- a/src/main/persistence/scheduling-automations/automation-run-operations.ts +++ b/src/main/persistence/scheduling-automations/automation-run-operations.ts @@ -94,6 +94,7 @@ export function createAutomationRun( workspaceId: automation.workspaceId, workspaceDisplayName: operations.getWorkspaceDisplayName(automation.workspaceId), sessionKind: 'terminal', + ...(automation.agentId === null ? { completionCondition: 'exit' as const } : {}), chatSessionId: null, terminalSessionId: null, terminalPaneKey: null, @@ -166,6 +167,13 @@ export function updateAutomationRun( } const now = Date.now() const current = operations.state.automationRuns[index] + if ( + current.completionCondition === 'exit' && + isFinalAutomationRunStatus(current.status) && + result.status !== current.status + ) { + return current + } const workspaceId = result.workspaceId ?? current.workspaceId const workspaceDisplayName = Object.hasOwn(result, 'workspaceDisplayName') ? normalizeAutomationRunWorkspaceDisplayName(result.workspaceDisplayName ?? null) @@ -178,6 +186,8 @@ export function updateAutomationRun( workspaceDisplayName ?? normalizeAutomationRunWorkspaceDisplayName(current.workspaceDisplayName ?? null) ?? operations.getWorkspaceDisplayName(workspaceId), + terminalIncarnationId: result.terminalIncarnationId ?? current.terminalIncarnationId, + terminalCommandExitCode: result.terminalCommandExitCode ?? current.terminalCommandExitCode, terminalSessionId: Object.hasOwn(result, 'terminalSessionId') ? (result.terminalSessionId ?? null) : current.terminalSessionId, diff --git a/src/main/runtime/orca-runtime-automations.test.ts b/src/main/runtime/orca-runtime-automations.test.ts index 00d5fbf70f6..31b6e6f515f 100644 --- a/src/main/runtime/orca-runtime-automations.test.ts +++ b/src/main/runtime/orca-runtime-automations.test.ts @@ -105,7 +105,9 @@ describe('OrcaRuntimeService automation methods', () => { }) it('launches a blank-terminal command on the resolved workspace host without an agent', async () => { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: This fixture supplies the store operations used by the launch path. const runtime = new OrcaRuntimeService(makeStore() as never) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: Expose the real protected resolver solely to stub the remote workspace in this test. const scope = runtime as unknown as { resolveTerminalWorkspaceLaunchScope: (selector: string) => Promise } @@ -117,6 +119,7 @@ describe('OrcaRuntimeService automation methods', () => { }) const createTerminal = vi .spyOn(runtime, 'createTerminal') + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: The launch path returns this create result unchanged; only its handle is asserted. .mockResolvedValue({ handle: 'term-shell' } as never) await runtime.launchAgentTerminal('id:folder:remote', { diff --git a/src/main/runtime/orca-runtime-register-pty.ts b/src/main/runtime/orca-runtime-register-pty.ts index dae2519ca9f..8d3abef0382 100644 --- a/src/main/runtime/orca-runtime-register-pty.ts +++ b/src/main/runtime/orca-runtime-register-pty.ts @@ -80,6 +80,14 @@ export class OrcaRuntimeWithRegisterPty extends OrcaRuntimeWithInvalidateAllHand ...(binding && paneKey ? { tabId: binding.tabId, paneKey } : {}), ...(binding?.incarnationId ? { incarnationId: binding.incarnationId } : {}) }) + if (paneKey && binding?.incarnationId) { + this.automation.bindShellRunTerminal({ + workspaceId: worktreeId, + paneKey, + ptyId, + incarnationId: binding.incarnationId + }) + } const hostScope = this.getOrchestrationCompatibilityHostScope(pty) if (paneKey && binding?.incarnationId && hostScope) { this._orchestrationDb?.retainReplacedWorkerTerminalResources({ diff --git a/src/main/runtime/orca-runtime-resolve-terminal-pane.ts b/src/main/runtime/orca-runtime-resolve-terminal-pane.ts index 51e81ee3d5d..854cb63478f 100644 --- a/src/main/runtime/orca-runtime-resolve-terminal-pane.ts +++ b/src/main/runtime/orca-runtime-resolve-terminal-pane.ts @@ -197,6 +197,9 @@ export class OrcaRuntimeWithResolveTerminalPane extends OrcaRuntimeWithGetTermin const pty = this.getLivePtyForHandle(handle) if (pty) { const read = this.readPtyTerminal(handle, pty.pty, opts) + if (providerSnapshot.streamOnly) { + return labelTerminalReadSource(read) + } const visibleRead = opts.screen ? await this.readRenderedScreen(pty.pty.ptyId, read, opts) : await this.withVisibleSnapshotFallback(pty.pty.ptyId, read, opts, providerSnapshot) @@ -216,7 +219,7 @@ export class OrcaRuntimeWithResolveTerminalPane extends OrcaRuntimeWithGetTermin cursor: opts.cursor, limit: opts.limit }) - if (!leaf.ptyId) { + if (!leaf.ptyId || providerSnapshot.streamOnly) { return { ...read, source: opts.screen ? 'screen-unavailable' : 'stream' } } const visibleRead = opts.screen diff --git a/src/main/runtime/orca-runtime-terminal-create-deduplication.ts b/src/main/runtime/orca-runtime-terminal-create-deduplication.ts index c12834cd07e..2240700964c 100644 --- a/src/main/runtime/orca-runtime-terminal-create-deduplication.ts +++ b/src/main/runtime/orca-runtime-terminal-create-deduplication.ts @@ -146,7 +146,14 @@ export class OrcaRuntimeWithTerminalCreateDeduplication extends OrcaRuntimeWithC async launchAgentTerminal( worktreeSelector: string, - opts: { agent: TuiAgent | null; prompt: string; title?: string } + opts: { + agent: TuiAgent | null + automationRunId?: string + prompt: string + title?: string + tabId?: string + leafId?: string + } ): Promise { if (opts.agent === null) { const workspace = await this.resolveTerminalWorkspaceLaunchScope(worktreeSelector) @@ -157,8 +164,14 @@ export class OrcaRuntimeWithTerminalCreateDeduplication extends OrcaRuntimeWithC terminalWindowsShell: this.store?.getSettings().terminalWindowsShell }) return await this.createTerminal(worktreeSelector, { - ...buildAutomationShellStartup(opts.prompt, resolveStartupShell(platform, shell)), - title: opts.title + ...buildAutomationShellStartup( + opts.prompt, + resolveStartupShell(platform, shell), + opts.automationRunId + ), + title: opts.title, + tabId: opts.tabId, + leafId: opts.leafId }) } const worktree = await this.resolveWorktreeSelector(worktreeSelector) diff --git a/src/main/runtime/rpc/methods/automations.test.ts b/src/main/runtime/rpc/methods/automations.test.ts index b3b849df9ba..a5ab3f8d28f 100644 --- a/src/main/runtime/rpc/methods/automations.test.ts +++ b/src/main/runtime/rpc/methods/automations.test.ts @@ -11,6 +11,7 @@ function makeRequest(method: string, params?: unknown): RpcRequest { describe('automation RPC methods', () => { it('routes automation CRUD and run operations to the runtime server', async () => { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: The fixture implements every runtime method exercised by these RPC requests. const runtime = { getRuntimeId: () => 'test-runtime', listAutomations: vi.fn().mockReturnValue([{ id: 'auto-1', name: 'Daily review' }]), @@ -108,6 +109,7 @@ describe('automation RPC methods', () => { }) it('returns a cursor page when the caller requests a bounded run history', async () => { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: The fixture implements every runtime method exercised by these RPC requests. const runtime = { getRuntimeId: () => 'test-runtime', listAutomationRunsPage: vi.fn().mockReturnValue({ @@ -127,6 +129,7 @@ describe('automation RPC methods', () => { }) it('preserves an explicit blank terminal when creating and updating an automation', async () => { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: The fixture implements every runtime method exercised by these RPC requests. const runtime = { getRuntimeId: () => 'test-runtime', createAutomation: vi.fn().mockResolvedValue({ id: 'auto-1', agentId: null }), @@ -169,6 +172,7 @@ describe('automation RPC methods', () => { automationId: automation.id, selector: { kind: 'self' as const } })) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: The fixture implements every runtime method exercised by these RPC requests. const runtime = { listAutomationsForScope: () => ({ automations, items }), showAutomation: () => automations[1], @@ -184,14 +188,18 @@ describe('automation RPC methods', () => { }) expect(() => show.handler({ id: 'shell' }, context)).toThrow('newer Orca client') expect( - list.handler({}, { - ...context, - clientCapabilities: [AUTOMATION_SHELL_RUNTIME_CAPABILITY] - }) + list.handler( + {}, + { + ...context, + clientCapabilities: [AUTOMATION_SHELL_RUNTIME_CAPABILITY] + } + ) ).toEqual({ automations, items }) }) it('rejects unknown providers and invalid schedules', async () => { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: The fixture implements every runtime method exercised by these RPC requests. const runtime = { getRuntimeId: () => 'test-runtime', createAutomation: vi.fn() @@ -226,6 +234,7 @@ describe('automation RPC methods', () => { }) it('preserves null baseBranch update values through the RPC boundary', async () => { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: The fixture implements every runtime method exercised by these RPC requests. const runtime = { getRuntimeId: () => 'test-runtime', updateAutomation: vi.fn().mockResolvedValue({ id: 'auto-1', baseBranch: null }) diff --git a/src/main/runtime/runtime-automation-controller.ts b/src/main/runtime/runtime-automation-controller.ts index d3ac55df438..79d9bcd50b8 100644 --- a/src/main/runtime/runtime-automation-controller.ts +++ b/src/main/runtime/runtime-automation-controller.ts @@ -55,6 +55,10 @@ export class RuntimeAutomationController { this.service = service } + bindShellRunTerminal(binding: Parameters[0]): void { + this.service?.bindShellRunTerminal(binding) + } + /** Keep runtime-owned automation work ahead of queued external probes. */ withExternalProbePriority(run: () => T): T { const wrap = this.service?.externalProbePriority diff --git a/src/main/runtime/runtime-terminal-contracts.ts b/src/main/runtime/runtime-terminal-contracts.ts index 680bec98227..9fd7d2b809a 100644 --- a/src/main/runtime/runtime-terminal-contracts.ts +++ b/src/main/runtime/runtime-terminal-contracts.ts @@ -178,6 +178,8 @@ export type TerminalWaiter = { /** How a provider-held screen should be fetched when runtime bytes are absent. */ export type RuntimeProviderSnapshotReadOptions = { + /** Internal observers need the host stream without waiting on a renderer. */ + streamOnly?: boolean timeoutMs?: number retireOnTimeout?: boolean visibleScreenOnly?: boolean diff --git a/src/main/startup/main-process-automations.ts b/src/main/startup/main-process-automations.ts index ee3e34ac995..ae8152fc811 100644 --- a/src/main/startup/main-process-automations.ts +++ b/src/main/startup/main-process-automations.ts @@ -1,3 +1,5 @@ +import { randomUUID } from 'node:crypto' +import { makePaneKey } from '../../shared/stable-pane-id' import { AutomationService } from '../automations/service' import { createHeadlessAutomationOutputSnapshotBuffer } from '../automations/headless-dispatch' import { buildHeadlessAutomationWorktreeCreateArgs } from '../automations/headless-workspace-create' @@ -49,10 +51,26 @@ export function initializeMainProcessAutomations(): AutomationService { if (!workspaceId) { throw new Error('The target workspace is no longer available.') } + const shellIdentity = + automation.agentId === null + ? { tabId: randomUUID(), leafId: randomUUID() } + : undefined + if (shellIdentity) { + await service.markDispatchResult({ + runId: run.id, + status: 'dispatching', + workspaceId, + workspaceDisplayName, + terminalSessionId: shellIdentity.tabId, + terminalPaneKey: makePaneKey(shellIdentity.tabId, shellIdentity.leafId) + }) + } const terminal = await runtime.launchAgentTerminal(`id:${workspaceId}`, { agent: automation.agentId, + automationRunId: run.id, prompt: automation.prompt, - title: run.title + title: run.title, + ...shellIdentity }) terminalHandle = terminal.handle terminalSessionId = terminal.tabId ?? null @@ -62,35 +80,33 @@ export function initializeMainProcessAutomations(): AutomationService { const worktree = await runtime.showManagedWorktree(`id:${workspaceId}`) workspaceDisplayName = worktree.displayName ?? null } - const completion = (async () => { - const wait = await runtime.waitForTerminal(terminalHandle, { - condition: automation.agentId === null ? 'exit' : 'tui-idle' - }) - const read = await runtime.readTerminal(terminalHandle, { - limit: terminalSnapshotLimit - }) - const snapshotBuffer = createHeadlessAutomationOutputSnapshotBuffer() - snapshotBuffer.append(read.tail.join('\n')) - if (wait.satisfied && (automation.agentId !== null || wait.exitCode === 0)) { - return { - status: 'completed' as const, - outputSnapshot: snapshotBuffer.snapshot(), - error: null - } - } - return { - status: 'dispatch_failed' as const, - outputSnapshot: snapshotBuffer.snapshot(), - error: - automation.agentId === null - ? wait.exitCode !== null - ? `Automation process exited with code ${wait.exitCode}.` - : 'Automation process did not report completion.' - : wait.blockedReason - ? `Automation agent is blocked: ${wait.blockedReason}.` - : 'Automation agent did not report completion.' - } - })() + const completion = + automation.agentId === null + ? undefined + : (async () => { + const wait = await runtime.waitForTerminal(terminalHandle, { + condition: 'tui-idle' + }) + const read = await runtime.readTerminal(terminalHandle, { + limit: terminalSnapshotLimit + }) + const snapshotBuffer = createHeadlessAutomationOutputSnapshotBuffer() + snapshotBuffer.append(read.tail.join('\n')) + if (wait.satisfied) { + return { + status: 'completed' as const, + outputSnapshot: snapshotBuffer.snapshot(), + error: null + } + } + return { + status: 'dispatch_failed' as const, + outputSnapshot: snapshotBuffer.snapshot(), + error: wait.blockedReason + ? `Automation agent is blocked: ${wait.blockedReason}.` + : 'Automation agent did not report completion.' + } + })() return { workspaceId, workspaceDisplayName, diff --git a/src/renderer/src/components/automations/AutomationEditorSettingsSidebar.test.tsx b/src/renderer/src/components/automations/AutomationEditorSettingsSidebar.test.tsx index 702697bf7ac..dacdb468fc3 100644 --- a/src/renderer/src/components/automations/AutomationEditorSettingsSidebar.test.tsx +++ b/src/renderer/src/components/automations/AutomationEditorSettingsSidebar.test.tsx @@ -23,6 +23,7 @@ const initialDraft: AutomationDraft = { time: '09:00', dayOfWeek: '1', customSchedule: '', + savedSchedule: null, missedRunGraceMinutes: '720', scheduleWarning: null } diff --git a/src/renderer/src/components/automations/AutomationSessionField.tsx b/src/renderer/src/components/automations/AutomationSessionField.tsx index 743bd137bd5..b1366596552 100644 --- a/src/renderer/src/components/automations/AutomationSessionField.tsx +++ b/src/renderer/src/components/automations/AutomationSessionField.tsx @@ -62,16 +62,13 @@ export function AutomationSessionField({ })) }} size="sm" + disabled={draft.agentId === null} className={toggleGroupClassName} > {translate('auto.components.automations.AutomationSessionField.c90888ee94', 'Fresh')} - + {translate('auto.components.automations.AutomationSessionField.f3c76dce51', 'Reuse')} diff --git a/src/renderer/src/hooks/automation-dispatch-handler.ts b/src/renderer/src/hooks/automation-dispatch-handler.ts index 49d59e7ac24..a8053a459c8 100644 --- a/src/renderer/src/hooks/automation-dispatch-handler.ts +++ b/src/renderer/src/hooks/automation-dispatch-handler.ts @@ -165,12 +165,26 @@ export async function handleAutomationDispatchRequest({ } } const result = await launchAgentBackgroundSession({ + automationRunId: run.id, agent: automation.agentId, worktreeId: worktree.id, prompt: automation.prompt, launchSource: 'unknown', title: run.title, - onData: completion.appendOutput, + onPrepared: + automation.agentId === null + ? async ({ tabId, paneKey }) => { + await markDispatchResult({ + runId: run.id, + status: 'dispatching', + workspaceId: worktree.id, + workspaceDisplayName: worktree.displayName, + terminalSessionId: tabId, + terminalPaneKey: paneKey + }) + } + : undefined, + onData: automation.agentId === null ? undefined : completion.appendOutput, onAgentStatus: (payload) => { completion.captureAssistantMessage(payload.lastAssistantMessage) // Why: session-boundary done = launch connect, not run completion (see observeAgentStatus). @@ -184,7 +198,11 @@ export async function handleAutomationDispatchRequest({ completion.handleAgentDone() }, onExit: (_ptyId, code) => { - completion.handleExit(code) + if (automation.agentId === null) { + releaseTerminalOwnership() + } else { + completion.handleExit(code) + } } }) if (!result) { diff --git a/src/renderer/src/hooks/useAutomationDispatchEvents.test.ts b/src/renderer/src/hooks/useAutomationDispatchEvents.test.ts index d2875dc57cf..e72e321519e 100644 --- a/src/renderer/src/hooks/useAutomationDispatchEvents.test.ts +++ b/src/renderer/src/hooks/useAutomationDispatchEvents.test.ts @@ -793,9 +793,15 @@ describe('useAutomationDispatchEvents setup launch', () => { await vi.waitFor(() => expect(mockReleaseTerminalOwnership).toHaveBeenCalledOnce()) expect(mockFinalizeTerminalOwnership).not.toHaveBeenCalled() - expect(mockMarkDispatchResult).toHaveBeenCalledWith( - expect.objectContaining({ status: 'dispatch_failed' }) - ) + if (agentId === null) { + expect(mockMarkDispatchResult).not.toHaveBeenCalledWith( + expect.objectContaining({ status: 'dispatch_failed' }) + ) + } else { + expect(mockMarkDispatchResult).toHaveBeenCalledWith( + expect.objectContaining({ status: 'dispatch_failed' }) + ) + } } ) diff --git a/src/renderer/src/lib/agent-background-session-contract.ts b/src/renderer/src/lib/agent-background-session-contract.ts index cd27e94d2be..a42d0879603 100644 --- a/src/renderer/src/lib/agent-background-session-contract.ts +++ b/src/renderer/src/lib/agent-background-session-contract.ts @@ -6,10 +6,12 @@ import type { AutomationTerminalOwnership } from '@/lib/automation-terminal-owne export type LaunchAgentBackgroundSessionArgs = { agent: TuiAgent | null + automationRunId?: string worktreeId: string prompt?: string launchSource?: LaunchSource title?: string + onPrepared?: (identity: { tabId: string; paneKey: string }) => Promise onData?: (chunk: string) => void onExit?: (ptyId: string, code: number) => void onAgentStatus?: (payload: ParsedAgentStatusPayload) => void diff --git a/src/renderer/src/lib/agent-background-session-startup.ts b/src/renderer/src/lib/agent-background-session-startup.ts index 296bc87acb4..935e9b901a7 100644 --- a/src/renderer/src/lib/agent-background-session-startup.ts +++ b/src/renderer/src/lib/agent-background-session-startup.ts @@ -16,6 +16,7 @@ import { resolveLocalWindowsAgentStartupShell } from '../../../shared/windows-te export function buildBackgroundSessionStartup(args: { agent: TuiAgent | null + automationRunId?: string prompt: string settings: GlobalSettings | null platform: NodeJS.Platform @@ -59,7 +60,11 @@ export function buildBackgroundSessionStartup(args: { launchConfig: startupPlan.launchConfig, startupCommandDelivery: startupPlan.startupCommandDelivery } - : buildAutomationShellStartup(prompt, resolveStartupShell(platform, shell)), + : buildAutomationShellStartup( + prompt, + resolveStartupShell(platform, shell), + args.automationRunId + ), pasteDraftAfterLaunch: hasPrompt && isFollowup ? prompt : null } } diff --git a/src/renderer/src/lib/launch-agent-background-session.ts b/src/renderer/src/lib/launch-agent-background-session.ts index 3b8582f9ce7..f427f52fc0c 100644 --- a/src/renderer/src/lib/launch-agent-background-session.ts +++ b/src/renderer/src/lib/launch-agent-background-session.ts @@ -73,6 +73,7 @@ export async function launchAgentBackgroundSession( const hasPrompt = trimmedPrompt.length > 0 const plan = buildBackgroundSessionStartup({ agent, + automationRunId: args.automationRunId, prompt: trimmedPrompt, settings: store.settings, platform: launchHost.platform, @@ -155,6 +156,7 @@ export async function launchAgentBackgroundSession( agentStatusConsumer?.consume(data) } try { + await args.onPrepared?.({ tabId: reservedTabId, paneKey }) if (runtimeTarget.kind === 'environment') { const created = await createRuntimeAgentBackgroundTerminal({ environmentId: runtimeTarget.environmentId, diff --git a/src/shared/automation-shell-startup.win32.test.ts b/src/shared/automation-shell-startup.win32.test.ts index b2ba39efc81..9356a0ef073 100644 --- a/src/shared/automation-shell-startup.win32.test.ts +++ b/src/shared/automation-shell-startup.win32.test.ts @@ -9,8 +9,8 @@ describe.skipIf(process.platform !== 'win32').each(['off', 'on'])( it.each([ { command: 'exit /b 0', expected: 0 }, { command: 'exit /b 7', expected: 7 }, - { command: 'orca_nonexistent_automation_command', expected: 9009 }, - { command: 'cmd.exe /d /c exit 7 & echo recovered', expected: 7 }, + { command: 'orca_nonexistent_automation_command', expected: 1 }, + { command: 'cmd.exe /d /c exit 7 & echo recovered', expected: 0 }, { command: 'echo literal!value! & exit /b 0', expected: 0, output: 'literal!value!' }, { command: `"${process.execPath}" -e "process.stdout.write('quoted path works'); process.exit(7)"`, @@ -18,6 +18,12 @@ describe.skipIf(process.platform !== 'win32').each(['off', 'on'])( output: 'quoted path works' } ])('preserves output and exit status: $command', async ({ command, expected, output }) => { + const direct = await runProcess({ + program: 'cmd.exe', + args: ['/d', '/v:off', '/s', '/c', `"${command}"`], + windowsVerbatimArguments: true, + timeoutMs: 15_000 + }) const startup = buildAutomationShellStartup(command, 'cmd', 'native-cmd-run') const result = await runProcess({ program: 'cmd.exe', @@ -31,6 +37,8 @@ describe.skipIf(process.platform !== 'win32').each(['off', 'on'])( result.stdout ) expect(result.timedOut).toBe(false) + expect(direct.timedOut).toBe(false) + expect(result.code).toBe(direct.code) expect(result.code).toBe(expected) expect(receipts).toEqual([expected]) if (output) { diff --git a/src/shared/automations-types.ts b/src/shared/automations-types.ts index 7052119d731..e9dc14e58bc 100644 --- a/src/shared/automations-types.ts +++ b/src/shared/automations-types.ts @@ -146,6 +146,10 @@ export type AutomationRun = { * is deleted and its live metadata is gone. */ workspaceDisplayName?: string | null sessionKind: 'terminal' + /** Captured at launch so editing the automation cannot change how this run settles. */ + completionCondition?: 'exit' + terminalIncarnationId?: string | null + terminalCommandExitCode?: number | null chatSessionId: string | null terminalSessionId: string | null /** Why: a terminal tab can later point at a different pane/PTY. Automation @@ -230,6 +234,8 @@ export type AutomationDispatchRequest = { } export type AutomationDispatchResult = { + terminalIncarnationId?: string | null + terminalCommandExitCode?: number | null runId: string status: AutomationRunStatus workspaceId?: string | null diff --git a/tests/e2e/automation-shell-command-recovery.spec.ts b/tests/e2e/automation-shell-command-recovery.spec.ts new file mode 100644 index 00000000000..01158df6dd0 --- /dev/null +++ b/tests/e2e/automation-shell-command-recovery.spec.ts @@ -0,0 +1,94 @@ +import { existsSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs' +import { rm } from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' +import { test, expect } from './helpers/orca-app' +import { waitForSessionReady } from './helpers/store' + +for (const workspaceMode of ['existing', 'new_per_run'] as const) { + test(`shell automation retains completion and output across renderer reload in ${workspaceMode}`, async ({ + orcaPage, + registerPostElectronShutdownCleanup + }, testInfo) => { + const fixturePath = mkdtempSync(path.join(os.tmpdir(), 'orca-automation-reload-')) + registerPostElectronShutdownCleanup(() => rm(fixturePath, { recursive: true, force: true })) + const counterPath = path.join(fixturePath, 'launches.txt') + const releasePath = path.join(fixturePath, 'release') + const finishedPath = path.join(fixturePath, 'finished') + const scriptPath = path.join(fixturePath, 'command.cjs') + const startMarker = 'ORCA_SHELL_BEFORE_RELOAD' + const endMarker = 'ORCA_SHELL_AFTER_RELOAD' + writeFileSync( + scriptPath, + [ + "const fs = require('node:fs')", + `fs.appendFileSync(${JSON.stringify(counterPath)}, 'launch\\n')`, + `console.log(${JSON.stringify(startMarker)})`, + 'const timer = setInterval(() => {', + ` if (!fs.existsSync(${JSON.stringify(releasePath)})) return`, + ' clearInterval(timer)', + ` console.log(${JSON.stringify(endMarker)})`, + ` fs.writeFileSync(${JSON.stringify(finishedPath)}, 'done')`, + '}, 50)' + ].join('\n') + ) + + await waitForSessionReady(orcaPage) + await orcaPage.evaluate(() => { + const store = window.__store + if (!store) { + throw new Error('Store unavailable') + } + store.getState().openAutomationsPage() + }) + await orcaPage.getByRole('button', { name: 'Add new', exact: true }).click() + const dialog = orcaPage.getByRole('dialog', { name: /^(Create|Edit) automation$/ }) + await dialog.locator('button[data-agent-combobox-root="true"]').click() + await orcaPage.getByRole('option', { name: 'Blank Terminal', exact: true }).click() + const name = `Shell reload ${workspaceMode}` + await dialog.getByRole('textbox', { name: 'Automation name' }).fill(name) + if (workspaceMode === 'new_per_run') { + await dialog.getByRole('radio', { name: 'New run', exact: true }).click() + } + const commandEditor = dialog.getByRole('textbox', { name: /^Shell command/ }) + await commandEditor.focus() + await commandEditor.pressSequentially(`node "${scriptPath}"`) + await dialog.getByRole('button', { name: 'Create', exact: true }).click() + await expect(dialog).toBeHidden() + await orcaPage.getByText(name, { exact: true }).click() + await orcaPage.getByRole('button', { name: 'Run Now', exact: true }).click() + await expect.poll(() => existsSync(counterPath), { timeout: 30_000 }).toBe(true) + await orcaPage.getByRole('tab', { name: /^Runs/ }).click() + await expect( + orcaPage.getByRole('tabpanel').getByRole('button', { name: /Launched/ }) + ).toBeVisible() + await orcaPage.screenshot({ path: testInfo.outputPath('shell-before-reload.png') }) + + await orcaPage.reload() + await waitForSessionReady(orcaPage) + await orcaPage.evaluate(() => { + const store = window.__store + if (!store) { + throw new Error('Store unavailable after reload') + } + store.getState().openAutomationsPage() + }) + await orcaPage.getByText(name, { exact: true }).click() + await orcaPage.getByRole('tab', { name: /^Runs/ }).click() + await expect( + orcaPage.getByRole('tabpanel').getByRole('button', { name: /Launched/ }) + ).toBeVisible() + writeFileSync(releasePath, 'finish') + await expect.poll(() => existsSync(finishedPath)).toBe(true) + expect(readFileSync(counterPath, 'utf8')).toBe('launch\n') + + await expect(orcaPage.getByText('1 run · 1 completed', { exact: true })).toBeVisible({ + timeout: 30_000 + }) + await orcaPage.getByRole('tabpanel').getByRole('button', { name: /Done/ }).click() + await expect(orcaPage.getByText(startMarker, { exact: false })).toBeVisible() + await expect(orcaPage.getByText(endMarker, { exact: false })).toBeVisible() + expect(readFileSync(counterPath, 'utf8')).toBe('launch\n') + await orcaPage.screenshot({ path: testInfo.outputPath('shell-after-reload.png') }) + }) +} diff --git a/tests/e2e/automation-shell-command-ssh-recovery.spec.ts b/tests/e2e/automation-shell-command-ssh-recovery.spec.ts new file mode 100644 index 00000000000..1e560e4c63e --- /dev/null +++ b/tests/e2e/automation-shell-command-ssh-recovery.spec.ts @@ -0,0 +1,109 @@ +import { test, expect } from './helpers/orca-app' +import { waitForSessionReady } from './helpers/store' +import { connectDockerSshRelayTarget } from './helpers/docker-ssh-relay-connection' +import { + cleanupDockerSshRelayTarget, + execDockerSshRelayTargetCommand, + shellQuote, + startDockerSshRelayTarget, + writeDockerSshRelayTargetFile +} from './helpers/docker-ssh-relay-target' + +test('SSH shell automation retains completion and output across renderer reload', async ({ + orcaPage, + registerPostElectronShutdownCleanup +}, testInfo) => { + test.skip(process.env.ORCA_E2E_SSH_DOCKER !== '1', 'Requires the Docker SSH fixture') + test.slow() + const target = startDockerSshRelayTarget(testInfo) + registerPostElectronShutdownCleanup(async () => cleanupDockerSshRelayTarget(target)) + const counterPath = '/tmp/orca-shell-automation-launches' + const releasePath = '/tmp/orca-shell-automation-release' + const finishedPath = '/tmp/orca-shell-automation-finished' + const scriptPath = '/tmp/orca-shell-automation-command.sh' + const startMarker = 'ORCA_SSH_SHELL_BEFORE_RELOAD' + const endMarker = 'ORCA_SSH_SHELL_AFTER_RELOAD' + writeDockerSshRelayTargetFile( + target, + scriptPath, + [ + `printf 'launch\\n' >> ${shellQuote(counterPath)}`, + `printf '%s\\n' ${shellQuote(startMarker)}`, + `while [ ! -f ${shellQuote(releasePath)} ]; do sleep 0.05; done`, + `printf '%s\\n' ${shellQuote(endMarker)}`, + `touch ${shellQuote(finishedPath)}` + ].join('\n') + ) + await waitForSessionReady(orcaPage) + await connectDockerSshRelayTarget(orcaPage, target, { relayGracePeriodSeconds: 0 }) + await orcaPage.evaluate(() => { + const store = window.__store + if (!store) { + throw new Error('Store unavailable') + } + store.getState().openAutomationsPage() + }) + await orcaPage.getByRole('button', { name: 'Add new', exact: true }).click() + const dialog = orcaPage.getByRole('dialog', { name: /^(Create|Edit) automation$/ }) + await dialog.locator('button[data-agent-combobox-root="true"]').click() + await orcaPage.getByRole('option', { name: 'Blank Terminal', exact: true }).click() + const name = 'SSH shell reload' + await dialog.getByRole('textbox', { name: 'Automation name' }).fill(name) + const commandEditor = dialog.getByRole('textbox', { name: /^Shell command/ }) + await commandEditor.focus() + await commandEditor.pressSequentially(`bash ${shellQuote(scriptPath)}`) + await dialog.getByRole('button', { name: 'Create', exact: true }).click() + await expect(dialog).toBeHidden() + await orcaPage.getByText(name, { exact: true }).click() + await orcaPage.getByRole('button', { name: 'Run Now', exact: true }).click() + await expect + .poll( + () => + execDockerSshRelayTargetCommand( + target, + `test -f ${shellQuote(counterPath)} && echo yes || echo no` + ) === 'yes', + { timeout: 30_000 } + ) + .toBe(true) + await orcaPage.getByRole('tab', { name: /^Runs/ }).click() + await expect( + orcaPage.getByRole('tabpanel').getByRole('button', { name: /Launched/ }) + ).toBeVisible() + await orcaPage.screenshot({ path: testInfo.outputPath('ssh-shell-before-reload.png') }) + + await orcaPage.reload() + await waitForSessionReady(orcaPage) + await orcaPage.evaluate(() => { + const store = window.__store + if (!store) { + throw new Error('Store unavailable after reload') + } + store.getState().openAutomationsPage() + }) + await orcaPage.getByText(name, { exact: true }).click() + await orcaPage.getByRole('tab', { name: /^Runs/ }).click() + await expect( + orcaPage.getByRole('tabpanel').getByRole('button', { name: /Launched/ }) + ).toBeVisible() + execDockerSshRelayTargetCommand(target, `touch ${shellQuote(releasePath)}`) + await expect + .poll( + () => + execDockerSshRelayTargetCommand( + target, + `test -f ${shellQuote(finishedPath)} && echo yes || echo no` + ) === 'yes' + ) + .toBe(true) + expect(execDockerSshRelayTargetCommand(target, `cat ${shellQuote(counterPath)}`)).toBe('launch') + + await expect(orcaPage.getByText('1 run · 1 completed', { exact: true })).toBeVisible({ + timeout: 30_000 + }) + await orcaPage.getByRole('tabpanel').getByRole('button', { name: /Done/ }).click() + await expect(orcaPage.getByText(startMarker, { exact: false })).toBeVisible() + await expect(orcaPage.getByText(endMarker, { exact: false })).toBeVisible() + expect(execDockerSshRelayTargetCommand(target, `cat ${shellQuote(counterPath)}`)).toBe('launch') + await orcaPage.screenshot({ path: testInfo.outputPath('ssh-shell-after-reload.png') }) +})