From 3134e5bbfd764da26e34cf281d91a701136d014a Mon Sep 17 00:00:00 2001 From: Jinjing <6427696+AmethystLiang@users.noreply.github.com> Date: Sun, 30 Aug 2026 17:29:59 -0700 Subject: [PATCH] feat(automations): display failed run states --- .../headless-completion-wait.test.ts | 57 ++++++ .../automations/headless-completion-wait.ts | 35 ++++ .../automations/headless-dispatch-runner.ts | 28 +-- src/main/automations/headless-dispatch.ts | 1 + .../headless-run-completion.test.ts | 118 +++++++++++++ .../automations/headless-run-completion.ts | 80 +++++++++ .../automations/potentially-live-run.test.ts | 67 +++++++ src/main/automations/potentially-live-run.ts | 59 +++++++ src/main/automations/service.test.ts | 166 ++++++++++++++++++ src/main/automations/service.ts | 19 ++ .../automation-run-operations.ts | 6 +- src/main/startup/main-process-automations.ts | 14 +- .../automations/AutomationRunDetailsPage.tsx | 9 +- .../automations/AutomationRunHistory.tsx | 6 +- .../automations/AutomationRunNoticeBand.tsx | 27 +++ .../AutomationRunPageFrame.test.tsx | 70 ++++++++ .../automations/AutomationRunPageFrame.tsx | 4 + .../automation-list-last-run.test.ts | 9 + .../automations/automation-list-last-run.ts | 12 +- .../automations/automation-page-parts.tsx | 14 +- .../automation-run-content.test.ts | 99 +++++++++++ .../automations/automation-run-content.ts | 25 ++- .../automation-run-view-state.test.ts | 21 +++ .../automations/automation-run-view-state.ts | 12 +- src/shared/automation-run-retention.test.ts | 13 ++ src/shared/automations-types.ts | 6 + 26 files changed, 941 insertions(+), 36 deletions(-) create mode 100644 src/main/automations/headless-completion-wait.test.ts create mode 100644 src/main/automations/headless-completion-wait.ts create mode 100644 src/main/automations/headless-run-completion.test.ts create mode 100644 src/main/automations/headless-run-completion.ts create mode 100644 src/main/automations/potentially-live-run.test.ts create mode 100644 src/main/automations/potentially-live-run.ts create mode 100644 src/renderer/src/components/automations/AutomationRunNoticeBand.tsx create mode 100644 src/renderer/src/components/automations/AutomationRunPageFrame.test.tsx create mode 100644 src/renderer/src/components/automations/automation-run-content.test.ts diff --git a/src/main/automations/headless-completion-wait.test.ts b/src/main/automations/headless-completion-wait.test.ts new file mode 100644 index 00000000000..78c8c95e1f8 --- /dev/null +++ b/src/main/automations/headless-completion-wait.test.ts @@ -0,0 +1,57 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { RuntimeTerminalWait } from '../../shared/runtime-types' +import { waitForHeadlessAutomationCompletion } from './headless-completion-wait' + +const blocked = { + handle: 'terminal-1', + condition: 'tui-idle', + satisfied: false, + status: 'running', + exitCode: null, + blockedReason: 'agent-approval-prompt' +} as RuntimeTerminalWait + +const completed = { + handle: 'terminal-1', + condition: 'tui-idle', + satisfied: true, + status: 'running', + exitCode: null +} as RuntimeTerminalWait + +afterEach(() => vi.useRealTimers()) + +describe('waitForHeadlessAutomationCompletion', () => { + it('keeps watching after a recoverable prompt', async () => { + vi.useFakeTimers() + vi.setSystemTime(1_000) + const waitForTerminal = vi.fn().mockResolvedValueOnce(blocked).mockResolvedValueOnce(completed) + + const resultPromise = waitForHeadlessAutomationCompletion( + { waitForTerminal }, + 'terminal-1', + 5_000 + ) + await vi.advanceTimersByTimeAsync(1_000) + + await expect(resultPromise).resolves.toBe(completed) + expect(waitForTerminal).toHaveBeenCalledTimes(2) + }) + + it('bounds prompt rechecks by the completion deadline', async () => { + vi.useFakeTimers() + vi.setSystemTime(1_000) + const waitForTerminal = vi.fn().mockResolvedValue(blocked) + + const resultPromise = waitForHeadlessAutomationCompletion( + { waitForTerminal }, + 'terminal-1', + 100 + ) + const rejection = expect(resultPromise).rejects.toThrow('timeout') + await vi.advanceTimersByTimeAsync(100) + + await rejection + expect(waitForTerminal).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/main/automations/headless-completion-wait.ts b/src/main/automations/headless-completion-wait.ts new file mode 100644 index 00000000000..f3894399965 --- /dev/null +++ b/src/main/automations/headless-completion-wait.ts @@ -0,0 +1,35 @@ +import type { RuntimeTerminalWait } from '../../shared/runtime-types' + +const HEADLESS_COMPLETION_TIMEOUT_MS = 5 * 60 * 1000 +const BLOCKED_PROMPT_RECHECK_MS = 1_000 + +type TerminalWaitRuntime = { + waitForTerminal: ( + handle: string, + options: { condition: 'tui-idle'; timeoutMs: number } + ) => Promise +} + +export async function waitForHeadlessAutomationCompletion( + runtime: TerminalWaitRuntime, + terminalHandle: string, + timeoutMs = HEADLESS_COMPLETION_TIMEOUT_MS +): Promise { + const deadline = Date.now() + timeoutMs + while (true) { + const remainingMs = deadline - Date.now() + if (remainingMs <= 0) { + throw new Error('timeout') + } + const result = await runtime.waitForTerminal(terminalHandle, { + condition: 'tui-idle', + timeoutMs: remainingMs + }) + if (!result.blockedReason) { + return result + } + await new Promise((resolve) => + setTimeout(resolve, Math.min(BLOCKED_PROMPT_RECHECK_MS, remainingMs)) + ) + } +} diff --git a/src/main/automations/headless-dispatch-runner.ts b/src/main/automations/headless-dispatch-runner.ts index 8d4e3aed290..a77f2dc4a9a 100644 --- a/src/main/automations/headless-dispatch-runner.ts +++ b/src/main/automations/headless-dispatch-runner.ts @@ -11,6 +11,7 @@ import { import type { HeadlessAutomationDispatcher } from './headless-dispatch' import type { AutomationRunTargetResult } from './run-target-resolution' import type { AutomationRunWriter } from './automation-run-writer' +import { observeHeadlessAutomationCompletion } from './headless-run-completion' export type HeadlessAutomationDispatchContext = { automation: Automation @@ -63,25 +64,14 @@ export async function runHeadlessAutomationDispatch( ctx.watchRun(updated) return updated } - void launch.completion - .then((completion) => - ctx.markDispatchResult({ - runId: run.id, - status: completion.status, - ...launchRunTarget, - precheckResult, - outputSnapshot: completion.outputSnapshot ?? null, - error: completion.error ?? null - }) - ) - .catch((error) => - ctx.markDispatchResult({ - runId: run.id, - status: 'dispatch_failed', - ...launchRunTarget, - error: describeDispatchError(error) - }) - ) + observeHeadlessAutomationCompletion({ + automation, + run, + launch, + target: launchRunTarget, + precheckResult, + markDispatchResult: ctx.markDispatchResult + }) return updated } catch (error) { return runs.updateRun({ diff --git a/src/main/automations/headless-dispatch.ts b/src/main/automations/headless-dispatch.ts index 359673bc573..fb4427df797 100644 --- a/src/main/automations/headless-dispatch.ts +++ b/src/main/automations/headless-dispatch.ts @@ -15,6 +15,7 @@ export type HeadlessAutomationDispatchLaunch = { terminalPtyId?: string | null completion?: Promise<{ status: 'completed' | 'dispatch_failed' + observationVerdict?: 'unverifiable' | null outputSnapshot?: AutomationRunOutputSnapshot | null error?: string | null }> diff --git a/src/main/automations/headless-run-completion.test.ts b/src/main/automations/headless-run-completion.test.ts new file mode 100644 index 00000000000..d4202c87401 --- /dev/null +++ b/src/main/automations/headless-run-completion.test.ts @@ -0,0 +1,118 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { + Automation, + AutomationDispatchResult, + AutomationRun +} from '../../shared/automations-types' +import type { HeadlessAutomationDispatchLaunch } from './headless-dispatch' +import { observeHeadlessAutomationCompletion } from './headless-run-completion' + +const target = { + workspaceId: 'workspace-1', + workspaceDisplayName: 'Workspace', + terminalSessionId: 'tab-1', + terminalPaneKey: 'pane-1', + terminalPtyId: 'pty-1' +} + +function observe( + workspaceMode: Automation['workspaceMode'], + completion: HeadlessAutomationDispatchLaunch['completion'] +) { + const markDispatchResult = vi.fn( + async (result: AutomationDispatchResult) => result as unknown as AutomationRun + ) + observeHeadlessAutomationCompletion({ + automation: { workspaceMode } as Automation, + run: { id: 'run-1' } as AutomationRun, + launch: { ...target, completion }, + target, + precheckResult: null, + markDispatchResult + }) + return markDispatchResult +} + +afterEach(() => vi.restoreAllMocks()) + +describe('observeHeadlessAutomationCompletion', () => { + it('keeps an unverifiable existing-workspace run non-final', async () => { + const mark = observe( + 'existing', + Promise.resolve({ + status: 'dispatch_failed', + observationVerdict: 'unverifiable' + }) + ) + await vi.waitFor(() => + expect(mark).toHaveBeenCalledWith(expect.objectContaining({ status: 'dispatched' })) + ) + }) + + it('keeps an unverifiable new-per-run failure bounded', async () => { + const mark = observe( + 'new_per_run', + Promise.resolve({ + status: 'dispatch_failed', + observationVerdict: 'unverifiable' + }) + ) + await vi.waitFor(() => + expect(mark).toHaveBeenCalledWith(expect.objectContaining({ status: 'dispatch_failed' })) + ) + }) + + it('leaves an observed existing-workspace failure final', async () => { + const mark = observe( + 'existing', + Promise.resolve({ status: 'dispatch_failed', error: 'Exited.' }) + ) + await vi.waitFor(() => + expect(mark).toHaveBeenCalledWith(expect.objectContaining({ status: 'dispatch_failed' })) + ) + }) + + it('treats observer rejection as unverifiable without leaking transport tokens', async () => { + vi.spyOn(console, 'error').mockImplementation(() => {}) + const mark = observe('existing', Promise.reject(new Error('terminal_handle_stale'))) + await vi.waitFor(() => + expect(mark).toHaveBeenCalledWith( + expect.objectContaining({ + status: 'dispatched', + observationVerdict: 'unverifiable', + error: 'Orca stopped watching this run before it reported completion.' + }) + ) + ) + }) + + it('keeps positive terminal exit evidence final', async () => { + const mark = observe('existing', Promise.reject(new Error('terminal_exited'))) + await vi.waitFor(() => + expect(mark).toHaveBeenCalledWith( + expect.objectContaining({ + status: 'dispatch_failed', + observationVerdict: null, + error: 'Automation terminal exited before the agent reported completion.' + }) + ) + ) + }) + + it('does not reinterpret persistence failure as observation loss', async () => { + vi.spyOn(console, 'error').mockImplementation(() => {}) + const markDispatchResult = vi.fn().mockRejectedValue(new Error('Automation run not found.')) + observeHeadlessAutomationCompletion({ + automation: { workspaceMode: 'existing' } as Automation, + run: { id: 'run-1' } as AutomationRun, + launch: { ...target, completion: Promise.resolve({ status: 'completed' }) }, + target, + precheckResult: null, + markDispatchResult + }) + await vi.waitFor(() => expect(markDispatchResult).toHaveBeenCalledTimes(1)) + expect(markDispatchResult).toHaveBeenCalledWith( + expect.objectContaining({ status: 'completed' }) + ) + }) +}) diff --git a/src/main/automations/headless-run-completion.ts b/src/main/automations/headless-run-completion.ts new file mode 100644 index 00000000000..c80ff04e03a --- /dev/null +++ b/src/main/automations/headless-run-completion.ts @@ -0,0 +1,80 @@ +import type { + Automation, + AutomationDispatchResult, + AutomationPrecheckResult, + AutomationRun +} from '../../shared/automations-types' +import type { HeadlessAutomationDispatchLaunch } from './headless-dispatch' + +export type HeadlessAutomationRunTarget = { + workspaceId: string + workspaceDisplayName: string | null + terminalSessionId: string | null + terminalPaneKey: string | null + terminalPtyId: string | null +} + +function persistHeadlessCompletion( + markDispatchResult: (result: AutomationDispatchResult) => Promise, + result: AutomationDispatchResult +): void { + void markDispatchResult(result).catch((error) => { + console.error('[automations] failed to persist run completion:', error) + }) +} + +export function observeHeadlessAutomationCompletion({ + automation, + run, + launch, + target, + precheckResult, + markDispatchResult +}: { + automation: Automation + run: AutomationRun + launch: HeadlessAutomationDispatchLaunch + target: HeadlessAutomationRunTarget + precheckResult: AutomationPrecheckResult | null + markDispatchResult: (result: AutomationDispatchResult) => Promise +}): void { + if (!launch.completion) { + return + } + void launch.completion.then( + (completion) => { + const retainPotentiallyLiveRun = + automation.workspaceMode === 'existing' && completion.observationVerdict === 'unverifiable' + persistHeadlessCompletion(markDispatchResult, { + runId: run.id, + status: retainPotentiallyLiveRun ? 'dispatched' : completion.status, + observationVerdict: completion.observationVerdict ?? null, + ...target, + precheckResult, + outputSnapshot: completion.outputSnapshot ?? null, + error: completion.error ?? null + }) + }, + (error) => { + const errorCode = error instanceof Error ? error.message.trim() : String(error).trim() + const observedExit = errorCode === 'terminal_exited' + if (!observedExit) { + // Why a fixed sentence: transport tokens belong in logs, not run history. + console.error('[automations] run completion observation failed:', error) + } + persistHeadlessCompletion(markDispatchResult, { + runId: run.id, + status: + !observedExit && automation.workspaceMode === 'existing' + ? 'dispatched' + : 'dispatch_failed', + observationVerdict: observedExit ? null : 'unverifiable', + ...target, + precheckResult, + error: observedExit + ? 'Automation terminal exited before the agent reported completion.' + : 'Orca stopped watching this run before it reported completion.' + }) + } + ) +} diff --git a/src/main/automations/potentially-live-run.test.ts b/src/main/automations/potentially-live-run.test.ts new file mode 100644 index 00000000000..11a9e23b966 --- /dev/null +++ b/src/main/automations/potentially-live-run.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'vitest' +import type { Automation, AutomationRun } from '../../shared/automations-types' +import { findPotentiallyLiveAutomationRun } from './potentially-live-run' + +const automation = { + id: 'automation-2', + workspaceMode: 'existing', + workspaceId: 'workspace-1' +} as Automation + +function run(overrides: Partial = {}): AutomationRun { + return { + id: 'run-1', + automationId: 'automation-1', + workspaceId: 'workspace-1', + status: 'dispatched', + ...overrides + } as AutomationRun +} + +describe('findPotentiallyLiveAutomationRun', () => { + it('finds an in-flight run from another automation in the same workspace', () => { + expect(findPotentiallyLiveAutomationRun(automation, 'current', [run()])?.id).toBe('run-1') + }) + + it('keeps an additive unverifiable verdict potentially live after a legacy final status', () => { + expect( + findPotentiallyLiveAutomationRun(automation, 'current', [ + run({ status: 'dispatch_failed', observationVerdict: 'unverifiable' }) + ])?.id + ).toBe('run-1') + }) + + it('does not infer liveness from terminal identity after an observed failure', () => { + expect( + findPotentiallyLiveAutomationRun(automation, 'current', [ + run({ + status: 'dispatch_failed', + terminalSessionId: 'tab-1', + terminalPtyId: 'pty-1' + }) + ]) + ).toBeNull() + }) + + it('recognizes a legacy completion-observer loss by its transport error', () => { + expect( + findPotentiallyLiveAutomationRun(automation, 'current', [ + run({ status: 'dispatch_failed', error: 'terminal_handle_stale' }) + ])?.id + ).toBe('run-1') + }) + + it('does not treat a persisted pre-dispatch row as live', () => { + expect( + findPotentiallyLiveAutomationRun(automation, 'current', [run({ status: 'pending' })]) + ).toBeNull() + }) + + it('does not block new-per-run workspaces', () => { + expect( + findPotentiallyLiveAutomationRun({ ...automation, workspaceMode: 'new_per_run' }, 'current', [ + run({ observationVerdict: 'unverifiable' }) + ]) + ).toBeNull() + }) +}) diff --git a/src/main/automations/potentially-live-run.ts b/src/main/automations/potentially-live-run.ts new file mode 100644 index 00000000000..d74f4d68b3b --- /dev/null +++ b/src/main/automations/potentially-live-run.ts @@ -0,0 +1,59 @@ +import { + isFinalAutomationRunStatus, + type Automation, + type AutomationRun +} from '../../shared/automations-types' +import type { Store } from '../persistence' + +const LEGACY_UNVERIFIABLE_ERRORS = new Set([ + 'terminal_handle_stale', + 'terminal_not_found', + 'timeout' +]) + +function isLegacyUnverifiableRun(run: AutomationRun): boolean { + return ( + run.status === 'dispatch_failed' && + typeof run.error === 'string' && + LEGACY_UNVERIFIABLE_ERRORS.has(run.error.trim()) + ) +} + +export function findPotentiallyLiveAutomationRun( + automation: Automation, + currentRunId: string, + runs: readonly AutomationRun[] +): AutomationRun | null { + if (automation.workspaceMode !== 'existing' || automation.workspaceId === null) { + return null + } + return ( + runs.find( + (run) => + run.id !== currentRunId && + run.workspaceId === automation.workspaceId && + (run.observationVerdict === 'unverifiable' || + run.status === 'dispatching' || + run.status === 'dispatched' || + isLegacyUnverifiableRun(run)) + ) ?? null + ) +} + +export function pinUnverifiableAutomationRun(store: Store, run: AutomationRun): void { + if ( + (run.observationVerdict !== 'unverifiable' && !isLegacyUnverifiableRun(run)) || + !isFinalAutomationRunStatus(run.status) + ) { + return + } + store.updateAutomationRun({ + runId: run.id, + status: 'dispatched', + observationVerdict: 'unverifiable', + workspaceId: run.workspaceId, + error: isLegacyUnverifiableRun(run) + ? 'Orca stopped watching this run before it reported completion.' + : run.error + }) +} diff --git a/src/main/automations/service.test.ts b/src/main/automations/service.test.ts index 773a3f3d9ba..9ad5bec21b0 100644 --- a/src/main/automations/service.test.ts +++ b/src/main/automations/service.test.ts @@ -132,6 +132,131 @@ describe('AutomationService', () => { ) }) + it('blocks a manual run when an existing workspace run is unverifiable', async () => { + vi.setSystemTime(new Date('2026-05-13T08:00:00Z')) + const store = await createStore() + store.addRepo(makeRepo()) + const automation = store.createAutomation({ + name: 'Manual check', + prompt: 'Check the repo', + agentId: 'claude', + projectId: 'r1', + workspaceMode: 'existing', + workspaceId: 'wt1', + timezone: 'UTC', + rrule: 'FREQ=DAILY;BYHOUR=9;BYMINUTE=0', + dtstart: new Date('2026-05-14T00:00:00Z').getTime() + }) + const previousRun = store.createAutomationRun(automation, Date.now() - 1_000, 'manual') + store.updateAutomationRun({ + runId: previousRun.id, + status: 'dispatch_failed', + workspaceId: 'wt1', + error: 'terminal_handle_stale' + }) + const send = vi.fn() + const service = new AutomationService(store, { tickMs: 60_000 }) + service.setWebContents({ isDestroyed: () => false, send } as never) + service.setRendererReady() + + const run = await service.runNow(automation.id) + + expect(run).toMatchObject({ + status: 'skipped_unavailable', + error: 'A previous automation run may still be live in this workspace.' + }) + expect( + store.listAutomationRuns(automation.id).find((entry) => entry.id === previousRun.id) + ).toMatchObject({ + status: 'dispatched', + observationVerdict: 'unverifiable', + error: 'Orca stopped watching this run before it reported completion.' + }) + expect(send).not.toHaveBeenCalled() + }) + + it('blocks a scheduled run when an existing workspace run is unverifiable', async () => { + const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone + const beforeRunAt = new Date(2026, 4, 13, 8, 59).getTime() + const scheduledRunAt = new Date(2026, 4, 13, 9, 0).getTime() + vi.setSystemTime(beforeRunAt) + const store = await createStore() + store.addRepo(makeRepo()) + const automation = store.createAutomation({ + name: 'Scheduled check', + prompt: 'Check the repo', + agentId: 'claude', + projectId: 'r1', + workspaceMode: 'existing', + workspaceId: 'wt1', + timezone, + rrule: 'FREQ=DAILY;BYHOUR=9;BYMINUTE=0', + dtstart: new Date(2026, 4, 12, 0, 0).getTime() + }) + const previousRun = store.createAutomationRun(automation, scheduledRunAt - 1_000, 'manual') + store.updateAutomationRun({ + runId: previousRun.id, + status: 'dispatch_failed', + observationVerdict: 'unverifiable', + workspaceId: 'wt1', + error: 'Orca stopped watching this run before it reported completion.' + }) + const headlessDispatcher = vi.fn() + const service = new AutomationService(store, { + tickMs: 60_000, + headlessDispatcher + }) + + vi.setSystemTime(new Date(2026, 4, 13, 9, 1).getTime()) + service.start() + await vi.waitFor(() => + expect(store.listAutomationRuns(automation.id)[0]).toMatchObject({ + status: 'skipped_unavailable', + error: 'A previous automation run may still be live in this workspace.' + }) + ) + service.stop() + expect( + store.listAutomationRuns(automation.id).find((entry) => entry.id === previousRun.id) + ).toMatchObject({ status: 'dispatched', observationVerdict: 'unverifiable' }) + expect(headlessDispatcher).not.toHaveBeenCalled() + }) + + it('blocks another automation targeting the same workspace', async () => { + vi.setSystemTime(new Date('2026-05-13T08:00:00Z')) + const store = await createStore() + store.addRepo(makeRepo()) + const base = { + prompt: 'Check the repo', + agentId: 'claude' as const, + projectId: 'r1', + workspaceMode: 'existing' as const, + workspaceId: 'wt1', + timezone: 'UTC', + rrule: 'FREQ=DAILY;BYHOUR=9;BYMINUTE=0', + dtstart: new Date('2026-05-14T00:00:00Z').getTime() + } + const first = store.createAutomation({ ...base, name: 'First check' }) + const second = store.createAutomation({ ...base, name: 'Second check' }) + const previousRun = store.createAutomationRun(first, Date.now() - 1_000, 'manual') + store.updateAutomationRun({ + runId: previousRun.id, + status: 'dispatched', + observationVerdict: 'unverifiable', + workspaceId: 'wt1', + error: 'Orca stopped watching this run before it reported completion.' + }) + const send = vi.fn() + const service = new AutomationService(store, { tickMs: 60_000 }) + service.setWebContents({ isDestroyed: () => false, send } as never) + service.setRendererReady() + + const run = await service.runNow(second.id) + + expect(run.status).toBe('skipped_unavailable') + expect(send).not.toHaveBeenCalled() + }) + it('skips dispatch when the selected project host setup is gone', async () => { vi.setSystemTime(new Date('2026-05-13T08:00:00Z')) const store = await createStore() @@ -365,6 +490,47 @@ describe('AutomationService', () => { ) }) + it('records a lost completion watch with an additive unverifiable verdict', async () => { + vi.setSystemTime(new Date('2026-05-13T08:00:00Z')) + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}) + const store = await createStore() + store.addRepo(makeRepo()) + const automation = store.createAutomation({ + name: 'Nightly triage', + prompt: 'Triage', + agentId: 'claude', + projectId: 'r1', + workspaceMode: 'new_per_run', + timezone: 'UTC', + rrule: 'FREQ=DAILY;BYHOUR=9;BYMINUTE=0', + dtstart: new Date('2026-05-14T00:00:00Z').getTime() + }) + const service = new AutomationService(store, { + tickMs: 60_000, + headlessDispatcher: vi.fn().mockResolvedValue({ + workspaceId: 'wt-1', + terminalSessionId: 'tab-1', + terminalPaneKey: 'tab-1:11111111-1111-4111-8111-111111111111', + terminalPtyId: 'pty-1', + completion: Promise.reject(new Error('terminal_handle_stale')) + }) + }) + + await service.runNow(automation.id) + + await vi.waitFor(() => + expect(store.listAutomationRuns(automation.id)[0]).toMatchObject({ + status: 'dispatch_failed', + observationVerdict: 'unverifiable', + error: 'Orca stopped watching this run before it reported completion.' + }) + ) + // Why: the transport token is log material, never user copy. + expect(store.listAutomationRuns(automation.id)[0]!.error).not.toContain('terminal_handle_stale') + expect(consoleError).toHaveBeenCalled() + consoleError.mockRestore() + }) + it('dispatches due scheduled automations headlessly', async () => { const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone const beforeRunAt = new Date(2026, 4, 13, 8, 59).getTime() diff --git a/src/main/automations/service.ts b/src/main/automations/service.ts index 9ed9564e5e5..5cd79d416e3 100644 --- a/src/main/automations/service.ts +++ b/src/main/automations/service.ts @@ -1,6 +1,7 @@ import type { WebContents } from 'electron' import type { Store } from '../persistence' import { + AUTOMATION_UNVERIFIABLE_WORKSPACE_ERROR, isFinalAutomationRunStatus, type Automation, type AutomationDispatchRequest, @@ -30,6 +31,10 @@ import type { AutomationsChangedPayload, PublishAutomationsChanged } from '../../shared/runtime-client-events' +import { + findPotentiallyLiveAutomationRun, + pinUnverifiableAutomationRun +} from './potentially-live-run' const DEFAULT_TICK_MS = 60 * 1000 @@ -297,6 +302,20 @@ export class AutomationService { run: AutomationRun, target: AutomationRunTargetResult ): Promise { + const potentiallyLiveRun = findPotentiallyLiveAutomationRun( + automation, + run.id, + this.store.listAutomationRuns() + ) + if (potentiallyLiveRun) { + pinUnverifiableAutomationRun(this.store, potentiallyLiveRun) + return this.store.updateAutomationRun({ + runId: run.id, + status: 'skipped_unavailable', + workspaceId: automation.workspaceId, + error: AUTOMATION_UNVERIFIABLE_WORKSPACE_ERROR + }) + } if (!target.ok) { return this.runs.updateRun({ runId: run.id, diff --git a/src/main/persistence/scheduling-automations/automation-run-operations.ts b/src/main/persistence/scheduling-automations/automation-run-operations.ts index 0dc9e61731a..990867a0a92 100644 --- a/src/main/persistence/scheduling-automations/automation-run-operations.ts +++ b/src/main/persistence/scheduling-automations/automation-run-operations.ts @@ -173,6 +173,9 @@ export function updateAutomationRun( const updated: AutomationRun = { ...current, status: result.status, + observationVerdict: Object.hasOwn(result, 'observationVerdict') + ? (result.observationVerdict ?? null) + : (current.observationVerdict ?? null), workspaceId, workspaceDisplayName: workspaceDisplayName ?? @@ -196,7 +199,8 @@ export function updateAutomationRun( usage: Object.hasOwn(result, 'usage') ? (result.usage ?? null) : (current.usage ?? null), error: result.error ?? null, startedAt: current.startedAt ?? now, - dispatchedAt: result.status === 'dispatched' ? now : current.dispatchedAt + dispatchedAt: + result.status === 'dispatched' ? (current.dispatchedAt ?? now) : current.dispatchedAt } // Replaced, not patched in place: the list projection caches on array identity. operations.state.automationRuns = operations.state.automationRuns.map((run) => diff --git a/src/main/startup/main-process-automations.ts b/src/main/startup/main-process-automations.ts index a980c272070..870d498a959 100644 --- a/src/main/startup/main-process-automations.ts +++ b/src/main/startup/main-process-automations.ts @@ -1,5 +1,6 @@ import { AutomationService } from '../automations/service' import { createHeadlessAutomationOutputSnapshotBuffer } from '../automations/headless-dispatch' +import { waitForHeadlessAutomationCompletion } from '../automations/headless-completion-wait' import { buildHeadlessAutomationWorktreeCreateArgs } from '../automations/headless-workspace-create' import { createRuntimeAutomationRunTerminalObserver } from '../automations/runtime-terminal-run-observer' import { mainProcessState as state } from './main-process-state' @@ -62,7 +63,13 @@ export function initializeMainProcessAutomations(): AutomationService { workspaceDisplayName = worktree.displayName ?? null } const completion = (async () => { - const wait = await runtime.waitForTerminal(terminalHandle, { condition: 'tui-idle' }) + const wait = await waitForHeadlessAutomationCompletion(runtime, terminalHandle) + if (wait.status === 'exited') { + return { + status: 'dispatch_failed' as const, + error: 'Automation terminal exited before the agent reported completion.' + } + } const read = await runtime.readTerminal(terminalHandle, { limit: terminalSnapshotLimit }) @@ -77,10 +84,9 @@ export function initializeMainProcessAutomations(): AutomationService { } return { status: 'dispatch_failed' as const, + observationVerdict: 'unverifiable' as const, outputSnapshot: snapshotBuffer.snapshot(), - error: wait.blockedReason - ? `Automation agent is blocked: ${wait.blockedReason}.` - : 'Automation agent did not report completion.' + error: 'Orca never saw this run report completion, so its result is unknown.' } })() return { diff --git a/src/renderer/src/components/automations/AutomationRunDetailsPage.tsx b/src/renderer/src/components/automations/AutomationRunDetailsPage.tsx index 63bdf933434..403c11aa22f 100644 --- a/src/renderer/src/components/automations/AutomationRunDetailsPage.tsx +++ b/src/renderer/src/components/automations/AutomationRunDetailsPage.tsx @@ -6,7 +6,8 @@ import { cn } from '@/lib/utils' import { translate } from '@/i18n/i18n' import type { Automation, AutomationRun } from '../../../../shared/automations-types' import { AutomationRunPageFrame } from './AutomationRunPageFrame' -import { getAutomationRunContent } from './automation-run-content' +import { getAutomationRunContent, getAutomationRunNotice } from './automation-run-content' +import { AutomationRunNoticeBand } from './AutomationRunNoticeBand' import type { AutomationRunViewState } from './automation-run-view-state' import type { AutomationRunWorkspaceDisplay } from './automation-run-workspace-display' import { @@ -38,6 +39,7 @@ export function AutomationRunDetailsPage({ onOpenWorkspace: () => void onBack: () => void }): React.JSX.Element { + const notice = getAutomationRunNotice(run) return (
: null} actions={ <> {canRerun && automation ? ( diff --git a/src/renderer/src/components/automations/AutomationRunHistory.tsx b/src/renderer/src/components/automations/AutomationRunHistory.tsx index cc36d416f14..79cb8a077a1 100644 --- a/src/renderer/src/components/automations/AutomationRunHistory.tsx +++ b/src/renderer/src/components/automations/AutomationRunHistory.tsx @@ -216,8 +216,10 @@ export function AutomationRunHistory({ )}
- - {getAutomationRunStatusLabel(run.status)} + + {getAutomationRunStatusLabel(run.status, run.observationVerdict)}
diff --git a/src/renderer/src/components/automations/AutomationRunNoticeBand.tsx b/src/renderer/src/components/automations/AutomationRunNoticeBand.tsx new file mode 100644 index 00000000000..6ea94e97849 --- /dev/null +++ b/src/renderer/src/components/automations/AutomationRunNoticeBand.tsx @@ -0,0 +1,27 @@ +import React from 'react' +import { AlertTriangle, Info } from 'lucide-react' +import { cn } from '@/lib/utils' +import type { AutomationRunNotice } from './automation-run-content' + +export function AutomationRunNoticeBand({ + notice +}: { + notice: AutomationRunNotice +}): React.JSX.Element { + const isError = notice.tone === 'error' + const Icon = isError ? AlertTriangle : Info + return ( +
+
+ ) +} diff --git a/src/renderer/src/components/automations/AutomationRunPageFrame.test.tsx b/src/renderer/src/components/automations/AutomationRunPageFrame.test.tsx new file mode 100644 index 00000000000..088f47fa5ee --- /dev/null +++ b/src/renderer/src/components/automations/AutomationRunPageFrame.test.tsx @@ -0,0 +1,70 @@ +// @vitest-environment happy-dom + +import { cleanup, render, screen } from '@testing-library/react' +import { afterEach, describe, expect, it } from 'vitest' +import { AutomationRunPageFrame } from './AutomationRunPageFrame' +import { AutomationRunNoticeBand } from './AutomationRunNoticeBand' + +afterEach(cleanup) + +describe('AutomationRunPageFrame notice', () => { + it('renders the run reason above the output body', () => { + render( + + } + onBack={() => {}} + > + 
{'{"id":"local-status","ok":true}'}
+ + ) + + const reason = screen.getByText('Orca stopped watching this run before it reported completion.') + const body = screen.getByText('{"id":"local-status","ok":true}') + expect(reason.closest('[role="status"]')).toBe(screen.getByRole('status')) + expect(reason).toBeTruthy() + // Why: the reason must precede the body in the DOM, not scroll with it. + expect(reason.compareDocumentPosition(body) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy() + }) + + it('adds no empty band when a run ended with no reason', () => { + const { container } = render( + {}} + > + 
report
+ + ) + + // Why count children: an always-rendered band would add a stray bordered strip + // between the header and the body on every healthy run. + expect(container.firstElementChild?.children).toHaveLength(2) + expect(screen.getByText('report')).toBeTruthy() + }) + + it('announces observed failures as alerts', () => { + render( + + ) + + expect(screen.getByRole('alert').textContent).toContain( + 'Automation process exited with code 1.' + ) + }) +}) diff --git a/src/renderer/src/components/automations/AutomationRunPageFrame.tsx b/src/renderer/src/components/automations/AutomationRunPageFrame.tsx index e51c94155e7..67f05ddfa38 100644 --- a/src/renderer/src/components/automations/AutomationRunPageFrame.tsx +++ b/src/renderer/src/components/automations/AutomationRunPageFrame.tsx @@ -11,6 +11,8 @@ type AutomationRunPageFrameProps = { statusVariant: React.ComponentProps['variant'] detail?: string | null actions?: React.ReactNode + /** Pinned under the header so the reason a run ended cannot scroll out of view. */ + notice?: React.ReactNode children: React.ReactNode onBack: () => void } @@ -22,6 +24,7 @@ export function AutomationRunPageFrame({ statusVariant, detail, actions, + notice, children, onBack }: AutomationRunPageFrameProps): React.JSX.Element { @@ -79,6 +82,7 @@ export function AutomationRunPageFrame({ {actions} + {notice}
{children}
) diff --git a/src/renderer/src/components/automations/automation-list-last-run.test.ts b/src/renderer/src/components/automations/automation-list-last-run.test.ts index 7eb54e123a5..2b56ea1ee91 100644 --- a/src/renderer/src/components/automations/automation-list-last-run.test.ts +++ b/src/renderer/src/components/automations/automation-list-last-run.test.ts @@ -104,6 +104,15 @@ describe('automation-list-last-run', () => { expect(getToneForAutomationRunStatus('completed')).toBe('succeeded') expect(getToneForAutomationRunStatus('dispatched')).toBe('running') expect(getToneForAutomationRunStatus('skipped_precheck')).toBe('skipped') + expect(getToneForAutomationRunStatus('dispatch_failed', 'unverifiable')).toBe('unknown') + }) + + it('renders additive unverifiable metadata without changing the legacy status', () => { + const snapshot = getLocalAutomationLastRunSnapshot( + makeAutomation(), + makeRun({ status: 'dispatch_failed', observationVerdict: 'unverifiable' }) + ) + expect(snapshot).toMatchObject({ tone: 'unknown', statusLabel: 'Unverifiable' }) }) it('prefers the latest run over lastRunAt-only metadata', () => { diff --git a/src/renderer/src/components/automations/automation-list-last-run.ts b/src/renderer/src/components/automations/automation-list-last-run.ts index b5f72275273..d36f8d3421a 100644 --- a/src/renderer/src/components/automations/automation-list-last-run.ts +++ b/src/renderer/src/components/automations/automation-list-last-run.ts @@ -52,7 +52,13 @@ export function getAutomationRunLastRunAt(run: AutomationRun): number { return run.dispatchedAt ?? run.startedAt ?? run.createdAt } -export function getToneForAutomationRunStatus(status: AutomationRunStatus): AutomationLastRunTone { +export function getToneForAutomationRunStatus( + status: AutomationRunStatus, + observationVerdict?: AutomationRun['observationVerdict'] +): AutomationLastRunTone { + if (observationVerdict === 'unverifiable') { + return 'unknown' + } if (status === 'dispatch_failed') { return 'failed' } @@ -95,8 +101,8 @@ export function getLocalAutomationLastRunSnapshot( if (lastRun) { return { at: getAutomationRunLastRunAt(lastRun), - tone: getToneForAutomationRunStatus(lastRun.status), - statusLabel: getAutomationRunStatusLabel(lastRun.status) + tone: getToneForAutomationRunStatus(lastRun.status, lastRun.observationVerdict), + statusLabel: getAutomationRunStatusLabel(lastRun.status, lastRun.observationVerdict) } } if (automation.lastRunAt) { diff --git a/src/renderer/src/components/automations/automation-page-parts.tsx b/src/renderer/src/components/automations/automation-page-parts.tsx index 4ffd81001bb..b9f61c2fe04 100644 --- a/src/renderer/src/components/automations/automation-page-parts.tsx +++ b/src/renderer/src/components/automations/automation-page-parts.tsx @@ -54,8 +54,12 @@ export function formatAutomationDateTimeWithRelative( } export function getAutomationRunStatusVariant( - status: AutomationRun['status'] + status: AutomationRun['status'], + observationVerdict?: AutomationRun['observationVerdict'] ): React.ComponentProps['variant'] { + if (observationVerdict === 'unverifiable') { + return 'outline' + } if (status === 'dispatched' || status === 'completed') { return 'secondary' } @@ -68,7 +72,13 @@ export function getAutomationRunStatusVariant( return 'dot' } -export function getAutomationRunStatusLabel(status: AutomationRun['status']): string { +export function getAutomationRunStatusLabel( + status: AutomationRun['status'], + observationVerdict?: AutomationRun['observationVerdict'] +): string { + if (observationVerdict === 'unverifiable') { + return 'Unverifiable' + } switch (status) { case 'pending': return 'Queued' diff --git a/src/renderer/src/components/automations/automation-run-content.test.ts b/src/renderer/src/components/automations/automation-run-content.test.ts new file mode 100644 index 00000000000..c3191ff9670 --- /dev/null +++ b/src/renderer/src/components/automations/automation-run-content.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from 'vitest' +import type { AutomationPrecheckResult, AutomationRun } from '../../../../shared/automations-types' +import { getAutomationRunContent, getAutomationRunNotice } from './automation-run-content' + +function makeRun(overrides: Partial = {}): AutomationRun { + return { + id: 'run-1', + automationId: 'automation-1', + title: 'Run 1', + scheduledFor: 1, + status: 'completed', + trigger: 'scheduled', + workspaceId: 'wt-1', + sessionKind: 'terminal', + chatSessionId: null, + terminalSessionId: 'tab-1', + terminalPaneKey: 'tab-1:pane-1', + terminalPtyId: 'pty-1', + outputSnapshot: null, + precheckResult: null, + usage: null, + error: null, + startedAt: 1, + dispatchedAt: 1, + createdAt: 1, + ...overrides + } +} + +function makePassingPrecheck(stdout: string): AutomationPrecheckResult { + return { + command: 'orca status --json', + exitCode: 0, + timedOut: false, + durationMs: 210, + stdout, + stderr: '', + stdoutTruncated: false, + stderrTruncated: false, + error: null, + startedAt: 1, + completedAt: 2 + } +} + +describe('getAutomationRunNotice', () => { + it('surfaces the run error even when a passing precheck fills the body', () => { + const run = makeRun({ + status: 'dispatched', + observationVerdict: 'unverifiable', + precheckResult: makePassingPrecheck('{"id":"local-status","ok":true}'), + error: 'Orca stopped watching this run before it reported completion.' + }) + // Why this pairing: the precheck stdout used to be the only thing the run page + // rendered, so the reason was invisible on exactly the runs that needed it. + expect(getAutomationRunContent(run)).toContain('local-status') + expect(getAutomationRunNotice(run)).toEqual({ + text: 'Orca stopped watching this run before it reported completion.', + tone: 'neutral' + }) + }) + + it('marks only an observed failure with the error tone', () => { + expect( + getAutomationRunNotice( + makeRun({ status: 'dispatch_failed', error: 'Automation process exited with code 1.' }) + ) + ).toEqual({ text: 'Automation process exited with code 1.', tone: 'error' }) + }) + + it('returns nothing for a run that ended without a reason', () => { + expect(getAutomationRunNotice(makeRun({ error: ' ' }))).toBeNull() + expect(getAutomationRunNotice(makeRun())).toBeNull() + }) +}) + +describe('getAutomationRunContent', () => { + it('prefers the saved output snapshot over the precheck output', () => { + expect( + getAutomationRunContent( + makeRun({ + outputSnapshot: { + format: 'plain_text', + content: '# Triage report', + capturedAt: 3, + truncated: false + }, + precheckResult: makePassingPrecheck('{"ok":true}') + }) + ) + ).toBe('# Triage report') + }) + + it('no longer repeats the error the notice already carries', () => { + expect(getAutomationRunContent(makeRun({ status: 'dispatch_failed', error: 'boom' }))).toBe( + 'No output content available.' + ) + }) +}) diff --git a/src/renderer/src/components/automations/automation-run-content.ts b/src/renderer/src/components/automations/automation-run-content.ts index 5ea2eddabb4..e66a04fedb1 100644 --- a/src/renderer/src/components/automations/automation-run-content.ts +++ b/src/renderer/src/components/automations/automation-run-content.ts @@ -1,5 +1,12 @@ import type { AutomationRun } from '../../../../shared/automations-types' +export type AutomationRunNoticeTone = 'error' | 'neutral' + +export type AutomationRunNotice = { + text: string + tone: AutomationRunNoticeTone +} + export function getAutomationRunContent(run: AutomationRun): string { const savedOutput = run.outputSnapshot?.content.trim() if (savedOutput) { @@ -13,5 +20,21 @@ export function getAutomationRunContent(run: AutomationRun): string { return output } } - return run.error ?? run.usage?.unavailableMessage ?? 'No output content available.' + return run.usage?.unavailableMessage ?? 'No output content available.' +} + +/** Why separate from the body: a passing precheck's stdout outranks `run.error` there, + * so the reason a run ended was invisible on exactly the runs that needed it. */ +export function getAutomationRunNotice(run: AutomationRun): AutomationRunNotice | null { + const text = run.error?.trim() + if (!text) { + return null + } + return { + text, + tone: + run.status === 'dispatch_failed' && run.observationVerdict !== 'unverifiable' + ? 'error' + : 'neutral' + } } diff --git a/src/renderer/src/components/automations/automation-run-view-state.test.ts b/src/renderer/src/components/automations/automation-run-view-state.test.ts index ce5477aa14a..1389550bf10 100644 --- a/src/renderer/src/components/automations/automation-run-view-state.test.ts +++ b/src/renderer/src/components/automations/automation-run-view-state.test.ts @@ -169,6 +169,27 @@ describe('canRerunAutomationRun', () => { } ) + it('hides rerun while the original run is unverifiable', () => { + expect( + canRerunAutomationRun({ + automation: makeAutomation(), + run: makeRun({ status: 'dispatch_failed', observationVerdict: 'unverifiable' }) + }) + ).toBe(false) + }) + + it('hides rerun for a host-blocked retry row', () => { + expect( + canRerunAutomationRun({ + automation: makeAutomation(), + run: makeRun({ + status: 'skipped_unavailable', + error: 'A previous automation run may still be live in this workspace.' + }) + }) + ).toBe(false) + }) + it.each([ 'pending', 'dispatching', diff --git a/src/renderer/src/components/automations/automation-run-view-state.ts b/src/renderer/src/components/automations/automation-run-view-state.ts index 56f55dec088..5308d07d65f 100644 --- a/src/renderer/src/components/automations/automation-run-view-state.ts +++ b/src/renderer/src/components/automations/automation-run-view-state.ts @@ -1,4 +1,8 @@ -import type { Automation, AutomationRun } from '../../../../shared/automations-types' +import { + AUTOMATION_UNVERIFIABLE_WORKSPACE_ERROR, + type Automation, + type AutomationRun +} from '../../../../shared/automations-types' export type AutomationRunViewAvailability = 'terminal' | 'workspace' | 'snapshot' | 'metadata' @@ -41,6 +45,12 @@ export function canRerunAutomationRun({ if (!automation || run.automationId !== automation.id) { return false } + if ( + run.observationVerdict === 'unverifiable' || + run.error === AUTOMATION_UNVERIFIABLE_WORKSPACE_ERROR + ) { + return false + } return ( run.status === 'dispatch_failed' || run.status === 'skipped_unavailable' || diff --git a/src/shared/automation-run-retention.test.ts b/src/shared/automation-run-retention.test.ts index c405c43f37b..f8f0e679f52 100644 --- a/src/shared/automation-run-retention.test.ts +++ b/src/shared/automation-run-retention.test.ts @@ -95,6 +95,19 @@ describe('pruneAutomationRuns', () => { ]) }) + it('retains an unverifiable observation as in-flight history', () => { + const unverifiable = run({ + id: 'unverifiable', + automationId: 'a', + status: 'dispatched', + observationVerdict: 'unverifiable', + createdAt: -1 + }) + const kept = pruneAutomationRuns([unverifiable, ...makeRuns('a', 120)]) + expect(kept).toContainEqual(unverifiable) + expect(kept.filter((entry) => entry.automationId === 'a')).toHaveLength(101) + }) + it('shrinks a realistic runaway history to the cap', () => { const runaway = [ ...makeRuns('a', 2796), diff --git a/src/shared/automations-types.ts b/src/shared/automations-types.ts index 80a56e3cdeb..db532398480 100644 --- a/src/shared/automations-types.ts +++ b/src/shared/automations-types.ts @@ -6,6 +6,9 @@ export type AutomationWorkspaceMode = 'existing' | 'new_per_run' export type AutomationExecutionTargetType = 'local' | 'ssh' export type AutomationSchedulerOwner = 'local_host_service' | 'ssh_bridge' | 'remote_host_service' export type AutomationMissedRunPolicy = 'run_once_within_grace' +export type AutomationRunObservationVerdict = 'unverifiable' +export const AUTOMATION_UNVERIFIABLE_WORKSPACE_ERROR = + 'A previous automation run may still be live in this workspace.' export type AutomationRunStatus = | 'pending' | 'dispatching' @@ -140,6 +143,8 @@ export type AutomationRun = { title: string scheduledFor: number status: AutomationRunStatus + /** Additive refinement for legacy `dispatch_failed`; old builds safely ignore it. */ + observationVerdict?: AutomationRunObservationVerdict | null trigger: AutomationRunTrigger workspaceId: string | null /** Why: run history must remain understandable after the backing workspace @@ -232,6 +237,7 @@ export type AutomationDispatchRequest = { export type AutomationDispatchResult = { runId: string status: AutomationRunStatus + observationVerdict?: AutomationRunObservationVerdict | null workspaceId?: string | null workspaceDisplayName?: string | null terminalSessionId?: string | null