diff --git a/src/main/agent-turn-idle-verdict.test.ts b/src/main/agent-turn-idle-verdict.test.ts new file mode 100644 index 00000000000..4132feee0b0 --- /dev/null +++ b/src/main/agent-turn-idle-verdict.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest' +import { + AGENT_START_GRACE_MS, + judgeIdleAgentTurn, + type AgentTurnStatusRow +} from './agent-turn-idle-verdict' + +const SENT_AT = 10_000 + +function judge(rows: AgentTurnStatusRow[], tail: string[] = [], now = SENT_AT + 1) { + return judgeIdleAgentTurn( + { rows, sentAt: SENT_AT, tail, readAgentCommands: () => ['claude'] }, + now + ) +} + +describe('judgeIdleAgentTurn', () => { + it('completes on a done reached since the turn was sent', () => { + expect(judge([{ state: 'done', stateStartedAt: SENT_AT }])).toEqual({ kind: 'completed' }) + }) + + it('waits past a previous turn, a connect-time or a resume-only done', () => { + expect(judge([{ state: 'done', stateStartedAt: SENT_AT - 1 }])).toEqual({ kind: 'wait' }) + expect(judge([{ state: 'done', stateStartedAt: SENT_AT, sessionBoundary: true }])).toEqual({ + kind: 'wait' + }) + expect(judge([{ state: 'done', stateStartedAt: SENT_AT, providerSessionOnly: true }])).toEqual({ + kind: 'wait' + }) + }) + + it('fails when the shell refused the agent command', () => { + expect(judge([], ['zsh: command not found: claude'])).toEqual({ + kind: 'failed', + missingCommandLine: 'zsh: command not found: claude' + }) + }) + + it('believes an idle pane only for an agent that never reported, after the grace', () => { + expect(judge([], [], SENT_AT + AGENT_START_GRACE_MS)).toEqual({ kind: 'completed' }) + expect(judge([], [], SENT_AT + AGENT_START_GRACE_MS - 1)).toEqual({ kind: 'wait' }) + expect(judge([{ state: 'working', stateStartedAt: SENT_AT }], [], SENT_AT * 100)).toEqual({ + kind: 'wait' + }) + }) +}) diff --git a/src/main/agent-turn-idle-verdict.ts b/src/main/agent-turn-idle-verdict.ts new file mode 100644 index 00000000000..b0e2bac67b0 --- /dev/null +++ b/src/main/agent-turn-idle-verdict.ts @@ -0,0 +1,89 @@ +/** + * Whether an agent's turn, sent at a known moment, is really over once its pane reads idle. A ready + * shell prompt satisfies `tui-idle` too, so a prompt typed at a shell whose agent is not installed + * ("command not found") would read as finished. The agent's own `done`, reached since the turn was + * sent, completes it; a reused pane's previous `done` is not this turn's. Not every agent reports + * status (no hooks on the host, no recognised title), so for a pane that never reported any an idle + * pane still means done, but only after the agent had time to start and only when no shell refused + * its command. Automations judge their runs by it; a connector judges each message's turn by it. + */ +import type { AgentStatusIpcPayload } from '../shared/agent-status-types' + +/** Covers agent spin-up over SSH before an idle pane without agent status is believed. */ +export const AGENT_START_GRACE_MS = 2 * 60 * 1000 +/** Only output after the prompt: a refusal further up predates this turn. */ +const MISSING_COMMAND_TAIL_LINES = 6 + +// bash, zsh, dash/sh, fish, PowerShell and cmd.exe refusing a command that does not exist. +const MISSING_COMMAND_PATTERNS = [ + /command not found/i, + /^\S+: \d+: \S+: not found$/i, + /unknown command/i, + /is not recognized as (?:the name of a cmdlet|an internal or external command)/i +] + +export type AgentTurnStatusRow = Pick< + AgentStatusIpcPayload, + 'state' | 'stateStartedAt' | 'sessionBoundary' | 'providerSessionOnly' +> + +export type IdleAgentTurnEvidence = { + /** The pane's agent status rows, from hooks, OSC and titles alike. */ + rows: readonly AgentTurnStatusRow[] + /** When the turn's prompt went in: status from before it is a previous turn's. */ + sentAt: number + tail: readonly string[] + /** Read only when no fresh `done` decides it: the names the agent command may run under. */ + readAgentCommands: () => readonly string[] +} + +export type IdleAgentTurnVerdict = + | { kind: 'completed' } + | { kind: 'failed'; missingCommandLine: string } + /** An idle shell that may not have started the agent yet. */ + | { kind: 'wait' } + +/** A shell refusing one of the agent's own commands; an agent's tool output never matches. */ +export function findMissingCommandLine( + tail: readonly string[], + commands: readonly string[] +): string | null { + const names = commands.map((command) => command.split(/\s+/)[0]).filter(Boolean) + const recent = tail + .map((line) => line.trim()) + .filter(Boolean) + .slice(-MISSING_COMMAND_TAIL_LINES) + return ( + recent.find( + (line) => + MISSING_COMMAND_PATTERNS.some((pattern) => pattern.test(line)) && + names.some((name) => line.includes(name)) + ) ?? null + ) +} + +export function judgeIdleAgentTurn( + evidence: IdleAgentTurnEvidence, + now: number +): IdleAgentTurnVerdict { + const { sentAt } = evidence + // Resume identity is no status report. + const rows = evidence.rows.filter((row) => !row.providerSessionOnly) + if ( + rows.some( + (row) => + // A connect-time `done` reports no turn. + row.state === 'done' && !row.sessionBoundary && row.stateStartedAt >= sentAt + ) + ) { + return { kind: 'completed' } + } + const missing = findMissingCommandLine(evidence.tail, evidence.readAgentCommands()) + if (missing) { + return { kind: 'failed', missingCommandLine: missing } + } + // Only an agent that never reports status is believed done by an idle pane. + return rows.length === 0 && now - sentAt >= AGENT_START_GRACE_MS + ? { kind: 'completed' } + : { kind: 'wait' } +} diff --git a/src/main/automations/automation-run-agent-evidence.ts b/src/main/automations/automation-run-agent-evidence.ts index bde6fafc437..5fb96189a63 100644 --- a/src/main/automations/automation-run-agent-evidence.ts +++ b/src/main/automations/automation-run-agent-evidence.ts @@ -1,35 +1,12 @@ -/** - * What an idle pane says about a run. A ready shell prompt satisfies `tui-idle` too, so a prompt - * typed at a shell whose agent is not installed ("command not found") would read as finished. - * The agent's own `done`, reached since the run was dispatched, completes it as on the desktop; a - * reused pane's previous `done` is not this run's. Not every agent reports status (no hooks on the - * host, no recognised title), so for a pane that never reported any an idle pane still means done, - * but only after the agent had time to start and only when no shell refused its command. - */ -import type { AgentStatusIpcPayload } from '../../shared/agent-status-types' +/** What an idle pane says about a run, judged as any agent turn sent at the run's dispatch. */ import type { AutomationRun } from '../../shared/automations-types' +import { judgeIdleAgentTurn, type AgentTurnStatusRow } from '../agent-turn-idle-verdict' -/** Covers agent spin-up over SSH before an idle pane without agent status is believed. */ -export const AGENT_START_GRACE_MS = 2 * 60 * 1000 -/** Only output after the prompt: a refusal further up predates this run. */ -const MISSING_COMMAND_TAIL_LINES = 6 - -// bash, zsh, dash/sh, fish, PowerShell and cmd.exe refusing a command that does not exist. -const MISSING_COMMAND_PATTERNS = [ - /command not found/i, - /^\S+: \d+: \S+: not found$/i, - /unknown command/i, - /is not recognized as (?:the name of a cmdlet|an internal or external command)/i -] +export { AGENT_START_GRACE_MS } from '../agent-turn-idle-verdict' export type AutomationRunAgentEvidence = { /** Agent status rows for a pane, from hooks, OSC and titles alike. */ - getAgentStatusRowsForPane( - paneKey: string - ): readonly Pick< - AgentStatusIpcPayload, - 'state' | 'stateStartedAt' | 'sessionBoundary' | 'providerSessionOnly' - >[] + getAgentStatusRowsForPane(paneKey: string): readonly AgentTurnStatusRow[] /** The names the run's agent command may run under, to tell its refusal from its output. */ agentCommandsForRun(run: AutomationRun): readonly string[] } @@ -40,25 +17,6 @@ export type IdleRunVerdict = /** An idle shell that may not have started the agent yet. */ | { kind: 'wait' } -/** A shell refusing one of the agent's own commands; an agent's tool output never matches. */ -export function findMissingCommandLine( - tail: readonly string[], - commands: readonly string[] -): string | null { - const names = commands.map((command) => command.split(/\s+/)[0]).filter(Boolean) - const recent = tail - .map((line) => line.trim()) - .filter(Boolean) - .slice(-MISSING_COMMAND_TAIL_LINES) - return ( - recent.find( - (line) => - MISSING_COMMAND_PATTERNS.some((pattern) => pattern.test(line)) && - names.some((name) => line.includes(name)) - ) ?? null - ) -} - export function judgeIdleRun( evidence: AutomationRunAgentEvidence, run: AutomationRun, @@ -66,32 +24,20 @@ export function judgeIdleRun( runStartedAt: number, now: number ): IdleRunVerdict { - // A reused pane's earlier `done` ended before this run's prompt went in. - const since = run.dispatchedAt ?? runStartedAt - // Resume identity is no status report. - const rows = run.terminalPaneKey - ? evidence - .getAgentStatusRowsForPane(run.terminalPaneKey) - .filter((row) => !row.providerSessionOnly) - : [] - if ( - rows.some( - (row) => - // A connect-time `done` reports no turn. - row.state === 'done' && !row.sessionBoundary && row.stateStartedAt >= since - ) - ) { - return { kind: 'completed' } - } - const missing = findMissingCommandLine(tail, evidence.agentCommandsForRun(run)) - if (missing) { - return { - kind: 'failed', - error: `Automation agent did not start; this host could not run its command (${missing}).` - } - } - // Only an agent that never reports status is believed done by an idle pane. - return rows.length === 0 && now - since >= AGENT_START_GRACE_MS - ? { kind: 'completed' } - : { kind: 'wait' } + const verdict = judgeIdleAgentTurn( + { + rows: run.terminalPaneKey ? evidence.getAgentStatusRowsForPane(run.terminalPaneKey) : [], + // A reused pane's earlier `done` ended before this run's prompt went in. + sentAt: run.dispatchedAt ?? runStartedAt, + tail, + readAgentCommands: () => evidence.agentCommandsForRun(run) + }, + now + ) + return verdict.kind === 'failed' + ? { + kind: 'failed', + error: `Automation agent did not start; this host could not run its command (${verdict.missingCommandLine}).` + } + : verdict }