mirror of
https://github.com/stablyai/orca.git
synced 2026-10-31 00:57:45 +00:00
refactor(automations): share the idle agent-turn completion rule (#32114)
judgeIdleRun's rule (fresh done since the prompt went in, shell refusal of the agent command, 2-minute grace for agents that never report status) moves into judgeIdleAgentTurn, keyed by rows and send time instead of an AutomationRun. Automations keep their wording and call order. No behaviour change.
This commit is contained in:
Jinwoo Hong
@@ -0,0 +1,46 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
AGENT_START_GRACE_MS,
|
||||
judgeIdleAgentTurn,
|
||||
type AgentTurnStatusRow
|
||||
} from './agent-turn-idle-verdict'
|
||||
|
||||
const SENT_AT = 10_000
|
||||
|
||||
function judge(rows: AgentTurnStatusRow[], tail: string[] = [], now = SENT_AT + 1) {
|
||||
return judgeIdleAgentTurn(
|
||||
{ rows, sentAt: SENT_AT, tail, readAgentCommands: () => ['claude'] },
|
||||
now
|
||||
)
|
||||
}
|
||||
|
||||
describe('judgeIdleAgentTurn', () => {
|
||||
it('completes on a done reached since the turn was sent', () => {
|
||||
expect(judge([{ state: 'done', stateStartedAt: SENT_AT }])).toEqual({ kind: 'completed' })
|
||||
})
|
||||
|
||||
it('waits past a previous turn, a connect-time or a resume-only done', () => {
|
||||
expect(judge([{ state: 'done', stateStartedAt: SENT_AT - 1 }])).toEqual({ kind: 'wait' })
|
||||
expect(judge([{ state: 'done', stateStartedAt: SENT_AT, sessionBoundary: true }])).toEqual({
|
||||
kind: 'wait'
|
||||
})
|
||||
expect(judge([{ state: 'done', stateStartedAt: SENT_AT, providerSessionOnly: true }])).toEqual({
|
||||
kind: 'wait'
|
||||
})
|
||||
})
|
||||
|
||||
it('fails when the shell refused the agent command', () => {
|
||||
expect(judge([], ['zsh: command not found: claude'])).toEqual({
|
||||
kind: 'failed',
|
||||
missingCommandLine: 'zsh: command not found: claude'
|
||||
})
|
||||
})
|
||||
|
||||
it('believes an idle pane only for an agent that never reported, after the grace', () => {
|
||||
expect(judge([], [], SENT_AT + AGENT_START_GRACE_MS)).toEqual({ kind: 'completed' })
|
||||
expect(judge([], [], SENT_AT + AGENT_START_GRACE_MS - 1)).toEqual({ kind: 'wait' })
|
||||
expect(judge([{ state: 'working', stateStartedAt: SENT_AT }], [], SENT_AT * 100)).toEqual({
|
||||
kind: 'wait'
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,89 @@
|
||||
/**
|
||||
* Whether an agent's turn, sent at a known moment, is really over once its pane reads idle. A ready
|
||||
* shell prompt satisfies `tui-idle` too, so a prompt typed at a shell whose agent is not installed
|
||||
* ("command not found") would read as finished. The agent's own `done`, reached since the turn was
|
||||
* sent, completes it; a reused pane's previous `done` is not this turn's. Not every agent reports
|
||||
* status (no hooks on the host, no recognised title), so for a pane that never reported any an idle
|
||||
* pane still means done, but only after the agent had time to start and only when no shell refused
|
||||
* its command. Automations judge their runs by it; a connector judges each message's turn by it.
|
||||
*/
|
||||
import type { AgentStatusIpcPayload } from '../shared/agent-status-types'
|
||||
|
||||
/** Covers agent spin-up over SSH before an idle pane without agent status is believed. */
|
||||
export const AGENT_START_GRACE_MS = 2 * 60 * 1000
|
||||
/** Only output after the prompt: a refusal further up predates this turn. */
|
||||
const MISSING_COMMAND_TAIL_LINES = 6
|
||||
|
||||
// bash, zsh, dash/sh, fish, PowerShell and cmd.exe refusing a command that does not exist.
|
||||
const MISSING_COMMAND_PATTERNS = [
|
||||
/command not found/i,
|
||||
/^\S+: \d+: \S+: not found$/i,
|
||||
/unknown command/i,
|
||||
/is not recognized as (?:the name of a cmdlet|an internal or external command)/i
|
||||
]
|
||||
|
||||
export type AgentTurnStatusRow = Pick<
|
||||
AgentStatusIpcPayload,
|
||||
'state' | 'stateStartedAt' | 'sessionBoundary' | 'providerSessionOnly'
|
||||
>
|
||||
|
||||
export type IdleAgentTurnEvidence = {
|
||||
/** The pane's agent status rows, from hooks, OSC and titles alike. */
|
||||
rows: readonly AgentTurnStatusRow[]
|
||||
/** When the turn's prompt went in: status from before it is a previous turn's. */
|
||||
sentAt: number
|
||||
tail: readonly string[]
|
||||
/** Read only when no fresh `done` decides it: the names the agent command may run under. */
|
||||
readAgentCommands: () => readonly string[]
|
||||
}
|
||||
|
||||
export type IdleAgentTurnVerdict =
|
||||
| { kind: 'completed' }
|
||||
| { kind: 'failed'; missingCommandLine: string }
|
||||
/** An idle shell that may not have started the agent yet. */
|
||||
| { kind: 'wait' }
|
||||
|
||||
/** A shell refusing one of the agent's own commands; an agent's tool output never matches. */
|
||||
export function findMissingCommandLine(
|
||||
tail: readonly string[],
|
||||
commands: readonly string[]
|
||||
): string | null {
|
||||
const names = commands.map((command) => command.split(/\s+/)[0]).filter(Boolean)
|
||||
const recent = tail
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.slice(-MISSING_COMMAND_TAIL_LINES)
|
||||
return (
|
||||
recent.find(
|
||||
(line) =>
|
||||
MISSING_COMMAND_PATTERNS.some((pattern) => pattern.test(line)) &&
|
||||
names.some((name) => line.includes(name))
|
||||
) ?? null
|
||||
)
|
||||
}
|
||||
|
||||
export function judgeIdleAgentTurn(
|
||||
evidence: IdleAgentTurnEvidence,
|
||||
now: number
|
||||
): IdleAgentTurnVerdict {
|
||||
const { sentAt } = evidence
|
||||
// Resume identity is no status report.
|
||||
const rows = evidence.rows.filter((row) => !row.providerSessionOnly)
|
||||
if (
|
||||
rows.some(
|
||||
(row) =>
|
||||
// A connect-time `done` reports no turn.
|
||||
row.state === 'done' && !row.sessionBoundary && row.stateStartedAt >= sentAt
|
||||
)
|
||||
) {
|
||||
return { kind: 'completed' }
|
||||
}
|
||||
const missing = findMissingCommandLine(evidence.tail, evidence.readAgentCommands())
|
||||
if (missing) {
|
||||
return { kind: 'failed', missingCommandLine: missing }
|
||||
}
|
||||
// Only an agent that never reports status is believed done by an idle pane.
|
||||
return rows.length === 0 && now - sentAt >= AGENT_START_GRACE_MS
|
||||
? { kind: 'completed' }
|
||||
: { kind: 'wait' }
|
||||
}
|
||||
@@ -1,35 +1,12 @@
|
||||
/**
|
||||
* What an idle pane says about a run. A ready shell prompt satisfies `tui-idle` too, so a prompt
|
||||
* typed at a shell whose agent is not installed ("command not found") would read as finished.
|
||||
* The agent's own `done`, reached since the run was dispatched, completes it as on the desktop; a
|
||||
* reused pane's previous `done` is not this run's. Not every agent reports status (no hooks on the
|
||||
* host, no recognised title), so for a pane that never reported any an idle pane still means done,
|
||||
* but only after the agent had time to start and only when no shell refused its command.
|
||||
*/
|
||||
import type { AgentStatusIpcPayload } from '../../shared/agent-status-types'
|
||||
/** What an idle pane says about a run, judged as any agent turn sent at the run's dispatch. */
|
||||
import type { AutomationRun } from '../../shared/automations-types'
|
||||
import { judgeIdleAgentTurn, type AgentTurnStatusRow } from '../agent-turn-idle-verdict'
|
||||
|
||||
/** Covers agent spin-up over SSH before an idle pane without agent status is believed. */
|
||||
export const AGENT_START_GRACE_MS = 2 * 60 * 1000
|
||||
/** Only output after the prompt: a refusal further up predates this run. */
|
||||
const MISSING_COMMAND_TAIL_LINES = 6
|
||||
|
||||
// bash, zsh, dash/sh, fish, PowerShell and cmd.exe refusing a command that does not exist.
|
||||
const MISSING_COMMAND_PATTERNS = [
|
||||
/command not found/i,
|
||||
/^\S+: \d+: \S+: not found$/i,
|
||||
/unknown command/i,
|
||||
/is not recognized as (?:the name of a cmdlet|an internal or external command)/i
|
||||
]
|
||||
export { AGENT_START_GRACE_MS } from '../agent-turn-idle-verdict'
|
||||
|
||||
export type AutomationRunAgentEvidence = {
|
||||
/** Agent status rows for a pane, from hooks, OSC and titles alike. */
|
||||
getAgentStatusRowsForPane(
|
||||
paneKey: string
|
||||
): readonly Pick<
|
||||
AgentStatusIpcPayload,
|
||||
'state' | 'stateStartedAt' | 'sessionBoundary' | 'providerSessionOnly'
|
||||
>[]
|
||||
getAgentStatusRowsForPane(paneKey: string): readonly AgentTurnStatusRow[]
|
||||
/** The names the run's agent command may run under, to tell its refusal from its output. */
|
||||
agentCommandsForRun(run: AutomationRun): readonly string[]
|
||||
}
|
||||
@@ -40,25 +17,6 @@ export type IdleRunVerdict =
|
||||
/** An idle shell that may not have started the agent yet. */
|
||||
| { kind: 'wait' }
|
||||
|
||||
/** A shell refusing one of the agent's own commands; an agent's tool output never matches. */
|
||||
export function findMissingCommandLine(
|
||||
tail: readonly string[],
|
||||
commands: readonly string[]
|
||||
): string | null {
|
||||
const names = commands.map((command) => command.split(/\s+/)[0]).filter(Boolean)
|
||||
const recent = tail
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.slice(-MISSING_COMMAND_TAIL_LINES)
|
||||
return (
|
||||
recent.find(
|
||||
(line) =>
|
||||
MISSING_COMMAND_PATTERNS.some((pattern) => pattern.test(line)) &&
|
||||
names.some((name) => line.includes(name))
|
||||
) ?? null
|
||||
)
|
||||
}
|
||||
|
||||
export function judgeIdleRun(
|
||||
evidence: AutomationRunAgentEvidence,
|
||||
run: AutomationRun,
|
||||
@@ -66,32 +24,20 @@ export function judgeIdleRun(
|
||||
runStartedAt: number,
|
||||
now: number
|
||||
): IdleRunVerdict {
|
||||
// A reused pane's earlier `done` ended before this run's prompt went in.
|
||||
const since = run.dispatchedAt ?? runStartedAt
|
||||
// Resume identity is no status report.
|
||||
const rows = run.terminalPaneKey
|
||||
? evidence
|
||||
.getAgentStatusRowsForPane(run.terminalPaneKey)
|
||||
.filter((row) => !row.providerSessionOnly)
|
||||
: []
|
||||
if (
|
||||
rows.some(
|
||||
(row) =>
|
||||
// A connect-time `done` reports no turn.
|
||||
row.state === 'done' && !row.sessionBoundary && row.stateStartedAt >= since
|
||||
)
|
||||
) {
|
||||
return { kind: 'completed' }
|
||||
}
|
||||
const missing = findMissingCommandLine(tail, evidence.agentCommandsForRun(run))
|
||||
if (missing) {
|
||||
return {
|
||||
kind: 'failed',
|
||||
error: `Automation agent did not start; this host could not run its command (${missing}).`
|
||||
}
|
||||
}
|
||||
// Only an agent that never reports status is believed done by an idle pane.
|
||||
return rows.length === 0 && now - since >= AGENT_START_GRACE_MS
|
||||
? { kind: 'completed' }
|
||||
: { kind: 'wait' }
|
||||
const verdict = judgeIdleAgentTurn(
|
||||
{
|
||||
rows: run.terminalPaneKey ? evidence.getAgentStatusRowsForPane(run.terminalPaneKey) : [],
|
||||
// A reused pane's earlier `done` ended before this run's prompt went in.
|
||||
sentAt: run.dispatchedAt ?? runStartedAt,
|
||||
tail,
|
||||
readAgentCommands: () => evidence.agentCommandsForRun(run)
|
||||
},
|
||||
now
|
||||
)
|
||||
return verdict.kind === 'failed'
|
||||
? {
|
||||
kind: 'failed',
|
||||
error: `Automation agent did not start; this host could not run its command (${verdict.missingCommandLine}).`
|
||||
}
|
||||
: verdict
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user