diff --git a/docs/reference/antigravity-readiness-evidence.md b/docs/reference/antigravity-readiness-evidence.md index 56605f4597c..e7774f57b4e 100644 --- a/docs/reference/antigravity-readiness-evidence.md +++ b/docs/reference/antigravity-readiness-evidence.md @@ -1,5 +1,24 @@ # Antigravity readiness: what the transcripts show +## 2026-09-19: CI replay corrections + +The full runtime suite still expected the old hand-written bare-caret screens to +be ready. It now replays the recorded 1.2.7 screen at 120×40, including the +background-handle, live-leaf, retained-trust-text and large retained-tail paths. +The waits allow 3.5 seconds so the 2-second idle poll can observe the asynchronous +screen projection. The no-whole-tail-split performance assertion remains. + +The older API-key, hidden-account and dismissed-dialog captures include shutdown. +Their unique `ESC[>4m ESC[=0;1u` trailer resets keyboard modes, moves down and clears +the shortcut footer with `ESC[J`; the dismissed dialog also prints a resume +command. Their live-phase tests stop before that recorded trailer. Full-file +screen checks require **not ready** after the footer is erased. No capture bytes +were edited and no production readiness condition was relaxed. + +The transcript suite now asserts the intended verdict directly, including rejecting +the model picker, instead of preserving historical defects as inverted expectations. + + ## 2026-09-19: host contact and live prompt submission A regression marked an SSH terminal `unverifiable` after caching a ready screen. diff --git a/src/main/runtime/agent-transcript-pane-test-harness.ts b/src/main/runtime/agent-transcript-pane-test-harness.ts index 4345e98fd93..1b2fac70b0f 100644 --- a/src/main/runtime/agent-transcript-pane-test-harness.ts +++ b/src/main/runtime/agent-transcript-pane-test-harness.ts @@ -11,6 +11,7 @@ export type TranscriptPaneOptions = { paneTitle: string foregroundProcess: string | null data: string + size?: { cols: number; rows: number } /** Set for a pane whose PTY lives on an SSH host or WSL distro rather than locally. */ connectionId?: string /** Simulates a PTY controller whose foreground probe never settles. */ @@ -71,6 +72,9 @@ export async function createTranscriptPane( } ] }) + if (options.size) { + runtime.seedHeadlessTerminal(TRANSCRIPT_PANE_PTY_ID, '\x1b[0m', options.size) + } // Why the guard: a restore seed is only applied to a never-written record, so the restore // cases must not write an empty chunk first. if (options.data.length > 0) { diff --git a/src/main/runtime/antigravity-readiness-transcripts.test.ts b/src/main/runtime/antigravity-readiness-transcripts.test.ts index 3ac7707565f..c7d51e502b5 100644 --- a/src/main/runtime/antigravity-readiness-transcripts.test.ts +++ b/src/main/runtime/antigravity-readiness-transcripts.test.ts @@ -1,17 +1,5 @@ -/** - * Pins Antigravity readiness to captured transcripts instead of hand-written fixtures. - * - * Five detector attempts were tuned against a five-line screen someone typed from memory, and - * three of them shipped worse behaviour than the bug they replaced. Nothing here asserts what - * Antigravity prints: the transcripts do. Six are recorded from a live `agy`; the rest name - * themselves as skipped until someone can reach them. - * - * Four cases are pinned as KNOWN DEFECT: on real output the shipped detector refuses the ready - * screen and accepts the live model picker. Those assert what it does, not what it should. - * - * Capture protocol: docs/reference/agent-pty-transcript-capture.md - * What each transcript decides: docs/reference/antigravity-readiness-evidence.md - */ +// Replays captured agy screens through the real runtime wait at their recorded 120×40 grid. +// Capture protocol: docs/reference/agent-pty-transcript-capture.md. import { existsSync, readFileSync } from 'node:fs' import { join } from 'node:path' import { describe, expect, it, vi } from 'vitest' @@ -35,10 +23,9 @@ const EVIDENCE_DOC = join( 'reference', 'antigravity-readiness-evidence.md' ) -// Why asymmetric: a ready verdict has to survive the settle window, while a refusal only has to -// hold for one poll. Keeping the refusal short keeps seven transcripts off the suite's clock. -const READY_TIMEOUT_MS = 2_000 -const REFUSAL_TIMEOUT_MS = 600 +// Allow the 2s idle poll to observe the asynchronously rendered snapshot before timing out. +const READY_TIMEOUT_MS = 3_500 +const REFUSAL_TIMEOUT_MS = READY_TIMEOUT_MS /** Antigravity's binary, as Orca launches and probes it (`tui-agent-config.ts` detectCmd). */ const ANTIGRAVITY_COMMAND = 'agy' // String.fromCharCode, not a literal: the formatter rewrites an escape sequence into a raw @@ -53,28 +40,23 @@ type TranscriptCase = { what: string /** What a correct detector must answer. Not what the shipped one answers. */ expectReady: boolean - /** - * Set where the shipped detector contradicts the transcript. The case then runs inverted, so - * CI pins the defect instead of going permanently red — and flips to failing the moment - * someone fixes it, which is exactly when these expectations need re-reading. - */ - knownDefect?: string + beforeShutdown?: boolean } const TRANSCRIPTS: readonly TranscriptCase[] = [ { name: 'antigravity-ready-api-key-gemini-model', + beforeShutdown: true, capture: 'B', what: 'ready screen, API-key identity — the account row reads "Gemini API key", not an email', - expectReady: true, - knownDefect: 'refused: the model row never starts a line, the logo shares it' + expectReady: true }, { name: 'antigravity-ready-account-info-hidden', + beforeShutdown: true, capture: 'B', what: 'ready screen with AGY_CLI_HIDE_ACCOUNT_INFO=1 — no account row at all', - expectReady: true, - knownDefect: 'refused: same line-start defect, and no account row exists to require' + expectReady: true }, { name: 'antigravity-dialog-trust-workspace', @@ -86,8 +68,7 @@ const TRANSCRIPTS: readonly TranscriptCase[] = [ name: 'antigravity-dialog-model-picker', capture: 'C', what: 'model picker owning the screen', - expectReady: false, - knownDefect: "accepted: the picker's own `Gemini 3.x Flash` rows satisfy the model rule" + expectReady: false }, { name: 'antigravity-dialog-command-palette', @@ -107,15 +88,14 @@ const TRANSCRIPTS: readonly TranscriptCase[] = [ name: 'antigravity-busy-turn-ended', capture: 'E', what: 'the turn has ended and the composer has returned, process still alive', - expectReady: true, - knownDefect: 'refused: the retained tail ends on the error block, with no composer row in it' + expectReady: true }, { name: 'antigravity-dialog-dismissed', + beforeShutdown: true, capture: 'D', what: 'the screen immediately after the model picker is dismissed', - expectReady: true, - knownDefect: 'refused: the banner is not reprinted and no model row starts a line' + expectReady: true }, // Not captured: this machine's agy has no OAuth session and offers only Gemini models, and // reaching the rest would mean signing the operator out or deleting their config. See @@ -156,6 +136,19 @@ function fixturePath(name: string): string { return join(FIXTURE_DIR, `${name}.txt`) } +function liveCapture(transcript: TranscriptCase): string { + const raw = readFileSync(fixturePath(transcript.name), 'utf8') + if (!transcript.beforeShutdown) { + return raw + } + // These recordings include process shutdown, which erases the live shortcut footer. + const shutdown = `${ESC}[>4m${ESC}[=0;1u` + const offset = raw.lastIndexOf(shutdown) + expect(offset).toBeGreaterThan(0) + expect(raw.slice(offset)).toContain(`${ESC}[?2004l`) + return raw.slice(0, offset) +} + /** * A `tui-idle` wait ends three ways, and only one of them is readiness: it resolves satisfied, it * resolves unsatisfied with a blocked reason, or it rejects with `timeout` because nothing ever @@ -171,6 +164,7 @@ async function readinessVerdict( // capture carries the OSC bytes, so the pane wears whatever the CLI actually set. paneTitle: extractLastOscTitle(transcript) ?? ANTIGRAVITY_COMMAND, foregroundProcess: ANTIGRAVITY_COMMAND, + size: { cols: 120, rows: 40 }, data: transcript }) try { @@ -194,29 +188,19 @@ describe('Antigravity readiness, decided by captured transcripts', () => { const captured = existsSync(path) const label = `capture ${transcript.capture}: ${transcript.what}` - // A pinned defect asserts what the detector DOES, so CI is honest rather than permanently - // red; fixing the detector flips this case to failing, which is when these expectations - // need re-reading. The correct answer stays in `expectReady` and in the test's name. - const shipped = - transcript.knownDefect === undefined ? transcript.expectReady : !transcript.expectReady - const verdictName = - transcript.knownDefect === undefined - ? `${label} → ${transcript.expectReady ? 'ready' : 'not ready'}` - : `${label} → must be ${transcript.expectReady ? 'ready' : 'not ready'}; KNOWN DEFECT, ${transcript.knownDefect}` + const verdictName = `${label} → ${transcript.expectReady ? 'ready' : 'not ready'}` it.skipIf(!captured)( verdictName, async () => { - // A refusal only has to hold for one poll; a ready verdict has to survive the settle - // window. Keeping the refusal short keeps eleven transcripts off the suite's clock. const verdict = await readinessVerdict( - readFileSync(path, 'utf8'), + liveCapture(transcript), transcript.expectReady ? READY_TIMEOUT_MS : REFUSAL_TIMEOUT_MS ) // A silent dialog carries no blocked-signal wording, so the assertion is only that Orca // does not call the pane ready and type a prompt into a dialog that owns the screen. expect({ ready: verdict.ready, outcome: verdict.outcome }).toMatchObject({ - ready: shipped + ready: transcript.expectReady }) }, READY_TIMEOUT_MS + 10_000 @@ -253,19 +237,10 @@ describe('Antigravity readiness, decided by captured transcripts', () => { }) describe('scaffold self-check', () => { - // Why these two live here: when a transcript lands and fails, the failure has to mean the - // capture disagreed with the detector — not that the harness or the timeouts are broken. - // Neither case is evidence about Antigravity; both are shapes the current detector already - // decides, used only to prove the plumbing reaches a verdict. + // Verify both completion paths independently of the capture inventory. it('reaches a ready verdict through the harness', async () => { const verdict = await readinessVerdict( - [ - 'Antigravity CLI 1.0.3', - 'user@example.com (Antigravity Business)', - 'Gemini 3.5 Flash (High)', - '~/orca/workspaces/orca/agy-dispatch-issue', - '>' - ].join('\n'), + readFileSync(fixturePath('antigravity-ready-default-127'), 'utf8'), READY_TIMEOUT_MS ) expect(verdict.ready).toBe(true) diff --git a/src/main/runtime/antigravity-visible-composer.test.ts b/src/main/runtime/antigravity-visible-composer.test.ts index a18cf637e92..a955a3babfd 100644 --- a/src/main/runtime/antigravity-visible-composer.test.ts +++ b/src/main/runtime/antigravity-visible-composer.test.ts @@ -22,6 +22,9 @@ async function capturedScreen(name: string): Promise { describe('Antigravity visible-screen readiness from real PTY captures', () => { it.each([ ['antigravity-ready-default-127', true], + ['antigravity-ready-api-key-gemini-model', false], + ['antigravity-ready-account-info-hidden', false], + ['antigravity-dialog-dismissed', false], ['antigravity-ready-plan-127', true], ['antigravity-ready-accept-edits-127', true], ['antigravity-plan-hint-as-draft-127', false], diff --git a/src/main/runtime/orca-runtime-tests/terminal-creation-and-readiness-part-06.spec.ts b/src/main/runtime/orca-runtime-tests/terminal-creation-and-readiness-part-06.spec.ts index 1d0841b5e1c..4f30e155dd3 100644 --- a/src/main/runtime/orca-runtime-tests/terminal-creation-and-readiness-part-06.spec.ts +++ b/src/main/runtime/orca-runtime-tests/terminal-creation-and-readiness-part-06.spec.ts @@ -1,14 +1,19 @@ +import { readFileSync } from 'node:fs' +import { join } from 'node:path' import { describe, expect, it, vi } from 'vitest' import { OrcaRuntimeService } from '../orca-runtime-test-mocks.spec' import { TEST_WORKTREE_ID, TEST_WORKTREE_PATH, - antigravityPromptBeforeModelReadyScreen, - antigravityReadyScreen, store, syncSinglePty } from '../orca-runtime-test-fixtures.spec' +const antigravityCapturedReady = readFileSync( + join(__dirname, '..', '__fixtures__', 'antigravity-ready-default-127.txt'), + 'utf8' +) + describe('OrcaRuntimeService', () => { it('waits for exit on background terminal handles', async () => { const runtime = new OrcaRuntimeService(store) @@ -532,10 +537,11 @@ describe('OrcaRuntimeService', () => { getForegroundProcess: async () => null }) const { handle } = await runtime.createTerminal(`path:${TEST_WORKTREE_PATH}`) - runtime.onPtyData('pty-bg', antigravityReadyScreen('Gemini 4 Experimental (High)'), Date.now()) + runtime.seedHeadlessTerminal('pty-bg', '\x1b[0m', { cols: 120, rows: 40 }) + runtime.onPtyData('pty-bg', antigravityCapturedReady, Date.now()) await expect( - runtime.waitForTerminal(handle, { condition: 'tui-idle', timeoutMs: 1_000 }) + runtime.waitForTerminal(handle, { condition: 'tui-idle', timeoutMs: 3_500 }) ).resolves.toMatchObject({ handle, condition: 'tui-idle', @@ -558,21 +564,11 @@ describe('OrcaRuntimeService', () => { } const splitSpy = vi.spyOn(String.prototype, 'split') - runtime.onPtyData( - 'pty-bg', - [ - 'Antigravity CLI 1.0.3\n', - 'user@example.com (Antigravity Business)\n', - pastedTail, - 'Gemini 4 Experimental (High)\n', - '~/orca/workspaces/orca/agy-dispatch-issue\n', - '>' - ].join(''), - Date.now() - ) + runtime.seedHeadlessTerminal('pty-bg', '\x1b[0m', { cols: 120, rows: 40 }) + runtime.onPtyData('pty-bg', [pastedTail, antigravityCapturedReady].join(''), Date.now()) await expect( - runtime.waitForTerminal(handle, { condition: 'tui-idle', timeoutMs: 1_000 }) + runtime.waitForTerminal(handle, { condition: 'tui-idle', timeoutMs: 3_500 }) ).resolves.toMatchObject({ handle, condition: 'tui-idle', @@ -581,12 +577,14 @@ describe('OrcaRuntimeService', () => { }) const splitReadyTail = splitSpy.mock.contexts.some((context) => { const value = typeof context === 'string' ? context : String(context) - return value.includes('antigravity cli') && value.includes('pasted text pasted text') + return ( + value.toLowerCase().includes('antigravity cli') && value.includes('pasted text pasted text') + ) }) expect(splitReadyTail).toBe(false) }) - it('resolves tui-idle from an Antigravity prompt before the model line', async () => { + it('resolves tui-idle after retained trust text is replaced by a captured Antigravity screen', async () => { const runtime = new OrcaRuntimeService(store) runtime.setPtyController({ spawn: vi.fn().mockResolvedValue({ id: 'pty-bg' }), @@ -595,18 +593,19 @@ describe('OrcaRuntimeService', () => { getForegroundProcess: async () => null }) const { handle } = await runtime.createTerminal(`path:${TEST_WORKTREE_PATH}`) + runtime.seedHeadlessTerminal('pty-bg', '\x1b[0m', { cols: 120, rows: 40 }) runtime.onPtyData( 'pty-bg', [ 'Do you trust this workspace directory?\n', 'Press t to trust\n', - antigravityPromptBeforeModelReadyScreen('Gemini 3.5 Flash (High)') + antigravityCapturedReady ].join(''), Date.now() ) await expect( - runtime.waitForTerminal(handle, { condition: 'tui-idle', timeoutMs: 1_000 }) + runtime.waitForTerminal(handle, { condition: 'tui-idle', timeoutMs: 3_500 }) ).resolves.toMatchObject({ handle, condition: 'tui-idle', @@ -639,11 +638,12 @@ describe('OrcaRuntimeService', () => { } ] }) - runtime.onPtyData('pty-1', antigravityReadyScreen(), Date.now()) + runtime.seedHeadlessTerminal('pty-1', '\x1b[0m', { cols: 120, rows: 40 }) + runtime.onPtyData('pty-1', antigravityCapturedReady, Date.now()) const [terminal] = (await runtime.listTerminals()).terminals await expect( - runtime.waitForTerminal(terminal.handle, { condition: 'tui-idle', timeoutMs: 1_000 }) + runtime.waitForTerminal(terminal.handle, { condition: 'tui-idle', timeoutMs: 3_500 }) ).resolves.toMatchObject({ handle: terminal.handle, condition: 'tui-idle', diff --git a/src/main/runtime/orca-runtime-tests/terminal-creation-and-readiness-part-07.spec.ts b/src/main/runtime/orca-runtime-tests/terminal-creation-and-readiness-part-07.spec.ts index 3ae648ab0ef..c28402e49d9 100644 --- a/src/main/runtime/orca-runtime-tests/terminal-creation-and-readiness-part-07.spec.ts +++ b/src/main/runtime/orca-runtime-tests/terminal-creation-and-readiness-part-07.spec.ts @@ -1,3 +1,5 @@ +import { readFileSync } from 'node:fs' +import { join } from 'node:path' import { describe, expect, it, vi } from 'vitest' import { AGENT_PROMPT_BRACKETED_PASTE_END, @@ -11,12 +13,16 @@ import { OrcaRuntimeService } from '../orca-runtime' import { acknowledgeAgentPromptSubmit } from '../orca-runtime-test-mocks.spec' import { TEST_WORKTREE_PATH, - antigravityReadyScreen, cursorBusyScreen, cursorReadyScreen, store } from '../orca-runtime-test-fixtures.spec' +const antigravityCapturedReady = readFileSync( + join(__dirname, '..', '__fixtures__', 'antigravity-ready-default-127.txt'), + 'utf8' +) + describe('OrcaRuntimeService', () => { it('resolves tui-idle from a Codex ready prompt even when stale startup lines remain', async () => { const runtime = new OrcaRuntimeService(store) @@ -64,19 +70,20 @@ describe('OrcaRuntimeService', () => { getForegroundProcess: async () => null }) const { handle } = await runtime.createTerminal(`path:${TEST_WORKTREE_PATH}`) + runtime.seedHeadlessTerminal('pty-bg', '\x1b[0m', { cols: 120, rows: 40 }) runtime.onPtyData( 'pty-bg', [ 'Do you trust this workspace directory?\n', 'Press t to trust\n', - antigravityReadyScreen(), + antigravityCapturedReady, '\n' ].join(''), Date.now() ) await expect( - runtime.waitForTerminal(handle, { condition: 'tui-idle', timeoutMs: 1_000 }) + runtime.waitForTerminal(handle, { condition: 'tui-idle', timeoutMs: 3_500 }) ).resolves.toMatchObject({ handle, condition: 'tui-idle',