From 5320cdc5315857b15b5cf4e396c3536b27bd396e Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Wed, 12 Aug 2026 23:58:27 -0400 Subject: [PATCH] test(terminal): harden startup readiness oracle --- config/reliability-gates.jsonc | 1 - .../helpers/startup-exec-readiness-oracle.ts | 20 +++++++---- .../e2e/paired-startup-exec-readiness.spec.ts | 35 +++++++++++++------ tests/e2e/ssh-startup-exec-readiness.spec.ts | 12 +++---- 4 files changed, 44 insertions(+), 24 deletions(-) diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 89ff8f2fe2a..0702c1948dd 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -32,7 +32,6 @@ "oracle": "Run the real generated wrappers under node-pty with disposable zsh and bash startup files. Plain and exec -a replacement shells must receive one queued command at their first completed line editor without waiting for the 15-second timeout. The identical oracle must time out without the startup identity and prompt fallback. Same-shell reads, child-process reads, a blocking zle-line-init hook, sqlite3, and sqlite3 renamed zsh must not trigger prompt fallback. Relay owner coverage must strip the identity and release through the same prompt contract.", "commands": [ "pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/repro-13767-shell-ready-marker-lost-to-exec.test.ts src/main/line-editor-ready-output-scanner.test.ts src/main/shell-startup-identity-scanner.test.ts src/main/shell-prompt-readiness-probe.test.ts src/shared/pty-slave-line-discipline-echo.test.ts src/shared/shell-process-readiness.test.ts src/main/daemon/session.test.ts src/main/daemon/shell-ready.test.ts src/main/providers/local-pty-shell-ready.test.ts src/main/providers/local-pty-provider.test.ts src/relay/pty-shell-launch.test.ts src/relay/pty-handler.test.ts", - "pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/repro-13767-shell-ready-marker-lost-to-exec.test.ts src/relay/pty-handler.test.ts", "ORCA_E2E_SSH_DOCKER=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/ssh-startup-exec-readiness.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1", "ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-startup-exec-readiness.spec.ts --config tests/playwright.config.ts --project electron-headless --project electron-headful --workers=1" ], diff --git a/tests/e2e/helpers/startup-exec-readiness-oracle.ts b/tests/e2e/helpers/startup-exec-readiness-oracle.ts index 82855cf43c6..48b2928ed6d 100644 --- a/tests/e2e/helpers/startup-exec-readiness-oracle.ts +++ b/tests/e2e/helpers/startup-exec-readiness-oracle.ts @@ -27,7 +27,14 @@ export type BashExecProfileBarrier = { startedPath: string } -export function bashExecProfileContents(runId: string, barrier?: BashExecProfileBarrier): string { +const BASH_EXEC_LINE = 'exec -a figterm-sta4067 /bin/bash --noprofile --norc -l -i' +const ZSH_EXEC_LINE = 'exec -a figterm-sta4067 /bin/zsh -o noglobalrcs -l -i' + +function execProfileContents( + runId: string, + execLine: string, + barrier?: BashExecProfileBarrier +): string { const guard = `ORCA_STA4067_EXEC_${runId.replaceAll(/[^A-Za-z0-9_]/g, '_')}` const barrierScript = barrier ? [ @@ -38,16 +45,17 @@ export function bashExecProfileContents(runId: string, barrier?: BashExecProfile return `if [[ -z "\${${guard}:-}" ]]; then export ${guard}=1 ${barrierScript} - exec -a figterm-sta4067 /bin/bash --noprofile --norc -l -i + ${execLine} fi ` } +export function bashExecProfileContents(runId: string, barrier?: BashExecProfileBarrier): string { + return execProfileContents(runId, BASH_EXEC_LINE, barrier) +} + export function zshExecProfileContents(runId: string, barrier?: BashExecProfileBarrier): string { - return bashExecProfileContents(runId, barrier).replace( - 'exec -a figterm-sta4067 /bin/bash --noprofile --norc -l -i', - 'exec -a figterm-sta4067 /bin/zsh -o noglobalrcs -l -i' - ) + return execProfileContents(runId, ZSH_EXEC_LINE, barrier) } function shellQuote(value: string): string { diff --git a/tests/e2e/paired-startup-exec-readiness.spec.ts b/tests/e2e/paired-startup-exec-readiness.spec.ts index 4f476b4c80d..cd96de038ce 100644 --- a/tests/e2e/paired-startup-exec-readiness.spec.ts +++ b/tests/e2e/paired-startup-exec-readiness.spec.ts @@ -15,6 +15,8 @@ import { installZshExecProfile } from './helpers/startup-exec-readiness-oracle' +test.skip(process.platform === 'win32', 'Paired startup-exec readiness uses POSIX shells.') + async function waitForWorktree(page: Page, id: string): Promise { await expect .poll( @@ -123,14 +125,18 @@ test('recovers the same startup exec through an isolated headless orca serve', a test.setTimeout(120_000) const runId = `headless_${Date.now()}` const host = await launchHeadlessPairedRuntimeHost() - const homePath = await host.app.evaluate(({ app }) => app.getPath('home')) - const ledgerPath = path.join(homePath, `.sta4067-${runId}.ledger`) - const startedPath = path.join(homePath, `.sta4067-${runId}.started`) - const releasePath = path.join(homePath, `.sta4067-${runId}.release`) - const removeProfile = installZshExecProfile(homePath, runId, { releasePath, startedPath }) - const client = await launchPairedWebClient(host.app, host.offer, { waitForWorkspace: false }) let terminal: string | null = null + let removeProfile: (() => void) | undefined + let client: Awaited> | undefined + let startedPath = '' + let releasePath = '' try { + const homePath = await host.app.evaluate(({ app }) => app.getPath('home')) + const ledgerPath = path.join(homePath, `.sta4067-${runId}.ledger`) + startedPath = path.join(homePath, `.sta4067-${runId}.started`) + releasePath = path.join(homePath, `.sta4067-${runId}.release`) + removeProfile = installZshExecProfile(homePath, runId, { releasePath, startedPath }) + client = await launchPairedWebClient(host.app, host.offer, { waitForWorkspace: false }) await host.client.call('repo.add', { path: testRepoPath, kind: 'git' }) await client.page.locator('[data-worktree-sidebar]').waitFor({ state: 'visible', @@ -162,10 +168,17 @@ test('recovers the same startup exec through an isolated headless orca serve', a await expectStartupExecRecovery(client.page, created, runId) expectLedger(ledgerPath) } finally { - await closeStartupExecTerminal(client.page, terminal) - await client.dispose() - removeProfile() - cleanupExecBarrier(startedPath, releasePath) - await host.dispose() + try { + if (client) { + await closeStartupExecTerminal(client.page, terminal) + await client.dispose() + } + removeProfile?.() + if (startedPath) { + cleanupExecBarrier(startedPath, releasePath) + } + } finally { + await host.dispose() + } } }) diff --git a/tests/e2e/ssh-startup-exec-readiness.spec.ts b/tests/e2e/ssh-startup-exec-readiness.spec.ts index 3f3069db778..e77c9afd67f 100644 --- a/tests/e2e/ssh-startup-exec-readiness.spec.ts +++ b/tests/e2e/ssh-startup-exec-readiness.spec.ts @@ -70,7 +70,10 @@ test.describe('startup exec readiness over live SSH', () => { .toBe('ready') await expectStartupCommandQueuedByCompatibilityFallback(orcaPage, created) expect( - execDockerSshRelayTargetControlCommand(target, `test ! -e '${ledgerPath}' && echo pending`) + execDockerSshRelayTargetControlCommand( + target, + `if test ! -e '${ledgerPath}'; then echo pending; else cat '${ledgerPath}'; fi` + ) ).toBe('pending') await reconnectDockerSshRelayTarget(orcaPage, remote.targetId) @@ -98,11 +101,8 @@ test.describe('startup exec readiness over live SSH', () => { /\/bash$/ ) expect( - execDockerSshRelayTargetControlCommand( - target, - `test "$(ps -o tpgid= -p '${pid}' | tr -d ' ')" = '${pid}' && echo foreground` - ) - ).toBe('foreground') + execDockerSshRelayTargetControlCommand(target, `ps -o tpgid= -p '${pid}' | tr -d ' '`) + ).toBe(String(pid)) } finally { await closeStartupExecTerminal(orcaPage, terminal) cleanupDockerSshRelayTarget(target)