test(e2e): gate the tab-bar agent launcher on Windows shells and WSL

The `+` menu agent launcher had no golden coverage in the Windows lane, so a
Windows-only break anywhere in its chain (detection row, startup-plan build,
tab create, PTY spawn, startup-command injection) could ship unnoticed.

Adds a golden spec that launches a stub agent from the menu and asserts the
agent's own banner reached the pane — a tab that spawned a bare shell instead
is indistinguishable at the store/tab layer. Runs two agents everywhere, and
on Windows also PowerShell, cmd, Git Bash and a WSL project runtime.
This commit is contained in:
Jinjing
2026-08-22 23:16:18 -07:00
parent 9ea2636b38
commit c170ac2317
7 changed files with 230 additions and 14 deletions
@@ -74,6 +74,7 @@ jobs:
xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:terminal-rendering-golden
xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:posix-profile-index-golden
xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:agent-tui-golden
xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:tab-bar-agent-launch-golden
xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:source-control-golden
- name: Run golden E2E tests on macOS
@@ -87,6 +88,7 @@ jobs:
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:terminal-rendering-golden
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:posix-profile-index-golden
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:agent-tui-golden
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:tab-bar-agent-launch-golden
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:source-control-golden
- name: Run golden E2E tests on Windows
@@ -97,6 +99,7 @@ jobs:
$env:ORCA_E2E_FORWARD_APP_LOGS = '1'
pnpm run --if-present test:e2e:workspace-session-golden
pnpm run --if-present test:e2e:windows-fresh-startup-golden
pnpm run --if-present test:e2e:tab-bar-agent-launch-golden
if (Test-Path tests/e2e/golden-fresh-profile-terminal.spec.ts) {
pnpm run test:e2e -- tests/e2e/golden-fresh-profile-terminal.spec.ts tests/e2e/golden-shell-command.spec.ts
}
+1
View File
@@ -100,6 +100,7 @@
"test:e2e:posix-profile-index-golden": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/golden-posix-fresh-startup.spec.ts tests/e2e/golden-posix-profile-index-fsync.spec.ts --grep @posix-profile-index-golden --config tests/playwright.config.ts --project electron-headless --workers=1",
"test:e2e:windows-fresh-startup-golden": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/golden-windows-fresh-startup.spec.ts --grep @windows-fresh-startup-golden --config tests/playwright.config.ts --project electron-headless --workers=1",
"test:e2e:agent-tui-golden": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/golden-agent-tui-launch.spec.ts tests/e2e/golden-shell-after-agent-exit.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1",
"test:e2e:tab-bar-agent-launch-golden": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/golden-tab-bar-agent-launch.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1",
"test:e2e:terminal-rendering-release-evidence": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/terminal-opencode-emoji-table-rendering.spec.ts tests/e2e/terminal-long-table-scroll-restore.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=2",
"test:e2e:terminal-perf": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/terminal-typing-latency.spec.ts tests/e2e/terminal-foreground-redraw-freeze.spec.ts tests/e2e/terminal-output-scheduler.spec.ts tests/e2e/terminal-hidden-tui-visual-restore.spec.ts tests/e2e/artificial-opencode-terminal-load.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=2",
"test:e2e:terminal-perf:scale": "pnpm run ensure:electron-runtime && node config/scripts/run-terminal-scale-perf-e2e.mjs",
+2
View File
@@ -0,0 +1,2 @@
#!/bin/sh
exec "$(dirname "$0")/golden-stub-agent" "$@"
@@ -0,0 +1,2 @@
@echo off
call "%~dp0golden-stub-agent.cmd" %*
@@ -0,0 +1,103 @@
import type { Page } from '@stablyai/playwright-test'
import { expect, test } from './helpers/orca-app'
import {
configureGoldenStubAgent,
getGoldenStubAgentLaunchEnv,
GOLDEN_STUB_AGENTS,
GOLDEN_STUB_READY_MARKER,
launchGoldenStubAgentFromNewTab
} from './helpers/golden-stub-agent'
import {
getFirstWslDistro,
removeWslGoldenStubAgent,
stageWslGoldenStubAgent,
useWslRuntimeForActiveProject
} from './helpers/wsl-golden-stub-agent'
import { ensureTerminalVisible, waitForActiveWorktree, waitForSessionReady } from './helpers/store'
import { getTerminalContent } from './helpers/terminal'
import type { BuiltInWindowsTerminalShell } from '../../src/shared/windows-terminal-shell'
// Covers the full tab-bar `+` agent launch chain — detection row, startup-plan
// build, tab create, PTY spawn under the configured runtime, startup-command
// injection. golden-agent-tui-launch.spec.ts covers one agent on the default
// shell and never runs in the Windows golden lane, so a Windows-only break
// anywhere in that chain ships as "clicking an agent does nothing".
test.use({ launchEnv: getGoldenStubAgentLaunchEnv() })
const WINDOWS_SHELLS: readonly BuiltInWindowsTerminalShell[] = [
'powershell.exe',
'cmd.exe',
'git-bash'
]
async function openWorkspaceTerminal(page: Page): Promise<void> {
await waitForSessionReady(page)
await waitForActiveWorktree(page)
await ensureTerminalVisible(page)
}
for (const { id, menuItemName } of GOLDEN_STUB_AGENTS) {
test(`tab-bar + menu launches ${id} into a live TUI @tab-bar-agent-launch-golden`, async ({
orcaPage
}) => {
await openWorkspaceTerminal(orcaPage)
await configureGoldenStubAgent(orcaPage, { agent: id })
await launchGoldenStubAgentFromNewTab(orcaPage, menuItemName)
const activeTab = orcaPage.locator('[data-testid="sortable-tab"][data-active="true"]')
await expect(activeTab).toHaveAttribute('data-tab-title', /Golden Stub Agent|Codex|Claude/i)
// Why the banner and not just the tab: a tab that spawned a bare shell
// instead of the agent looks identical at the store/tab layer.
expect(await getTerminalContent(orcaPage)).toContain(GOLDEN_STUB_READY_MARKER)
})
}
// Why describe-scoped and not a body-level skip: `orcaPage` builds before the
// test body, so a body-level skip still pays a full Electron launch per case.
test.describe('Windows runtimes', () => {
test.skip(process.platform !== 'win32', 'Windows agent launch matrix is Windows-only')
for (const shell of WINDOWS_SHELLS) {
test(`tab-bar + menu launches an agent under ${shell} @tab-bar-agent-launch-golden`, async ({
orcaPage
}) => {
await openWorkspaceTerminal(orcaPage)
// Why each shell: the queued launch command is quoted for the shell family
// the tab actually spawns (posix for Git Bash, cmd, PowerShell). A mismatch
// injects a command the shell rejects, and the agent never starts.
await configureGoldenStubAgent(orcaPage, { agent: 'codex', windowsShell: shell })
await launchGoldenStubAgentFromNewTab(orcaPage)
expect(await getTerminalContent(orcaPage)).toContain(GOLDEN_STUB_READY_MARKER)
})
}
test('tab-bar + menu launches an agent inside WSL @tab-bar-agent-launch-golden', async ({
orcaPage
}) => {
await openWorkspaceTerminal(orcaPage)
const distro = await getFirstWslDistro(orcaPage)
test.skip(!distro, 'No WSL distro is available on this Windows host')
test.skip(
!stageWslGoldenStubAgent(distro!),
'WSL distro would not accept the staged stub agent'
)
try {
// Why the runtime switch and not terminalWindowsShell: WSL is a project
// runtime, so it must retarget agent detection into the distro as well as
// the PTY. A launch quoted for a Windows shell never starts there.
await useWslRuntimeForActiveProject(orcaPage, distro!)
await configureGoldenStubAgent(orcaPage, { agent: 'codex' })
await launchGoldenStubAgentFromNewTab(orcaPage)
// The stub only exists inside the distro, so its marker proves the agent
// ran in WSL rather than on the Windows host.
expect(await getTerminalContent(orcaPage)).toContain(GOLDEN_STUB_READY_MARKER)
} finally {
removeWslGoldenStubAgent(distro!)
}
})
})
+37 -14
View File
@@ -2,10 +2,18 @@ import path from 'node:path'
import type { Page } from '@stablyai/playwright-test'
import { expect } from '@stablyai/playwright-test'
import { focusActiveTerminalInput, waitForTerminalOutput } from './terminal'
import type { BuiltInWindowsTerminalShell } from '../../../src/shared/windows-terminal-shell'
export const GOLDEN_STUB_READY_MARKER = 'GOLDEN_STUB_AGENT_READY'
export const GOLDEN_STUB_EXIT_MARKER = 'GOLDEN_STUB_AGENT_EXITED'
/** Agents the fixture dir shims onto PATH, so detection lists them and the
* tab-bar `+` menu renders a launch row for each. */
export const GOLDEN_STUB_AGENTS = [
{ id: 'codex', menuItemName: /^Codex(?:\s|$)/i },
{ id: 'claude', menuItemName: /^Claude(?:\s|$)/i }
] as const
const fixtureDir = path.join(process.cwd(), 'tests', 'e2e', 'fixtures', 'golden-stub-agent')
export function getGoldenStubAgentLaunchEnv(): NodeJS.ProcessEnv {
@@ -15,23 +23,38 @@ export function getGoldenStubAgentLaunchEnv(): NodeJS.ProcessEnv {
}
}
export async function configureGoldenStubAgent(page: Page): Promise<void> {
await page.evaluate(async () => {
const store = window.__store
if (!store) {
throw new Error('Orca store is unavailable')
}
await store.getState().updateSettings({
defaultTuiAgent: 'codex',
agentCmdOverrides: { codex: 'golden-stub-agent' },
agentDefaultArgs: { codex: '' }
})
})
export async function configureGoldenStubAgent(
page: Page,
options: {
agent?: (typeof GOLDEN_STUB_AGENTS)[number]['id']
/** Windows default shell the launch command must survive; ignored elsewhere. */
windowsShell?: BuiltInWindowsTerminalShell
} = {}
): Promise<void> {
const agent = options.agent ?? 'codex'
await page.evaluate(
async ({ agent, windowsShell }) => {
const store = window.__store
if (!store) {
throw new Error('Orca store is unavailable')
}
await store.getState().updateSettings({
defaultTuiAgent: agent,
agentCmdOverrides: { [agent]: 'golden-stub-agent' },
agentDefaultArgs: { [agent]: '' },
...(windowsShell ? { terminalWindowsShell: windowsShell } : {})
})
},
{ agent, windowsShell: options.windowsShell ?? null }
)
}
export async function launchGoldenStubAgentFromNewTab(page: Page): Promise<void> {
export async function launchGoldenStubAgentFromNewTab(
page: Page,
menuItemName: RegExp = /^Codex(?:\s|$)/i
): Promise<void> {
await page.getByRole('button', { name: 'New tab' }).click({ force: true })
const launchOption = page.getByRole('menuitem', { name: /^Codex(?:\s|$)/i }).first()
const launchOption = page.getByRole('menuitem', { name: menuItemName }).first()
await expect(launchOption).toBeVisible({ timeout: 15_000 })
await launchOption.click({ force: true })
await focusActiveTerminalInput(page)
@@ -0,0 +1,82 @@
import { execFileSync } from 'node:child_process'
import type { Page } from '@stablyai/playwright-test'
import { buildWslExecArgs } from '../../../src/shared/wsl-login-shell-command'
/** Distro path the stub is staged at. Only reachable from inside WSL, so the
* stub's marker in a pane is itself proof the pane executed in the distro. */
const WSL_STUB_PATH = '/usr/local/bin/golden-stub-agent'
const WSL_STUB_AGENT_LINK = '/usr/local/bin/codex'
// Why printf and not a heredoc: the whole script crosses to wsl.exe as one argv
// element, and keeping it newline-free avoids Windows argv-encoding surprises.
const STAGE_SCRIPT =
`mkdir -p /usr/local/bin && ` +
`printf '#!/bin/sh\\necho GOLDEN_STUB_AGENT_READY\\nexec sleep 3600\\n' > ${WSL_STUB_PATH} && ` +
`chmod 0755 ${WSL_STUB_PATH} && ` +
`ln -sf ${WSL_STUB_PATH} ${WSL_STUB_AGENT_LINK}`
const UNSTAGE_SCRIPT = `rm -f ${WSL_STUB_AGENT_LINK} ${WSL_STUB_PATH}`
// Why --exec (via buildWslExecArgs): under `--`, wsl.exe expands `$name` in
// every argument and silently rewrites the script.
function runInWslAsRoot(distro: string, script: string): void {
execFileSync('wsl.exe', ['-u', 'root', ...buildWslExecArgs(distro, ['sh', '-c', script])], {
stdio: 'pipe',
windowsHide: true
})
}
export async function getFirstWslDistro(page: Page): Promise<string | null> {
const wsl = await page.evaluate(async () => ({
available: await window.api.wsl.isAvailable(),
distros: await window.api.wsl.listDistros()
}))
return wsl.available ? (wsl.distros[0] ?? null) : null
}
/** Returns false when the distro won't take the stub (no root interop, read-only
* rootfs); callers skip rather than fail, since that is a host limitation. */
export function stageWslGoldenStubAgent(distro: string): boolean {
try {
runInWslAsRoot(distro, STAGE_SCRIPT)
return true
} catch {
return false
}
}
export function removeWslGoldenStubAgent(distro: string): void {
try {
runInWslAsRoot(distro, UNSTAGE_SCRIPT)
} catch {
// Best-effort cleanup; a leftover stub only affects this fixture's own name.
}
}
/** Points the active worktree's project at the WSL runtime, so agent detection
* and terminal spawn both target the distro instead of the Windows host. */
export async function useWslRuntimeForActiveProject(page: Page, distro: string): Promise<void> {
await page.evaluate(async (wslDistro) => {
const store = window.__store
if (!store) {
throw new Error('Orca store is unavailable')
}
const state = store.getState()
const worktreeId = state.activeWorktreeId
if (!worktreeId) {
throw new Error('No active worktree')
}
const activeWorktree = Object.values(state.worktreesByRepo)
.flat()
.find((worktree) => worktree.id === worktreeId)
const activeProject = state.projects.find((project) =>
activeWorktree ? project.sourceRepoIds.includes(activeWorktree.repoId) : false
)
if (!activeProject) {
throw new Error('No active project')
}
await state.updateProject(activeProject.id, {
localWindowsRuntimePreference: { kind: 'wsl', distro: wslDistro }
})
}, distro)
}