From c170ac23171898542cbf3f2dc442d5bf067b29db Mon Sep 17 00:00:00 2001 From: Jinjing <6427696+AmethystLiang@users.noreply.github.com> Date: Sat, 22 Aug 2026 23:16:18 -0700 Subject: [PATCH] test(e2e): gate the tab-bar agent launcher on Windows shells and WSL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `+` menu agent launcher had no golden coverage in the Windows lane, so a Windows-only break anywhere in its chain (detection row, startup-plan build, tab create, PTY spawn, startup-command injection) could ship unnoticed. Adds a golden spec that launches a stub agent from the menu and asserts the agent's own banner reached the pane — a tab that spawned a bare shell instead is indistinguishable at the store/tab layer. Runs two agents everywhere, and on Windows also PowerShell, cmd, Git Bash and a WSL project runtime. --- .github/workflows/golden-e2e-experiment.yml | 3 + package.json | 1 + tests/e2e/fixtures/golden-stub-agent/claude | 2 + .../e2e/fixtures/golden-stub-agent/claude.cmd | 2 + tests/e2e/golden-tab-bar-agent-launch.spec.ts | 103 ++++++++++++++++++ tests/e2e/helpers/golden-stub-agent.ts | 51 ++++++--- tests/e2e/helpers/wsl-golden-stub-agent.ts | 82 ++++++++++++++ 7 files changed, 230 insertions(+), 14 deletions(-) create mode 100755 tests/e2e/fixtures/golden-stub-agent/claude create mode 100644 tests/e2e/fixtures/golden-stub-agent/claude.cmd create mode 100644 tests/e2e/golden-tab-bar-agent-launch.spec.ts create mode 100644 tests/e2e/helpers/wsl-golden-stub-agent.ts diff --git a/.github/workflows/golden-e2e-experiment.yml b/.github/workflows/golden-e2e-experiment.yml index 1f0a2b42755..040996489d1 100644 --- a/.github/workflows/golden-e2e-experiment.yml +++ b/.github/workflows/golden-e2e-experiment.yml @@ -74,6 +74,7 @@ jobs: xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:terminal-rendering-golden xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:posix-profile-index-golden xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:agent-tui-golden + xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:tab-bar-agent-launch-golden xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:source-control-golden - name: Run golden E2E tests on macOS @@ -87,6 +88,7 @@ jobs: env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:terminal-rendering-golden env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:posix-profile-index-golden env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:agent-tui-golden + env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:tab-bar-agent-launch-golden env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:source-control-golden - name: Run golden E2E tests on Windows @@ -97,6 +99,7 @@ jobs: $env:ORCA_E2E_FORWARD_APP_LOGS = '1' pnpm run --if-present test:e2e:workspace-session-golden pnpm run --if-present test:e2e:windows-fresh-startup-golden + pnpm run --if-present test:e2e:tab-bar-agent-launch-golden if (Test-Path tests/e2e/golden-fresh-profile-terminal.spec.ts) { pnpm run test:e2e -- tests/e2e/golden-fresh-profile-terminal.spec.ts tests/e2e/golden-shell-command.spec.ts } diff --git a/package.json b/package.json index a026bdb7824..66d0d0bf53e 100644 --- a/package.json +++ b/package.json @@ -100,6 +100,7 @@ "test:e2e:posix-profile-index-golden": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/golden-posix-fresh-startup.spec.ts tests/e2e/golden-posix-profile-index-fsync.spec.ts --grep @posix-profile-index-golden --config tests/playwright.config.ts --project electron-headless --workers=1", "test:e2e:windows-fresh-startup-golden": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/golden-windows-fresh-startup.spec.ts --grep @windows-fresh-startup-golden --config tests/playwright.config.ts --project electron-headless --workers=1", "test:e2e:agent-tui-golden": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/golden-agent-tui-launch.spec.ts tests/e2e/golden-shell-after-agent-exit.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + "test:e2e:tab-bar-agent-launch-golden": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/golden-tab-bar-agent-launch.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", "test:e2e:terminal-rendering-release-evidence": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/terminal-opencode-emoji-table-rendering.spec.ts tests/e2e/terminal-long-table-scroll-restore.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=2", "test:e2e:terminal-perf": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/terminal-typing-latency.spec.ts tests/e2e/terminal-foreground-redraw-freeze.spec.ts tests/e2e/terminal-output-scheduler.spec.ts tests/e2e/terminal-hidden-tui-visual-restore.spec.ts tests/e2e/artificial-opencode-terminal-load.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=2", "test:e2e:terminal-perf:scale": "pnpm run ensure:electron-runtime && node config/scripts/run-terminal-scale-perf-e2e.mjs", diff --git a/tests/e2e/fixtures/golden-stub-agent/claude b/tests/e2e/fixtures/golden-stub-agent/claude new file mode 100755 index 00000000000..d16a99de266 --- /dev/null +++ b/tests/e2e/fixtures/golden-stub-agent/claude @@ -0,0 +1,2 @@ +#!/bin/sh +exec "$(dirname "$0")/golden-stub-agent" "$@" diff --git a/tests/e2e/fixtures/golden-stub-agent/claude.cmd b/tests/e2e/fixtures/golden-stub-agent/claude.cmd new file mode 100644 index 00000000000..894a9216aa2 --- /dev/null +++ b/tests/e2e/fixtures/golden-stub-agent/claude.cmd @@ -0,0 +1,2 @@ +@echo off +call "%~dp0golden-stub-agent.cmd" %* diff --git a/tests/e2e/golden-tab-bar-agent-launch.spec.ts b/tests/e2e/golden-tab-bar-agent-launch.spec.ts new file mode 100644 index 00000000000..98904ad9920 --- /dev/null +++ b/tests/e2e/golden-tab-bar-agent-launch.spec.ts @@ -0,0 +1,103 @@ +import type { Page } from '@stablyai/playwright-test' +import { expect, test } from './helpers/orca-app' +import { + configureGoldenStubAgent, + getGoldenStubAgentLaunchEnv, + GOLDEN_STUB_AGENTS, + GOLDEN_STUB_READY_MARKER, + launchGoldenStubAgentFromNewTab +} from './helpers/golden-stub-agent' +import { + getFirstWslDistro, + removeWslGoldenStubAgent, + stageWslGoldenStubAgent, + useWslRuntimeForActiveProject +} from './helpers/wsl-golden-stub-agent' +import { ensureTerminalVisible, waitForActiveWorktree, waitForSessionReady } from './helpers/store' +import { getTerminalContent } from './helpers/terminal' +import type { BuiltInWindowsTerminalShell } from '../../src/shared/windows-terminal-shell' + +// Covers the full tab-bar `+` agent launch chain — detection row, startup-plan +// build, tab create, PTY spawn under the configured runtime, startup-command +// injection. golden-agent-tui-launch.spec.ts covers one agent on the default +// shell and never runs in the Windows golden lane, so a Windows-only break +// anywhere in that chain ships as "clicking an agent does nothing". + +test.use({ launchEnv: getGoldenStubAgentLaunchEnv() }) + +const WINDOWS_SHELLS: readonly BuiltInWindowsTerminalShell[] = [ + 'powershell.exe', + 'cmd.exe', + 'git-bash' +] + +async function openWorkspaceTerminal(page: Page): Promise { + await waitForSessionReady(page) + await waitForActiveWorktree(page) + await ensureTerminalVisible(page) +} + +for (const { id, menuItemName } of GOLDEN_STUB_AGENTS) { + test(`tab-bar + menu launches ${id} into a live TUI @tab-bar-agent-launch-golden`, async ({ + orcaPage + }) => { + await openWorkspaceTerminal(orcaPage) + await configureGoldenStubAgent(orcaPage, { agent: id }) + await launchGoldenStubAgentFromNewTab(orcaPage, menuItemName) + + const activeTab = orcaPage.locator('[data-testid="sortable-tab"][data-active="true"]') + await expect(activeTab).toHaveAttribute('data-tab-title', /Golden Stub Agent|Codex|Claude/i) + // Why the banner and not just the tab: a tab that spawned a bare shell + // instead of the agent looks identical at the store/tab layer. + expect(await getTerminalContent(orcaPage)).toContain(GOLDEN_STUB_READY_MARKER) + }) +} + +// Why describe-scoped and not a body-level skip: `orcaPage` builds before the +// test body, so a body-level skip still pays a full Electron launch per case. +test.describe('Windows runtimes', () => { + test.skip(process.platform !== 'win32', 'Windows agent launch matrix is Windows-only') + + for (const shell of WINDOWS_SHELLS) { + test(`tab-bar + menu launches an agent under ${shell} @tab-bar-agent-launch-golden`, async ({ + orcaPage + }) => { + await openWorkspaceTerminal(orcaPage) + // Why each shell: the queued launch command is quoted for the shell family + // the tab actually spawns (posix for Git Bash, cmd, PowerShell). A mismatch + // injects a command the shell rejects, and the agent never starts. + await configureGoldenStubAgent(orcaPage, { agent: 'codex', windowsShell: shell }) + await launchGoldenStubAgentFromNewTab(orcaPage) + + expect(await getTerminalContent(orcaPage)).toContain(GOLDEN_STUB_READY_MARKER) + }) + } + + test('tab-bar + menu launches an agent inside WSL @tab-bar-agent-launch-golden', async ({ + orcaPage + }) => { + await openWorkspaceTerminal(orcaPage) + + const distro = await getFirstWslDistro(orcaPage) + test.skip(!distro, 'No WSL distro is available on this Windows host') + test.skip( + !stageWslGoldenStubAgent(distro!), + 'WSL distro would not accept the staged stub agent' + ) + + try { + // Why the runtime switch and not terminalWindowsShell: WSL is a project + // runtime, so it must retarget agent detection into the distro as well as + // the PTY. A launch quoted for a Windows shell never starts there. + await useWslRuntimeForActiveProject(orcaPage, distro!) + await configureGoldenStubAgent(orcaPage, { agent: 'codex' }) + await launchGoldenStubAgentFromNewTab(orcaPage) + + // The stub only exists inside the distro, so its marker proves the agent + // ran in WSL rather than on the Windows host. + expect(await getTerminalContent(orcaPage)).toContain(GOLDEN_STUB_READY_MARKER) + } finally { + removeWslGoldenStubAgent(distro!) + } + }) +}) diff --git a/tests/e2e/helpers/golden-stub-agent.ts b/tests/e2e/helpers/golden-stub-agent.ts index 62e44c9b091..1b61c547f80 100644 --- a/tests/e2e/helpers/golden-stub-agent.ts +++ b/tests/e2e/helpers/golden-stub-agent.ts @@ -2,10 +2,18 @@ import path from 'node:path' import type { Page } from '@stablyai/playwright-test' import { expect } from '@stablyai/playwright-test' import { focusActiveTerminalInput, waitForTerminalOutput } from './terminal' +import type { BuiltInWindowsTerminalShell } from '../../../src/shared/windows-terminal-shell' export const GOLDEN_STUB_READY_MARKER = 'GOLDEN_STUB_AGENT_READY' export const GOLDEN_STUB_EXIT_MARKER = 'GOLDEN_STUB_AGENT_EXITED' +/** Agents the fixture dir shims onto PATH, so detection lists them and the + * tab-bar `+` menu renders a launch row for each. */ +export const GOLDEN_STUB_AGENTS = [ + { id: 'codex', menuItemName: /^Codex(?:\s|$)/i }, + { id: 'claude', menuItemName: /^Claude(?:\s|$)/i } +] as const + const fixtureDir = path.join(process.cwd(), 'tests', 'e2e', 'fixtures', 'golden-stub-agent') export function getGoldenStubAgentLaunchEnv(): NodeJS.ProcessEnv { @@ -15,23 +23,38 @@ export function getGoldenStubAgentLaunchEnv(): NodeJS.ProcessEnv { } } -export async function configureGoldenStubAgent(page: Page): Promise { - await page.evaluate(async () => { - const store = window.__store - if (!store) { - throw new Error('Orca store is unavailable') - } - await store.getState().updateSettings({ - defaultTuiAgent: 'codex', - agentCmdOverrides: { codex: 'golden-stub-agent' }, - agentDefaultArgs: { codex: '' } - }) - }) +export async function configureGoldenStubAgent( + page: Page, + options: { + agent?: (typeof GOLDEN_STUB_AGENTS)[number]['id'] + /** Windows default shell the launch command must survive; ignored elsewhere. */ + windowsShell?: BuiltInWindowsTerminalShell + } = {} +): Promise { + const agent = options.agent ?? 'codex' + await page.evaluate( + async ({ agent, windowsShell }) => { + const store = window.__store + if (!store) { + throw new Error('Orca store is unavailable') + } + await store.getState().updateSettings({ + defaultTuiAgent: agent, + agentCmdOverrides: { [agent]: 'golden-stub-agent' }, + agentDefaultArgs: { [agent]: '' }, + ...(windowsShell ? { terminalWindowsShell: windowsShell } : {}) + }) + }, + { agent, windowsShell: options.windowsShell ?? null } + ) } -export async function launchGoldenStubAgentFromNewTab(page: Page): Promise { +export async function launchGoldenStubAgentFromNewTab( + page: Page, + menuItemName: RegExp = /^Codex(?:\s|$)/i +): Promise { await page.getByRole('button', { name: 'New tab' }).click({ force: true }) - const launchOption = page.getByRole('menuitem', { name: /^Codex(?:\s|$)/i }).first() + const launchOption = page.getByRole('menuitem', { name: menuItemName }).first() await expect(launchOption).toBeVisible({ timeout: 15_000 }) await launchOption.click({ force: true }) await focusActiveTerminalInput(page) diff --git a/tests/e2e/helpers/wsl-golden-stub-agent.ts b/tests/e2e/helpers/wsl-golden-stub-agent.ts new file mode 100644 index 00000000000..7451b7aaba0 --- /dev/null +++ b/tests/e2e/helpers/wsl-golden-stub-agent.ts @@ -0,0 +1,82 @@ +import { execFileSync } from 'node:child_process' +import type { Page } from '@stablyai/playwright-test' +import { buildWslExecArgs } from '../../../src/shared/wsl-login-shell-command' + +/** Distro path the stub is staged at. Only reachable from inside WSL, so the + * stub's marker in a pane is itself proof the pane executed in the distro. */ +const WSL_STUB_PATH = '/usr/local/bin/golden-stub-agent' +const WSL_STUB_AGENT_LINK = '/usr/local/bin/codex' + +// Why printf and not a heredoc: the whole script crosses to wsl.exe as one argv +// element, and keeping it newline-free avoids Windows argv-encoding surprises. +const STAGE_SCRIPT = + `mkdir -p /usr/local/bin && ` + + `printf '#!/bin/sh\\necho GOLDEN_STUB_AGENT_READY\\nexec sleep 3600\\n' > ${WSL_STUB_PATH} && ` + + `chmod 0755 ${WSL_STUB_PATH} && ` + + `ln -sf ${WSL_STUB_PATH} ${WSL_STUB_AGENT_LINK}` + +const UNSTAGE_SCRIPT = `rm -f ${WSL_STUB_AGENT_LINK} ${WSL_STUB_PATH}` + +// Why --exec (via buildWslExecArgs): under `--`, wsl.exe expands `$name` in +// every argument and silently rewrites the script. +function runInWslAsRoot(distro: string, script: string): void { + execFileSync('wsl.exe', ['-u', 'root', ...buildWslExecArgs(distro, ['sh', '-c', script])], { + stdio: 'pipe', + windowsHide: true + }) +} + +export async function getFirstWslDistro(page: Page): Promise { + const wsl = await page.evaluate(async () => ({ + available: await window.api.wsl.isAvailable(), + distros: await window.api.wsl.listDistros() + })) + return wsl.available ? (wsl.distros[0] ?? null) : null +} + +/** Returns false when the distro won't take the stub (no root interop, read-only + * rootfs); callers skip rather than fail, since that is a host limitation. */ +export function stageWslGoldenStubAgent(distro: string): boolean { + try { + runInWslAsRoot(distro, STAGE_SCRIPT) + return true + } catch { + return false + } +} + +export function removeWslGoldenStubAgent(distro: string): void { + try { + runInWslAsRoot(distro, UNSTAGE_SCRIPT) + } catch { + // Best-effort cleanup; a leftover stub only affects this fixture's own name. + } +} + +/** Points the active worktree's project at the WSL runtime, so agent detection + * and terminal spawn both target the distro instead of the Windows host. */ +export async function useWslRuntimeForActiveProject(page: Page, distro: string): Promise { + await page.evaluate(async (wslDistro) => { + const store = window.__store + if (!store) { + throw new Error('Orca store is unavailable') + } + const state = store.getState() + const worktreeId = state.activeWorktreeId + if (!worktreeId) { + throw new Error('No active worktree') + } + const activeWorktree = Object.values(state.worktreesByRepo) + .flat() + .find((worktree) => worktree.id === worktreeId) + const activeProject = state.projects.find((project) => + activeWorktree ? project.sourceRepoIds.includes(activeWorktree.repoId) : false + ) + if (!activeProject) { + throw new Error('No active project') + } + await state.updateProject(activeProject.id, { + localWindowsRuntimePreference: { kind: 'wsl', distro: wslDistro } + }) + }, distro) +}