mirror of
https://github.com/stablyai/orca.git
synced 2026-10-23 08:03:39 +00:00
test(e2e): gate the tab-bar agent launcher on Windows shells and WSL
The `+` menu agent launcher had no golden coverage in the Windows lane, so a Windows-only break anywhere in its chain (detection row, startup-plan build, tab create, PTY spawn, startup-command injection) could ship unnoticed. Adds a golden spec that launches a stub agent from the menu and asserts the agent's own banner reached the pane — a tab that spawned a bare shell instead is indistinguishable at the store/tab layer. Runs two agents everywhere, and on Windows also PowerShell, cmd, Git Bash and a WSL project runtime.
This commit is contained in:
@@ -74,6 +74,7 @@ jobs:
|
||||
xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:terminal-rendering-golden
|
||||
xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:posix-profile-index-golden
|
||||
xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:agent-tui-golden
|
||||
xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:tab-bar-agent-launch-golden
|
||||
xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:source-control-golden
|
||||
|
||||
- name: Run golden E2E tests on macOS
|
||||
@@ -87,6 +88,7 @@ jobs:
|
||||
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:terminal-rendering-golden
|
||||
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:posix-profile-index-golden
|
||||
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:agent-tui-golden
|
||||
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:tab-bar-agent-launch-golden
|
||||
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:source-control-golden
|
||||
|
||||
- name: Run golden E2E tests on Windows
|
||||
@@ -97,6 +99,7 @@ jobs:
|
||||
$env:ORCA_E2E_FORWARD_APP_LOGS = '1'
|
||||
pnpm run --if-present test:e2e:workspace-session-golden
|
||||
pnpm run --if-present test:e2e:windows-fresh-startup-golden
|
||||
pnpm run --if-present test:e2e:tab-bar-agent-launch-golden
|
||||
if (Test-Path tests/e2e/golden-fresh-profile-terminal.spec.ts) {
|
||||
pnpm run test:e2e -- tests/e2e/golden-fresh-profile-terminal.spec.ts tests/e2e/golden-shell-command.spec.ts
|
||||
}
|
||||
|
||||
@@ -100,6 +100,7 @@
|
||||
"test:e2e:posix-profile-index-golden": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/golden-posix-fresh-startup.spec.ts tests/e2e/golden-posix-profile-index-fsync.spec.ts --grep @posix-profile-index-golden --config tests/playwright.config.ts --project electron-headless --workers=1",
|
||||
"test:e2e:windows-fresh-startup-golden": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/golden-windows-fresh-startup.spec.ts --grep @windows-fresh-startup-golden --config tests/playwright.config.ts --project electron-headless --workers=1",
|
||||
"test:e2e:agent-tui-golden": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/golden-agent-tui-launch.spec.ts tests/e2e/golden-shell-after-agent-exit.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1",
|
||||
"test:e2e:tab-bar-agent-launch-golden": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/golden-tab-bar-agent-launch.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1",
|
||||
"test:e2e:terminal-rendering-release-evidence": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/terminal-opencode-emoji-table-rendering.spec.ts tests/e2e/terminal-long-table-scroll-restore.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=2",
|
||||
"test:e2e:terminal-perf": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/terminal-typing-latency.spec.ts tests/e2e/terminal-foreground-redraw-freeze.spec.ts tests/e2e/terminal-output-scheduler.spec.ts tests/e2e/terminal-hidden-tui-visual-restore.spec.ts tests/e2e/artificial-opencode-terminal-load.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=2",
|
||||
"test:e2e:terminal-perf:scale": "pnpm run ensure:electron-runtime && node config/scripts/run-terminal-scale-perf-e2e.mjs",
|
||||
|
||||
Executable
+2
@@ -0,0 +1,2 @@
|
||||
#!/bin/sh
|
||||
exec "$(dirname "$0")/golden-stub-agent" "$@"
|
||||
@@ -0,0 +1,2 @@
|
||||
@echo off
|
||||
call "%~dp0golden-stub-agent.cmd" %*
|
||||
@@ -0,0 +1,103 @@
|
||||
import type { Page } from '@stablyai/playwright-test'
|
||||
import { expect, test } from './helpers/orca-app'
|
||||
import {
|
||||
configureGoldenStubAgent,
|
||||
getGoldenStubAgentLaunchEnv,
|
||||
GOLDEN_STUB_AGENTS,
|
||||
GOLDEN_STUB_READY_MARKER,
|
||||
launchGoldenStubAgentFromNewTab
|
||||
} from './helpers/golden-stub-agent'
|
||||
import {
|
||||
getFirstWslDistro,
|
||||
removeWslGoldenStubAgent,
|
||||
stageWslGoldenStubAgent,
|
||||
useWslRuntimeForActiveProject
|
||||
} from './helpers/wsl-golden-stub-agent'
|
||||
import { ensureTerminalVisible, waitForActiveWorktree, waitForSessionReady } from './helpers/store'
|
||||
import { getTerminalContent } from './helpers/terminal'
|
||||
import type { BuiltInWindowsTerminalShell } from '../../src/shared/windows-terminal-shell'
|
||||
|
||||
// Covers the full tab-bar `+` agent launch chain — detection row, startup-plan
|
||||
// build, tab create, PTY spawn under the configured runtime, startup-command
|
||||
// injection. golden-agent-tui-launch.spec.ts covers one agent on the default
|
||||
// shell and never runs in the Windows golden lane, so a Windows-only break
|
||||
// anywhere in that chain ships as "clicking an agent does nothing".
|
||||
|
||||
test.use({ launchEnv: getGoldenStubAgentLaunchEnv() })
|
||||
|
||||
const WINDOWS_SHELLS: readonly BuiltInWindowsTerminalShell[] = [
|
||||
'powershell.exe',
|
||||
'cmd.exe',
|
||||
'git-bash'
|
||||
]
|
||||
|
||||
async function openWorkspaceTerminal(page: Page): Promise<void> {
|
||||
await waitForSessionReady(page)
|
||||
await waitForActiveWorktree(page)
|
||||
await ensureTerminalVisible(page)
|
||||
}
|
||||
|
||||
for (const { id, menuItemName } of GOLDEN_STUB_AGENTS) {
|
||||
test(`tab-bar + menu launches ${id} into a live TUI @tab-bar-agent-launch-golden`, async ({
|
||||
orcaPage
|
||||
}) => {
|
||||
await openWorkspaceTerminal(orcaPage)
|
||||
await configureGoldenStubAgent(orcaPage, { agent: id })
|
||||
await launchGoldenStubAgentFromNewTab(orcaPage, menuItemName)
|
||||
|
||||
const activeTab = orcaPage.locator('[data-testid="sortable-tab"][data-active="true"]')
|
||||
await expect(activeTab).toHaveAttribute('data-tab-title', /Golden Stub Agent|Codex|Claude/i)
|
||||
// Why the banner and not just the tab: a tab that spawned a bare shell
|
||||
// instead of the agent looks identical at the store/tab layer.
|
||||
expect(await getTerminalContent(orcaPage)).toContain(GOLDEN_STUB_READY_MARKER)
|
||||
})
|
||||
}
|
||||
|
||||
// Why describe-scoped and not a body-level skip: `orcaPage` builds before the
|
||||
// test body, so a body-level skip still pays a full Electron launch per case.
|
||||
test.describe('Windows runtimes', () => {
|
||||
test.skip(process.platform !== 'win32', 'Windows agent launch matrix is Windows-only')
|
||||
|
||||
for (const shell of WINDOWS_SHELLS) {
|
||||
test(`tab-bar + menu launches an agent under ${shell} @tab-bar-agent-launch-golden`, async ({
|
||||
orcaPage
|
||||
}) => {
|
||||
await openWorkspaceTerminal(orcaPage)
|
||||
// Why each shell: the queued launch command is quoted for the shell family
|
||||
// the tab actually spawns (posix for Git Bash, cmd, PowerShell). A mismatch
|
||||
// injects a command the shell rejects, and the agent never starts.
|
||||
await configureGoldenStubAgent(orcaPage, { agent: 'codex', windowsShell: shell })
|
||||
await launchGoldenStubAgentFromNewTab(orcaPage)
|
||||
|
||||
expect(await getTerminalContent(orcaPage)).toContain(GOLDEN_STUB_READY_MARKER)
|
||||
})
|
||||
}
|
||||
|
||||
test('tab-bar + menu launches an agent inside WSL @tab-bar-agent-launch-golden', async ({
|
||||
orcaPage
|
||||
}) => {
|
||||
await openWorkspaceTerminal(orcaPage)
|
||||
|
||||
const distro = await getFirstWslDistro(orcaPage)
|
||||
test.skip(!distro, 'No WSL distro is available on this Windows host')
|
||||
test.skip(
|
||||
!stageWslGoldenStubAgent(distro!),
|
||||
'WSL distro would not accept the staged stub agent'
|
||||
)
|
||||
|
||||
try {
|
||||
// Why the runtime switch and not terminalWindowsShell: WSL is a project
|
||||
// runtime, so it must retarget agent detection into the distro as well as
|
||||
// the PTY. A launch quoted for a Windows shell never starts there.
|
||||
await useWslRuntimeForActiveProject(orcaPage, distro!)
|
||||
await configureGoldenStubAgent(orcaPage, { agent: 'codex' })
|
||||
await launchGoldenStubAgentFromNewTab(orcaPage)
|
||||
|
||||
// The stub only exists inside the distro, so its marker proves the agent
|
||||
// ran in WSL rather than on the Windows host.
|
||||
expect(await getTerminalContent(orcaPage)).toContain(GOLDEN_STUB_READY_MARKER)
|
||||
} finally {
|
||||
removeWslGoldenStubAgent(distro!)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -2,10 +2,18 @@ import path from 'node:path'
|
||||
import type { Page } from '@stablyai/playwright-test'
|
||||
import { expect } from '@stablyai/playwright-test'
|
||||
import { focusActiveTerminalInput, waitForTerminalOutput } from './terminal'
|
||||
import type { BuiltInWindowsTerminalShell } from '../../../src/shared/windows-terminal-shell'
|
||||
|
||||
export const GOLDEN_STUB_READY_MARKER = 'GOLDEN_STUB_AGENT_READY'
|
||||
export const GOLDEN_STUB_EXIT_MARKER = 'GOLDEN_STUB_AGENT_EXITED'
|
||||
|
||||
/** Agents the fixture dir shims onto PATH, so detection lists them and the
|
||||
* tab-bar `+` menu renders a launch row for each. */
|
||||
export const GOLDEN_STUB_AGENTS = [
|
||||
{ id: 'codex', menuItemName: /^Codex(?:\s|$)/i },
|
||||
{ id: 'claude', menuItemName: /^Claude(?:\s|$)/i }
|
||||
] as const
|
||||
|
||||
const fixtureDir = path.join(process.cwd(), 'tests', 'e2e', 'fixtures', 'golden-stub-agent')
|
||||
|
||||
export function getGoldenStubAgentLaunchEnv(): NodeJS.ProcessEnv {
|
||||
@@ -15,23 +23,38 @@ export function getGoldenStubAgentLaunchEnv(): NodeJS.ProcessEnv {
|
||||
}
|
||||
}
|
||||
|
||||
export async function configureGoldenStubAgent(page: Page): Promise<void> {
|
||||
await page.evaluate(async () => {
|
||||
const store = window.__store
|
||||
if (!store) {
|
||||
throw new Error('Orca store is unavailable')
|
||||
}
|
||||
await store.getState().updateSettings({
|
||||
defaultTuiAgent: 'codex',
|
||||
agentCmdOverrides: { codex: 'golden-stub-agent' },
|
||||
agentDefaultArgs: { codex: '' }
|
||||
})
|
||||
})
|
||||
export async function configureGoldenStubAgent(
|
||||
page: Page,
|
||||
options: {
|
||||
agent?: (typeof GOLDEN_STUB_AGENTS)[number]['id']
|
||||
/** Windows default shell the launch command must survive; ignored elsewhere. */
|
||||
windowsShell?: BuiltInWindowsTerminalShell
|
||||
} = {}
|
||||
): Promise<void> {
|
||||
const agent = options.agent ?? 'codex'
|
||||
await page.evaluate(
|
||||
async ({ agent, windowsShell }) => {
|
||||
const store = window.__store
|
||||
if (!store) {
|
||||
throw new Error('Orca store is unavailable')
|
||||
}
|
||||
await store.getState().updateSettings({
|
||||
defaultTuiAgent: agent,
|
||||
agentCmdOverrides: { [agent]: 'golden-stub-agent' },
|
||||
agentDefaultArgs: { [agent]: '' },
|
||||
...(windowsShell ? { terminalWindowsShell: windowsShell } : {})
|
||||
})
|
||||
},
|
||||
{ agent, windowsShell: options.windowsShell ?? null }
|
||||
)
|
||||
}
|
||||
|
||||
export async function launchGoldenStubAgentFromNewTab(page: Page): Promise<void> {
|
||||
export async function launchGoldenStubAgentFromNewTab(
|
||||
page: Page,
|
||||
menuItemName: RegExp = /^Codex(?:\s|$)/i
|
||||
): Promise<void> {
|
||||
await page.getByRole('button', { name: 'New tab' }).click({ force: true })
|
||||
const launchOption = page.getByRole('menuitem', { name: /^Codex(?:\s|$)/i }).first()
|
||||
const launchOption = page.getByRole('menuitem', { name: menuItemName }).first()
|
||||
await expect(launchOption).toBeVisible({ timeout: 15_000 })
|
||||
await launchOption.click({ force: true })
|
||||
await focusActiveTerminalInput(page)
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import type { Page } from '@stablyai/playwright-test'
|
||||
import { buildWslExecArgs } from '../../../src/shared/wsl-login-shell-command'
|
||||
|
||||
/** Distro path the stub is staged at. Only reachable from inside WSL, so the
|
||||
* stub's marker in a pane is itself proof the pane executed in the distro. */
|
||||
const WSL_STUB_PATH = '/usr/local/bin/golden-stub-agent'
|
||||
const WSL_STUB_AGENT_LINK = '/usr/local/bin/codex'
|
||||
|
||||
// Why printf and not a heredoc: the whole script crosses to wsl.exe as one argv
|
||||
// element, and keeping it newline-free avoids Windows argv-encoding surprises.
|
||||
const STAGE_SCRIPT =
|
||||
`mkdir -p /usr/local/bin && ` +
|
||||
`printf '#!/bin/sh\\necho GOLDEN_STUB_AGENT_READY\\nexec sleep 3600\\n' > ${WSL_STUB_PATH} && ` +
|
||||
`chmod 0755 ${WSL_STUB_PATH} && ` +
|
||||
`ln -sf ${WSL_STUB_PATH} ${WSL_STUB_AGENT_LINK}`
|
||||
|
||||
const UNSTAGE_SCRIPT = `rm -f ${WSL_STUB_AGENT_LINK} ${WSL_STUB_PATH}`
|
||||
|
||||
// Why --exec (via buildWslExecArgs): under `--`, wsl.exe expands `$name` in
|
||||
// every argument and silently rewrites the script.
|
||||
function runInWslAsRoot(distro: string, script: string): void {
|
||||
execFileSync('wsl.exe', ['-u', 'root', ...buildWslExecArgs(distro, ['sh', '-c', script])], {
|
||||
stdio: 'pipe',
|
||||
windowsHide: true
|
||||
})
|
||||
}
|
||||
|
||||
export async function getFirstWslDistro(page: Page): Promise<string | null> {
|
||||
const wsl = await page.evaluate(async () => ({
|
||||
available: await window.api.wsl.isAvailable(),
|
||||
distros: await window.api.wsl.listDistros()
|
||||
}))
|
||||
return wsl.available ? (wsl.distros[0] ?? null) : null
|
||||
}
|
||||
|
||||
/** Returns false when the distro won't take the stub (no root interop, read-only
|
||||
* rootfs); callers skip rather than fail, since that is a host limitation. */
|
||||
export function stageWslGoldenStubAgent(distro: string): boolean {
|
||||
try {
|
||||
runInWslAsRoot(distro, STAGE_SCRIPT)
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
export function removeWslGoldenStubAgent(distro: string): void {
|
||||
try {
|
||||
runInWslAsRoot(distro, UNSTAGE_SCRIPT)
|
||||
} catch {
|
||||
// Best-effort cleanup; a leftover stub only affects this fixture's own name.
|
||||
}
|
||||
}
|
||||
|
||||
/** Points the active worktree's project at the WSL runtime, so agent detection
|
||||
* and terminal spawn both target the distro instead of the Windows host. */
|
||||
export async function useWslRuntimeForActiveProject(page: Page, distro: string): Promise<void> {
|
||||
await page.evaluate(async (wslDistro) => {
|
||||
const store = window.__store
|
||||
if (!store) {
|
||||
throw new Error('Orca store is unavailable')
|
||||
}
|
||||
const state = store.getState()
|
||||
const worktreeId = state.activeWorktreeId
|
||||
if (!worktreeId) {
|
||||
throw new Error('No active worktree')
|
||||
}
|
||||
const activeWorktree = Object.values(state.worktreesByRepo)
|
||||
.flat()
|
||||
.find((worktree) => worktree.id === worktreeId)
|
||||
const activeProject = state.projects.find((project) =>
|
||||
activeWorktree ? project.sourceRepoIds.includes(activeWorktree.repoId) : false
|
||||
)
|
||||
if (!activeProject) {
|
||||
throw new Error('No active project')
|
||||
}
|
||||
await state.updateProject(activeProject.id, {
|
||||
localWindowsRuntimePreference: { kind: 'wsl', distro: wslDistro }
|
||||
})
|
||||
}, distro)
|
||||
}
|
||||
Reference in New Issue
Block a user