test: run real WSL terminal launch and paste in PR CI (#19072)

* test: continuously exercise real WSL terminal launch and paste

* test: establish live WSL reader before changing default shell

* ci: pin WSL kernel installer and participation selectors

* ci: route deleted WSL paths and record immutable run evidence

* test: require exactly three WSL repetitions in lane contract
This commit is contained in:
Neil
2026-09-06 05:24:54 -07:00
committed by GitHub
parent adcc30be3b
commit f952f1ac96
10 changed files with 415 additions and 4 deletions
@@ -0,0 +1,8 @@
name: Set up WSL test runtime
description: Install a checksum-pinned Ubuntu WSL1 guest with executable Node and Git for real terminal tests.
runs:
using: composite
steps:
- name: Provision Ubuntu WSL1
shell: pwsh
run: '& "${{ github.action_path }}/setup.ps1"'
@@ -0,0 +1,32 @@
$ErrorActionPreference = 'Stop'
if (-not $IsWindows) { throw 'WSL test provisioning requires a Windows runner' }
$rootfs = Join-Path $env:RUNNER_TEMP 'noble-rootfs.tar.gz'
Invoke-WebRequest 'https://releases.ubuntu.com/24.04.4/ubuntu-24.04.4-wsl-amd64.wsl' -OutFile $rootfs
if ((Get-FileHash $rootfs -Algorithm SHA256).Hash.ToLowerInvariant() -ne '9b2f7730dc68227dd04a9f3e5eab86ad85caf556b8606ad94f1f29ff5c4fd3f5') { throw 'Ubuntu rootfs checksum mismatch' }
$distroDir = Join-Path $env:RUNNER_TEMP 'orca-wsl-ubuntu'
wsl.exe --import Ubuntu $distroDir $rootfs --version 1
if ($LASTEXITCODE -ne 0) { throw "WSL import failed: $LASTEXITCODE" }
wsl.exe --distribution Ubuntu --user root --exec /usr/bin/true
if ($LASTEXITCODE -ne 0) { throw "WSL guest did not start: $LASTEXITCODE" }
wsl.exe --distribution Ubuntu --user root --exec /usr/bin/apt-get update
if ($LASTEXITCODE -ne 0) { throw "WSL apt update failed: $LASTEXITCODE" }
wsl.exe --distribution Ubuntu --user root --exec /usr/bin/apt-get install --yes git curl xz-utils
if ($LASTEXITCODE -ne 0) { throw "WSL git install failed: $LASTEXITCODE" }
$kernelMsi = Join-Path $env:RUNNER_TEMP 'wsl_update_x64.msi'
Invoke-WebRequest 'https://wslstorestorage.blob.core.windows.net/wslblob/wsl_update_x64.msi' -OutFile $kernelMsi
if ((Get-FileHash $kernelMsi -Algorithm SHA256).Hash.ToLowerInvariant() -ne '4d09c776c8d45f70a202281d18e19be1118f53159b0c217a5274a31ce18525fe') { throw 'WSL kernel installer checksum mismatch' }
$installer = Start-Process msiexec.exe -ArgumentList @('/i', $kernelMsi, '/quiet', '/norestart') -Wait -PassThru
if ($installer.ExitCode -ne 0) { throw "WSL kernel installation failed: $($installer.ExitCode)" }
wsl.exe --status
if ($LASTEXITCODE -ne 0) { throw "WSL status failed: $LASTEXITCODE" }
wsl.exe --distribution Ubuntu --user root --exec /usr/bin/curl --fail --silent --show-error --location https://nodejs.org/dist/v22.14.0/node-v22.14.0-linux-x64.tar.xz --output /tmp/orca-node.tar.xz
if ($LASTEXITCODE -ne 0) { throw 'Node download failed' }
$nodeHash = wsl.exe --distribution Ubuntu --user root --exec /usr/bin/sha256sum /tmp/orca-node.tar.xz
if ($LASTEXITCODE -ne 0 -or -not ($nodeHash -match '^69b09dba5c8dcb05c4e4273a4340db1005abeafe3927efda2bc5b249e80437ec')) { throw 'Node checksum mismatch' }
wsl.exe --distribution Ubuntu --user root --exec /usr/bin/tar -xJf /tmp/orca-node.tar.xz -C /usr/local --strip-components=1
if ($LASTEXITCODE -ne 0) { throw 'Node extraction failed' }
wsl.exe --distribution Ubuntu --user root --exec /usr/local/bin/node --version
if ($LASTEXITCODE -ne 0) { throw 'Node cannot execute in WSL' }
wsl.exe --list --verbose
if ($LASTEXITCODE -ne 0) { throw "WSL enumeration failed: $LASTEXITCODE" }
+14
View File
@@ -45,6 +45,7 @@ jobs:
test_files: ${{ steps.e2e_filter.outputs.test_files }}
ssh_source_changed: ${{ steps.e2e_filter.outputs.ssh_source_changed }}
native_ime_source_changed: ${{ steps.e2e_filter.outputs.native_ime_source_changed }}
wsl_source_changed: ${{ steps.e2e_filter.outputs.wsl_source_changed }}
steps:
- name: Checkout
uses: actions/checkout@v6
@@ -92,6 +93,9 @@ jobs:
# trigger on IME source rather than on a spec name in some route's list.
NATIVE_IME_SOURCE_CHANGED="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --native-ime-source)"
echo "native_ime_source_changed=$NATIVE_IME_SOURCE_CHANGED" >> "$GITHUB_OUTPUT"
WSL_CHANGED="$(git diff --name-only --no-renames --diff-filter=ACDMR --merge-base "$BASE" "$HEAD")"
WSL_SOURCE_CHANGED="$(printf '%s\n' "$WSL_CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --wsl-source)"
echo "wsl_source_changed=$WSL_SOURCE_CHANGED" >> "$GITHUB_OUTPUT"
echo "Native IME source changed: $NATIVE_IME_SOURCE_CHANGED"
SHOULD_RUN="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --reusable-workflow)"
if [ "$SHOULD_RUN" = true ]; then
@@ -942,6 +946,16 @@ jobs:
contents: read
uses: ./.github/workflows/terminal-ime-e2e.yml
windows_wsl:
name: real WSL terminal
needs: code_paths
if: needs.code_paths.outputs.wsl_source_changed == 'true'
permissions:
contents: read
uses: ./.github/workflows/windows-wsl-e2e.yml
with:
ref: ${{ github.event.pull_request.head.sha }}
verify:
if: always()
needs:
+74
View File
@@ -0,0 +1,74 @@
name: Windows WSL terminal E2E
on:
workflow_dispatch:
inputs:
ref:
description: Commit to validate
type: string
required: false
workflow_call:
inputs:
ref:
type: string
required: false
permissions:
contents: read
concurrency:
group: windows-wsl-e2e-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
wsl-terminal:
runs-on: windows-2022
timeout-minutes: 30
env:
NODE_OPTIONS: --max-old-space-size=4096
steps:
- uses: actions/checkout@v6
with:
ref: ${{ inputs.ref || github.sha }}
persist-credentials: false
- uses: ./.github/actions/setup-wsl-test-runtime
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: electron
- name: Build relay and Electron
run: |
pnpm run build:relay
if ($LASTEXITCODE -ne 0) { throw 'Relay build failed' }
pnpm exec electron-vite build --mode e2e
if ($LASTEXITCODE -ne 0) { throw 'Electron build failed' }
- name: Exercise real WSL launch and paste
env:
SKIP_BUILD: '1'
ORCA_E2E_FORWARD_APP_LOGS: '1'
PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/wsl-results.json
run: >-
pnpm exec playwright test
tests/e2e/golden-tab-bar-agent-launch.spec.ts
tests/e2e/terminal-windows-shell-paste-ownership.spec.ts
--config tests/playwright.config.ts
--project=electron-headless
--grep "WSL"
--repeat-each=3
--workers=1
--reporter=list,json
- name: Require all nine WSL executions
if: always()
run: node config/scripts/verify-wsl-e2e-participation.mjs test-results/wsl-results.json
- name: Upload WSL participation report
uses: actions/upload-artifact@v7
if: always()
with:
name: windows-wsl-participation-report
path: test-results/wsl-results.json
retention-days: 3
- uses: actions/upload-artifact@v7
if: failure()
with:
name: windows-wsl-terminal-traces
path: test-results/
retention-days: 7
+84
View File
@@ -18151,6 +18151,90 @@
"No p95 CI history or full product mutation proof."
],
"demotionRule": "Keep experimental while any recovery reproduction fails or any teardown, identity, resource-count, or rendered oracle flakes; never promote by extending sleeps or retries."
},
{
"id": "terminal.windows-wsl-launch-and-paste",
"title": "Real WSL terminal agent launch and paste ownership",
"maturity": "experimental",
"protection": "partial",
"owner": "terminal-runtime",
"layer": "electron-windows-wsl",
"surfaces": ["agent tab launch", "keyboard paste", "terminal runtime retention"],
"platforms": ["windows"],
"providers": ["wsl1", "wsl2"],
"coveredPlatforms": ["windows"],
"coveredProviders": ["wsl1"],
"coverageNotes": "Real WSL1 coverage: three scenarios each passed three times with no skips or retries; exact JSON report verified. Latest PR routing and installer-checksum follow-ups await CI. WSL2 remains untested.",
"motivatingLinks": ["https://github.com/stablyai/orca/actions/runs/34030832614"],
"invariant": "An agent launched into WSL runs in the guest; keyboard paste reaches exactly one owning PTY and preserves Linux content even after the default shell changes.",
"oracle": "Run the existing real WSL launch and two paste cases three times; require nine passes and zero skipped, unexpected, or flaky results in the Playwright JSON report.",
"commands": [
"gh workflow run windows-wsl-e2e.yml",
"pnpm exec playwright test tests/e2e/golden-tab-bar-agent-launch.spec.ts tests/e2e/terminal-windows-shell-paste-ownership.spec.ts --config tests/playwright.config.ts --project=electron-headless --workers=1",
"node_modules/.bin/vitest run --config config/vitest.config.ts config/scripts/wsl-e2e-lane-contract.test.mjs config/scripts/verify-wsl-e2e-participation.test.mjs",
"gh run view 34031806291 --log"
],
"testFiles": [
"tests/e2e/golden-tab-bar-agent-launch.spec.ts",
"tests/e2e/terminal-windows-shell-paste-ownership.spec.ts",
"config/scripts/wsl-e2e-lane-contract.test.mjs",
"config/scripts/verify-wsl-e2e-participation.test.mjs"
],
"assertionRefs": [
{
"file": "tests/e2e/golden-tab-bar-agent-launch.spec.ts",
"assertions": ["requires a distro-only marker from the launched agent"]
},
{
"file": "tests/e2e/terminal-windows-shell-paste-ownership.spec.ts",
"assertions": [
"requires exact Linux pasted content and exactly one PTY write",
"retains WSL paste ownership after changing the default shell"
]
},
{
"file": "config/scripts/verify-wsl-e2e-participation.test.mjs",
"assertions": ["rejects skipped, missing, substituted and retried scenarios"]
}
],
"evidenceRuns": [
{
"date": "2026-09-06",
"runner": "ci",
"platform": "windows",
"result": "passed",
"command": "gh run view 34031806291 --log",
"durationSeconds": 210,
"summary": "Immutable run34031806291 at92fc5152: WSL1 launch3 and paste6 passed after reader-readiness correction; named-scenario verifier accepted actual JSON report with0skips0retries. Command retrieves recorded evidence; workflow_dispatch command above reruns current coverage."
}
],
"runtimeBudget": {
"p95Seconds": 1800,
"scope": "CI job timeout; measured p95 is not established"
},
"flakeHistory": {
"status": "soaking",
"evidence": "Initial permanent-lane diagnostic8passed1failed on missing PTY before changing settings. After requiring guest-reader readiness before mutation, run34031806291 passed9/9. Two earlier setup validations also passed9/9. Long-term CI history remains missing."
},
"redGreenEvidence": {
"status": "partial",
"evidence": "Verifier rejects actual8pass1fail CI report and accepts actual9pass report. Unit contracts reject skips, missing or substituted scenarios and retried passes. No full application fault-mutation proof."
},
"performanceBudget": {
"required": false,
"evidence": "CI-only provisioning and routing; no application runtime changes."
},
"promotionCriteria": [
"Require all nine real WSL executions on the final workflow head.",
"Demonstrate missing or skipped WSL execution fails participation.",
"Collect repeated CI history before adding this experimental lane to required verification."
],
"knownGaps": [
"WSL2 is not provisioned.",
"No SSH, folder-only workspace, packaged mixed-version, or live-service claim.",
"The new PR lane is outside verify until reliability is established."
],
"demotionRule": "Keep experimental if provisioning or an execution flakes; never promote by skipping a case, raising timeouts, or retrying until green."
}
]
}
+21
View File
@@ -13,6 +13,18 @@ const NATIVE_IME_HARNESS =
/^(?:config\/scripts\/(?:run-terminal-ibus-hangul-e2e|terminal-ime-engagement-receipt)\.mjs$|tests\/e2e\/terminal-ime-(?:boundary-probe|byte-reader|engagement-receipt)\.ts$|tests\/e2e\/terminal-(?:ibus-hangul|hangul-terminating-digit|macos-2set-korean)-native\.spec\.ts$)/
export const PR_E2E_SOURCE_ROUTES = [
{
id: 'terminal.windows-wsl-launch-and-paste',
specs: [
'tests/e2e/golden-tab-bar-agent-launch.spec.ts',
'tests/e2e/terminal-windows-shell-paste-ownership.spec.ts'
],
matches: (file) =>
isProductSource(file) &&
/^(?:config\/scripts\/verify-wsl-e2e-participation\.mjs$|src\/main\/(?:wsl[/-]|pty\/.*wsl|providers\/wsl)|src\/shared\/(?:wsl-|windows-terminal-shell)|src\/renderer\/src\/.*(?:terminal-paste|pty-paste)|tests\/e2e\/(?:golden-tab-bar-agent-launch\.spec|terminal-windows-shell-paste-ownership\.spec|helpers\/(?:wsl-golden-stub-agent|golden-stub-agent))|\.github\/(?:actions\/setup-wsl-test-runtime\/|workflows\/windows-wsl-e2e\.yml))/.test(
file
)
},
{
id: 'ephemeral-vm-runtime.rollback-readable-sidecar',
specs: ['tests/e2e/ephemeral-vm-provisioned-root.spec.ts'],
@@ -227,6 +239,13 @@ export function shouldRunReusablePrE2e(changedPaths) {
)
}
export function hasWslSourceChange(changedPaths) {
const route = PR_E2E_SOURCE_ROUTES.find(
(candidate) => candidate.id === 'terminal.windows-wsl-launch-and-paste'
)
return changedPaths.some(route.matches)
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
let input = ''
process.stdin.setEncoding('utf8')
@@ -238,6 +257,8 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)
process.stdout.write(`${hasSshSourceChange(changedPaths)}\n`)
} else if (process.argv.includes('--reusable-workflow')) {
process.stdout.write(`${shouldRunReusablePrE2e(changedPaths)}\n`)
} else if (process.argv.includes('--wsl-source')) {
process.stdout.write(`${hasWslSourceChange(changedPaths)}\n`)
} else if (process.argv.includes('--native-ime-source')) {
process.stdout.write(`${hasNativeImeSourceChange(changedPaths)}\n`)
} else {
@@ -0,0 +1,54 @@
import { readFileSync } from 'node:fs'
import { pathToFileURL } from 'node:url'
export const WSL_TEST_TITLES = [
'tab-bar + menu launches an agent inside WSL @tab-bar-agent-launch-golden',
'WSL terminal keyboard paste preserves Linux shell content with one PTY owner',
'existing WSL terminal keeps paste runtime after default shell changes'
]
export function verifyWslParticipation(report) {
const stats = report?.stats
if (
!stats ||
stats.expected !== 9 ||
stats.skipped !== 0 ||
stats.unexpected !== 0 ||
stats.flaky !== 0 ||
report.errors?.length
) {
throw new Error(`WSL participation failed: ${JSON.stringify(stats)}`)
}
const counts = new Map(WSL_TEST_TITLES.map((title) => [title, 0]))
const visit = (suites) => {
for (const suite of suites ?? []) {
for (const spec of suite.specs ?? []) {
if (!counts.has(spec.title)) {
throw new Error(`Unexpected WSL scenario: ${spec.title}`)
}
for (const test of spec.tests ?? []) {
if (
test.expectedStatus !== 'passed' ||
test.results?.length !== 1 ||
test.results[0].status !== 'passed'
) {
throw new Error(`WSL scenario did not pass without retries: ${spec.title}`)
}
counts.set(spec.title, counts.get(spec.title) + 1)
}
}
visit(suite.suites)
}
}
visit(report.suites)
for (const [title, count] of counts) {
if (count !== 3) {
throw new Error(`WSL scenario requires three executions: ${title} (${count})`)
}
}
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
verifyWslParticipation(JSON.parse(readFileSync(process.argv[2], 'utf8')))
console.log('All three WSL scenarios passed three times without skips or retries.')
}
@@ -0,0 +1,52 @@
import { describe, expect, it } from 'vitest'
import { verifyWslParticipation, WSL_TEST_TITLES } from './verify-wsl-e2e-participation.mjs'
function report() {
return {
stats: { expected: 9, skipped: 0, unexpected: 0, flaky: 0 },
suites: [
{
suites: [
{
specs: WSL_TEST_TITLES.map((title) => ({
title,
tests: Array.from({ length: 3 }, () => ({
expectedStatus: 'passed',
results: [{ status: 'passed' }]
}))
}))
}
]
}
]
}
}
describe('WSL participation', () => {
it('accepts all three named scenarios executed three times', () => {
expect(() => verifyWslParticipation(report())).not.toThrow()
})
it.each(['skipped', 'unexpected', 'flaky'])('rejects a nonzero %s result', (key) => {
const value = report()
value.stats[key] = 1
expect(() => verifyWslParticipation(value)).toThrow('participation failed')
})
it('rejects missing scenarios even when aggregate counts claim nine passes', () => {
const value = report()
value.suites[0].suites[0].specs.pop()
expect(() => verifyWslParticipation(value)).toThrow('requires three executions')
})
it('rejects an unrelated scenario substituted for an expected scenario', () => {
const value = report()
value.suites[0].suites[0].specs[0].title = 'native shell passes'
expect(() => verifyWslParticipation(value)).toThrow('Unexpected WSL scenario')
})
it('rejects a pass obtained after a failed attempt', () => {
const value = report()
value.suites[0].suites[0].specs[0].tests[0].results.unshift({ status: 'failed' })
expect(() => verifyWslParticipation(value)).toThrow('without retries')
})
it('rejects missing report content', () => {
expect(() => verifyWslParticipation({})).toThrow('participation failed')
})
})
@@ -0,0 +1,69 @@
import { readFileSync } from 'node:fs'
import { describe, expect, it } from 'vitest'
import { parse } from 'yaml'
import { hasWslSourceChange, selectPrE2eSpecs } from './pr-e2e-source-routing.mjs'
const read = (path) => readFileSync(new URL(`../../${path}`, import.meta.url), 'utf8')
describe('real WSL terminal lane', () => {
it.each([
'config/scripts/verify-wsl-e2e-participation.mjs',
'src/main/wsl-availability.ts',
'src/main/wsl/wsl-runner.ts',
'src/main/pty/wsl-orca-env.ts',
'src/shared/wsl-login-shell-command.ts',
'src/shared/windows-terminal-shell.ts',
'tests/e2e/helpers/wsl-golden-stub-agent.ts',
'tests/e2e/golden-tab-bar-agent-launch.spec.ts',
'tests/e2e/terminal-windows-shell-paste-ownership.spec.ts',
'.github/actions/setup-wsl-test-runtime/setup.ps1',
'.github/workflows/windows-wsl-e2e.yml'
])('routes %s to both WSL sentinels', (path) => {
expect(hasWslSourceChange([path])).toBe(true)
expect(selectPrE2eSpecs([path])).toEqual(
expect.arrayContaining([
'tests/e2e/golden-tab-bar-agent-launch.spec.ts',
'tests/e2e/terminal-windows-shell-paste-ownership.spec.ts'
])
)
})
it.each([
'docs/reference/wsl-command-execution.md',
'src/main/wsl-availability.test.ts',
'src/main/ssh/connection.ts'
])('excludes unrelated or unit-only change %s', (path) => {
expect(hasWslSourceChange([path])).toBe(false)
})
it('runs the reusable lane at the immutable PR head', () => {
const pr = parse(read('.github/workflows/pr.yml'))
expect(pr.jobs.windows_wsl.if).toBe("needs.code_paths.outputs.wsl_source_changed == 'true'")
expect(pr.jobs.windows_wsl.with.ref).toBe('${{ github.event.pull_request.head.sha }}')
const detector = pr.jobs['code_paths'].steps.find(
(step) => step.name === 'Filter changed E2E specs'
)
expect(detector.run).toContain(
'WSL_CHANGED="$(git diff --name-only --no-renames --diff-filter=ACDMR'
)
expect(detector.run).toContain(
'"$WSL_CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --wsl-source'
)
const workflow = parse(read('.github/workflows/windows-wsl-e2e.yml'))
const steps = workflow.jobs['wsl-terminal'].steps
expect(steps[0].with.ref).toBe('${{ inputs.ref || github.sha }}')
expect(steps.some((step) => step.uses === './.github/actions/setup-wsl-test-runtime')).toBe(
true
)
const exercise = steps.find((step) => step.name === 'Exercise real WSL launch and paste')
expect(exercise.run.split(/\s+/).filter((arg) => arg.startsWith('--repeat-each='))).toEqual([
'--repeat-each=3'
])
expect(exercise.run).toContain('--grep "WSL"')
const receipt = steps.find((step) => step.name === 'Require all nine WSL executions')
expect(receipt.if).toBe('always()')
expect(receipt.run).toBe(
'node config/scripts/verify-wsl-e2e-participation.mjs test-results/wsl-results.json'
)
})
})
@@ -423,10 +423,6 @@ test.describe('Windows terminal shell paste ownership', () => {
const wslDistro = await configureActiveProjectWslRuntime(orcaPage)
test.skip(!wslDistro, 'No WSL distro is available on this Windows host')
const tabId = await createWindowsProjectRuntimeTerminalTab(orcaPage, 'wsl.exe')
await updateWindowsDefaultShellSetting(orcaPage, 'cmd.exe')
await expect(
orcaPage.locator(`[data-testid="sortable-tab"][data-tab-id="${tabId}"] [data-shell-icon]`)
).toHaveAttribute('data-shell-icon', 'wsl.exe')
await waitForActiveTerminalManager(orcaPage, 30_000)
await installTerminalPtyWriteSpy(electronApp)
@@ -454,6 +450,13 @@ test.describe('Windows terminal shell paste ownership', () => {
scriptStarted = true
await waitForTerminalOutput(orcaPage, `PASTE_READY_${runId}`, 10_000)
// Exercise a live WSL process across the settings change.
await updateWindowsDefaultShellSetting(orcaPage, 'cmd.exe')
await expect(
orcaPage.locator(`[data-testid="sortable-tab"][data-tab-id="${tabId}"] [data-shell-icon]`)
).toHaveAttribute('data-shell-icon', 'wsl.exe')
expect(await waitForActivePanePtyId(orcaPage)).toBe(ptyId)
await clearTerminalPtyWriteLog(electronApp)
await orcaPage.evaluate((text) => window.api.ui.writeClipboardText(text), payload)
await focusActiveTerminalInput(orcaPage)