test: continuously exercise real WSL terminal launch and paste

This commit is contained in:
Neil
2026-09-06 04:48:58 -07:00
parent adcc30be3b
commit 72d4b6dc1b
9 changed files with 411 additions and 0 deletions
@@ -0,0 +1,8 @@
name: Set up WSL test runtime
description: Install a checksum-pinned Ubuntu WSL1 guest with executable Node and Git for real terminal tests.
runs:
using: composite
steps:
- name: Provision Ubuntu WSL1
shell: pwsh
run: '& "${{ github.action_path }}/setup.ps1"'
@@ -0,0 +1,31 @@
$ErrorActionPreference = 'Stop'
if (-not $IsWindows) { throw 'WSL test provisioning requires a Windows runner' }
$rootfs = Join-Path $env:RUNNER_TEMP 'noble-rootfs.tar.gz'
Invoke-WebRequest 'https://releases.ubuntu.com/24.04.4/ubuntu-24.04.4-wsl-amd64.wsl' -OutFile $rootfs
if ((Get-FileHash $rootfs -Algorithm SHA256).Hash.ToLowerInvariant() -ne '9b2f7730dc68227dd04a9f3e5eab86ad85caf556b8606ad94f1f29ff5c4fd3f5') { throw 'Ubuntu rootfs checksum mismatch' }
$distroDir = Join-Path $env:RUNNER_TEMP 'orca-wsl-ubuntu'
wsl.exe --import Ubuntu $distroDir $rootfs --version 1
if ($LASTEXITCODE -ne 0) { throw "WSL import failed: $LASTEXITCODE" }
wsl.exe --distribution Ubuntu --user root --exec /usr/bin/true
if ($LASTEXITCODE -ne 0) { throw "WSL guest did not start: $LASTEXITCODE" }
wsl.exe --distribution Ubuntu --user root --exec /usr/bin/apt-get update
if ($LASTEXITCODE -ne 0) { throw "WSL apt update failed: $LASTEXITCODE" }
wsl.exe --distribution Ubuntu --user root --exec /usr/bin/apt-get install --yes git curl xz-utils
if ($LASTEXITCODE -ne 0) { throw "WSL git install failed: $LASTEXITCODE" }
$kernelMsi = Join-Path $env:RUNNER_TEMP 'wsl_update_x64.msi'
Invoke-WebRequest 'https://wslstorestorage.blob.core.windows.net/wslblob/wsl_update_x64.msi' -OutFile $kernelMsi
$installer = Start-Process msiexec.exe -ArgumentList @('/i', $kernelMsi, '/quiet', '/norestart') -Wait -PassThru
if ($installer.ExitCode -ne 0) { throw "WSL kernel installation failed: $($installer.ExitCode)" }
wsl.exe --status
if ($LASTEXITCODE -ne 0) { throw "WSL status failed: $LASTEXITCODE" }
wsl.exe --distribution Ubuntu --user root --exec /usr/bin/curl --fail --silent --show-error --location https://nodejs.org/dist/v22.14.0/node-v22.14.0-linux-x64.tar.xz --output /tmp/orca-node.tar.xz
if ($LASTEXITCODE -ne 0) { throw 'Node download failed' }
$nodeHash = wsl.exe --distribution Ubuntu --user root --exec /usr/bin/sha256sum /tmp/orca-node.tar.xz
if ($LASTEXITCODE -ne 0 -or -not ($nodeHash -match '^69b09dba5c8dcb05c4e4273a4340db1005abeafe3927efda2bc5b249e80437ec')) { throw 'Node checksum mismatch' }
wsl.exe --distribution Ubuntu --user root --exec /usr/bin/tar -xJf /tmp/orca-node.tar.xz -C /usr/local --strip-components=1
if ($LASTEXITCODE -ne 0) { throw 'Node extraction failed' }
wsl.exe --distribution Ubuntu --user root --exec /usr/local/bin/node --version
if ($LASTEXITCODE -ne 0) { throw 'Node cannot execute in WSL' }
wsl.exe --list --verbose
if ($LASTEXITCODE -ne 0) { throw "WSL enumeration failed: $LASTEXITCODE" }
+13
View File
@@ -45,6 +45,7 @@ jobs:
test_files: ${{ steps.e2e_filter.outputs.test_files }}
ssh_source_changed: ${{ steps.e2e_filter.outputs.ssh_source_changed }}
native_ime_source_changed: ${{ steps.e2e_filter.outputs.native_ime_source_changed }}
wsl_source_changed: ${{ steps.e2e_filter.outputs.wsl_source_changed }}
steps:
- name: Checkout
uses: actions/checkout@v6
@@ -92,6 +93,8 @@ jobs:
# trigger on IME source rather than on a spec name in some route's list.
NATIVE_IME_SOURCE_CHANGED="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --native-ime-source)"
echo "native_ime_source_changed=$NATIVE_IME_SOURCE_CHANGED" >> "$GITHUB_OUTPUT"
WSL_SOURCE_CHANGED="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --wsl-source)"
echo "wsl_source_changed=$WSL_SOURCE_CHANGED" >> "$GITHUB_OUTPUT"
echo "Native IME source changed: $NATIVE_IME_SOURCE_CHANGED"
SHOULD_RUN="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --reusable-workflow)"
if [ "$SHOULD_RUN" = true ]; then
@@ -942,6 +945,16 @@ jobs:
contents: read
uses: ./.github/workflows/terminal-ime-e2e.yml
windows_wsl:
name: real WSL terminal
needs: code_paths
if: needs.code_paths.outputs.wsl_source_changed == 'true'
permissions:
contents: read
uses: ./.github/workflows/windows-wsl-e2e.yml
with:
ref: ${{ github.event.pull_request.head.sha }}
verify:
if: always()
needs:
+74
View File
@@ -0,0 +1,74 @@
name: Windows WSL terminal E2E
on:
workflow_dispatch:
inputs:
ref:
description: Commit to validate
type: string
required: false
workflow_call:
inputs:
ref:
type: string
required: false
permissions:
contents: read
concurrency:
group: windows-wsl-e2e-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
wsl-terminal:
runs-on: windows-2022
timeout-minutes: 30
env:
NODE_OPTIONS: --max-old-space-size=4096
steps:
- uses: actions/checkout@v6
with:
ref: ${{ inputs.ref || github.sha }}
persist-credentials: false
- uses: ./.github/actions/setup-wsl-test-runtime
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: electron
- name: Build relay and Electron
run: |
pnpm run build:relay
if ($LASTEXITCODE -ne 0) { throw 'Relay build failed' }
pnpm exec electron-vite build --mode e2e
if ($LASTEXITCODE -ne 0) { throw 'Electron build failed' }
- name: Exercise real WSL launch and paste
env:
SKIP_BUILD: '1'
ORCA_E2E_FORWARD_APP_LOGS: '1'
PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/wsl-results.json
run: >-
pnpm exec playwright test
tests/e2e/golden-tab-bar-agent-launch.spec.ts
tests/e2e/terminal-windows-shell-paste-ownership.spec.ts
--config tests/playwright.config.ts
--project=electron-headless
--grep "WSL"
--repeat-each=3
--workers=1
--reporter=list,json
- name: Require all nine WSL executions
if: always()
run: node config/scripts/verify-wsl-e2e-participation.mjs test-results/wsl-results.json
- name: Upload WSL participation report
uses: actions/upload-artifact@v7
if: always()
with:
name: windows-wsl-participation-report
path: test-results/wsl-results.json
retention-days: 3
- uses: actions/upload-artifact@v7
if: failure()
with:
name: windows-wsl-terminal-traces
path: test-results/
retention-days: 7
+103
View File
@@ -18151,6 +18151,109 @@
"No p95 CI history or full product mutation proof."
],
"demotionRule": "Keep experimental while any recovery reproduction fails or any teardown, identity, resource-count, or rendered oracle flakes; never promote by extending sleeps or retries."
},
{
"id": "terminal.windows-wsl-launch-and-paste",
"title": "Real WSL terminal agent launch and paste ownership",
"maturity": "experimental",
"protection": "partial",
"owner": "terminal-runtime",
"layer": "electron-windows-wsl",
"surfaces": [
"agent tab launch",
"keyboard paste",
"terminal runtime retention"
],
"platforms": [
"windows"
],
"providers": [
"wsl1",
"wsl2"
],
"coveredPlatforms": [
"windows"
],
"coveredProviders": [
"wsl1"
],
"coverageNotes": "Two Windows CI runs each passed the three WSL1 cases three times. The final per-scenario participation verifier and PR routing still await CI validation. WSL2 remains untested.",
"motivatingLinks": [
"https://github.com/stablyai/orca/actions/runs/34030832614"
],
"invariant": "An agent launched into WSL runs in the guest; keyboard paste reaches exactly one owning PTY and preserves Linux content even after the default shell changes.",
"oracle": "Run the existing real WSL launch and two paste cases three times; require nine passes and zero skipped, unexpected, or flaky results in the Playwright JSON report.",
"commands": [
"gh workflow run windows-terminal-restart-e2e.yml --ref nwparker/validate-reusable-wsl-lane",
"gh workflow run windows-wsl-e2e.yml",
"pnpm exec playwright test tests/e2e/golden-tab-bar-agent-launch.spec.ts tests/e2e/terminal-windows-shell-paste-ownership.spec.ts --config tests/playwright.config.ts --project=electron-headless --workers=1",
"node_modules/.bin/vitest run --config config/vitest.config.ts config/scripts/wsl-e2e-lane-contract.test.mjs config/scripts/verify-wsl-e2e-participation.test.mjs"
],
"testFiles": [
"tests/e2e/golden-tab-bar-agent-launch.spec.ts",
"tests/e2e/terminal-windows-shell-paste-ownership.spec.ts",
"config/scripts/wsl-e2e-lane-contract.test.mjs",
"config/scripts/verify-wsl-e2e-participation.test.mjs"
],
"assertionRefs": [
{
"file": "tests/e2e/golden-tab-bar-agent-launch.spec.ts",
"assertions": [
"requires a distro-only marker from the launched agent"
]
},
{
"file": "tests/e2e/terminal-windows-shell-paste-ownership.spec.ts",
"assertions": [
"requires exact Linux pasted content and exactly one PTY write",
"retains WSL paste ownership after changing the default shell"
]
},
{
"file": "config/scripts/verify-wsl-e2e-participation.test.mjs",
"assertions": [
"rejects skipped, missing, substituted and retried scenarios"
]
}
],
"evidenceRuns": [
{
"date": "2026-09-06",
"runner": "ci",
"platform": "windows",
"result": "passed",
"command": "gh workflow run windows-terminal-restart-e2e.yml --ref nwparker/validate-reusable-wsl-lane",
"durationSeconds": 234,
"summary": "Run34030832614 at4de05e5: real WSL1 launch3 and paste6 passed; aggregate receipt required9 expected and0 skipped/unexpected/flaky. Final per-scenario verifier is newer and pending."
}
],
"runtimeBudget": {
"p95Seconds": 1800,
"scope": "CI job timeout; measured p95 is not established"
},
"flakeHistory": {
"status": "soaking",
"evidence": "Pinned-image run34030695826 and reusable-action run34030832614 each passed9/9 with no skips; final verifier head pending."
},
"redGreenEvidence": {
"status": "partial",
"evidence": "Verifier tests reject absent or skipped scenarios, substituted titles, and passes after retries. Actual nine-pass report verification on the final verifier head is pending."
},
"performanceBudget": {
"required": false,
"evidence": "CI-only provisioning and routing; no application runtime changes."
},
"promotionCriteria": [
"Require all nine real WSL executions on the final workflow head.",
"Demonstrate missing or skipped WSL execution fails participation.",
"Collect repeated CI history before adding this experimental lane to required verification."
],
"knownGaps": [
"WSL2 is not provisioned.",
"No SSH, folder-only workspace, packaged mixed-version, or live-service claim.",
"The new PR lane is outside verify until reliability is established."
],
"demotionRule": "Keep experimental if provisioning or an execution flakes; never promote by skipping a case, raising timeouts, or retrying until green."
}
]
}
+21
View File
@@ -13,6 +13,18 @@ const NATIVE_IME_HARNESS =
/^(?:config\/scripts\/(?:run-terminal-ibus-hangul-e2e|terminal-ime-engagement-receipt)\.mjs$|tests\/e2e\/terminal-ime-(?:boundary-probe|byte-reader|engagement-receipt)\.ts$|tests\/e2e\/terminal-(?:ibus-hangul|hangul-terminating-digit|macos-2set-korean)-native\.spec\.ts$)/
export const PR_E2E_SOURCE_ROUTES = [
{
id: 'terminal.windows-wsl-launch-and-paste',
specs: [
'tests/e2e/golden-tab-bar-agent-launch.spec.ts',
'tests/e2e/terminal-windows-shell-paste-ownership.spec.ts'
],
matches: (file) =>
isProductSource(file) &&
/^(?:config\/scripts\/verify-wsl-e2e-participation\.mjs$|src\/main\/(?:wsl[/-]|pty\/.*wsl|providers\/wsl)|src\/shared\/(?:wsl-|windows-terminal-shell)|src\/renderer\/src\/.*(?:terminal-paste|pty-paste)|tests\/e2e\/(?:golden-tab-bar-agent-launch\.spec|terminal-windows-shell-paste-ownership\.spec|helpers\/(?:wsl-golden-stub-agent|golden-stub-agent))|\.github\/(?:actions\/setup-wsl-test-runtime\/|workflows\/windows-wsl-e2e\.yml))/.test(
file
)
},
{
id: 'ephemeral-vm-runtime.rollback-readable-sidecar',
specs: ['tests/e2e/ephemeral-vm-provisioned-root.spec.ts'],
@@ -227,6 +239,13 @@ export function shouldRunReusablePrE2e(changedPaths) {
)
}
export function hasWslSourceChange(changedPaths) {
const route = PR_E2E_SOURCE_ROUTES.find(
(candidate) => candidate.id === 'terminal.windows-wsl-launch-and-paste'
)
return changedPaths.some(route.matches)
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
let input = ''
process.stdin.setEncoding('utf8')
@@ -238,6 +257,8 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)
process.stdout.write(`${hasSshSourceChange(changedPaths)}\n`)
} else if (process.argv.includes('--reusable-workflow')) {
process.stdout.write(`${shouldRunReusablePrE2e(changedPaths)}\n`)
} else if (process.argv.includes('--wsl-source')) {
process.stdout.write(`${hasWslSourceChange(changedPaths)}\n`)
} else if (process.argv.includes('--native-ime-source')) {
process.stdout.write(`${hasNativeImeSourceChange(changedPaths)}\n`)
} else {
@@ -0,0 +1,54 @@
import { readFileSync } from 'node:fs'
import { pathToFileURL } from 'node:url'
export const WSL_TEST_TITLES = [
'tab-bar + menu launches an agent inside WSL @tab-bar-agent-launch-golden',
'WSL terminal keyboard paste preserves Linux shell content with one PTY owner',
'existing WSL terminal keeps paste runtime after default shell changes'
]
export function verifyWslParticipation(report) {
const stats = report?.stats
if (
!stats ||
stats.expected !== 9 ||
stats.skipped !== 0 ||
stats.unexpected !== 0 ||
stats.flaky !== 0 ||
report.errors?.length
) {
throw new Error(`WSL participation failed: ${JSON.stringify(stats)}`)
}
const counts = new Map(WSL_TEST_TITLES.map((title) => [title, 0]))
const visit = (suites) => {
for (const suite of suites ?? []) {
for (const spec of suite.specs ?? []) {
if (!counts.has(spec.title)) {
throw new Error(`Unexpected WSL scenario: ${spec.title}`)
}
for (const test of spec.tests ?? []) {
if (
test.expectedStatus !== 'passed' ||
test.results?.length !== 1 ||
test.results[0].status !== 'passed'
) {
throw new Error(`WSL scenario did not pass without retries: ${spec.title}`)
}
counts.set(spec.title, counts.get(spec.title) + 1)
}
}
visit(suite.suites)
}
}
visit(report.suites)
for (const [title, count] of counts) {
if (count !== 3) {
throw new Error(`WSL scenario requires three executions: ${title} (${count})`)
}
}
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
verifyWslParticipation(JSON.parse(readFileSync(process.argv[2], 'utf8')))
console.log('All three WSL scenarios passed three times without skips or retries.')
}
@@ -0,0 +1,52 @@
import { describe, expect, it } from 'vitest'
import { verifyWslParticipation, WSL_TEST_TITLES } from './verify-wsl-e2e-participation.mjs'
function report() {
return {
stats: { expected: 9, skipped: 0, unexpected: 0, flaky: 0 },
suites: [
{
suites: [
{
specs: WSL_TEST_TITLES.map((title) => ({
title,
tests: Array.from({ length: 3 }, () => ({
expectedStatus: 'passed',
results: [{ status: 'passed' }]
}))
}))
}
]
}
]
}
}
describe('WSL participation', () => {
it('accepts all three named scenarios executed three times', () => {
expect(() => verifyWslParticipation(report())).not.toThrow()
})
it.each(['skipped', 'unexpected', 'flaky'])('rejects a nonzero %s result', (key) => {
const value = report()
value.stats[key] = 1
expect(() => verifyWslParticipation(value)).toThrow('participation failed')
})
it('rejects missing scenarios even when aggregate counts claim nine passes', () => {
const value = report()
value.suites[0].suites[0].specs.pop()
expect(() => verifyWslParticipation(value)).toThrow('requires three executions')
})
it('rejects an unrelated scenario substituted for an expected scenario', () => {
const value = report()
value.suites[0].suites[0].specs[0].title = 'native shell passes'
expect(() => verifyWslParticipation(value)).toThrow('Unexpected WSL scenario')
})
it('rejects a pass obtained after a failed attempt', () => {
const value = report()
value.suites[0].suites[0].specs[0].tests[0].results.unshift({ status: 'failed' })
expect(() => verifyWslParticipation(value)).toThrow('without retries')
})
it('rejects missing report content', () => {
expect(() => verifyWslParticipation({})).toThrow('participation failed')
})
})
@@ -0,0 +1,55 @@
import { readFileSync } from 'node:fs'
import { describe, expect, it } from 'vitest'
import { parse } from 'yaml'
import { hasWslSourceChange, selectPrE2eSpecs } from './pr-e2e-source-routing.mjs'
const read = (path) => readFileSync(new URL(`../../${path}`, import.meta.url), 'utf8')
describe('real WSL terminal lane', () => {
it.each([
'config/scripts/verify-wsl-e2e-participation.mjs',
'src/main/wsl-availability.ts',
'src/main/wsl/wsl-runner.ts',
'src/main/pty/wsl-orca-env.ts',
'src/shared/wsl-login-shell-command.ts',
'src/shared/windows-terminal-shell.ts',
'tests/e2e/helpers/wsl-golden-stub-agent.ts',
'tests/e2e/golden-tab-bar-agent-launch.spec.ts',
'tests/e2e/terminal-windows-shell-paste-ownership.spec.ts',
'.github/actions/setup-wsl-test-runtime/setup.ps1',
'.github/workflows/windows-wsl-e2e.yml'
])('routes %s to both WSL sentinels', (path) => {
expect(hasWslSourceChange([path])).toBe(true)
expect(selectPrE2eSpecs([path])).toEqual(
expect.arrayContaining([
'tests/e2e/golden-tab-bar-agent-launch.spec.ts',
'tests/e2e/terminal-windows-shell-paste-ownership.spec.ts'
])
)
})
it.each([
'docs/reference/wsl-command-execution.md',
'src/main/wsl-availability.test.ts',
'src/main/ssh/connection.ts'
])('excludes unrelated or unit-only change %s', (path) => {
expect(hasWslSourceChange([path])).toBe(false)
})
it('runs the reusable lane at the immutable PR head', () => {
const pr = parse(read('.github/workflows/pr.yml'))
expect(pr.jobs.windows_wsl.if).toBe("needs.code_paths.outputs.wsl_source_changed == 'true'")
expect(pr.jobs.windows_wsl.with.ref).toBe('${{ github.event.pull_request.head.sha }}')
const workflow = parse(read('.github/workflows/windows-wsl-e2e.yml'))
const steps = workflow.jobs['wsl-terminal'].steps
expect(steps[0].with.ref).toBe('${{ inputs.ref || github.sha }}')
expect(steps.some((step) => step.uses === './.github/actions/setup-wsl-test-runtime')).toBe(
true
)
const receipt = steps.find((step) => step.name === 'Require all nine WSL executions')
expect(receipt.if).toBe('always()')
expect(receipt.run).toBe(
'node config/scripts/verify-wsl-e2e-participation.mjs test-results/wsl-results.json'
)
})
})