Add Windows golden E2E test for fresh-startup regression

Windows terminal rendering golden is flaky on CI runners. Re-enable
Windows in the golden E2E gate with a scoped test for the fresh-profile
startup regression from #14130. Terminal rendering continues on Linux
and macOS; Windows runs fresh-startup only.
This commit is contained in:
m4air
2026-08-12 21:28:32 -07:00
committed by OrcaWin
parent aeb7183afe
commit b115f8d256
6 changed files with 83 additions and 28 deletions
+11 -4
View File
@@ -23,10 +23,9 @@ jobs:
platform: linux
- os: macos-15
platform: mac
# Why: Windows golden E2E is temporarily disabled on CI while its
# flaky runner-only failures are investigated.
# - os: windows-latest
# platform: windows
# Match the release gate: Windows runs the fresh-startup golden only.
- os: windows-2022
platform: windows
steps:
- name: Checkout
@@ -74,6 +73,14 @@ jobs:
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e -- tests/e2e/golden-core-flows.spec.ts
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:terminal-rendering-golden
- name: Run golden E2E tests on Windows
if: runner.os == 'Windows'
shell: pwsh
run: |
$env:SKIP_BUILD = '1'
$env:ORCA_E2E_FORWARD_APP_LOGS = '1'
pnpm run test:e2e:windows-fresh-startup-golden
- name: Upload Playwright traces
if: failure()
uses: actions/upload-artifact@v7
+16 -8
View File
@@ -783,7 +783,7 @@ jobs:
{
echo "## Release E2E Signal"
echo ""
echo "- Terminal rendering golden is release-blocking."
echo "- Platform golden E2E is release-blocking: terminal rendering on Linux/macOS and fresh startup on Windows."
echo "- Full E2E runs separately after publication and cannot change the release result."
echo "- Terminal rendering release evidence is diagnostic/non-blocking."
echo ""
@@ -817,7 +817,7 @@ jobs:
terminal-rendering-golden:
needs: cut
if: needs.cut.outputs.should_release == 'true'
name: terminal rendering golden ${{ matrix.platform }}
name: golden e2e ${{ matrix.platform }}
runs-on: ${{ matrix.os }}
timeout-minutes: 30
env:
@@ -830,10 +830,10 @@ jobs:
platform: linux
- os: macos-15
platform: mac
# Why: Windows terminal rendering golden is temporarily disabled on
# CI while its flaky runner-only failures are investigated.
# - os: windows-latest
# platform: windows
# Windows terminal rendering remains flaky; keep its blocking signal
# scoped to the fresh-profile startup regression from #14130.
- os: windows-2022
platform: windows
steps:
- name: Checkout
@@ -866,7 +866,7 @@ jobs:
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Build Electron app for terminal rendering golden
- name: Build Electron app for platform golden
run: npx electron-vite build --mode e2e
- name: Run terminal rendering golden on Linux
@@ -877,11 +877,19 @@ jobs:
if: runner.os == 'macOS'
run: env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:terminal-rendering-golden
- name: Run fresh-startup golden on Windows
if: runner.os == 'Windows'
shell: pwsh
run: |
$env:SKIP_BUILD = '1'
$env:ORCA_E2E_FORWARD_APP_LOGS = '1'
pnpm run test:e2e:windows-fresh-startup-golden
- name: Upload Playwright traces
if: failure()
uses: actions/upload-artifact@v7
with:
name: terminal-rendering-golden-${{ matrix.platform }}-playwright-traces
name: golden-e2e-${{ matrix.platform }}-playwright-traces
path: test-results/
retention-days: 7
if-no-files-found: ignore
@@ -492,7 +492,7 @@ describe('Electron runtime package contract', () => {
expect(uploadStep.with.path).toBe('${{ env.ORCA_E2E_TERMINAL_PERF_REPORT_PATH }}')
})
it('keeps terminal rendering regressions in the manual golden E2E workflow', () => {
it('keeps platform golden regressions in the manual and release workflows', () => {
const packageScripts = packageJson.scripts
const goldenWorkflow = parse(
readFileSync(join(projectDir, '.github/workflows/golden-e2e-experiment.yml'), 'utf8')
@@ -506,17 +506,19 @@ describe('Electron runtime package contract', () => {
['mac', 'macOS'],
['windows', 'Windows']
])
const goldenPlatforms = goldenWorkflow.jobs['golden-e2e'].strategy.matrix.include
.map(({ platform }) => platform)
.sort()
const goldenRunSteps = goldenPlatforms.map((platform) => {
const label = goldenPlatformLabels.get(platform)
const goldenMatrix = goldenWorkflow.jobs['golden-e2e'].strategy.matrix.include
const goldenPlatforms = goldenMatrix.map(({ platform }) => platform).sort()
const goldenRunSteps = new Map(
goldenPlatforms.map((platform) => {
const label = goldenPlatformLabels.get(platform)
expect(label, platform).toBeDefined()
expect(label, platform).toBeDefined()
return steps.find((step) => step.name === `Run golden E2E tests on ${label}`)
})
return [platform, steps.find((step) => step.name === `Run golden E2E tests on ${label}`)]
})
)
const releaseGoldenJob = releaseWorkflow.jobs['terminal-rendering-golden']
const releaseGoldenMatrix = releaseGoldenJob.strategy.matrix.include
const releaseEvidenceJob = releaseWorkflow.jobs['terminal-rendering-release-evidence']
const releaseBuildNeeds = releaseWorkflow.jobs.build.needs
const publishReleaseNeeds = releaseWorkflow.jobs['publish-release'].needs
@@ -535,15 +537,34 @@ describe('Electron runtime package contract', () => {
expect(packageScripts['test:e2e:terminal-rendering-golden']).not.toContain(
'terminal-long-table-scroll-restore.spec.ts'
)
expect(packageScripts['test:e2e:windows-fresh-startup-golden']).toContain(
'golden-windows-fresh-startup.spec.ts'
)
expect(packageScripts['test:e2e:windows-fresh-startup-golden']).toContain(
'@windows-fresh-startup-golden'
)
expect(packageScripts['test:e2e:terminal-rendering-release-evidence']).toContain(
'terminal-opencode-emoji-table-rendering.spec.ts'
)
expect(packageScripts['test:e2e:terminal-rendering-release-evidence']).toContain(
'terminal-long-table-scroll-restore.spec.ts'
)
for (const runStep of goldenRunSteps) {
expect(runStep?.run).toContain('pnpm run test:e2e:terminal-rendering-golden')
}
expect(goldenMatrix).toEqual([
{ os: 'ubuntu-latest', platform: 'linux' },
{ os: 'macos-15', platform: 'mac' },
{ os: 'windows-2022', platform: 'windows' }
])
expect(goldenRunSteps.get('linux')?.run).toContain(
'pnpm run test:e2e:terminal-rendering-golden'
)
expect(goldenRunSteps.get('mac')?.run).toContain('pnpm run test:e2e:terminal-rendering-golden')
expect(goldenRunSteps.get('windows')).toMatchObject({
if: "runner.os == 'Windows'",
shell: 'pwsh'
})
expect(goldenRunSteps.get('windows').run).toContain(
'pnpm run test:e2e:windows-fresh-startup-golden'
)
expect(goldenWorkflow.on.pull_request).toBeUndefined()
expect(goldenWorkflow.on.workflow_dispatch).toBeDefined()
expect(releaseBuildNeeds).not.toContain('terminal-rendering-golden')
@@ -552,12 +573,18 @@ describe('Electron runtime package contract', () => {
expect(publishReleaseNeeds).toContain('build')
expect(publishReleaseNeeds).not.toContain('terminal-rendering-release-evidence')
expect(releaseGoldenJob['continue-on-error']).toBeUndefined()
expect(releaseGoldenJob.strategy.matrix.include.map(({ platform }) => platform).sort()).toEqual(
goldenPlatforms
)
expect(releaseGoldenMatrix).toEqual(goldenMatrix)
expect(releaseGoldenJob.steps.map((step) => step.run ?? '')).toContain(
'xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:terminal-rendering-golden'
)
const releaseWindowsRunStep = releaseGoldenJob.steps.find(
(step) => step.name === 'Run fresh-startup golden on Windows'
)
expect(releaseWindowsRunStep).toMatchObject({
if: "runner.os == 'Windows'",
shell: 'pwsh'
})
expect(releaseWindowsRunStep.run).toContain('pnpm run test:e2e:windows-fresh-startup-golden')
expect(releaseEvidenceJob['continue-on-error']).toBe(true)
expect(
releaseEvidenceJob.strategy.matrix.include.map(({ platform }) => platform).sort()
+1
View File
@@ -87,6 +87,7 @@
"test:e2e:multi-client-navigation": "node config/scripts/run-multi-client-navigation-e2e.mjs",
"test:e2e:floating-mobile-emulator": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/floating-mobile-emulator-tab.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1",
"test:e2e:terminal-rendering-golden": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/terminal-raw-emoji-table-scroll-restore.spec.ts tests/e2e/terminal-webgl-atlas-budget.spec.ts --grep @terminal-rendering-golden --config tests/playwright.config.ts --project electron-headless --workers=1",
"test:e2e:windows-fresh-startup-golden": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/golden-windows-fresh-startup.spec.ts --grep @windows-fresh-startup-golden --config tests/playwright.config.ts --project electron-headless --workers=1",
"test:e2e:terminal-rendering-release-evidence": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/terminal-opencode-emoji-table-rendering.spec.ts tests/e2e/terminal-long-table-scroll-restore.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=2",
"test:e2e:terminal-perf": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/terminal-typing-latency.spec.ts tests/e2e/terminal-foreground-redraw-freeze.spec.ts tests/e2e/terminal-output-scheduler.spec.ts tests/e2e/terminal-hidden-tui-visual-restore.spec.ts tests/e2e/artificial-opencode-terminal-load.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=2",
"test:e2e:terminal-perf:scale": "pnpm run ensure:electron-runtime && node config/scripts/run-terminal-scale-perf-e2e.mjs",
@@ -0,0 +1,12 @@
import { expect, test } from './helpers/orca-app'
test.use({ dismissOnboarding: false, seedTestRepo: false })
test.skip(process.platform !== 'win32', 'Fresh-profile fsync regression is Windows-only')
test('fresh Windows profile reaches onboarding @windows-fresh-startup-golden', async ({
orcaPage
}) => {
await expect(orcaPage.getByRole('heading', { name: /Pick your default agent/i })).toBeVisible({
timeout: 30_000
})
})
+1 -1
View File
@@ -252,6 +252,7 @@ export const test = base.extend<OrcaTestFixtures, OrcaWorkerFixtures>({
...(headful ? { ORCA_E2E_HEADFUL: '1' } : { ORCA_E2E_HEADLESS: '1' })
}
})
forwardElectronProcessLogs(app, testInfo)
try {
const resolvedHome = await app.evaluate(({ app }) => app.getPath('home'))
assertElectronResolvedIsolatedHome(resolvedHome, homeIsolation)
@@ -261,7 +262,6 @@ export const test = base.extend<OrcaTestFixtures, OrcaWorkerFixtures>({
await removeUserDataDirAfterShutdown(userDataDir)
throw error
}
forwardElectronProcessLogs(app, testInfo)
await provideFixture(app)
// Why: the Playwright close promise can settle before all Electron and PTY
// descendants are gone in CI; worker teardown then hangs on open handles.