From b115f8d256cd75d34ee05e2f60e704e372f8600c Mon Sep 17 00:00:00 2001 From: m4air Date: Wed, 12 Aug 2026 20:17:01 -0700 Subject: [PATCH] Add Windows golden E2E test for fresh-startup regression Windows terminal rendering golden is flaky on CI runners. Re-enable Windows in the golden E2E gate with a scoped test for the fresh-profile startup regression from #14130. Terminal rendering continues on Linux and macOS; Windows runs fresh-startup only. --- .github/workflows/golden-e2e-experiment.yml | 15 +++-- .github/workflows/release-cut.yml | 24 +++++--- ...package-electron-runtime-contract.test.mjs | 57 ++++++++++++++----- package.json | 1 + .../e2e/golden-windows-fresh-startup.spec.ts | 12 ++++ tests/e2e/helpers/orca-app.ts | 2 +- 6 files changed, 83 insertions(+), 28 deletions(-) create mode 100644 tests/e2e/golden-windows-fresh-startup.spec.ts diff --git a/.github/workflows/golden-e2e-experiment.yml b/.github/workflows/golden-e2e-experiment.yml index 89c1c86096f..4591514322e 100644 --- a/.github/workflows/golden-e2e-experiment.yml +++ b/.github/workflows/golden-e2e-experiment.yml @@ -23,10 +23,9 @@ jobs: platform: linux - os: macos-15 platform: mac - # Why: Windows golden E2E is temporarily disabled on CI while its - # flaky runner-only failures are investigated. - # - os: windows-latest - # platform: windows + # Match the release gate: Windows runs the fresh-startup golden only. + - os: windows-2022 + platform: windows steps: - name: Checkout @@ -74,6 +73,14 @@ jobs: env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e -- tests/e2e/golden-core-flows.spec.ts env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:terminal-rendering-golden + - name: Run golden E2E tests on Windows + if: runner.os == 'Windows' + shell: pwsh + run: | + $env:SKIP_BUILD = '1' + $env:ORCA_E2E_FORWARD_APP_LOGS = '1' + pnpm run test:e2e:windows-fresh-startup-golden + - name: Upload Playwright traces if: failure() uses: actions/upload-artifact@v7 diff --git a/.github/workflows/release-cut.yml b/.github/workflows/release-cut.yml index 6dba5ac1ff6..5d9e4f29b72 100644 --- a/.github/workflows/release-cut.yml +++ b/.github/workflows/release-cut.yml @@ -783,7 +783,7 @@ jobs: { echo "## Release E2E Signal" echo "" - echo "- Terminal rendering golden is release-blocking." + echo "- Platform golden E2E is release-blocking: terminal rendering on Linux/macOS and fresh startup on Windows." echo "- Full E2E runs separately after publication and cannot change the release result." echo "- Terminal rendering release evidence is diagnostic/non-blocking." echo "" @@ -817,7 +817,7 @@ jobs: terminal-rendering-golden: needs: cut if: needs.cut.outputs.should_release == 'true' - name: terminal rendering golden ${{ matrix.platform }} + name: golden e2e ${{ matrix.platform }} runs-on: ${{ matrix.os }} timeout-minutes: 30 env: @@ -830,10 +830,10 @@ jobs: platform: linux - os: macos-15 platform: mac - # Why: Windows terminal rendering golden is temporarily disabled on - # CI while its flaky runner-only failures are investigated. - # - os: windows-latest - # platform: windows + # Windows terminal rendering remains flaky; keep its blocking signal + # scoped to the fresh-profile startup regression from #14130. + - os: windows-2022 + platform: windows steps: - name: Checkout @@ -866,7 +866,7 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - - name: Build Electron app for terminal rendering golden + - name: Build Electron app for platform golden run: npx electron-vite build --mode e2e - name: Run terminal rendering golden on Linux @@ -877,11 +877,19 @@ jobs: if: runner.os == 'macOS' run: env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:terminal-rendering-golden + - name: Run fresh-startup golden on Windows + if: runner.os == 'Windows' + shell: pwsh + run: | + $env:SKIP_BUILD = '1' + $env:ORCA_E2E_FORWARD_APP_LOGS = '1' + pnpm run test:e2e:windows-fresh-startup-golden + - name: Upload Playwright traces if: failure() uses: actions/upload-artifact@v7 with: - name: terminal-rendering-golden-${{ matrix.platform }}-playwright-traces + name: golden-e2e-${{ matrix.platform }}-playwright-traces path: test-results/ retention-days: 7 if-no-files-found: ignore diff --git a/config/scripts/package-electron-runtime-contract.test.mjs b/config/scripts/package-electron-runtime-contract.test.mjs index 1b695086808..2cf5e18af73 100644 --- a/config/scripts/package-electron-runtime-contract.test.mjs +++ b/config/scripts/package-electron-runtime-contract.test.mjs @@ -492,7 +492,7 @@ describe('Electron runtime package contract', () => { expect(uploadStep.with.path).toBe('${{ env.ORCA_E2E_TERMINAL_PERF_REPORT_PATH }}') }) - it('keeps terminal rendering regressions in the manual golden E2E workflow', () => { + it('keeps platform golden regressions in the manual and release workflows', () => { const packageScripts = packageJson.scripts const goldenWorkflow = parse( readFileSync(join(projectDir, '.github/workflows/golden-e2e-experiment.yml'), 'utf8') @@ -506,17 +506,19 @@ describe('Electron runtime package contract', () => { ['mac', 'macOS'], ['windows', 'Windows'] ]) - const goldenPlatforms = goldenWorkflow.jobs['golden-e2e'].strategy.matrix.include - .map(({ platform }) => platform) - .sort() - const goldenRunSteps = goldenPlatforms.map((platform) => { - const label = goldenPlatformLabels.get(platform) + const goldenMatrix = goldenWorkflow.jobs['golden-e2e'].strategy.matrix.include + const goldenPlatforms = goldenMatrix.map(({ platform }) => platform).sort() + const goldenRunSteps = new Map( + goldenPlatforms.map((platform) => { + const label = goldenPlatformLabels.get(platform) - expect(label, platform).toBeDefined() + expect(label, platform).toBeDefined() - return steps.find((step) => step.name === `Run golden E2E tests on ${label}`) - }) + return [platform, steps.find((step) => step.name === `Run golden E2E tests on ${label}`)] + }) + ) const releaseGoldenJob = releaseWorkflow.jobs['terminal-rendering-golden'] + const releaseGoldenMatrix = releaseGoldenJob.strategy.matrix.include const releaseEvidenceJob = releaseWorkflow.jobs['terminal-rendering-release-evidence'] const releaseBuildNeeds = releaseWorkflow.jobs.build.needs const publishReleaseNeeds = releaseWorkflow.jobs['publish-release'].needs @@ -535,15 +537,34 @@ describe('Electron runtime package contract', () => { expect(packageScripts['test:e2e:terminal-rendering-golden']).not.toContain( 'terminal-long-table-scroll-restore.spec.ts' ) + expect(packageScripts['test:e2e:windows-fresh-startup-golden']).toContain( + 'golden-windows-fresh-startup.spec.ts' + ) + expect(packageScripts['test:e2e:windows-fresh-startup-golden']).toContain( + '@windows-fresh-startup-golden' + ) expect(packageScripts['test:e2e:terminal-rendering-release-evidence']).toContain( 'terminal-opencode-emoji-table-rendering.spec.ts' ) expect(packageScripts['test:e2e:terminal-rendering-release-evidence']).toContain( 'terminal-long-table-scroll-restore.spec.ts' ) - for (const runStep of goldenRunSteps) { - expect(runStep?.run).toContain('pnpm run test:e2e:terminal-rendering-golden') - } + expect(goldenMatrix).toEqual([ + { os: 'ubuntu-latest', platform: 'linux' }, + { os: 'macos-15', platform: 'mac' }, + { os: 'windows-2022', platform: 'windows' } + ]) + expect(goldenRunSteps.get('linux')?.run).toContain( + 'pnpm run test:e2e:terminal-rendering-golden' + ) + expect(goldenRunSteps.get('mac')?.run).toContain('pnpm run test:e2e:terminal-rendering-golden') + expect(goldenRunSteps.get('windows')).toMatchObject({ + if: "runner.os == 'Windows'", + shell: 'pwsh' + }) + expect(goldenRunSteps.get('windows').run).toContain( + 'pnpm run test:e2e:windows-fresh-startup-golden' + ) expect(goldenWorkflow.on.pull_request).toBeUndefined() expect(goldenWorkflow.on.workflow_dispatch).toBeDefined() expect(releaseBuildNeeds).not.toContain('terminal-rendering-golden') @@ -552,12 +573,18 @@ describe('Electron runtime package contract', () => { expect(publishReleaseNeeds).toContain('build') expect(publishReleaseNeeds).not.toContain('terminal-rendering-release-evidence') expect(releaseGoldenJob['continue-on-error']).toBeUndefined() - expect(releaseGoldenJob.strategy.matrix.include.map(({ platform }) => platform).sort()).toEqual( - goldenPlatforms - ) + expect(releaseGoldenMatrix).toEqual(goldenMatrix) expect(releaseGoldenJob.steps.map((step) => step.run ?? '')).toContain( 'xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:terminal-rendering-golden' ) + const releaseWindowsRunStep = releaseGoldenJob.steps.find( + (step) => step.name === 'Run fresh-startup golden on Windows' + ) + expect(releaseWindowsRunStep).toMatchObject({ + if: "runner.os == 'Windows'", + shell: 'pwsh' + }) + expect(releaseWindowsRunStep.run).toContain('pnpm run test:e2e:windows-fresh-startup-golden') expect(releaseEvidenceJob['continue-on-error']).toBe(true) expect( releaseEvidenceJob.strategy.matrix.include.map(({ platform }) => platform).sort() diff --git a/package.json b/package.json index 6a0e4f33814..c9cf4a72891 100644 --- a/package.json +++ b/package.json @@ -87,6 +87,7 @@ "test:e2e:multi-client-navigation": "node config/scripts/run-multi-client-navigation-e2e.mjs", "test:e2e:floating-mobile-emulator": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/floating-mobile-emulator-tab.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", "test:e2e:terminal-rendering-golden": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/terminal-raw-emoji-table-scroll-restore.spec.ts tests/e2e/terminal-webgl-atlas-budget.spec.ts --grep @terminal-rendering-golden --config tests/playwright.config.ts --project electron-headless --workers=1", + "test:e2e:windows-fresh-startup-golden": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/golden-windows-fresh-startup.spec.ts --grep @windows-fresh-startup-golden --config tests/playwright.config.ts --project electron-headless --workers=1", "test:e2e:terminal-rendering-release-evidence": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/terminal-opencode-emoji-table-rendering.spec.ts tests/e2e/terminal-long-table-scroll-restore.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=2", "test:e2e:terminal-perf": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/terminal-typing-latency.spec.ts tests/e2e/terminal-foreground-redraw-freeze.spec.ts tests/e2e/terminal-output-scheduler.spec.ts tests/e2e/terminal-hidden-tui-visual-restore.spec.ts tests/e2e/artificial-opencode-terminal-load.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=2", "test:e2e:terminal-perf:scale": "pnpm run ensure:electron-runtime && node config/scripts/run-terminal-scale-perf-e2e.mjs", diff --git a/tests/e2e/golden-windows-fresh-startup.spec.ts b/tests/e2e/golden-windows-fresh-startup.spec.ts new file mode 100644 index 00000000000..f153f765dcd --- /dev/null +++ b/tests/e2e/golden-windows-fresh-startup.spec.ts @@ -0,0 +1,12 @@ +import { expect, test } from './helpers/orca-app' + +test.use({ dismissOnboarding: false, seedTestRepo: false }) +test.skip(process.platform !== 'win32', 'Fresh-profile fsync regression is Windows-only') + +test('fresh Windows profile reaches onboarding @windows-fresh-startup-golden', async ({ + orcaPage +}) => { + await expect(orcaPage.getByRole('heading', { name: /Pick your default agent/i })).toBeVisible({ + timeout: 30_000 + }) +}) diff --git a/tests/e2e/helpers/orca-app.ts b/tests/e2e/helpers/orca-app.ts index b083e37f36d..434a3297df3 100644 --- a/tests/e2e/helpers/orca-app.ts +++ b/tests/e2e/helpers/orca-app.ts @@ -252,6 +252,7 @@ export const test = base.extend({ ...(headful ? { ORCA_E2E_HEADFUL: '1' } : { ORCA_E2E_HEADLESS: '1' }) } }) + forwardElectronProcessLogs(app, testInfo) try { const resolvedHome = await app.evaluate(({ app }) => app.getPath('home')) assertElectronResolvedIsolatedHome(resolvedHome, homeIsolation) @@ -261,7 +262,6 @@ export const test = base.extend({ await removeUserDataDirAfterShutdown(userDataDir) throw error } - forwardElectronProcessLogs(app, testInfo) await provideFixture(app) // Why: the Playwright close promise can settle before all Electron and PTY // descendants are gone in CI; worker teardown then hangs on open handles.