mirror of
https://github.com/stablyai/orca.git
synced 2026-09-21 16:02:20 +00:00
Add Windows golden E2E test for fresh-startup regression
Windows terminal rendering golden is flaky on CI runners. Re-enable Windows in the golden E2E gate with a scoped test for the fresh-profile startup regression from #14130. Terminal rendering continues on Linux and macOS; Windows runs fresh-startup only.
This commit is contained in:
@@ -23,10 +23,9 @@ jobs:
|
||||
platform: linux
|
||||
- os: macos-15
|
||||
platform: mac
|
||||
# Why: Windows golden E2E is temporarily disabled on CI while its
|
||||
# flaky runner-only failures are investigated.
|
||||
# - os: windows-latest
|
||||
# platform: windows
|
||||
# Match the release gate: Windows runs the fresh-startup golden only.
|
||||
- os: windows-2022
|
||||
platform: windows
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
@@ -74,6 +73,14 @@ jobs:
|
||||
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e -- tests/e2e/golden-core-flows.spec.ts
|
||||
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:terminal-rendering-golden
|
||||
|
||||
- name: Run golden E2E tests on Windows
|
||||
if: runner.os == 'Windows'
|
||||
shell: pwsh
|
||||
run: |
|
||||
$env:SKIP_BUILD = '1'
|
||||
$env:ORCA_E2E_FORWARD_APP_LOGS = '1'
|
||||
pnpm run test:e2e:windows-fresh-startup-golden
|
||||
|
||||
- name: Upload Playwright traces
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
|
||||
@@ -783,7 +783,7 @@ jobs:
|
||||
{
|
||||
echo "## Release E2E Signal"
|
||||
echo ""
|
||||
echo "- Terminal rendering golden is release-blocking."
|
||||
echo "- Platform golden E2E is release-blocking: terminal rendering on Linux/macOS and fresh startup on Windows."
|
||||
echo "- Full E2E runs separately after publication and cannot change the release result."
|
||||
echo "- Terminal rendering release evidence is diagnostic/non-blocking."
|
||||
echo ""
|
||||
@@ -817,7 +817,7 @@ jobs:
|
||||
terminal-rendering-golden:
|
||||
needs: cut
|
||||
if: needs.cut.outputs.should_release == 'true'
|
||||
name: terminal rendering golden ${{ matrix.platform }}
|
||||
name: golden e2e ${{ matrix.platform }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
@@ -830,10 +830,10 @@ jobs:
|
||||
platform: linux
|
||||
- os: macos-15
|
||||
platform: mac
|
||||
# Why: Windows terminal rendering golden is temporarily disabled on
|
||||
# CI while its flaky runner-only failures are investigated.
|
||||
# - os: windows-latest
|
||||
# platform: windows
|
||||
# Windows terminal rendering remains flaky; keep its blocking signal
|
||||
# scoped to the fresh-profile startup regression from #14130.
|
||||
- os: windows-2022
|
||||
platform: windows
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
@@ -866,7 +866,7 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Build Electron app for terminal rendering golden
|
||||
- name: Build Electron app for platform golden
|
||||
run: npx electron-vite build --mode e2e
|
||||
|
||||
- name: Run terminal rendering golden on Linux
|
||||
@@ -877,11 +877,19 @@ jobs:
|
||||
if: runner.os == 'macOS'
|
||||
run: env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:terminal-rendering-golden
|
||||
|
||||
- name: Run fresh-startup golden on Windows
|
||||
if: runner.os == 'Windows'
|
||||
shell: pwsh
|
||||
run: |
|
||||
$env:SKIP_BUILD = '1'
|
||||
$env:ORCA_E2E_FORWARD_APP_LOGS = '1'
|
||||
pnpm run test:e2e:windows-fresh-startup-golden
|
||||
|
||||
- name: Upload Playwright traces
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: terminal-rendering-golden-${{ matrix.platform }}-playwright-traces
|
||||
name: golden-e2e-${{ matrix.platform }}-playwright-traces
|
||||
path: test-results/
|
||||
retention-days: 7
|
||||
if-no-files-found: ignore
|
||||
|
||||
@@ -492,7 +492,7 @@ describe('Electron runtime package contract', () => {
|
||||
expect(uploadStep.with.path).toBe('${{ env.ORCA_E2E_TERMINAL_PERF_REPORT_PATH }}')
|
||||
})
|
||||
|
||||
it('keeps terminal rendering regressions in the manual golden E2E workflow', () => {
|
||||
it('keeps platform golden regressions in the manual and release workflows', () => {
|
||||
const packageScripts = packageJson.scripts
|
||||
const goldenWorkflow = parse(
|
||||
readFileSync(join(projectDir, '.github/workflows/golden-e2e-experiment.yml'), 'utf8')
|
||||
@@ -506,17 +506,19 @@ describe('Electron runtime package contract', () => {
|
||||
['mac', 'macOS'],
|
||||
['windows', 'Windows']
|
||||
])
|
||||
const goldenPlatforms = goldenWorkflow.jobs['golden-e2e'].strategy.matrix.include
|
||||
.map(({ platform }) => platform)
|
||||
.sort()
|
||||
const goldenRunSteps = goldenPlatforms.map((platform) => {
|
||||
const label = goldenPlatformLabels.get(platform)
|
||||
const goldenMatrix = goldenWorkflow.jobs['golden-e2e'].strategy.matrix.include
|
||||
const goldenPlatforms = goldenMatrix.map(({ platform }) => platform).sort()
|
||||
const goldenRunSteps = new Map(
|
||||
goldenPlatforms.map((platform) => {
|
||||
const label = goldenPlatformLabels.get(platform)
|
||||
|
||||
expect(label, platform).toBeDefined()
|
||||
expect(label, platform).toBeDefined()
|
||||
|
||||
return steps.find((step) => step.name === `Run golden E2E tests on ${label}`)
|
||||
})
|
||||
return [platform, steps.find((step) => step.name === `Run golden E2E tests on ${label}`)]
|
||||
})
|
||||
)
|
||||
const releaseGoldenJob = releaseWorkflow.jobs['terminal-rendering-golden']
|
||||
const releaseGoldenMatrix = releaseGoldenJob.strategy.matrix.include
|
||||
const releaseEvidenceJob = releaseWorkflow.jobs['terminal-rendering-release-evidence']
|
||||
const releaseBuildNeeds = releaseWorkflow.jobs.build.needs
|
||||
const publishReleaseNeeds = releaseWorkflow.jobs['publish-release'].needs
|
||||
@@ -535,15 +537,34 @@ describe('Electron runtime package contract', () => {
|
||||
expect(packageScripts['test:e2e:terminal-rendering-golden']).not.toContain(
|
||||
'terminal-long-table-scroll-restore.spec.ts'
|
||||
)
|
||||
expect(packageScripts['test:e2e:windows-fresh-startup-golden']).toContain(
|
||||
'golden-windows-fresh-startup.spec.ts'
|
||||
)
|
||||
expect(packageScripts['test:e2e:windows-fresh-startup-golden']).toContain(
|
||||
'@windows-fresh-startup-golden'
|
||||
)
|
||||
expect(packageScripts['test:e2e:terminal-rendering-release-evidence']).toContain(
|
||||
'terminal-opencode-emoji-table-rendering.spec.ts'
|
||||
)
|
||||
expect(packageScripts['test:e2e:terminal-rendering-release-evidence']).toContain(
|
||||
'terminal-long-table-scroll-restore.spec.ts'
|
||||
)
|
||||
for (const runStep of goldenRunSteps) {
|
||||
expect(runStep?.run).toContain('pnpm run test:e2e:terminal-rendering-golden')
|
||||
}
|
||||
expect(goldenMatrix).toEqual([
|
||||
{ os: 'ubuntu-latest', platform: 'linux' },
|
||||
{ os: 'macos-15', platform: 'mac' },
|
||||
{ os: 'windows-2022', platform: 'windows' }
|
||||
])
|
||||
expect(goldenRunSteps.get('linux')?.run).toContain(
|
||||
'pnpm run test:e2e:terminal-rendering-golden'
|
||||
)
|
||||
expect(goldenRunSteps.get('mac')?.run).toContain('pnpm run test:e2e:terminal-rendering-golden')
|
||||
expect(goldenRunSteps.get('windows')).toMatchObject({
|
||||
if: "runner.os == 'Windows'",
|
||||
shell: 'pwsh'
|
||||
})
|
||||
expect(goldenRunSteps.get('windows').run).toContain(
|
||||
'pnpm run test:e2e:windows-fresh-startup-golden'
|
||||
)
|
||||
expect(goldenWorkflow.on.pull_request).toBeUndefined()
|
||||
expect(goldenWorkflow.on.workflow_dispatch).toBeDefined()
|
||||
expect(releaseBuildNeeds).not.toContain('terminal-rendering-golden')
|
||||
@@ -552,12 +573,18 @@ describe('Electron runtime package contract', () => {
|
||||
expect(publishReleaseNeeds).toContain('build')
|
||||
expect(publishReleaseNeeds).not.toContain('terminal-rendering-release-evidence')
|
||||
expect(releaseGoldenJob['continue-on-error']).toBeUndefined()
|
||||
expect(releaseGoldenJob.strategy.matrix.include.map(({ platform }) => platform).sort()).toEqual(
|
||||
goldenPlatforms
|
||||
)
|
||||
expect(releaseGoldenMatrix).toEqual(goldenMatrix)
|
||||
expect(releaseGoldenJob.steps.map((step) => step.run ?? '')).toContain(
|
||||
'xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:terminal-rendering-golden'
|
||||
)
|
||||
const releaseWindowsRunStep = releaseGoldenJob.steps.find(
|
||||
(step) => step.name === 'Run fresh-startup golden on Windows'
|
||||
)
|
||||
expect(releaseWindowsRunStep).toMatchObject({
|
||||
if: "runner.os == 'Windows'",
|
||||
shell: 'pwsh'
|
||||
})
|
||||
expect(releaseWindowsRunStep.run).toContain('pnpm run test:e2e:windows-fresh-startup-golden')
|
||||
expect(releaseEvidenceJob['continue-on-error']).toBe(true)
|
||||
expect(
|
||||
releaseEvidenceJob.strategy.matrix.include.map(({ platform }) => platform).sort()
|
||||
|
||||
@@ -87,6 +87,7 @@
|
||||
"test:e2e:multi-client-navigation": "node config/scripts/run-multi-client-navigation-e2e.mjs",
|
||||
"test:e2e:floating-mobile-emulator": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/floating-mobile-emulator-tab.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1",
|
||||
"test:e2e:terminal-rendering-golden": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/terminal-raw-emoji-table-scroll-restore.spec.ts tests/e2e/terminal-webgl-atlas-budget.spec.ts --grep @terminal-rendering-golden --config tests/playwright.config.ts --project electron-headless --workers=1",
|
||||
"test:e2e:windows-fresh-startup-golden": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/golden-windows-fresh-startup.spec.ts --grep @windows-fresh-startup-golden --config tests/playwright.config.ts --project electron-headless --workers=1",
|
||||
"test:e2e:terminal-rendering-release-evidence": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/terminal-opencode-emoji-table-rendering.spec.ts tests/e2e/terminal-long-table-scroll-restore.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=2",
|
||||
"test:e2e:terminal-perf": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/terminal-typing-latency.spec.ts tests/e2e/terminal-foreground-redraw-freeze.spec.ts tests/e2e/terminal-output-scheduler.spec.ts tests/e2e/terminal-hidden-tui-visual-restore.spec.ts tests/e2e/artificial-opencode-terminal-load.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=2",
|
||||
"test:e2e:terminal-perf:scale": "pnpm run ensure:electron-runtime && node config/scripts/run-terminal-scale-perf-e2e.mjs",
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import { expect, test } from './helpers/orca-app'
|
||||
|
||||
test.use({ dismissOnboarding: false, seedTestRepo: false })
|
||||
test.skip(process.platform !== 'win32', 'Fresh-profile fsync regression is Windows-only')
|
||||
|
||||
test('fresh Windows profile reaches onboarding @windows-fresh-startup-golden', async ({
|
||||
orcaPage
|
||||
}) => {
|
||||
await expect(orcaPage.getByRole('heading', { name: /Pick your default agent/i })).toBeVisible({
|
||||
timeout: 30_000
|
||||
})
|
||||
})
|
||||
@@ -252,6 +252,7 @@ export const test = base.extend<OrcaTestFixtures, OrcaWorkerFixtures>({
|
||||
...(headful ? { ORCA_E2E_HEADFUL: '1' } : { ORCA_E2E_HEADLESS: '1' })
|
||||
}
|
||||
})
|
||||
forwardElectronProcessLogs(app, testInfo)
|
||||
try {
|
||||
const resolvedHome = await app.evaluate(({ app }) => app.getPath('home'))
|
||||
assertElectronResolvedIsolatedHome(resolvedHome, homeIsolation)
|
||||
@@ -261,7 +262,6 @@ export const test = base.extend<OrcaTestFixtures, OrcaWorkerFixtures>({
|
||||
await removeUserDataDirAfterShutdown(userDataDir)
|
||||
throw error
|
||||
}
|
||||
forwardElectronProcessLogs(app, testInfo)
|
||||
await provideFixture(app)
|
||||
// Why: the Playwright close promise can settle before all Electron and PTY
|
||||
// descendants are gone in CI; worker teardown then hangs on open handles.
|
||||
|
||||
Reference in New Issue
Block a user