Stop running the experimental layout suite on every matching PR (#30907)

This commit is contained in:
Neil
2026-10-19 16:16:50 -07:00
committed by GitHub
parent f3a1238a34
commit cbb9f369b8
7 changed files with 105 additions and 238 deletions
+32
View File
@@ -0,0 +1,32 @@
name: Layout owner E2E (manual)
run-name: Layout owner E2E ${{ inputs.ref || github.ref }}
on:
workflow_dispatch:
inputs:
ref:
description: Commit or ref to test (defaults to the workflow commit)
required: false
type: string
permissions:
contents: read
# This experiment costs about an hour of runner time; keep one suite active repository-wide.
concurrency:
group: layout-owner-e2e
cancel-in-progress: false
jobs:
layout_owner_switch_on:
name: layout owner (switch on)
permissions:
contents: read
uses: ./.github/workflows/e2e.yml
with:
ref: ${{ inputs.ref || github.sha }}
test_files: '["tests/e2e/workspace-layout-oracle-headless.spec.ts","tests/e2e/workspace-layout-oracle.spec.ts","tests/e2e/cli-file-open-position.spec.ts","tests/e2e/relaunch-first-terminal.spec.ts","tests/e2e/relaunch-sleeping-agents-resume-on-view.spec.ts","tests/e2e/terminal-cold-activation-deferral.spec.ts","tests/e2e/cli-first-worktree-single-terminal.spec.ts","tests/e2e/unopened-workspace-agent-single-resume.spec.ts","tests/e2e/terminal-pane-close-layout-consistency.spec.ts","tests/e2e/golden-file-open-edit-save.spec.ts","tests/e2e/golden-source-control-open-diff.spec.ts","tests/e2e/paired-cli-terminal-graph-sync-tab-retention.spec.ts","tests/e2e/floating-workspace-pane-height.spec.ts","tests/e2e/editor-model-host-isolation.spec.ts","tests/e2e/editor-tab-selection-restore.spec.ts","tests/e2e/finished-agent-ghost-resume.spec.ts","tests/e2e/live-background-terminal-mount-authority.spec.ts","tests/e2e/workspace-back-forward-navigation.spec.ts","tests/e2e/worktree-lineage-toggle-shortcut.spec.ts","tests/e2e/local-cli-terminal-retention.spec.ts","tests/e2e/default-tabs-mark-crash-relaunch.spec.ts","tests/e2e/terminal-cli-split-closed-before-mount.spec.ts","tests/e2e/layout-stream-loss-reopen.spec.ts","tests/e2e/terminal-tab-close-restart-persistence.spec.ts","tests/e2e/artificial-opencode-terminal-load.spec.ts","tests/e2e/ssh-orcad-"]'
run_changed_e2e: true
needs_build: true
artifact_suffix: -layout-owner-switch-on
layout_owner: main
-26
View File
@@ -60,7 +60,6 @@ jobs:
e2e_needs_build: ${{ steps.e2e_filter.outputs.e2e_needs_build }}
native_ime_source_changed: ${{ steps.e2e_filter.outputs.native_ime_source_changed }}
wsl_source_changed: ${{ steps.e2e_filter.outputs.wsl_source_changed }}
layout_owner_switch_on: ${{ steps.e2e_filter.outputs.layout_owner_switch_on }}
steps:
- name: Checkout
uses: actions/checkout@v6
@@ -86,7 +85,6 @@ jobs:
/.oxfmtrc.json
/config/scripts/pr-package-test-paths.mjs
/config/scripts/pr-layout-oracle-route.mjs
/config/scripts/pr-layout-owner-switch-on-route.mjs
/config/scripts/pr-e2e-source-routing.mjs
/config/scripts/ci-e2e-job-selection.mjs
/config/scripts/pr-ready-check-reuse.mjs
@@ -167,8 +165,6 @@ jobs:
WSL_CHANGED="$(git diff --name-only --no-renames --diff-filter=ACDMR "$DIFF_BASE" HEAD)"
WSL_SOURCE_CHANGED="$(printf '%s\n' "$WSL_CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --wsl-source)"
echo "wsl_source_changed=$WSL_SOURCE_CHANGED" >> "$GITHUB_OUTPUT"
LAYOUT_OWNER_SWITCH_ON="$(printf '%s\n' "$WSL_CHANGED" | node config/scripts/pr-layout-owner-switch-on-route.mjs)"
echo "layout_owner_switch_on=$LAYOUT_OWNER_SWITCH_ON" >> "$GITHUB_OUTPUT"
echo "Native IME source changed: $NATIVE_IME_SOURCE_CHANGED"
SHOULD_RUN="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --reusable-workflow "${E2E_SCOPE_ARGS[@]}")"
if [ "$SHOULD_RUN" = true ]; then
@@ -1586,28 +1582,6 @@ jobs:
needs_build: true
artifact_suffix: -layout-oracle
# Why not in verify's needs: the layout-owner switch is off in shipped builds, and some of these
# specs fail with it on until that work lands. It reports a red X without blocking the merge.
layout_owner_switch_on:
name: layout owner (switch on)
needs: [code_paths, preflight]
if: >-
!cancelled() &&
needs.code_paths.result == 'success' &&
needs.code_paths.outputs.layout_owner_switch_on == 'true' &&
needs.preflight.result == 'success'
permissions:
contents: read
uses: ./.github/workflows/e2e.yml
with:
ref: ${{ github.sha }}
# LAYOUT_OWNER_SWITCH_ON_E2E_SPECS in config/scripts/pr-layout-owner-switch-on-route.mjs.
test_files: '["tests/e2e/workspace-layout-oracle-headless.spec.ts","tests/e2e/workspace-layout-oracle.spec.ts","tests/e2e/cli-file-open-position.spec.ts","tests/e2e/relaunch-first-terminal.spec.ts","tests/e2e/relaunch-sleeping-agents-resume-on-view.spec.ts","tests/e2e/terminal-cold-activation-deferral.spec.ts","tests/e2e/cli-first-worktree-single-terminal.spec.ts","tests/e2e/unopened-workspace-agent-single-resume.spec.ts","tests/e2e/terminal-pane-close-layout-consistency.spec.ts","tests/e2e/golden-file-open-edit-save.spec.ts","tests/e2e/golden-source-control-open-diff.spec.ts","tests/e2e/paired-cli-terminal-graph-sync-tab-retention.spec.ts","tests/e2e/floating-workspace-pane-height.spec.ts","tests/e2e/editor-model-host-isolation.spec.ts","tests/e2e/editor-tab-selection-restore.spec.ts","tests/e2e/finished-agent-ghost-resume.spec.ts","tests/e2e/live-background-terminal-mount-authority.spec.ts","tests/e2e/workspace-back-forward-navigation.spec.ts","tests/e2e/worktree-lineage-toggle-shortcut.spec.ts","tests/e2e/local-cli-terminal-retention.spec.ts","tests/e2e/default-tabs-mark-crash-relaunch.spec.ts","tests/e2e/terminal-cli-split-closed-before-mount.spec.ts","tests/e2e/layout-stream-loss-reopen.spec.ts","tests/e2e/terminal-tab-close-restart-persistence.spec.ts","tests/e2e/artificial-opencode-terminal-load.spec.ts","tests/e2e/ssh-orcad-"]'
run_changed_e2e: true
needs_build: true
artifact_suffix: -layout-owner-switch-on
layout_owner: main
# Why this is not in verify's needs: it is the first PR-gate run of a harness whose reliability
# is only known from nightly main runs (20/20 green, 2026-08-09..2026-08-29, p50 3m25s). It
# reports a red X on the PR without blocking, exactly like `e2e` above. Deliberately no
@@ -0,0 +1,61 @@
import { describe, expect, it } from 'vitest'
import { divergenceKind, summarizeDivergence } from './layout-save-divergence-summary.mjs'
describe('layout save divergence summary', () => {
it('counts reports by kind and by the spec running when each was written', () => {
const id = 'c47bcbc2-ec0c-458b-9d54-a2252e8f747a'
const report = {
suites: [
{
file: 'a.spec.ts',
specs: [
{
line: 8,
title: 'opens',
tests: [{ results: [{ startTime: '2026-10-19T00:00:00.000Z', duration: 1000 }] }]
}
]
}
]
}
const start = Date.parse('2026-10-19T00:00:00.000Z')
const lines = [
JSON.stringify({
at: start + 500,
paths: [`group ${id} changed: tabOrder`, 'workspace added']
}),
// Written after the test ended, as delayed saves are.
JSON.stringify({ at: start + 5000, paths: [`terminal ${id} tab-bar entry removed`] }),
JSON.stringify({ at: start + 60_000, paths: ['1 pane(s) rebound'] }),
'{"at":',
''
]
expect(summarizeDivergence(lines, [report])).toEqual({
reports: 3,
paths: 4,
unreadable: 1,
byKind: {
'group <id> changed: tabOrder': 1,
'workspace added': 1,
'terminal <id> tab-bar entry removed': 1,
'<n> pane(s) rebound': 1
},
bySpec: { 'a.spec.ts:8 opens': 2, unattributed: 1 },
bySpecAndKind: {
'a.spec.ts:8 opens': {
'group <id> changed: tabOrder': 1,
'workspace added': 1,
'terminal <id> tab-bar entry removed': 1
},
unattributed: { '<n> pane(s) rebound': 1 }
}
})
expect(divergenceKind('tab picker-agent-tab changed: createdAt, terminal')).toBe(
'tab <id> changed: createdAt, terminal'
)
expect(divergenceKind('editor view /tmp/a b/c.ts changed: viewState')).toBe(
'editor view <id> changed: viewState'
)
expect(divergenceKind('file /tmp/a b/c.ts added')).toBe('file <id> added')
})
})
@@ -312,7 +312,6 @@ function createMobileCiScopeRepository(
'config/scripts/pr-code-change-scope.mjs',
'config/scripts/pr-package-test-paths.mjs',
'config/scripts/pr-layout-oracle-route.mjs',
'config/scripts/pr-layout-owner-switch-on-route.mjs',
'config/scripts/git-pull-request-diff-base.mjs',
'config/scripts/pr-e2e-source-routing.mjs',
'config/scripts/ci-e2e-job-selection.mjs'
@@ -1,69 +0,0 @@
// Paths whose change runs the advisory `layout_owner_switch_on` job in pr.yml.
import process from 'node:process'
import { pathToFileURL } from 'node:url'
import { MANAGED_SSH_E2E_SPEC_PREFIX } from './ci-e2e-job-selection.mjs'
import { LAYOUT_ORACLE_E2E_SPECS } from './pr-layout-oracle-route.mjs'
/** The oracle, its positive control, and the layout-owner acceptance specs. */
export const LAYOUT_OWNER_SWITCH_ON_E2E_SPECS = [
...LAYOUT_ORACLE_E2E_SPECS,
// Positive control: its direct store write must leave at least one divergence report.
'tests/e2e/cli-file-open-position.spec.ts',
'tests/e2e/relaunch-first-terminal.spec.ts',
'tests/e2e/relaunch-sleeping-agents-resume-on-view.spec.ts',
'tests/e2e/terminal-cold-activation-deferral.spec.ts',
'tests/e2e/cli-first-worktree-single-terminal.spec.ts',
'tests/e2e/unopened-workspace-agent-single-resume.spec.ts',
'tests/e2e/terminal-pane-close-layout-consistency.spec.ts',
// Saves that drop a terminal's tab-bar entry. Not diff-word-wrap.spec.ts: its file-diff case
// fails with the switch off too (main's scheduled e2e run 39148564778, shard 2).
'tests/e2e/golden-file-open-edit-save.spec.ts',
'tests/e2e/golden-source-control-open-diff.spec.ts',
'tests/e2e/paired-cli-terminal-graph-sync-tab-retention.spec.ts',
// Saves whose tab order differs from main's while a frame is held.
'tests/e2e/floating-workspace-pane-height.spec.ts',
'tests/e2e/editor-model-host-isolation.spec.ts',
'tests/e2e/editor-tab-selection-restore.spec.ts',
'tests/e2e/finished-agent-ghost-resume.spec.ts',
'tests/e2e/live-background-terminal-mount-authority.spec.ts',
'tests/e2e/workspace-back-forward-navigation.spec.ts',
'tests/e2e/worktree-lineage-toggle-shortcut.spec.ts',
'tests/e2e/local-cli-terminal-retention.spec.ts',
'tests/e2e/default-tabs-mark-crash-relaunch.spec.ts',
'tests/e2e/terminal-cli-split-closed-before-mount.spec.ts',
'tests/e2e/layout-stream-loss-reopen.spec.ts',
// A tab close by the handle a split returned.
'tests/e2e/terminal-tab-close-restart-persistence.spec.ts',
// A split main first sees at its terminal binding keeps its direction (fails only at CI's window size).
'tests/e2e/artificial-opencode-terminal-load.spec.ts',
// Every managed-SSH spec, in e2e.yml's Docker lane; its orcad template stays switch-off.
MANAGED_SSH_E2E_SPEC_PREFIX
]
const LAYOUT_OWNER_SWITCH_ON_PREFIXES = [
'src/shared/workspace-layout',
'src/shared/layout-owner-switch.ts',
'src/main/persistence/',
'src/main/runtime/',
'src/renderer/src/store/',
'tests/e2e/',
// The job itself.
'.github/workflows/pr.yml',
'.github/workflows/e2e.yml',
'config/scripts/pr-layout-owner-switch-on-route.mjs',
'config/scripts/layout-save-divergence-summary.mjs'
]
export function isLayoutOwnerSwitchOnSource(file) {
return LAYOUT_OWNER_SWITCH_ON_PREFIXES.some((prefix) => file.startsWith(prefix))
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
let input = ''
process.stdin.setEncoding('utf8')
for await (const chunk of process.stdin) {
input += chunk
}
const changedPaths = input.split(/\r?\n/).filter(Boolean)
process.stdout.write(`${changedPaths.some(isLayoutOwnerSwitchOnSource)}\n`)
}
@@ -1,142 +0,0 @@
import { describe, expect, it } from 'vitest'
import { existsSync, readdirSync, readFileSync } from 'node:fs'
import { join, resolve } from 'node:path'
import { parse } from 'yaml'
import { MANAGED_SSH_E2E_SPEC_PREFIX, selectGeneralE2eSpecs } from './ci-e2e-job-selection.mjs'
import { divergenceKind, summarizeDivergence } from './layout-save-divergence-summary.mjs'
import {
isLayoutOwnerSwitchOnSource,
LAYOUT_OWNER_SWITCH_ON_E2E_SPECS
} from './pr-layout-owner-switch-on-route.mjs'
const projectDir = resolve(import.meta.dirname, '../..')
const read = (file) => parse(readFileSync(join(projectDir, file), 'utf8'))
const prWorkflow = read('.github/workflows/pr.yml')
const e2eWorkflow = read('.github/workflows/e2e.yml')
describe('layout owner switch-on job', () => {
const job = prWorkflow.jobs.layout_owner_switch_on
it('runs the listed specs on a switch-on build of the merge commit', () => {
expect(job.uses).toBe('./.github/workflows/e2e.yml')
expect(job.with.ref).toBe('${{ github.sha }}')
expect(job.with.layout_owner).toBe('main')
expect(JSON.parse(job.with.test_files)).toEqual(LAYOUT_OWNER_SWITCH_ON_E2E_SPECS)
for (const spec of LAYOUT_OWNER_SWITCH_ON_E2E_SPECS) {
if (spec !== MANAGED_SSH_E2E_SPEC_PREFIX) {
expect(existsSync(join(projectDir, spec)), spec).toBe(true)
}
}
// Every spec that gates on the switch-on build marker runs here.
const gated = readdirSync(join(projectDir, 'tests/e2e'))
.filter((file) => file.endsWith('.spec.ts'))
.map((file) => `tests/e2e/${file}`)
.filter((spec) =>
readFileSync(join(projectDir, spec), 'utf8').includes('ORCA_LAYOUT_OWNER_MAIN')
)
expect(gated.length).toBeGreaterThan(0)
expect(LAYOUT_OWNER_SWITCH_ON_E2E_SPECS).toEqual(expect.arrayContaining(gated))
expect(selectGeneralE2eSpecs(LAYOUT_OWNER_SWITCH_ON_E2E_SPECS)).not.toContain(
'tests/e2e/workspace-layout-oracle-headless.spec.ts'
)
})
it('does not block the merge and keeps its artifacts apart', () => {
expect(prWorkflow.jobs.verify.needs).not.toContain('layout_owner_switch_on')
const suffixes = Object.values(prWorkflow.jobs)
.filter((other) => other.uses === './.github/workflows/e2e.yml')
.map((other) => other.with.artifact_suffix ?? '')
expect(new Set(suffixes).size).toBe(suffixes.length)
})
it('sets the switch only on the app builds, never job-wide', () => {
for (const other of Object.values(e2eWorkflow.jobs)) {
expect(JSON.stringify(other.env ?? {})).not.toContain('layout_owner')
}
const build = e2eWorkflow.jobs.build.steps.find((step) => step.name === 'Build E2E outputs')
expect(build.run).toContain('export ORCA_LAYOUT_OWNER="$LAYOUT_OWNER"')
expect(e2eWorkflow.on.workflow_call.inputs.layout_owner.default).toBe('')
})
it('routes layout sources, the e2e tree and the job itself', () => {
for (const file of [
'src/shared/workspace-layout/workspace-layout-load.ts',
'src/shared/layout-owner-switch.ts',
'src/main/persistence/window-save-divergence.ts',
'src/main/runtime/terminal-surface-close.ts',
'src/renderer/src/store/slices/terminals.ts',
'tests/e2e/helpers/orca-app.ts',
'.github/workflows/pr.yml',
'.github/workflows/e2e.yml'
]) {
expect(isLayoutOwnerSwitchOnSource(file), file).toBe(true)
}
for (const file of ['src/renderer/src/components/ui/button.tsx', 'docs/STYLEGUIDE.md']) {
expect(isLayoutOwnerSwitchOnSource(file), file).toBe(false)
}
const filter = prWorkflow.jobs.code_paths.steps.find((step) => step.id === 'e2e_filter')
expect(filter.run).toContain('node config/scripts/pr-layout-owner-switch-on-route.mjs')
expect(prWorkflow.jobs.code_paths.steps[0].with['sparse-checkout']).toContain(
'/config/scripts/pr-layout-owner-switch-on-route.mjs'
)
})
})
describe('layout save divergence summary', () => {
it('counts reports by kind and by the spec running when each was written', () => {
const id = 'c47bcbc2-ec0c-458b-9d54-a2252e8f747a'
const report = {
suites: [
{
file: 'a.spec.ts',
specs: [
{
line: 8,
title: 'opens',
tests: [{ results: [{ startTime: '2026-10-19T00:00:00.000Z', duration: 1000 }] }]
}
]
}
]
}
const start = Date.parse('2026-10-19T00:00:00.000Z')
const lines = [
JSON.stringify({
at: start + 500,
paths: [`group ${id} changed: tabOrder`, 'workspace added']
}),
// Written after the test ended, as delayed saves are.
JSON.stringify({ at: start + 5000, paths: [`terminal ${id} tab-bar entry removed`] }),
JSON.stringify({ at: start + 60_000, paths: ['1 pane(s) rebound'] }),
'{"at":',
''
]
expect(summarizeDivergence(lines, [report])).toEqual({
reports: 3,
paths: 4,
unreadable: 1,
byKind: {
'group <id> changed: tabOrder': 1,
'workspace added': 1,
'terminal <id> tab-bar entry removed': 1,
'<n> pane(s) rebound': 1
},
bySpec: { 'a.spec.ts:8 opens': 2, unattributed: 1 },
bySpecAndKind: {
'a.spec.ts:8 opens': {
'group <id> changed: tabOrder': 1,
'workspace added': 1,
'terminal <id> tab-bar entry removed': 1
},
unattributed: { '<n> pane(s) rebound': 1 }
}
})
expect(divergenceKind('tab picker-agent-tab changed: createdAt, terminal')).toBe(
'tab <id> changed: createdAt, terminal'
)
expect(divergenceKind('editor view /tmp/a b/c.ts changed: viewState')).toBe(
'editor view <id> changed: viewState'
)
expect(divergenceKind('file /tmp/a b/c.ts added')).toBe('file <id> added')
})
})
@@ -78,3 +78,15 @@ describe('PR workflow parallelism', () => {
expect(env['INPUT_GITHUB-TOKEN']).toBe('test-token')
})
})
it('keeps the costly layout-owner experiment manual and limits it to one active suite', () => {
const experiment = parse(readFileSync('.github/workflows/layout-owner-e2e.yml', 'utf8'))
expect(Object.keys(experiment.on)).toEqual(['workflow_dispatch'])
expect(experiment.concurrency).toEqual({
group: 'layout-owner-e2e',
'cancel-in-progress': false
})
for (const job of Object.values(workflow.jobs)) {
expect(job.with?.layout_owner).toBeUndefined()
}
})