diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a94a7ea2ba5..abafb5cd6c8 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -227,6 +227,7 @@ jobs: mapfile -t TEST_FILES < <(jq -r '.[] | select( . != "tests/e2e/ssh-startup-exec-readiness.spec.ts" and . != "tests/e2e/paired-startup-exec-readiness.spec.ts" and + . != "tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts" and . != "tests/e2e/terminal-ibus-hangul-native.spec.ts" )' <<<"$TEST_FILES_JSON") if [ "${#TEST_FILES[@]}" -eq 0 ]; then @@ -262,12 +263,13 @@ jobs: needs: [build, prepare-native-cache] # effect of one route listing a startup-readiness spec — pruning that spec would have # silently retired the whole lane. The signal is now derived from the SSH routes directly. - # The two spec clauses stay for their honest purpose: changed-e2e hands these specs to this + # The explicit spec clauses stay for their honest purpose: changed-e2e hands these specs to this # lane, so editing one must still run it here. if: >- inputs.test_files == '' || inputs.ssh_source_changed == 'true' || contains(inputs.test_files, 'tests/e2e/ssh-startup-exec-readiness.spec.ts') || + contains(inputs.test_files, 'tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts') || contains(inputs.test_files, 'tests/e2e/paired-startup-exec-readiness.spec.ts') runs-on: ubuntu-latest # Why 60: this lane now also runs the remaining Docker-SSH specs serially. They average diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 1153293f77c..76ba45e0432 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -18035,19 +18035,27 @@ ], "platforms": ["macos", "linux", "windows"], "providers": ["ssh"], - "coveredPlatforms": ["macos"], + "coveredPlatforms": [ + "macos", + "linux" + ], "coveredProviders": ["ssh"], - "coverageNotes": "A macOS Electron client drives a Linux Docker SSH execution host. The six-spec suite passed ten enabled cases with clean worker exit (5.2m). The formerly skipped frozen-host input case now waits for recovered authority before sending input and passed four separate executions (one initial and three repetitions). The existing flooded-shell fixme remains an explicitly reproduced application gap.", + "coverageNotes": "A macOS Electron client drives a Linux Docker SSH execution host. The six-spec suite passed ten enabled cases with clean worker exit (5.2m). The formerly skipped frozen-host input case now waits for recovered authority before sending input and passed four separate executions (one initial and three repetitions). The existing flooded-shell fixme remains an explicitly reproduced application gap. The bulk-open freeze reproduction runs in Linux headed CI with SwiftShader on Xvfb: headless Linux schedules idle animation frames about 1s apart, invalidating the foreground interaction measurement. Original uninstrumented five-pane workload passed all ten repetitions with zero retries/skips in 6.6m; bulk-open lag 79.3–147.8ms and interaction 127.1–155.9ms, unchanged 2500ms/5000ms budgets. Run 34037669843, head f25eab3fd7d723509ced026633f80b193a139b76, excludes unmerged replay-input application fix #19075.", "motivatingLinks": [ "https://github.com/stablyai/orca/issues/18018", "https://github.com/stablyai/orca/pull/18546", - "https://github.com/stablyai/orca/issues/12547" + "https://github.com/stablyai/orca/issues/12547", + "https://github.com/stablyai/orca/issues/16764", + "https://github.com/stablyai/orca/actions/runs/34037450427", + "https://github.com/stablyai/orca/actions/runs/34037669843" ], "invariant": "Transport loss and frozen-host silence must preserve the remote session; host relay loss may rebind a pane without accumulating reattachable leases. Reconnects must preserve usable terminal content, bounded PTYs/fds/processes, complete large listings, and independently recoverable watcher processes. Electron test shutdown must release inherited pipes after confirmed root exit without closing live-process pipes.", "oracle": "Poll a changed connected SSH authority after injected faults, then require terminal output and appropriate PTY identity. Read remote process/fd state, listFiles replies, and rendered explorer rows. Resolve Playwright cleanup only after the root process exits and its inherited pipes close; live-process pipes remain untouched.", "commands": [ "ORCA_E2E_SSH_DOCKER=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/ssh-docker-transport-drop-recovery.spec.ts tests/e2e/ssh-docker-half-open-link.spec.ts tests/e2e/ssh-docker-quick-open-large-listing.spec.ts tests/e2e/ssh-docker-reconnect-pane-restore.spec.ts tests/e2e/ssh-docker-resource-accumulation.spec.ts tests/e2e/ssh-docker-watcher-isolation.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", - "pnpm exec vitest run --config config/vitest.config.ts tests/e2e/helpers/electron-process-shutdown.unit.test.ts" + "pnpm exec vitest run --config config/vitest.config.ts tests/e2e/helpers/electron-process-shutdown.unit.test.ts", + "ORCA_E2E_SSH_DOCKER=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts --config tests/playwright.config.ts --project=electron-headful --workers=1", + "ORCA_E2E_SSH_DOCKER=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts --config tests/playwright.config.ts --project=electron-headful --workers=1 --repeat-each=10" ], "testFiles": [ "tests/e2e/ssh-docker-transport-drop-recovery.spec.ts", @@ -18056,7 +18064,8 @@ "tests/e2e/ssh-docker-reconnect-pane-restore.spec.ts", "tests/e2e/ssh-docker-resource-accumulation.spec.ts", "tests/e2e/ssh-docker-watcher-isolation.spec.ts", - "tests/e2e/helpers/electron-process-shutdown.unit.test.ts" + "tests/e2e/helpers/electron-process-shutdown.unit.test.ts", + "tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts" ], "assertionRefs": [ { @@ -18101,6 +18110,12 @@ "releases inherited pipes after confirmed exit, including prior exit", "retains live-process pipes on shutdown timeout" ] + }, + { + "file": "tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts", + "assertions": [ + "five flooding SSH panes remain below unchanged 2500ms soft and 5000ms hard freeze budgets during bulk reopen and two double-animation-frame view changes" + ] } ], "evidenceRuns": [ @@ -18121,6 +18136,15 @@ "command": "ORCA_E2E_SSH_DOCKER=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/ssh-docker-transport-drop-recovery.spec.ts tests/e2e/ssh-docker-half-open-link.spec.ts tests/e2e/ssh-docker-quick-open-large-listing.spec.ts tests/e2e/ssh-docker-reconnect-pane-restore.spec.ts tests/e2e/ssh-docker-resource-accumulation.spec.ts tests/e2e/ssh-docker-watcher-isolation.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", "durationSeconds": 312, "summary": "Six specs: ten passed, two existing fixme skipped, clean worker shutdown. Baseline same enabled suite: ten passed but worker teardown timed out (7.3m)." + }, + { + "date": "2026-09-06", + "runner": "ci", + "platform": "linux", + "result": "passed", + "command": "ORCA_E2E_SSH_DOCKER=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts --config tests/playwright.config.ts --project=electron-headful --workers=1 --repeat-each=10", + "durationSeconds": 396, + "summary": "Original uninstrumented five-pane workload passed all ten repetitions with zero retries/skips in 6.6m; bulk-open lag 79.3–147.8ms and interaction 127.1–155.9ms, unchanged 2500ms/5000ms budgets. Run 34037669843, head f25eab3fd7d723509ced026633f80b193a139b76, excludes unmerged replay-input application fix #19075." } ], "runtimeBudget": { @@ -18146,9 +18170,10 @@ ], "knownGaps": [ "The disconnected 48MB flood still loses its relay channel: original post-flood input marker failed in 60s, and waiting for the finite producer completion marker failed in 120s. It remains an explicit #18018 fixme reproduction; frozen-host input is re-enabled after four successful runs.", - "Linux and Windows desktop clients, WSL, folder workspaces, paired runtimes and live agent CLIs are not exercised by these Docker specs.", + "Linux headed CI covers the bulk-open freeze reproduction; Windows clients, WSL, folder workspaces, paired runtimes and live agent CLIs are not covered by that result.", "Some legacy assertions inspect terminal serialization or backing state rather than rendered DOM; no blanket visual coverage claim.", - "No p95 CI history or full product mutation proof." + "No p95 CI history or full product mutation proof.", + "One headless bulk-open probe reached 6478.6ms in run 34035957303; animation-frame scheduling explains the consistent interaction failures, but does not directly explain that isolated timer-lag outlier. Long-term headed CI soak remains outstanding." ], "demotionRule": "Keep experimental while any recovery reproduction fails or any teardown, identity, resource-count, or rendered oracle flakes; never promote by extending sleeps or retries." }, diff --git a/config/scripts/pr-e2e-gate-contract.test.mjs b/config/scripts/pr-e2e-gate-contract.test.mjs index 41f9338ab75..f5295faf1ad 100644 --- a/config/scripts/pr-e2e-gate-contract.test.mjs +++ b/config/scripts/pr-e2e-gate-contract.test.mjs @@ -168,6 +168,7 @@ describe('PR E2E gate contract', () => { expect(changedRun.env.TEST_FILES_JSON).toBe('${{ inputs.test_files }}') expect(changedRun.run).toContain('. != "tests/e2e/ssh-startup-exec-readiness.spec.ts"') expect(changedRun.run).toContain('. != "tests/e2e/paired-startup-exec-readiness.spec.ts"') + expect(changedRun.run).toContain('. != "tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts"') expect(changedRun.run).toContain('if [ "${#TEST_FILES[@]}" -eq 0 ]') expect(changedRun.run).toContain('grep -l \'@headful\' "${TEST_FILES[@]}"') expect(changedRun.run).toContain('E2E_PROJECT_ARGS+=(--project=electron-headful)') @@ -379,8 +380,7 @@ describe('PR E2E gate contract', () => { // run-ssh-docker-e2e.mjs so the gap stays legible rather than looking like coverage. const unreachableSpecs = new Set([ 'tests/e2e/ssh-docker-relay-perf.spec.ts', - 'tests/e2e/ssh-codex-display-artifacts-repro.spec.ts', - 'tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts' + 'tests/e2e/ssh-codex-display-artifacts-repro.spec.ts' ]) // Why comments are stripped: this file's own runner lists the two exempt specs by name in a // prose comment. A substring scan over raw text would count any spec merely *discussed* in a diff --git a/config/scripts/run-ssh-docker-bulk-open-freeze-e2e.mjs b/config/scripts/run-ssh-docker-bulk-open-freeze-e2e.mjs index 153f3fb5bc6..294bf7e2c7b 100644 --- a/config/scripts/run-ssh-docker-bulk-open-freeze-e2e.mjs +++ b/config/scripts/run-ssh-docker-bulk-open-freeze-e2e.mjs @@ -29,7 +29,7 @@ const result = spawnSync( '--config', 'tests/playwright.config.ts', '--project', - 'electron-headless', + 'electron-headful', '--workers=1', ...extraArgs ], diff --git a/config/scripts/run-ssh-docker-e2e.mjs b/config/scripts/run-ssh-docker-e2e.mjs index 9ab44b8457e..faa354689cb 100644 --- a/config/scripts/run-ssh-docker-e2e.mjs +++ b/config/scripts/run-ssh-docker-e2e.mjs @@ -33,31 +33,8 @@ if (runtime.status !== 0) { // all. Recorded as a real gap, not as coverage living somewhere else. // ssh-codex-display-artifacts-repro.spec.ts — installs a real remote codex binary that CI // runners do not have (observed as `spawn codex ENOENT`). Runs in no CI lane at all. -// ssh-docker-bulk-open-freeze-repro.spec.ts — un-rotted and now measurable, and marked -// `test.fixme` because its oracle cannot gate. Absent from this list AND skipped, so the -// two cannot drift: it is also reachable from the changed-specs lane whenever the spec -// itself is edited, and a wall-clock oracle that fails there is worth no more than one -// that fails here. -// The rot (#16764) is fixed: the stale call sites are repaired, it connects after session -// restore instead of before, and readiness keys on the repeating flood marker rather than -// a one-shot READY line the flood buries within ~16ms. It runs end to end and prints a -// measurement instead of dying on a call site. -// What it is NOT is portable. Three runs of the same measurement path: -// developer workstation: hiddenFlood 2.1ms bulkOpen 41.5ms interaction 53.6ms -// GitHub ubuntu runner A: hiddenFlood 1.5ms bulkOpen 2575.6ms interaction 3464.2ms -// GitHub ubuntu runner B: hiddenFlood 0.2ms bulkOpen 397.4ms interaction 3386.7ms -// bulkOpen swings 6.5x between two CI runs of the same code, so a fixed threshold on it is -// a coin flip; interaction sits stably ~64x over the workstation figure because it times a -// view remount, not the renderer freeze the issue reports, and only shares the budget -// constant because both are milliseconds. Every failure so far is the soft budget; hard -// has never tripped, and the relay was still streaming each time — the budget failed, not -// the product. Same rule as ssh-docker-relay-perf above. Gating needs a distribution -// first, then a host-relative oracle; a bigger constant, or a ratio picked from three -// samples, is the same arbitrary number in different clothes. -// COVERAGE GAP, recorded as such: 5 simultaneously flooding SSH panes exercise writer -// saturation, ACK/credit accounting and per-pane polling together, and nothing else covers -// that combination. Flip `test.fixme` back to `test` to run it. Tracked in -// stablyai/orca#16764. +// The bulk-open frame probe runs headed: headless Linux compositing schedules idle RAFs +// roughly 1s apart, so it cannot measure foreground interaction against the same budget. // // Why both projects: ssh-port-forward-lifecycle is @headful, which the headless project // grep-inverts away. @@ -87,6 +64,7 @@ const result = spawnSync( 'tests/e2e/ssh-ai-vault-session-history.spec.ts', 'tests/e2e/ssh-cold-activation-restore.spec.ts', 'tests/e2e/ssh-cold-hydration-gap-tab-seeding.spec.ts', + 'tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts', 'tests/e2e/ssh-docker-half-open-link.spec.ts', 'tests/e2e/ssh-docker-quick-open-large-listing.spec.ts', 'tests/e2e/ssh-docker-reconnect-pane-restore.spec.ts', diff --git a/tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts b/tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts index 2de70c199d3..4f51b346b91 100644 --- a/tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts +++ b/tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts @@ -53,32 +53,9 @@ function continuousFloodCommand(runId: string, index: number): string { test.describe('R2 Docker SSH bulk-open freeze', () => { test.skip(!RUN_DOCKER_SSH, 'Set ORCA_E2E_SSH_DOCKER=1 to run Docker SSH freeze repro') - // Fixme: un-rotted and measurable, but its oracle is wall-clock and does not survive a change of - // host, so it cannot gate. Three runs of the same measurement path: - // - // host hiddenFlood bulkOpen interaction - // developer workstation 2.1ms 41.5ms 53.6ms - // GitHub ubuntu runner A 1.5ms 2575.6ms 3464.2ms - // GitHub ubuntu runner B 0.2ms 397.4ms 3386.7ms - // - // Two separate problems, and neither is the product. `bulkOpenMaxLagMs` swings 6.5x between two - // CI runs of the same code, so a fixed threshold on it is a coin flip; `interactionProbeMs` sits - // stably ~64x over the workstation figure, because it times two `setActiveView` round trips - // through a double rAF — a view remount cost, not the renderer freeze #16764 reports. It shares - // SOFT/HARD_FREEZE_LAG_MS with the lag probe only because both are milliseconds. `hardFreeze` - // has never tripped on any host; the failure is always the soft budget. - // - // Not converted to a ratio against a calibration run: with a 6.5x within-host swing on the very - // quantity that would be normalized, a threshold picked from three samples is the same arbitrary - // constant in dimensionless clothing. Gating needs a distribution first. - // - // Kept executable rather than deleted: flip `test.fixme` back to `test` to run it, which is how - // the numbers above were taken. Tracked in stablyai/orca#16764. - // - // The cost is real and is recorded in run-ssh-docker-e2e.mjs: 5 simultaneously flooding SSH panes - // exercise writer saturation, ACK/credit accounting and per-pane polling together, and nothing - // else covers that combination. It is a gap, not coverage living somewhere else. - test.fixme('bulk-open many flooding SSH terminals and measure renderer lag @freeze-repro', async ({ + // Headless Linux disables compositing and schedules idle RAFs ~1s apart; use headed CI. + // Headed SwiftShader restores ~16ms frames without changing the freeze budgets. + test('bulk-open many flooding SSH terminals and measure renderer lag @freeze-repro @headful', async ({ orcaPage, registerPostElectronShutdownCleanup }, testInfo) => {