mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 00:02:31 +00:00
test: restore SSH bulk-open freeze coverage in headed CI (#19081)
* test: restore SSH bulk-open freeze coverage in headed CI * test: record ten passing headed SSH freeze repetitions * test: record ten passing headed SSH freeze repetitions * test: route changed SSH freeze spec only to its dedicated lane
This commit is contained in:
Neil
@@ -227,6 +227,7 @@ jobs:
|
||||
mapfile -t TEST_FILES < <(jq -r '.[] | select(
|
||||
. != "tests/e2e/ssh-startup-exec-readiness.spec.ts" and
|
||||
. != "tests/e2e/paired-startup-exec-readiness.spec.ts" and
|
||||
. != "tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts" and
|
||||
. != "tests/e2e/terminal-ibus-hangul-native.spec.ts"
|
||||
)' <<<"$TEST_FILES_JSON")
|
||||
if [ "${#TEST_FILES[@]}" -eq 0 ]; then
|
||||
@@ -262,12 +263,13 @@ jobs:
|
||||
needs: [build, prepare-native-cache]
|
||||
# effect of one route listing a startup-readiness spec — pruning that spec would have
|
||||
# silently retired the whole lane. The signal is now derived from the SSH routes directly.
|
||||
# The two spec clauses stay for their honest purpose: changed-e2e hands these specs to this
|
||||
# The explicit spec clauses stay for their honest purpose: changed-e2e hands these specs to this
|
||||
# lane, so editing one must still run it here.
|
||||
if: >-
|
||||
inputs.test_files == '' ||
|
||||
inputs.ssh_source_changed == 'true' ||
|
||||
contains(inputs.test_files, 'tests/e2e/ssh-startup-exec-readiness.spec.ts') ||
|
||||
contains(inputs.test_files, 'tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts') ||
|
||||
contains(inputs.test_files, 'tests/e2e/paired-startup-exec-readiness.spec.ts')
|
||||
runs-on: ubuntu-latest
|
||||
# Why 60: this lane now also runs the remaining Docker-SSH specs serially. They average
|
||||
|
||||
@@ -18035,19 +18035,27 @@
|
||||
],
|
||||
"platforms": ["macos", "linux", "windows"],
|
||||
"providers": ["ssh"],
|
||||
"coveredPlatforms": ["macos"],
|
||||
"coveredPlatforms": [
|
||||
"macos",
|
||||
"linux"
|
||||
],
|
||||
"coveredProviders": ["ssh"],
|
||||
"coverageNotes": "A macOS Electron client drives a Linux Docker SSH execution host. The six-spec suite passed ten enabled cases with clean worker exit (5.2m). The formerly skipped frozen-host input case now waits for recovered authority before sending input and passed four separate executions (one initial and three repetitions). The existing flooded-shell fixme remains an explicitly reproduced application gap.",
|
||||
"coverageNotes": "A macOS Electron client drives a Linux Docker SSH execution host. The six-spec suite passed ten enabled cases with clean worker exit (5.2m). The formerly skipped frozen-host input case now waits for recovered authority before sending input and passed four separate executions (one initial and three repetitions). The existing flooded-shell fixme remains an explicitly reproduced application gap. The bulk-open freeze reproduction runs in Linux headed CI with SwiftShader on Xvfb: headless Linux schedules idle animation frames about 1s apart, invalidating the foreground interaction measurement. Original uninstrumented five-pane workload passed all ten repetitions with zero retries/skips in 6.6m; bulk-open lag 79.3–147.8ms and interaction 127.1–155.9ms, unchanged 2500ms/5000ms budgets. Run 34037669843, head f25eab3fd7d723509ced026633f80b193a139b76, excludes unmerged replay-input application fix #19075.",
|
||||
"motivatingLinks": [
|
||||
"https://github.com/stablyai/orca/issues/18018",
|
||||
"https://github.com/stablyai/orca/pull/18546",
|
||||
"https://github.com/stablyai/orca/issues/12547"
|
||||
"https://github.com/stablyai/orca/issues/12547",
|
||||
"https://github.com/stablyai/orca/issues/16764",
|
||||
"https://github.com/stablyai/orca/actions/runs/34037450427",
|
||||
"https://github.com/stablyai/orca/actions/runs/34037669843"
|
||||
],
|
||||
"invariant": "Transport loss and frozen-host silence must preserve the remote session; host relay loss may rebind a pane without accumulating reattachable leases. Reconnects must preserve usable terminal content, bounded PTYs/fds/processes, complete large listings, and independently recoverable watcher processes. Electron test shutdown must release inherited pipes after confirmed root exit without closing live-process pipes.",
|
||||
"oracle": "Poll a changed connected SSH authority after injected faults, then require terminal output and appropriate PTY identity. Read remote process/fd state, listFiles replies, and rendered explorer rows. Resolve Playwright cleanup only after the root process exits and its inherited pipes close; live-process pipes remain untouched.",
|
||||
"commands": [
|
||||
"ORCA_E2E_SSH_DOCKER=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/ssh-docker-transport-drop-recovery.spec.ts tests/e2e/ssh-docker-half-open-link.spec.ts tests/e2e/ssh-docker-quick-open-large-listing.spec.ts tests/e2e/ssh-docker-reconnect-pane-restore.spec.ts tests/e2e/ssh-docker-resource-accumulation.spec.ts tests/e2e/ssh-docker-watcher-isolation.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts tests/e2e/helpers/electron-process-shutdown.unit.test.ts"
|
||||
"pnpm exec vitest run --config config/vitest.config.ts tests/e2e/helpers/electron-process-shutdown.unit.test.ts",
|
||||
"ORCA_E2E_SSH_DOCKER=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts --config tests/playwright.config.ts --project=electron-headful --workers=1",
|
||||
"ORCA_E2E_SSH_DOCKER=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts --config tests/playwright.config.ts --project=electron-headful --workers=1 --repeat-each=10"
|
||||
],
|
||||
"testFiles": [
|
||||
"tests/e2e/ssh-docker-transport-drop-recovery.spec.ts",
|
||||
@@ -18056,7 +18064,8 @@
|
||||
"tests/e2e/ssh-docker-reconnect-pane-restore.spec.ts",
|
||||
"tests/e2e/ssh-docker-resource-accumulation.spec.ts",
|
||||
"tests/e2e/ssh-docker-watcher-isolation.spec.ts",
|
||||
"tests/e2e/helpers/electron-process-shutdown.unit.test.ts"
|
||||
"tests/e2e/helpers/electron-process-shutdown.unit.test.ts",
|
||||
"tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts"
|
||||
],
|
||||
"assertionRefs": [
|
||||
{
|
||||
@@ -18101,6 +18110,12 @@
|
||||
"releases inherited pipes after confirmed exit, including prior exit",
|
||||
"retains live-process pipes on shutdown timeout"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts",
|
||||
"assertions": [
|
||||
"five flooding SSH panes remain below unchanged 2500ms soft and 5000ms hard freeze budgets during bulk reopen and two double-animation-frame view changes"
|
||||
]
|
||||
}
|
||||
],
|
||||
"evidenceRuns": [
|
||||
@@ -18121,6 +18136,15 @@
|
||||
"command": "ORCA_E2E_SSH_DOCKER=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/ssh-docker-transport-drop-recovery.spec.ts tests/e2e/ssh-docker-half-open-link.spec.ts tests/e2e/ssh-docker-quick-open-large-listing.spec.ts tests/e2e/ssh-docker-reconnect-pane-restore.spec.ts tests/e2e/ssh-docker-resource-accumulation.spec.ts tests/e2e/ssh-docker-watcher-isolation.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1",
|
||||
"durationSeconds": 312,
|
||||
"summary": "Six specs: ten passed, two existing fixme skipped, clean worker shutdown. Baseline same enabled suite: ten passed but worker teardown timed out (7.3m)."
|
||||
},
|
||||
{
|
||||
"date": "2026-09-06",
|
||||
"runner": "ci",
|
||||
"platform": "linux",
|
||||
"result": "passed",
|
||||
"command": "ORCA_E2E_SSH_DOCKER=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts --config tests/playwright.config.ts --project=electron-headful --workers=1 --repeat-each=10",
|
||||
"durationSeconds": 396,
|
||||
"summary": "Original uninstrumented five-pane workload passed all ten repetitions with zero retries/skips in 6.6m; bulk-open lag 79.3–147.8ms and interaction 127.1–155.9ms, unchanged 2500ms/5000ms budgets. Run 34037669843, head f25eab3fd7d723509ced026633f80b193a139b76, excludes unmerged replay-input application fix #19075."
|
||||
}
|
||||
],
|
||||
"runtimeBudget": {
|
||||
@@ -18146,9 +18170,10 @@
|
||||
],
|
||||
"knownGaps": [
|
||||
"The disconnected 48MB flood still loses its relay channel: original post-flood input marker failed in 60s, and waiting for the finite producer completion marker failed in 120s. It remains an explicit #18018 fixme reproduction; frozen-host input is re-enabled after four successful runs.",
|
||||
"Linux and Windows desktop clients, WSL, folder workspaces, paired runtimes and live agent CLIs are not exercised by these Docker specs.",
|
||||
"Linux headed CI covers the bulk-open freeze reproduction; Windows clients, WSL, folder workspaces, paired runtimes and live agent CLIs are not covered by that result.",
|
||||
"Some legacy assertions inspect terminal serialization or backing state rather than rendered DOM; no blanket visual coverage claim.",
|
||||
"No p95 CI history or full product mutation proof."
|
||||
"No p95 CI history or full product mutation proof.",
|
||||
"One headless bulk-open probe reached 6478.6ms in run 34035957303; animation-frame scheduling explains the consistent interaction failures, but does not directly explain that isolated timer-lag outlier. Long-term headed CI soak remains outstanding."
|
||||
],
|
||||
"demotionRule": "Keep experimental while any recovery reproduction fails or any teardown, identity, resource-count, or rendered oracle flakes; never promote by extending sleeps or retries."
|
||||
},
|
||||
|
||||
@@ -168,6 +168,7 @@ describe('PR E2E gate contract', () => {
|
||||
expect(changedRun.env.TEST_FILES_JSON).toBe('${{ inputs.test_files }}')
|
||||
expect(changedRun.run).toContain('. != "tests/e2e/ssh-startup-exec-readiness.spec.ts"')
|
||||
expect(changedRun.run).toContain('. != "tests/e2e/paired-startup-exec-readiness.spec.ts"')
|
||||
expect(changedRun.run).toContain('. != "tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts"')
|
||||
expect(changedRun.run).toContain('if [ "${#TEST_FILES[@]}" -eq 0 ]')
|
||||
expect(changedRun.run).toContain('grep -l \'@headful\' "${TEST_FILES[@]}"')
|
||||
expect(changedRun.run).toContain('E2E_PROJECT_ARGS+=(--project=electron-headful)')
|
||||
@@ -379,8 +380,7 @@ describe('PR E2E gate contract', () => {
|
||||
// run-ssh-docker-e2e.mjs so the gap stays legible rather than looking like coverage.
|
||||
const unreachableSpecs = new Set([
|
||||
'tests/e2e/ssh-docker-relay-perf.spec.ts',
|
||||
'tests/e2e/ssh-codex-display-artifacts-repro.spec.ts',
|
||||
'tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts'
|
||||
'tests/e2e/ssh-codex-display-artifacts-repro.spec.ts'
|
||||
])
|
||||
// Why comments are stripped: this file's own runner lists the two exempt specs by name in a
|
||||
// prose comment. A substring scan over raw text would count any spec merely *discussed* in a
|
||||
|
||||
@@ -29,7 +29,7 @@ const result = spawnSync(
|
||||
'--config',
|
||||
'tests/playwright.config.ts',
|
||||
'--project',
|
||||
'electron-headless',
|
||||
'electron-headful',
|
||||
'--workers=1',
|
||||
...extraArgs
|
||||
],
|
||||
|
||||
@@ -33,31 +33,8 @@ if (runtime.status !== 0) {
|
||||
// all. Recorded as a real gap, not as coverage living somewhere else.
|
||||
// ssh-codex-display-artifacts-repro.spec.ts — installs a real remote codex binary that CI
|
||||
// runners do not have (observed as `spawn codex ENOENT`). Runs in no CI lane at all.
|
||||
// ssh-docker-bulk-open-freeze-repro.spec.ts — un-rotted and now measurable, and marked
|
||||
// `test.fixme` because its oracle cannot gate. Absent from this list AND skipped, so the
|
||||
// two cannot drift: it is also reachable from the changed-specs lane whenever the spec
|
||||
// itself is edited, and a wall-clock oracle that fails there is worth no more than one
|
||||
// that fails here.
|
||||
// The rot (#16764) is fixed: the stale call sites are repaired, it connects after session
|
||||
// restore instead of before, and readiness keys on the repeating flood marker rather than
|
||||
// a one-shot READY line the flood buries within ~16ms. It runs end to end and prints a
|
||||
// measurement instead of dying on a call site.
|
||||
// What it is NOT is portable. Three runs of the same measurement path:
|
||||
// developer workstation: hiddenFlood 2.1ms bulkOpen 41.5ms interaction 53.6ms
|
||||
// GitHub ubuntu runner A: hiddenFlood 1.5ms bulkOpen 2575.6ms interaction 3464.2ms
|
||||
// GitHub ubuntu runner B: hiddenFlood 0.2ms bulkOpen 397.4ms interaction 3386.7ms
|
||||
// bulkOpen swings 6.5x between two CI runs of the same code, so a fixed threshold on it is
|
||||
// a coin flip; interaction sits stably ~64x over the workstation figure because it times a
|
||||
// view remount, not the renderer freeze the issue reports, and only shares the budget
|
||||
// constant because both are milliseconds. Every failure so far is the soft budget; hard
|
||||
// has never tripped, and the relay was still streaming each time — the budget failed, not
|
||||
// the product. Same rule as ssh-docker-relay-perf above. Gating needs a distribution
|
||||
// first, then a host-relative oracle; a bigger constant, or a ratio picked from three
|
||||
// samples, is the same arbitrary number in different clothes.
|
||||
// COVERAGE GAP, recorded as such: 5 simultaneously flooding SSH panes exercise writer
|
||||
// saturation, ACK/credit accounting and per-pane polling together, and nothing else covers
|
||||
// that combination. Flip `test.fixme` back to `test` to run it. Tracked in
|
||||
// stablyai/orca#16764.
|
||||
// The bulk-open frame probe runs headed: headless Linux compositing schedules idle RAFs
|
||||
// roughly 1s apart, so it cannot measure foreground interaction against the same budget.
|
||||
//
|
||||
// Why both projects: ssh-port-forward-lifecycle is @headful, which the headless project
|
||||
// grep-inverts away.
|
||||
@@ -87,6 +64,7 @@ const result = spawnSync(
|
||||
'tests/e2e/ssh-ai-vault-session-history.spec.ts',
|
||||
'tests/e2e/ssh-cold-activation-restore.spec.ts',
|
||||
'tests/e2e/ssh-cold-hydration-gap-tab-seeding.spec.ts',
|
||||
'tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts',
|
||||
'tests/e2e/ssh-docker-half-open-link.spec.ts',
|
||||
'tests/e2e/ssh-docker-quick-open-large-listing.spec.ts',
|
||||
'tests/e2e/ssh-docker-reconnect-pane-restore.spec.ts',
|
||||
|
||||
@@ -53,32 +53,9 @@ function continuousFloodCommand(runId: string, index: number): string {
|
||||
test.describe('R2 Docker SSH bulk-open freeze', () => {
|
||||
test.skip(!RUN_DOCKER_SSH, 'Set ORCA_E2E_SSH_DOCKER=1 to run Docker SSH freeze repro')
|
||||
|
||||
// Fixme: un-rotted and measurable, but its oracle is wall-clock and does not survive a change of
|
||||
// host, so it cannot gate. Three runs of the same measurement path:
|
||||
//
|
||||
// host hiddenFlood bulkOpen interaction
|
||||
// developer workstation 2.1ms 41.5ms 53.6ms
|
||||
// GitHub ubuntu runner A 1.5ms 2575.6ms 3464.2ms
|
||||
// GitHub ubuntu runner B 0.2ms 397.4ms 3386.7ms
|
||||
//
|
||||
// Two separate problems, and neither is the product. `bulkOpenMaxLagMs` swings 6.5x between two
|
||||
// CI runs of the same code, so a fixed threshold on it is a coin flip; `interactionProbeMs` sits
|
||||
// stably ~64x over the workstation figure, because it times two `setActiveView` round trips
|
||||
// through a double rAF — a view remount cost, not the renderer freeze #16764 reports. It shares
|
||||
// SOFT/HARD_FREEZE_LAG_MS with the lag probe only because both are milliseconds. `hardFreeze`
|
||||
// has never tripped on any host; the failure is always the soft budget.
|
||||
//
|
||||
// Not converted to a ratio against a calibration run: with a 6.5x within-host swing on the very
|
||||
// quantity that would be normalized, a threshold picked from three samples is the same arbitrary
|
||||
// constant in dimensionless clothing. Gating needs a distribution first.
|
||||
//
|
||||
// Kept executable rather than deleted: flip `test.fixme` back to `test` to run it, which is how
|
||||
// the numbers above were taken. Tracked in stablyai/orca#16764.
|
||||
//
|
||||
// The cost is real and is recorded in run-ssh-docker-e2e.mjs: 5 simultaneously flooding SSH panes
|
||||
// exercise writer saturation, ACK/credit accounting and per-pane polling together, and nothing
|
||||
// else covers that combination. It is a gap, not coverage living somewhere else.
|
||||
test.fixme('bulk-open many flooding SSH terminals and measure renderer lag @freeze-repro', async ({
|
||||
// Headless Linux disables compositing and schedules idle RAFs ~1s apart; use headed CI.
|
||||
// Headed SwiftShader restores ~16ms frames without changing the freeze budgets.
|
||||
test('bulk-open many flooding SSH terminals and measure renderer lag @freeze-repro @headful', async ({
|
||||
orcaPage,
|
||||
registerPostElectronShutdownCleanup
|
||||
}, testInfo) => {
|
||||
|
||||
Reference in New Issue
Block a user