starred/orca
1
0
Fork 0
mirror of https://github.com/stablyai/orca.git synced 2026-10-02 08:02:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

docs(reliability): record exact split benchmark runs

Browse Source
This commit is contained in:
Neil
2026-08-31 02:10:31 -07:00
parent 962faacec8
commit a16161f41d
1 changed files with 24 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/reliability-gates.jsonc
+24 -3
View File
@@ -5745,7 +5745,10 @@
"pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/terminal-pane-split-with-inherited-cwd.test.ts src/renderer/src/lib/pane-manager/pane-split-close.test.ts src/renderer/src/components/terminal-pane/resolve-split-cwd.test.ts src/renderer/src/components/terminal-pane/pty-connection-split-cwd-resolution.test.ts src/renderer/src/components/terminal-pane/terminal-pane-tab-detach.test.ts src/renderer/src/components/terminal-pane/pty-preconnect-input-buffer.test.ts src/renderer/src/components/terminal-pane/pty-transport-input-write.test.ts --reporter=dot",
// Historical evidence records only; these labels do not verify the checkout or harness.
"ORCA_TERMINAL_SPLIT_LATENCY_BENCH=1 ORCA_TERMINAL_SPLIT_LATENCY_LABEL=baseline-df14d1a2983d8339e788d0e521f1c4affd9c6d5f-headful-run1 ORCA_TERMINAL_SPLIT_LATENCY_OUTPUT=/private/tmp/orca-terminal-split-activation-baseline-df14d1a2983d8339e788d0e521f1c4affd9c6d5f-headful-run1.json pnpm exec playwright test tests/e2e/terminal-split-activation-latency.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1",
"ORCA_TERMINAL_SPLIT_LATENCY_BENCH=1 ORCA_TERMINAL_SPLIT_LATENCY_LABEL=candidate-d453ffcdb704764daced1b2917fddee7224389f0-headful-run2 ORCA_TERMINAL_SPLIT_LATENCY_OUTPUT=/private/tmp/orca-terminal-split-activation-candidate-d453ffcdb704764daced1b2917fddee7224389f0-headful-run2.json pnpm exec playwright test tests/e2e/terminal-split-activation-latency.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1"
"ORCA_TERMINAL_SPLIT_LATENCY_BENCH=1 ORCA_TERMINAL_SPLIT_LATENCY_LABEL=candidate-d453ffcdb704764daced1b2917fddee7224389f0-headful-run2 ORCA_TERMINAL_SPLIT_LATENCY_OUTPUT=/private/tmp/orca-terminal-split-activation-candidate-d453ffcdb704764daced1b2917fddee7224389f0-headful-run2.json pnpm exec playwright test tests/e2e/terminal-split-activation-latency.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1",
// Exact-HEAD evidence records use the committed benchmark harness; they remain opt-in audit commands.
"ORCA_TERMINAL_SPLIT_LATENCY_BENCH=1 ORCA_TERMINAL_SPLIT_LATENCY_LABEL=candidate-962faacec8c-headful-current ORCA_TERMINAL_SPLIT_LATENCY_OUTPUT=/private/tmp/orca-terminal-split-activation-962faacec8c-headful.json pnpm exec playwright test tests/e2e/terminal-split-activation-latency.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1",
"ORCA_TERMINAL_SPLIT_LATENCY_BENCH=1 ORCA_TERMINAL_SPLIT_LATENCY_LABEL=candidate-962faacec8c-headful-run2 ORCA_TERMINAL_SPLIT_LATENCY_OUTPUT=/private/tmp/orca-terminal-split-activation-962faacec8c-headful-run2.json pnpm exec playwright test tests/e2e/terminal-split-activation-latency.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1"
],
"testFiles": [
"src/renderer/src/components/terminal-pane/terminal-pane-split-with-inherited-cwd.test.ts",
@@ -5881,16 +5884,34 @@
"command": "ORCA_TERMINAL_SPLIT_LATENCY_BENCH=1 ORCA_TERMINAL_SPLIT_LATENCY_LABEL=candidate-d453ffcdb704764daced1b2917fddee7224389f0-headful-run2 ORCA_TERMINAL_SPLIT_LATENCY_OUTPUT=/private/tmp/orca-terminal-split-activation-candidate-d453ffcdb704764daced1b2917fddee7224389f0-headful-run2.json pnpm exec playwright test tests/e2e/terminal-split-activation-latency.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1",
"durationSeconds": 72,
"summary": "At candidate d453ffcdb704764daced1b2917fddee7224389f0, the visible BrowserWindow and document completed 3/3 warmups followed by 20/20 measured cold-CWD cycles with every event present, distinct child PTYs, and observed child exits. Shortcut-to-focus p50/p95/max was 12.8/14.4/16.5 ms, PTY bind was 177.0/316.1/333.3 ms, and first echo was 268.6/627.4/710.7 ms. Artifact SHA-256: 9aef7fa842c732eb74f0066a77b2a0336e8f956a06ca61387f9999d6e76213cc."
},
{
"date": "2026-08-31",
"runner": "local",
"platform": "macos",
"result": "passed",
"command": "ORCA_TERMINAL_SPLIT_LATENCY_BENCH=1 ORCA_TERMINAL_SPLIT_LATENCY_LABEL=candidate-962faacec8c-headful-current ORCA_TERMINAL_SPLIT_LATENCY_OUTPUT=/private/tmp/orca-terminal-split-activation-962faacec8c-headful.json pnpm exec playwright test tests/e2e/terminal-split-activation-latency.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1",
"durationSeconds": 72,
"summary": "At exact HEAD 962faacec8c, the visible BrowserWindow and document completed 3/3 warmups followed by 20/20 measured cold-CWD cycles with every event present, distinct child PTYs, and observed child exits. Shortcut-to-focus p50/p95/max was 12.6/13.7/13.7 ms, PTY bind was 140.5/399.1/464.1 ms, and first echo was 192.0/519.3/2343.1 ms. Artifact SHA-256: 875e9d37dc711472a81438e4bbdbc8cbc7aada8c961d14195c024d8da350b9e2."
},
{
"date": "2026-08-31",
"runner": "local",
"platform": "macos",
"result": "passed",
"command": "ORCA_TERMINAL_SPLIT_LATENCY_BENCH=1 ORCA_TERMINAL_SPLIT_LATENCY_LABEL=candidate-962faacec8c-headful-run2 ORCA_TERMINAL_SPLIT_LATENCY_OUTPUT=/private/tmp/orca-terminal-split-activation-962faacec8c-headful-run2.json pnpm exec playwright test tests/e2e/terminal-split-activation-latency.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1",
"durationSeconds": 72,
"summary": "At exact HEAD 962faacec8c, the visible BrowserWindow and document completed 3/3 warmups followed by 20/20 measured cold-CWD cycles with every event present, distinct child PTYs, and observed child exits. Shortcut-to-focus p50/p95/max was 12.8/14.0/14.3 ms, PTY bind was 236.5/654.0/687.3 ms, and first echo was 566.8/1191.5/1219.7 ms. Artifact SHA-256: 4f66b93e5c936ade05f880010f4ec027385d5c89893430169af91c7fdfbe1d06."
}
],
"evidenceProcedure": "Historical artifact procedure: run the benchmark spec in a visible macOS Electron project from the primary worktree, complete 3 warmups followed by 20 measured cold-CWD cycles, save the JSON report, and record its SHA-256. The recorded command strings are audit records, not rerunnable gate commands: their operator-supplied labels name the claimed product SHA but did not verify the checkout, and the uncommitted harness revision was not captured. Rerun both product revisions with one committed harness before promotion.",
"evidenceProcedure": "Run the benchmark spec in a visible macOS Electron project from the primary worktree, complete 3 warmups followed by 20 measured cold-CWD cycles, save the JSON report, and record its SHA-256. The older baseline/candidate records are historical audit records whose labels do not verify the checkout; the two 2026-08-31 exact-HEAD records use the committed final harness. A paired rerun of the baseline and final product revisions with one committed harness remains required before promotion.",
"runtimeBudget": {
"p95Seconds": 240,
"scope": "the full listed gate command set: eight deterministic renderer/local-transport unit files plus two opt-in 3-warmup/20-measured visible Electron benchmark invocations"
},
"flakeHistory": {
"status": "not-started",
"evidence": "The focused promise-barrier and remount-handoff suite now passes locally in 93 tests; one visible baseline/candidate benchmark pair also passes locally. Routed CI and soak history have not started. The focused unit command runs the checkout from which it is invoked. Historical benchmark commands are audit records whose labels do not verify the product checkout or harness revision, and the stored pair predates the final harness hardening."
"evidence": "The focused promise-barrier and remount-handoff suite now passes locally in 93 tests; the visible benchmark passes for the historical pair and for two exact-HEAD runs. Routed CI and soak history have not started. The focused unit command runs the checkout from which it is invoked. Historical benchmark labels do not verify the product checkout; the exact-HEAD records use the committed final harness."
},
"redGreenEvidence": {
"status": "partial",