From 8cf0e81ced556b775606e4d40d355d4d2e90558e Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Mon, 21 Sep 2026 14:13:09 -0700 Subject: [PATCH] fix(perf): calibrate report budgets without masking latency stalls (#22075) * fix(perf): calibrate report budgets without masking latency stalls * docs(perf): record historical evidence for report limits --- .../check-terminal-perf-report-budgets.mjs | 50 ++++------------ ...heck-terminal-perf-report-budgets.test.mjs | 43 ++++++++++++-- ...enerate-terminal-perf-html-report.test.mjs | 15 ++++- .../scripts/terminal-perf-report-budgets.mjs | 37 ++++++++++++ config/scripts/terminal-perf-report-rows.mjs | 33 ++++++----- .../reference/terminal-perf-report-budgets.md | 59 +++++++++++++++++++ 6 files changed, 177 insertions(+), 60 deletions(-) create mode 100644 config/scripts/terminal-perf-report-budgets.mjs create mode 100644 docs/reference/terminal-perf-report-budgets.md diff --git a/config/scripts/check-terminal-perf-report-budgets.mjs b/config/scripts/check-terminal-perf-report-budgets.mjs index 93747b01cc1..3570da907f5 100644 --- a/config/scripts/check-terminal-perf-report-budgets.mjs +++ b/config/scripts/check-terminal-perf-report-budgets.mjs @@ -1,5 +1,6 @@ import { basename } from 'node:path' import { collectTerminalPerfRows, readJsonReport } from './terminal-perf-report-annotations.mjs' +import { reportBudgetsForScenario } from './terminal-perf-report-budgets.mjs' const reportPaths = process.argv.slice(2) if (reportPaths[0] === '--') { @@ -13,34 +14,6 @@ if (reportPaths.length === 0) { process.exit(1) } -// Why: these mirror the e2e regression ceilings so saved JSON reports can fail -// in automation without rerunning Electron or changing the human summary table. -const BUDGETS = { - maxMedianKeyLatencyMs: 75, - maxWorstKeyLatencyMs: 300, - maxRevisitLatencyMs: 300, - maxTimerDriftMs: 150, - // Why: mirrors MAX_TIMER_DRIFT_UNDER_LOAD_MS in artificial-opencode-terminal-load.spec.ts - // so injected multi-pane redraw rows are not judged against the unloaded ceiling. - maxTimerDriftUnderLoadMs: 3_500, - maxScrollLatencyMs: 150, - maxRestoreLatencyMs: 1000, - maxRendererQueuedChars: 2 * 1024 * 1024, - maxRendererPeakQueuedChars: 2 * 1024 * 1024, - maxRendererDroppedBacklogs: 0 -} - -// Why: only these annotation types assert against MAX_TIMER_DRIFT_UNDER_LOAD_MS -// in the e2e suite; other rows keep the unloaded smoke ceiling. -function isUnderLoadTimerDriftScenario(scenario) { - return ( - scenario === 'opencode-same-workspace-typing' || - scenario === 'opencode-cross-workspace-typing' || - scenario.startsWith('opencode-scale-same-workspace-') || - scenario.startsWith('opencode-scale-cross-workspace-') - ) -} - function parseMs(value, fieldName, row, failures) { if (value == null || value === '') { return null @@ -77,6 +50,7 @@ function addMaxFailure(failures, row, label, actual, budget, unit = '') { function validateRow(row) { const failures = [] let checkedMetricCount = 0 + const budgets = reportBudgetsForScenario(row.scenario) const addBudgetCheck = (label, actual, budget, unit = '') => { if (actual != null) { checkedMetricCount += 1 @@ -86,55 +60,53 @@ function validateRow(row) { addBudgetCheck( 'median typing latency', parseMs(row.median, 'median', row, failures), - BUDGETS.maxMedianKeyLatencyMs, + budgets.median, 'ms' ) addBudgetCheck( 'worst typing latency', parseMs(row.worst, 'worst', row, failures), - BUDGETS.maxWorstKeyLatencyMs, + budgets.worst, 'ms' ) addBudgetCheck( 'revisit latency', parseMs(row.revisit, 'revisit', row, failures), - BUDGETS.maxRevisitLatencyMs, + budgets.revisit, 'ms' ) addBudgetCheck( 'timer drift', parseMs(row.maxTimerDrift, 'maxTimerDrift', row, failures), - isUnderLoadTimerDriftScenario(row.scenario) - ? BUDGETS.maxTimerDriftUnderLoadMs - : BUDGETS.maxTimerDriftMs, + budgets.maxTimerDrift, 'ms' ) addBudgetCheck( 'scroll latency', parseMs(row.scroll, 'scroll', row, failures), - BUDGETS.maxScrollLatencyMs, + budgets.scroll, 'ms' ) addBudgetCheck( 'restore latency', parseMs(row.restore, 'restore', row, failures), - BUDGETS.maxRestoreLatencyMs, + budgets.restore, 'ms' ) addBudgetCheck( 'renderer queued chars', parseCount(row.rendererQueuedChars, 'rendererQueuedChars', row, failures), - BUDGETS.maxRendererQueuedChars + budgets.rendererQueuedChars ) addBudgetCheck( 'renderer peak queued chars', parseCount(row.rendererPeakQueuedChars, 'rendererPeakQueuedChars', row, failures), - BUDGETS.maxRendererPeakQueuedChars + budgets.rendererPeakQueuedChars ) addBudgetCheck( 'renderer dropped backlogs', parseCount(row.rendererDroppedBacklogs, 'rendererDroppedBacklogs', row, failures), - BUDGETS.maxRendererDroppedBacklogs + budgets.rendererDroppedBacklogs ) // Why: parked-memory rows carry heap/view-count metrics with no latency // budget; recognize them so memory-only scenarios pass the gate instead of diff --git a/config/scripts/check-terminal-perf-report-budgets.test.mjs b/config/scripts/check-terminal-perf-report-budgets.test.mjs index f0ae7981795..95d6093b52d 100644 --- a/config/scripts/check-terminal-perf-report-budgets.test.mjs +++ b/config/scripts/check-terminal-perf-report-budgets.test.mjs @@ -81,7 +81,7 @@ describe('check-terminal-perf-report-budgets', () => { [ 'panes=101', 'frames=60', - 'median=76.0ms', + 'median=26.0ms', 'worst=301.0ms', 'revisit=301.0ms', 'maxTimerDrift=151.0ms', @@ -96,7 +96,7 @@ describe('check-terminal-perf-report-budgets', () => { const result = runChecker(reportPath) expect(result.status).toBe(1) - expect(result.stderr).toContain('median typing latency 76ms exceeded budget 75ms') + expect(result.stderr).toContain('median typing latency 26ms exceeded budget 25ms') expect(result.stderr).toContain('worst typing latency 301ms exceeded budget 300ms') expect(result.stderr).toContain('revisit latency 301ms exceeded budget 300ms') expect(result.stderr).toContain('timer drift 151ms exceeded budget 150ms') @@ -107,9 +107,7 @@ describe('check-terminal-perf-report-budgets', () => { expect(result.stderr).toContain('renderer dropped backlogs 1 exceeded budget 0') }) - // Why: covers every isUnderLoadTimerDriftScenario branch (two exact + two - // prefix matches) so a predicate regression cannot silently re-apply the - // unloaded 150ms ceiling to multi-pane redraw rows. + // Redraw scenarios must not inherit the unloaded drift ceiling. it.each([ 'opencode-same-workspace-typing', 'opencode-cross-workspace-typing', @@ -130,6 +128,41 @@ describe('check-terminal-perf-report-budgets', () => { expect(passOutput).toContain('Terminal perf budget check passed for 1 annotation row(s).') }) + // Historical outliers must not become the new latency baseline. + it.each([ + ['opencode-baseline-typing', 'median=13.8ms worst=83.5ms', 0], + ['opencode-baseline-typing', 'maxTimerDrift=189.0ms', 1], + ['opencode-hidden-real-pty-restore-latin', 'restore=1640.7ms', 1], + ['opencode-hidden-real-pty-pressure-typing-25', 'worst=2090.6ms', 1], + ['opencode-hidden-real-pty-pressure-typing-25', 'maxTimerDrift=2032.6ms', 1], + ['opencode-main-pressure-worktree-revisit-marker', 'revisit=1030.2ms', 1], + ['opencode-main-pressure-worktree-revisit-typing', 'worst=1045.5ms', 1], + ['opencode-scale-same-workspace-25', 'worst=2053.2ms', 1], + ['opencode-scale-same-workspace-25', 'median=26.0ms', 1], + ['opencode-hidden-real-pty-pressure-typing-25', 'median=26.0ms', 1], + ['opencode-main-pressure-active-typing-25', 'rendererDroppedBacklogs=1', 1], + ['opencode-hidden-real-pty-pressure-typing-25', 'rendererPeakQueuedChars=2097153', 1], + ['opencode-main-pressure-active-scroll-25', 'scroll=151.0ms', 1], + ['opencode-baseline-typing', 'worst=301.0ms', 1] + ])('%s (%s) exits %s', (scenario, description, status) => { + const result = runChecker(writeReport(description, scenario)) + expect(result.status, result.stderr).toBe(status) + }) + + it.each([ + 'opencode-main-pressure-active-typing', + 'opencode-main-pressure-active-typing-25', + 'opencode-main-pressure-active-typing-50', + 'opencode-main-pressure-worktree-revisit-typing', + 'opencode-main-pressure-worktree-revisit-drain' + ])('allows measured transient peaks but preserves backlog and loss limits for %s', (scenario) => { + expect(runChecker(writeReport('rendererPeakQueuedChars=3227648', scenario)).status).toBe(0) + expect(runChecker(writeReport('rendererPeakQueuedChars=3670016', scenario)).status).toBe(0) + expect(runChecker(writeReport('rendererPeakQueuedChars=3670017', scenario)).status).toBe(1) + expect(runChecker(writeReport('rendererQueuedChars=2097153', scenario)).status).toBe(1) + expect(runChecker(writeReport('rendererDroppedBacklogs=1', scenario)).status).toBe(1) + }) + it('fails multi-pane redraw scenarios that exceed the under-load timer-drift budget', () => { const failPath = writeReport( ['panes=50', 'frames=60', 'median=12.0ms', 'worst=40.0ms', 'maxTimerDrift=3501.0ms'].join( diff --git a/config/scripts/generate-terminal-perf-html-report.test.mjs b/config/scripts/generate-terminal-perf-html-report.test.mjs index a804bb6639d..0b7f51fb48f 100644 --- a/config/scripts/generate-terminal-perf-html-report.test.mjs +++ b/config/scripts/generate-terminal-perf-html-report.test.mjs @@ -164,10 +164,23 @@ describe('generate-terminal-perf-html-report', () => { const html = readFileSync(outputPath, 'utf8') expect(result.budgetFailureCount).toBe(5) expect(html).toContain('Fail') - expect(html).toContain('medianMs 80 > 75') + expect(html).toContain('medianMs 80 > 25') + expect(html).toContain('worstMs 301 > 300') expect(html).toContain('Cross-workspace hidden panes') }) + it('continues to flag slow hidden restores', () => { + const reportPath = writeReport( + 'panes=18 restore=1640.7ms', + 'opencode-hidden-real-pty-restore-latin' + ) + const outputPath = join(makeTempDir(), 'report.html') + + const result = generateTerminalPerfHtmlReport({ inputPaths: [reportPath], outputPath }) + + expect(result.budgetFailureCount).toBe(1) + }) + it('renders ordered revisions with trend charts and baseline deltas', () => { const mainReport = writeReport( 'panes=25 median=50.0ms worst=120.0ms rendererDroppedBacklogs=0', diff --git a/config/scripts/terminal-perf-report-budgets.mjs b/config/scripts/terminal-perf-report-budgets.mjs new file mode 100644 index 00000000000..7b34005c6ce --- /dev/null +++ b/config/scripts/terminal-perf-report-budgets.mjs @@ -0,0 +1,37 @@ +// Historical measurements and rationale: docs/reference/terminal-perf-report-budgets.md. +const MIB = 1024 * 1024 + +const DEFAULT_BUDGETS = { + median: 25, + worst: 300, + revisit: 300, + maxTimerDrift: 150, + scroll: 150, + restore: 1_000, + rendererQueuedChars: 2 * MIB, + rendererPeakQueuedChars: 2 * MIB, + rendererDroppedBacklogs: 0 +} + +export function reportBudgetsForScenario(scenario) { + const budgets = { ...DEFAULT_BUDGETS } + // Preserve the existing report gate's injected-redraw drift allowance. + if ( + scenario === 'opencode-same-workspace-typing' || + scenario === 'opencode-cross-workspace-typing' || + scenario.startsWith('opencode-scale-same-workspace-') || + scenario.startsWith('opencode-scale-cross-workspace-') + ) { + budgets.maxTimerDrift = 3_500 + } + if ( + scenario === 'opencode-main-pressure-active-typing' || + scenario.startsWith('opencode-main-pressure-active-typing-') || + scenario === 'opencode-main-pressure-worktree-revisit-typing' || + scenario === 'opencode-main-pressure-worktree-revisit-drain' + ) { + // Only the transient peak gets headroom; current backlog must still drain below 2 MiB. + budgets.rendererPeakQueuedChars = 3.5 * MIB + } + return budgets +} diff --git a/config/scripts/terminal-perf-report-rows.mjs b/config/scripts/terminal-perf-report-rows.mjs index c1f562f3b9e..f7bd48eab79 100644 --- a/config/scripts/terminal-perf-report-rows.mjs +++ b/config/scripts/terminal-perf-report-rows.mjs @@ -1,16 +1,17 @@ import { readFileSync } from 'node:fs' +import { reportBudgetsForScenario } from './terminal-perf-report-budgets.mjs' -const BUDGETS = { - medianMs: 75, - worstMs: 300, - revisitMs: 300, - maxTimerDriftMs: 150, - scrollMs: 150, - restoreMs: 1000, - rendererQueuedChars: 2 * 1024 * 1024, - rendererPeakQueuedChars: 2 * 1024 * 1024, - rendererDroppedBacklogs: 0 -} +const ROW_BUDGET_FIELDS = [ + ['medianMs', 'median'], + ['worstMs', 'worst'], + ['revisitMs', 'revisit'], + ['maxTimerDriftMs', 'maxTimerDrift'], + ['scrollMs', 'scroll'], + ['restoreMs', 'restore'], + ['rendererQueuedChars', 'rendererQueuedChars'], + ['rendererPeakQueuedChars', 'rendererPeakQueuedChars'], + ['rendererDroppedBacklogs', 'rendererDroppedBacklogs'] +] const SCENARIO_LABELS = [ ['opencode-scale-same-workspace', 'Same workspace panes'], @@ -157,14 +158,16 @@ export function scenarioTitle(scenario, row) { } export function budgetFailures(row) { + const budgets = reportBudgetsForScenario(row.scenario) const failures = [] - for (const [key, budget] of Object.entries(BUDGETS)) { - const value = row[key] - if (value == null) { + for (const [rowKey, budgetKey] of ROW_BUDGET_FIELDS) { + const value = row[rowKey] + const budget = budgets[budgetKey] + if (value == null || budget == null) { continue } if (value > budget) { - failures.push(`${key} ${value} > ${budget}`) + failures.push(`${rowKey} ${value} > ${budget}`) } } return failures diff --git a/docs/reference/terminal-perf-report-budgets.md b/docs/reference/terminal-perf-report-budgets.md new file mode 100644 index 00000000000..54d26c3d2a5 --- /dev/null +++ b/docs/reference/terminal-perf-report-budgets.md @@ -0,0 +1,59 @@ +# Terminal performance report budgets + +The saved-report gate is a performance regression gate, deliberately stricter than +some Electron test timeouts. Passing the Electron suite does not establish that a +slow sample is acceptable. CLI and HTML reports use the same policy in +`config/scripts/terminal-perf-report-budgets.mjs`. + +## Historical evidence (2026-09-21) + +Sample: 11 full scheduled Ubuntu runs, 32 annotation rows per run, August 1 through +September 21 (352 rows). Runs include failures, rather than selecting only green +runs. These are samples across different revisions and shared runners, not a +controlled A/B experiment or a statistical tail-latency estimate. Original June +logs returned HTTP 410 and could not establish an original baseline. + +Values below are milliseconds except peak queue chars (JavaScript character +counts, not process memory bytes). Maximum median spans every typing scenario; +peak queue spans the active/revisit ACK-pressure scenarios. + +| Date / run | Baseline median | Maximum typing median | Latin restore | Hidden 25-pane worst key | Peak queue chars | +| ----------------------------------------------------------------------- | --------------: | --------------------: | ------------: | -----------------------: | ---------------: | +| [2026-08-01](https://github.com/stablyai/orca/actions/runs/30693362353) | 11.7 | 12.7 | 1492.0 | 61.2 | 1867776 | +| [2026-08-03](https://github.com/stablyai/orca/actions/runs/30802935899) | 7.1 | 9.4 | 414.0 | 12.8 | 294912 | +| [2026-08-15](https://github.com/stablyai/orca/actions/runs/31875140053) | 8.9 | 13.8 | 1297.5 | 15.4 | 2523136 | +| [2026-08-24](https://github.com/stablyai/orca/actions/runs/32708128219) | 12.2 | 12.2 | 463.6 | 16.3 | 3227648 | +| [2026-09-01](https://github.com/stablyai/orca/actions/runs/33488510218) | 6.6 | 6.8 | 302.2 | 11.5 | 360448 | +| [2026-09-07](https://github.com/stablyai/orca/actions/runs/34102348134) | 7.5 | 11.3 | 328.2 | 269.6 | 2818048 | +| [2026-09-11](https://github.com/stablyai/orca/actions/runs/34580494139) | 9.5 | 10.4 | 233.5 | 186.2 | 2441216 | +| [2026-09-15](https://github.com/stablyai/orca/actions/runs/34948597774) | 10.3 | 12.6 | 282.2 | 1178.4 | 2818048 | +| [2026-09-17](https://github.com/stablyai/orca/actions/runs/35201162712) | 10.3 | 12.7 | 1182.2 | 15.7 | 2998272 | +| [2026-09-19](https://github.com/stablyai/orca/actions/runs/35432611564) | 7.6 | 10.3 | 236.7 | 1435.5 | 2588672 | +| [2026-09-21](https://github.com/stablyai/orca/actions/runs/35579708852) | 7.8 | 11.6 | 1640.7 | 2090.6 | 2523136 | + +## Decisions + +- Median typing: **25 ms**, tightened from 75 ms. The largest observed median was + 13.8 ms, leaving about 81% headroom without accepting a sustained 5x slowdown. +- Worst key: retain **300 ms**, including stress scenarios. Typical per-scenario + worst-key samples were tens of milliseconds; isolated 1–3 second samples are + failures to investigate, not a reason to adopt the e2e 3–3.5 second ceiling. +- Revisit: retain **300 ms**. Median of the 11 revisit samples was 160.7 ms; + the 1,030.2 ms outlier remains a failure. +- Restore: retain **1,000 ms**. Median restore per scenario ranged from 123 to + 493.9 ms. Observed outliers up to 1,706.4 ms do not justify a 4 second budget. +- Timer drift: retain **150 ms** and the pre-existing **3,500 ms** allowance only + for injected same/cross-workspace redraw scenarios. No new scenario receives + the broad allowance. This retains the CLI gate's existing policy in HTML too. +- Scroll: retain **150 ms**. Dropped backlogs: retain **zero**. +- Current queue: retain **2 Mi characters** everywhere. Only the transient peak + in active/revisit ACK-pressure scenarios gets **3.5 Mi characters** (3,670,016). + The maximum observed peak was 3,227,648 (3.08 Mi), leaving about 14% headroom. + The old 2 Mi peak budget rejects ordinary deliberately held-ACK bursts; the + proposed 5 Mi e2e ceiling was unnecessarily loose. Other scenarios keep 2 Mi. + +For the linked September 21 report, the two peak-queue failures are corrected; +the three slow restores, worst-key stall, and timer stall still fail. This change +does not claim to fix those stalls or make that run green. Re-evaluate future +budget changes against recorded measurements; do not set limits just above a new +failure or mirror relaxed test timeouts.