fix(perf): calibrate report budgets without masking latency stalls (#22075)

* fix(perf): calibrate report budgets without masking latency stalls * docs(perf): record historical evidence for report limits
2026-09-22 00:02:31 +00:00 · 2026-09-21 14:13:09 -07:00
parent 72a1b148c1
commit 8cf0e81ced
6 changed files with 177 additions and 60 deletions
@@ -1,5 +1,6 @@
 import { basename } from 'node:path'
 import { collectTerminalPerfRows, readJsonReport } from './terminal-perf-report-annotations.mjs'
+import { reportBudgetsForScenario } from './terminal-perf-report-budgets.mjs'

 const reportPaths = process.argv.slice(2)
 if (reportPaths[0] === '--') {
@@ -13,34 +14,6 @@ if (reportPaths.length === 0) {
  process.exit(1)
 }

-// Why: these mirror the e2e regression ceilings so saved JSON reports can fail
-// in automation without rerunning Electron or changing the human summary table.
-const BUDGETS = {
-  maxMedianKeyLatencyMs: 75,
-  maxWorstKeyLatencyMs: 300,
-  maxRevisitLatencyMs: 300,
-  maxTimerDriftMs: 150,
-  // Why: mirrors MAX_TIMER_DRIFT_UNDER_LOAD_MS in artificial-opencode-terminal-load.spec.ts
-  // so injected multi-pane redraw rows are not judged against the unloaded ceiling.
-  maxTimerDriftUnderLoadMs: 3_500,
-  maxScrollLatencyMs: 150,
-  maxRestoreLatencyMs: 1000,
-  maxRendererQueuedChars: 2 * 1024 * 1024,
-  maxRendererPeakQueuedChars: 2 * 1024 * 1024,
-  maxRendererDroppedBacklogs: 0
-}
-
-// Why: only these annotation types assert against MAX_TIMER_DRIFT_UNDER_LOAD_MS
-// in the e2e suite; other rows keep the unloaded smoke ceiling.
-function isUnderLoadTimerDriftScenario(scenario) {
-  return (
-    scenario === 'opencode-same-workspace-typing' ||
-    scenario === 'opencode-cross-workspace-typing' ||
-    scenario.startsWith('opencode-scale-same-workspace-') ||
-    scenario.startsWith('opencode-scale-cross-workspace-')
-  )
-}
-
 function parseMs(value, fieldName, row, failures) {
  if (value == null || value === '') {
    return null
@@ -77,6 +50,7 @@ function addMaxFailure(failures, row, label, actual, budget, unit = '') {
 function validateRow(row) {
  const failures = []
  let checkedMetricCount = 0
+  const budgets = reportBudgetsForScenario(row.scenario)
  const addBudgetCheck = (label, actual, budget, unit = '') => {
    if (actual != null) {
      checkedMetricCount += 1
@@ -86,55 +60,53 @@ function validateRow(row) {
  addBudgetCheck(
    'median typing latency',
    parseMs(row.median, 'median', row, failures),
-    BUDGETS.maxMedianKeyLatencyMs,
+    budgets.median,
    'ms'
  )
  addBudgetCheck(
    'worst typing latency',
    parseMs(row.worst, 'worst', row, failures),
-    BUDGETS.maxWorstKeyLatencyMs,
+    budgets.worst,
    'ms'
  )
  addBudgetCheck(
    'revisit latency',
    parseMs(row.revisit, 'revisit', row, failures),
-    BUDGETS.maxRevisitLatencyMs,
+    budgets.revisit,
    'ms'
  )
  addBudgetCheck(
    'timer drift',
    parseMs(row.maxTimerDrift, 'maxTimerDrift', row, failures),
-    isUnderLoadTimerDriftScenario(row.scenario)
-      ? BUDGETS.maxTimerDriftUnderLoadMs
-      : BUDGETS.maxTimerDriftMs,
+    budgets.maxTimerDrift,
    'ms'
  )
  addBudgetCheck(
    'scroll latency',
    parseMs(row.scroll, 'scroll', row, failures),
-    BUDGETS.maxScrollLatencyMs,
+    budgets.scroll,
    'ms'
  )
  addBudgetCheck(
    'restore latency',
    parseMs(row.restore, 'restore', row, failures),
-    BUDGETS.maxRestoreLatencyMs,
+    budgets.restore,
    'ms'
  )
  addBudgetCheck(
    'renderer queued chars',
    parseCount(row.rendererQueuedChars, 'rendererQueuedChars', row, failures),
-    BUDGETS.maxRendererQueuedChars
+    budgets.rendererQueuedChars
  )
  addBudgetCheck(
    'renderer peak queued chars',
    parseCount(row.rendererPeakQueuedChars, 'rendererPeakQueuedChars', row, failures),
-    BUDGETS.maxRendererPeakQueuedChars
+    budgets.rendererPeakQueuedChars
  )
  addBudgetCheck(
    'renderer dropped backlogs',
    parseCount(row.rendererDroppedBacklogs, 'rendererDroppedBacklogs', row, failures),
-    BUDGETS.maxRendererDroppedBacklogs
+    budgets.rendererDroppedBacklogs
  )
  // Why: parked-memory rows carry heap/view-count metrics with no latency
  // budget; recognize them so memory-only scenarios pass the gate instead of
@@ -81,7 +81,7 @@ describe('check-terminal-perf-report-budgets', () => {
      [
        'panes=101',
        'frames=60',
-        'median=76.0ms',
+        'median=26.0ms',
        'worst=301.0ms',
        'revisit=301.0ms',
        'maxTimerDrift=151.0ms',
@@ -96,7 +96,7 @@ describe('check-terminal-perf-report-budgets', () => {
    const result = runChecker(reportPath)

    expect(result.status).toBe(1)
-    expect(result.stderr).toContain('median typing latency 76ms exceeded budget 75ms')
+    expect(result.stderr).toContain('median typing latency 26ms exceeded budget 25ms')
    expect(result.stderr).toContain('worst typing latency 301ms exceeded budget 300ms')
    expect(result.stderr).toContain('revisit latency 301ms exceeded budget 300ms')
    expect(result.stderr).toContain('timer drift 151ms exceeded budget 150ms')
@@ -107,9 +107,7 @@ describe('check-terminal-perf-report-budgets', () => {
    expect(result.stderr).toContain('renderer dropped backlogs 1 exceeded budget 0')
  })

-  // Why: covers every isUnderLoadTimerDriftScenario branch (two exact + two
-  // prefix matches) so a predicate regression cannot silently re-apply the
-  // unloaded 150ms ceiling to multi-pane redraw rows.
+  // Redraw scenarios must not inherit the unloaded drift ceiling.
  it.each([
    'opencode-same-workspace-typing',
    'opencode-cross-workspace-typing',
@@ -130,6 +128,41 @@ describe('check-terminal-perf-report-budgets', () => {
    expect(passOutput).toContain('Terminal perf budget check passed for 1 annotation row(s).')
  })

+  // Historical outliers must not become the new latency baseline.
+  it.each([
+    ['opencode-baseline-typing', 'median=13.8ms worst=83.5ms', 0],
+    ['opencode-baseline-typing', 'maxTimerDrift=189.0ms', 1],
+    ['opencode-hidden-real-pty-restore-latin', 'restore=1640.7ms', 1],
+    ['opencode-hidden-real-pty-pressure-typing-25', 'worst=2090.6ms', 1],
+    ['opencode-hidden-real-pty-pressure-typing-25', 'maxTimerDrift=2032.6ms', 1],
+    ['opencode-main-pressure-worktree-revisit-marker', 'revisit=1030.2ms', 1],
+    ['opencode-main-pressure-worktree-revisit-typing', 'worst=1045.5ms', 1],
+    ['opencode-scale-same-workspace-25', 'worst=2053.2ms', 1],
+    ['opencode-scale-same-workspace-25', 'median=26.0ms', 1],
+    ['opencode-hidden-real-pty-pressure-typing-25', 'median=26.0ms', 1],
+    ['opencode-main-pressure-active-typing-25', 'rendererDroppedBacklogs=1', 1],
+    ['opencode-hidden-real-pty-pressure-typing-25', 'rendererPeakQueuedChars=2097153', 1],
+    ['opencode-main-pressure-active-scroll-25', 'scroll=151.0ms', 1],
+    ['opencode-baseline-typing', 'worst=301.0ms', 1]
+  ])('%s (%s) exits %s', (scenario, description, status) => {
+    const result = runChecker(writeReport(description, scenario))
+    expect(result.status, result.stderr).toBe(status)
+  })
+
+  it.each([
+    'opencode-main-pressure-active-typing',
+    'opencode-main-pressure-active-typing-25',
+    'opencode-main-pressure-active-typing-50',
+    'opencode-main-pressure-worktree-revisit-typing',
+    'opencode-main-pressure-worktree-revisit-drain'
+  ])('allows measured transient peaks but preserves backlog and loss limits for %s', (scenario) => {
+    expect(runChecker(writeReport('rendererPeakQueuedChars=3227648', scenario)).status).toBe(0)
+    expect(runChecker(writeReport('rendererPeakQueuedChars=3670016', scenario)).status).toBe(0)
+    expect(runChecker(writeReport('rendererPeakQueuedChars=3670017', scenario)).status).toBe(1)
+    expect(runChecker(writeReport('rendererQueuedChars=2097153', scenario)).status).toBe(1)
+    expect(runChecker(writeReport('rendererDroppedBacklogs=1', scenario)).status).toBe(1)
+  })
+
  it('fails multi-pane redraw scenarios that exceed the under-load timer-drift budget', () => {
    const failPath = writeReport(
      ['panes=50', 'frames=60', 'median=12.0ms', 'worst=40.0ms', 'maxTimerDrift=3501.0ms'].join(
@@ -164,10 +164,23 @@ describe('generate-terminal-perf-html-report', () => {
    const html = readFileSync(outputPath, 'utf8')
    expect(result.budgetFailureCount).toBe(5)
    expect(html).toContain('Fail')
-    expect(html).toContain('medianMs 80 &gt; 75')
+    expect(html).toContain('medianMs 80 &gt; 25')
+    expect(html).toContain('worstMs 301 &gt; 300')
    expect(html).toContain('Cross-workspace hidden panes')
  })

+  it('continues to flag slow hidden restores', () => {
+    const reportPath = writeReport(
+      'panes=18 restore=1640.7ms',
+      'opencode-hidden-real-pty-restore-latin'
+    )
+    const outputPath = join(makeTempDir(), 'report.html')
+
+    const result = generateTerminalPerfHtmlReport({ inputPaths: [reportPath], outputPath })
+
+    expect(result.budgetFailureCount).toBe(1)
+  })
+
  it('renders ordered revisions with trend charts and baseline deltas', () => {
    const mainReport = writeReport(
      'panes=25 median=50.0ms worst=120.0ms rendererDroppedBacklogs=0',
@@ -0,0 +1,37 @@
+// Historical measurements and rationale: docs/reference/terminal-perf-report-budgets.md.
+const MIB = 1024 * 1024
+
+const DEFAULT_BUDGETS = {
+  median: 25,
+  worst: 300,
+  revisit: 300,
+  maxTimerDrift: 150,
+  scroll: 150,
+  restore: 1_000,
+  rendererQueuedChars: 2 * MIB,
+  rendererPeakQueuedChars: 2 * MIB,
+  rendererDroppedBacklogs: 0
+}
+
+export function reportBudgetsForScenario(scenario) {
+  const budgets = { ...DEFAULT_BUDGETS }
+  // Preserve the existing report gate's injected-redraw drift allowance.
+  if (
+    scenario === 'opencode-same-workspace-typing' ||
+    scenario === 'opencode-cross-workspace-typing' ||
+    scenario.startsWith('opencode-scale-same-workspace-') ||
+    scenario.startsWith('opencode-scale-cross-workspace-')
+  ) {
+    budgets.maxTimerDrift = 3_500
+  }
+  if (
+    scenario === 'opencode-main-pressure-active-typing' ||
+    scenario.startsWith('opencode-main-pressure-active-typing-') ||
+    scenario === 'opencode-main-pressure-worktree-revisit-typing' ||
+    scenario === 'opencode-main-pressure-worktree-revisit-drain'
+  ) {
+    // Only the transient peak gets headroom; current backlog must still drain below 2 MiB.
+    budgets.rendererPeakQueuedChars = 3.5 * MIB
+  }
+  return budgets
+}
@@ -1,16 +1,17 @@
 import { readFileSync } from 'node:fs'
+import { reportBudgetsForScenario } from './terminal-perf-report-budgets.mjs'

-const BUDGETS = {
-  medianMs: 75,
-  worstMs: 300,
-  revisitMs: 300,
-  maxTimerDriftMs: 150,
-  scrollMs: 150,
-  restoreMs: 1000,
-  rendererQueuedChars: 2 * 1024 * 1024,
-  rendererPeakQueuedChars: 2 * 1024 * 1024,
-  rendererDroppedBacklogs: 0
-}
+const ROW_BUDGET_FIELDS = [
+  ['medianMs', 'median'],
+  ['worstMs', 'worst'],
+  ['revisitMs', 'revisit'],
+  ['maxTimerDriftMs', 'maxTimerDrift'],
+  ['scrollMs', 'scroll'],
+  ['restoreMs', 'restore'],
+  ['rendererQueuedChars', 'rendererQueuedChars'],
+  ['rendererPeakQueuedChars', 'rendererPeakQueuedChars'],
+  ['rendererDroppedBacklogs', 'rendererDroppedBacklogs']
+]

 const SCENARIO_LABELS = [
  ['opencode-scale-same-workspace', 'Same workspace panes'],
@@ -157,14 +158,16 @@ export function scenarioTitle(scenario, row) {
 }

 export function budgetFailures(row) {
+  const budgets = reportBudgetsForScenario(row.scenario)
  const failures = []
-  for (const [key, budget] of Object.entries(BUDGETS)) {
-    const value = row[key]
-    if (value == null) {
+  for (const [rowKey, budgetKey] of ROW_BUDGET_FIELDS) {
+    const value = row[rowKey]
+    const budget = budgets[budgetKey]
+    if (value == null || budget == null) {
      continue
    }
    if (value > budget) {
-      failures.push(`${key} ${value} > ${budget}`)
+      failures.push(`${rowKey} ${value} > ${budget}`)
    }
  }
  return failures
@@ -0,0 +1,59 @@
+# Terminal performance report budgets
+
+The saved-report gate is a performance regression gate, deliberately stricter than
+some Electron test timeouts. Passing the Electron suite does not establish that a
+slow sample is acceptable. CLI and HTML reports use the same policy in
+`config/scripts/terminal-perf-report-budgets.mjs`.
+
+## Historical evidence (2026-09-21)
+
+Sample: 11 full scheduled Ubuntu runs, 32 annotation rows per run, August 1 through
+September 21 (352 rows). Runs include failures, rather than selecting only green
+runs. These are samples across different revisions and shared runners, not a
+controlled A/B experiment or a statistical tail-latency estimate. Original June
+logs returned HTTP 410 and could not establish an original baseline.
+
+Values below are milliseconds except peak queue chars (JavaScript character
+counts, not process memory bytes). Maximum median spans every typing scenario;
+peak queue spans the active/revisit ACK-pressure scenarios.
+
+| Date / run                                                              | Baseline median | Maximum typing median | Latin restore | Hidden 25-pane worst key | Peak queue chars |
+| ----------------------------------------------------------------------- | --------------: | --------------------: | ------------: | -----------------------: | ---------------: |
+| [2026-08-01](https://github.com/stablyai/orca/actions/runs/30693362353) |            11.7 |                  12.7 |        1492.0 |                     61.2 |          1867776 |
+| [2026-08-03](https://github.com/stablyai/orca/actions/runs/30802935899) |             7.1 |                   9.4 |         414.0 |                     12.8 |           294912 |
+| [2026-08-15](https://github.com/stablyai/orca/actions/runs/31875140053) |             8.9 |                  13.8 |        1297.5 |                     15.4 |          2523136 |
+| [2026-08-24](https://github.com/stablyai/orca/actions/runs/32708128219) |            12.2 |                  12.2 |         463.6 |                     16.3 |          3227648 |
+| [2026-09-01](https://github.com/stablyai/orca/actions/runs/33488510218) |             6.6 |                   6.8 |         302.2 |                     11.5 |           360448 |
+| [2026-09-07](https://github.com/stablyai/orca/actions/runs/34102348134) |             7.5 |                  11.3 |         328.2 |                    269.6 |          2818048 |
+| [2026-09-11](https://github.com/stablyai/orca/actions/runs/34580494139) |             9.5 |                  10.4 |         233.5 |                    186.2 |          2441216 |
+| [2026-09-15](https://github.com/stablyai/orca/actions/runs/34948597774) |            10.3 |                  12.6 |         282.2 |                   1178.4 |          2818048 |
+| [2026-09-17](https://github.com/stablyai/orca/actions/runs/35201162712) |            10.3 |                  12.7 |        1182.2 |                     15.7 |          2998272 |
+| [2026-09-19](https://github.com/stablyai/orca/actions/runs/35432611564) |             7.6 |                  10.3 |         236.7 |                   1435.5 |          2588672 |
+| [2026-09-21](https://github.com/stablyai/orca/actions/runs/35579708852) |             7.8 |                  11.6 |        1640.7 |                   2090.6 |          2523136 |
+
+## Decisions
+
+- Median typing: **25 ms**, tightened from 75 ms. The largest observed median was
+  13.8 ms, leaving about 81% headroom without accepting a sustained 5x slowdown.
+- Worst key: retain **300 ms**, including stress scenarios. Typical per-scenario
+  worst-key samples were tens of milliseconds; isolated 1–3 second samples are
+  failures to investigate, not a reason to adopt the e2e 3–3.5 second ceiling.
+- Revisit: retain **300 ms**. Median of the 11 revisit samples was 160.7 ms;
+  the 1,030.2 ms outlier remains a failure.
+- Restore: retain **1,000 ms**. Median restore per scenario ranged from 123 to
+  493.9 ms. Observed outliers up to 1,706.4 ms do not justify a 4 second budget.
+- Timer drift: retain **150 ms** and the pre-existing **3,500 ms** allowance only
+  for injected same/cross-workspace redraw scenarios. No new scenario receives
+  the broad allowance. This retains the CLI gate's existing policy in HTML too.
+- Scroll: retain **150 ms**. Dropped backlogs: retain **zero**.
+- Current queue: retain **2 Mi characters** everywhere. Only the transient peak
+  in active/revisit ACK-pressure scenarios gets **3.5 Mi characters** (3,670,016).
+  The maximum observed peak was 3,227,648 (3.08 Mi), leaving about 14% headroom.
+  The old 2 Mi peak budget rejects ordinary deliberately held-ACK bursts; the
+  proposed 5 Mi e2e ceiling was unnecessarily loose. Other scenarios keep 2 Mi.
+
+For the linked September 21 report, the two peak-queue failures are corrected;
+the three slow restores, worst-key stall, and timer stall still fail. This change
+does not claim to fix those stalls or make that run green. Re-evaluate future
+budget changes against recorded measurements; do not set limits just above a new
+failure or mirror relaxed test timeouts.