fix(perf): present benchmark frames on isolated CI display

.github/workflows/terminal-perf.yml

@@ -55,6 +55,8 @@ jobs:
     timeout-minutes: 45
     env:
       NODE_OPTIONS: --max-old-space-size=4096
+      ORCA_BACKGROUND_LAUNCH: '1'
+      ORCA_E2E_TERMINAL_PERF_XVFB: '1'
       ORCA_E2E_FORWARD_APP_LOGS: '1'
       ORCA_E2E_TERMINAL_PERF_REPORT_PATH: ${{ inputs.report_path || 'test-results/terminal-scale-perf-report.json' }}
 
@@ -91,30 +93,59 @@ jobs:
       - name: Build Electron app for terminal perf
         run: npx electron-vite build --mode e2e
 
-      - name: Compare hidden and Xvfb-visible windows
+      - name: Run terminal scale perf report gate
         env:
-          ORCA_BACKGROUND_LAUNCH: '1'
-          ORCA_PERF_DIAGNOSTIC_TRACE: '1'
-          SKIP_BUILD: '1'
-          ORCA_E2E_OPENCODE_FRAME_COUNT: '60'
-          ORCA_E2E_OPENCODE_SCALE_HIDDEN_PRESSURE_PANES: '25'
+          ORCA_TERMINAL_PERF_GREP: ${{ inputs.grep }}
+          ORCA_TERMINAL_PERF_FRAME_COUNT: ${{ inputs.frame_count }}
+          ORCA_TERMINAL_PERF_FRAME_INTERVAL_MS: ${{ inputs.frame_interval_ms }}
+          ORCA_TERMINAL_PERF_PRESSURE_OUTPUT_CHARS: ${{ inputs.pressure_output_chars }}
+          ORCA_TERMINAL_PERF_SCALE_PANES: ${{ inputs.scale_panes }}
+          ORCA_TERMINAL_PERF_SCALE_CROSS_WORKSPACE_PANES: ${{ inputs.scale_cross_workspace_panes }}
+          ORCA_TERMINAL_PERF_SCALE_PRESSURE_PANES: ${{ inputs.scale_pressure_panes }}
+          ORCA_TERMINAL_PERF_SCALE_HIDDEN_PRESSURE_PANES: ${{ inputs.scale_hidden_pressure_panes }}
         run: |
-          mkdir -p "$RUNNER_TEMP/perf-evidence"
-          for mode in hidden visible; do
-            export ORCA_PERF_DIAGNOSTIC_RENDERER=swiftshader
-            export ORCA_PERF_DIAGNOSTIC_XVFB_VISIBLE=0
-            if [ "$mode" = visible ]; then
-              export ORCA_PERF_DIAGNOSTIC_XVFB_VISIBLE=1
-            fi
-            export ORCA_PERF_DIAGNOSTIC_DIR="$RUNNER_TEMP/perf-evidence/mode-$mode"
-            mkdir -p "$ORCA_PERF_DIAGNOSTIC_DIR"
-            xvfb-run --auto-servernum pnpm exec playwright test tests/e2e/artificial-opencode-terminal-load.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1 --grep 'Latin hidden|hidden restore responsive with 25' --repeat-each=5 --reporter=json > "$ORCA_PERF_DIAGNOSTIC_DIR/report.json" 2> "$ORCA_PERF_DIAGNOSTIC_DIR/stderr.log" || echo "mode $mode had a test failure"
-          done
+          set -euo pipefail
+          args=()
+          if [ -n "${ORCA_TERMINAL_PERF_GREP:-}" ]; then
+            args+=(--grep "$ORCA_TERMINAL_PERF_GREP")
+          fi
+          if [ -n "${ORCA_TERMINAL_PERF_FRAME_COUNT:-}" ]; then
+            export ORCA_E2E_OPENCODE_FRAME_COUNT="$ORCA_TERMINAL_PERF_FRAME_COUNT"
+          fi
+          if [ -n "${ORCA_TERMINAL_PERF_FRAME_INTERVAL_MS:-}" ]; then
+            export ORCA_E2E_OPENCODE_FRAME_INTERVAL_MS="$ORCA_TERMINAL_PERF_FRAME_INTERVAL_MS"
+          fi
+          if [ -n "${ORCA_TERMINAL_PERF_PRESSURE_OUTPUT_CHARS:-}" ]; then
+            export ORCA_E2E_OPENCODE_PRESSURE_OUTPUT_CHARS="$ORCA_TERMINAL_PERF_PRESSURE_OUTPUT_CHARS"
+          fi
+          if [ -n "${ORCA_TERMINAL_PERF_SCALE_PANES:-}" ]; then
+            export ORCA_E2E_OPENCODE_SCALE_PANES="$ORCA_TERMINAL_PERF_SCALE_PANES"
+          fi
+          if [ -n "${ORCA_TERMINAL_PERF_SCALE_CROSS_WORKSPACE_PANES:-}" ]; then
+            export ORCA_E2E_OPENCODE_SCALE_CROSS_WORKSPACE_PANES="$ORCA_TERMINAL_PERF_SCALE_CROSS_WORKSPACE_PANES"
+          fi
+          if [ -n "${ORCA_TERMINAL_PERF_SCALE_PRESSURE_PANES:-}" ]; then
+            export ORCA_E2E_OPENCODE_SCALE_PRESSURE_PANES="$ORCA_TERMINAL_PERF_SCALE_PRESSURE_PANES"
+          fi
+          if [ -n "${ORCA_TERMINAL_PERF_SCALE_HIDDEN_PRESSURE_PANES:-}" ]; then
+            export ORCA_E2E_OPENCODE_SCALE_HIDDEN_PRESSURE_PANES="$ORCA_TERMINAL_PERF_SCALE_HIDDEN_PRESSURE_PANES"
+          fi
+          xvfb-run --auto-servernum env SKIP_BUILD=1 pnpm run test:e2e:terminal-perf:scale:report -- "${args[@]}"
 
-      - name: Upload profiles and reports
+      - name: Upload terminal perf report
         if: always()
         uses: actions/upload-artifact@v7
         with:
-          name: terminal-perf-diagnostic
-          path: ${{ runner.temp }}/perf-evidence
+          name: terminal-scale-perf-report
+          path: ${{ env.ORCA_E2E_TERMINAL_PERF_REPORT_PATH }}
           retention-days: 14
+          if-no-files-found: warn
+
+      - name: Upload Playwright traces
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: terminal-scale-perf-traces
+          path: test-results/
+          retention-days: 7
+          if-no-files-found: ignore

docs/reference/terminal-perf-latency-investigation.md

@@ -58,11 +58,31 @@ this is not a one-second xterm parse or proof of a second of CPU consumption.
 The same throttle is present in Chromium 148.0.7778.218 (Electron 42.3.3) and
 146.0.7680.177. This source comparison does not prove identical runtime behavior.
 
+## Visibility control
+
+[Run 35660847455](https://github.com/stablyai/orca/actions/runs/35660847455)
+compared ten hidden-window samples with ten visible-window samples on the same
+isolated Xvfb runner, using SwiftShader in both modes. Actual window visibility
+was recorded. The background terminal panes remained hidden in both modes.
+
+| Measurement | Hidden window | Visible window |
+| --- | ---: | ---: |
+| Undrawn-frame throttle decisions | 1,251 | 0 |
+| Largest worst-key latency | 3,062.8 ms | 30.5 ms |
+| Largest timer drift | 3,111.8 ms | 67.0 ms |
+| Restore range | 213.8–1,862.2 ms (9 completed) | 223.4–734.3 ms (10 completed) |
+| Electron tests passed | 9/10 | 10/10 |
+
+All ten visible-window samples satisfy the existing latency limits. This isolates
+the never-presented Linux test window as the trigger for the reproduced native
+stalls. It does not establish a newly introduced application-code regression or
+prove that every historical outlier had the same cause.
+
 ## Remaining verification
 
-Compare hidden and visible application windows on an isolated Linux CI Xvfb
-display, retaining the same hidden-terminal workload and graphics flags. Verify
-actual window visibility and native throttling, not only measured restore time.
-A supported correction must then pass the full terminal scale report gate with
-unchanged latency limits. Keep desktop automation windowless and remove temporary
-diagnostic workflow changes before merging a fix.
+Run the full terminal scale report gate with the original graphics flags, no
+profiling, and unchanged latency limits. The correction presents the benchmark
+window only when explicitly enabled on an isolated GitHub Actions Linux display;
+ordinary local automation stays windowless. Terminal workloads, hidden-pane gates,
+and production launch policy are unchanged. Temporary diagnostic hooks and the
+comparison workflow have been removed.

tests/e2e/artificial-opencode-hidden-pressure-scenario.ts

@@ -1,4 +1,3 @@
-import { withTerminalPerfProfile } from './terminal-perf-diagnostics'
 import type { Page, TestInfo } from '@stablyai/playwright-test'
 import { expect } from '@stablyai/playwright-test'
 import { randomUUID } from 'node:crypto'
@@ -217,8 +216,10 @@ export async function runHiddenRealPtyPressureScenario<
     expect(measurement.maxTimerDriftMs).toBeLessThan(MAX_HIDDEN_PRESSURE_TIMER_DRIFT_MS)
 
     await deps.releaseTerminalAckGate(orcaPage)
-    const restoreLatencyMs = await withTerminalPerfProfile(orcaPage, 'restore', runId, () =>
-      measureHiddenOutputRestoreLatency(orcaPage, secondWorktreeId, runId)
+    const restoreLatencyMs = await measureHiddenOutputRestoreLatency(
+      orcaPage,
+      secondWorktreeId,
+      runId
     )
     testInfo.annotations.push({
       type: `opencode-hidden-real-pty-restore${annotationSuffix ?? ''}`,

tests/e2e/artificial-opencode-terminal-load.spec.ts

@@ -1,7 +1,4 @@
-import {
-  configureTerminalPerfDiagnostics,
-  withTerminalPerfProfile
-} from './terminal-perf-diagnostics'
+import { presentTerminalPerfWindow } from './terminal-perf-presentation'
 import type { Page, TestInfo } from '@stablyai/playwright-test'
 import { randomUUID } from 'node:crypto'
 import { mkdirSync, rmSync, writeFileSync } from 'node:fs'
@@ -33,6 +30,10 @@ import { runMainPressureScenario } from './artificial-opencode-main-pressure-sce
 import { runRendererBackpressureRevisitScenario } from './artificial-opencode-revisit-pressure-scenario'
 import { startSyntheticOpenCodeInjection } from './artificial-opencode-synthetic-injection'
 
+test.beforeEach(async ({ electronApp }, testInfo) => {
+  await presentTerminalPerfWindow(electronApp, testInfo)
+})
+
 type TypingMeasurement = {
   latencies: number[]
   medianLatencyMs: number
@@ -240,24 +241,11 @@ function median(values: number[]): number {
   return sorted[Math.floor(sorted.length / 2)] ?? 0
 }
 
-configureTerminalPerfDiagnostics()
-
 async function measureTypingDuringLoad(
   page: Page,
   scriptPath: string,
   ptyId: string,
   runId: string
-): Promise<TypingMeasurement> {
-  return withTerminalPerfProfile(page, 'typing', runId, () =>
-    measureTypingDuringLoadProfiled(page, scriptPath, ptyId, runId)
-  )
-}
-
-async function measureTypingDuringLoadProfiled(
-  page: Page,
-  scriptPath: string,
-  ptyId: string,
-  runId: string
 ): Promise<TypingMeasurement> {
   await sendToTerminal(page, ptyId, `node ${JSON.stringify(scriptPath)}\r`)
   await waitForTerminalOutputForPtyId(page, ptyId, `OPENCODE_TYPING_READY_${runId}`, 10_000)

tests/e2e/helpers/electron-launch-args.ts

@@ -11,14 +11,7 @@ export function getOrcaElectronLaunchArgs(mainPath: string, headful: boolean): s
     // Crash tests must not block later launches on AppKit's saved-window recovery dialog.
     return [...keychainArgs, appPath, '-ApplePersistenceIgnoreState', 'YES']
   }
-  if (process.platform === 'linux' && process.env.ORCA_PERF_DIAGNOSTIC_RENDERER === 'before-c647') {
-    return ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage', appPath]
-  }
-  if (
-    (headful || process.env.ORCA_PERF_DIAGNOSTIC_RENDERER === 'swiftshader') &&
-    process.platform === 'linux' &&
-    process.env.CI
-  ) {
+  if (headful && process.platform === 'linux' && process.env.CI) {
     // Hosted runners have no GPU; SwiftShader keeps WebGL assertions from silently skipping.
     return [
       '--use-gl=angle',

tests/e2e/terminal-perf-browser-trace.ts

@@ -1,35 +0,0 @@
-import type { Page } from '@stablyai/playwright-test'
-import { mkdirSync, writeFileSync } from 'node:fs'
-import path from 'node:path'
-
-export async function withTerminalBrowserTrace<T>(
-  page: Page,
-  outputPath: string,
-  measure: () => Promise<T>
-): Promise<T> {
-  const session = await page.context().newCDPSession(page)
-  const events: unknown[] = []
-  session.on('Tracing.dataCollected', ({ value }) => events.push(...value))
-  try {
-    await session.send('Tracing.start', {
-      categories:
-        '-*,devtools.timeline,disabled-by-default-devtools.timeline,blink,blink.user_timing,cc,toplevel,v8,renderer.scheduler,gpu,viz',
-      transferMode: 'ReportEvents'
-    })
-    try {
-      await page.evaluate(() => performance.mark('orca-perf-measure-start'))
-      return await measure()
-    } finally {
-      await page.evaluate(() => performance.mark('orca-perf-measure-end')).catch(() => undefined)
-      const completed = new Promise<void>((resolve) =>
-        session.once('Tracing.tracingComplete', () => resolve())
-      )
-      await session.send('Tracing.end')
-      await completed
-      mkdirSync(path.dirname(outputPath), { recursive: true })
-      writeFileSync(outputPath, JSON.stringify({ traceEvents: events }))
-    }
-  } finally {
-    await session.detach()
-  }
-}

tests/e2e/terminal-perf-diagnostics.ts

@@ -1,104 +0,0 @@
-import { withTerminalBrowserTrace } from './terminal-perf-browser-trace'
-import type { Page } from '@stablyai/playwright-test'
-import path from 'node:path'
-import { test } from './helpers/orca-app'
-import { withTypingRendererCpuProfile } from './typing-renderer-cpu-profile'
-
-export function configureTerminalPerfDiagnostics(): void {
-  test.beforeEach(async ({ electronApp, orcaPage }, testInfo) => {
-    if (!process.env.ORCA_PERF_DIAGNOSTIC_DIR) {
-      return
-    }
-    const visibility = await orcaPage.evaluate(() => document.visibilityState)
-    const unthrottle = process.env.ORCA_PERF_DIAGNOSTIC_UNTHROTTLE === '1'
-    const visibleOnXvfb = process.env.ORCA_PERF_DIAGNOSTIC_XVFB_VISIBLE === '1'
-    if (visibleOnXvfb) {
-      if (process.platform !== 'linux' || !process.env.GITHUB_ACTIONS || !process.env.DISPLAY) {
-        throw new Error('Visible diagnostic requires an isolated GitHub Actions Xvfb display')
-      }
-      await electronApp.evaluate(({ BrowserWindow }) => {
-        for (const window of BrowserWindow.getAllWindows()) {
-          window.showInactive()
-        }
-      })
-    }
-    const windows = await electronApp.evaluate(({ BrowserWindow }, unthrottle) => {
-      return BrowserWindow.getAllWindows().map((window) => {
-        if (unthrottle) {
-          window.webContents.setBackgroundThrottling(false)
-        }
-        return {
-          visible: window.isVisible(),
-          throttled: window.webContents.getBackgroundThrottling()
-        }
-      })
-    }, unthrottle)
-    testInfo.annotations.push({
-      type: 'perf-diagnostic-window',
-      description: JSON.stringify({
-        windows,
-        visibility,
-        renderer: process.env.ORCA_PERF_DIAGNOSTIC_RENDERER ?? 'current'
-      })
-    })
-  })
-}
-
-export async function withTerminalPerfProfile<T>(
-  page: Page,
-  phase: string,
-  runId: string,
-  measure: () => Promise<T>
-): Promise<T> {
-  const directory = process.env.ORCA_PERF_DIAGNOSTIC_DIR
-  if (!directory) {
-    return measure()
-  }
-  const watcher = await page.evaluateHandle(() => {
-    const longTasks: { start: number; duration: number }[] = []
-    const observer = new PerformanceObserver((list) => {
-      for (const entry of list.getEntries()) {
-        longTasks.push({ start: entry.startTime, duration: entry.duration })
-      }
-    })
-    observer.observe({ type: 'longtask' })
-    const started = performance.now()
-    return {
-      stop: () => {
-        observer.disconnect()
-        return {
-          started,
-          finished: performance.now(),
-          visibility: document.visibilityState,
-          longTasks
-        }
-      }
-    }
-  })
-  try {
-    const result =
-      process.env.ORCA_PERF_DIAGNOSTIC_TRACE === '1'
-        ? await withTerminalBrowserTrace(
-            page,
-            path.join(directory, `${phase}-${runId}.trace.json`),
-            measure
-          )
-        : await withTypingRendererCpuProfile(
-            page,
-            path.join(directory, `${phase}-${runId}.cpuprofile`),
-            measure
-          )
-    console.error(
-      JSON.stringify({
-        phase,
-        runId,
-        result,
-        renderer: await watcher.evaluate((watcher) => watcher.stop())
-      })
-    )
-    return result
-  } finally {
-    await watcher.evaluate((watcher) => watcher.stop()).catch(() => undefined)
-    await watcher.dispose()
-  }
-}

tests/e2e/terminal-perf-presentation.ts

@@ -0,0 +1,35 @@
+import type { ElectronApplication, TestInfo } from '@stablyai/playwright-test'
+
+export function shouldPresentTerminalPerfWindow(
+  env: Readonly<Record<string, string | undefined>> = process.env,
+  platform: string = process.platform
+): boolean {
+  if (env.ORCA_E2E_TERMINAL_PERF_XVFB !== '1') {
+    return false
+  }
+  if (platform !== 'linux' || env.GITHUB_ACTIONS !== 'true' || !env.DISPLAY) {
+    throw new Error('Terminal perf presentation requires an isolated GitHub Actions Xvfb display')
+  }
+  return true
+}
+
+export async function presentTerminalPerfWindow(
+  electronApp: ElectronApplication,
+  testInfo: TestInfo
+): Promise<void> {
+  if (!shouldPresentTerminalPerfWindow()) {
+    return
+  }
+  // An unpresented Linux window triggers Chromium's one-second undrawn-frame throttle.
+  const visible = await electronApp.evaluate(({ BrowserWindow }) => {
+    const windows = BrowserWindow.getAllWindows()
+    for (const window of windows) {
+      window.showInactive()
+    }
+    return windows.length > 0 && windows.every((window) => window.isVisible())
+  })
+  if (!visible) {
+    throw new Error('Terminal perf window was not presented on the isolated display')
+  }
+  testInfo.annotations.push({ type: 'terminal-perf-presentation', description: 'isolated-xvfb' })
+}

tests/e2e/terminal-perf-presentation.unit.test.ts

@@ -0,0 +1,38 @@
+import { describe, expect, it } from 'vitest'
+import { shouldPresentTerminalPerfWindow } from './terminal-perf-presentation'
+
+describe('terminal perf window presentation', () => {
+  const isolatedDisplay = {
+    ORCA_E2E_TERMINAL_PERF_XVFB: '1',
+    ORCA_BACKGROUND_LAUNCH: '1',
+    GITHUB_ACTIONS: 'true',
+    DISPLAY: ':99'
+  }
+
+  it.each(['darwin', 'linux', 'win32'])('keeps ordinary %s runs hidden', (platform) => {
+    expect(shouldPresentTerminalPerfWindow({}, platform)).toBe(false)
+    expect(
+      shouldPresentTerminalPerfWindow(
+        { ...isolatedDisplay, ORCA_E2E_TERMINAL_PERF_XVFB: '0' },
+        platform
+      )
+    ).toBe(false)
+  })
+
+  it('permits the explicit Linux CI display without changing background launch policy', () => {
+    expect(shouldPresentTerminalPerfWindow(isolatedDisplay, 'linux')).toBe(true)
+  })
+
+  it.each(['darwin', 'win32'])('rejects visible diagnostics on %s', (platform) => {
+    expect(() => shouldPresentTerminalPerfWindow(isolatedDisplay, platform)).toThrow('isolated')
+  })
+
+  it.each([
+    { ...isolatedDisplay, GITHUB_ACTIONS: undefined },
+    { ...isolatedDisplay, GITHUB_ACTIONS: 'false' },
+    { ...isolatedDisplay, DISPLAY: undefined },
+    { ...isolatedDisplay, DISPLAY: '' }
+  ])('rejects a missing isolated display: %j', (env) => {
+    expect(() => shouldPresentTerminalPerfWindow(env, 'linux')).toThrow('isolated')
+  })
+})