From c11181e445520ea67a504dcdd6c3f36ccf4e87e2 Mon Sep 17 00:00:00 2001 From: Neil Date: Mon, 21 Sep 2026 15:20:32 -0700 Subject: [PATCH] fix(perf): present benchmark frames on isolated CI display --- .github/workflows/terminal-perf.yml | 71 ++++++++---- .../terminal-perf-latency-investigation.md | 32 +++++- ...icial-opencode-hidden-pressure-scenario.ts | 7 +- .../artificial-opencode-terminal-load.spec.ts | 22 +--- tests/e2e/helpers/electron-launch-args.ts | 9 +- tests/e2e/terminal-perf-browser-trace.ts | 35 ------ tests/e2e/terminal-perf-diagnostics.ts | 104 ------------------ tests/e2e/terminal-perf-presentation.ts | 35 ++++++ .../terminal-perf-presentation.unit.test.ts | 38 +++++++ 9 files changed, 160 insertions(+), 193 deletions(-) delete mode 100644 tests/e2e/terminal-perf-browser-trace.ts delete mode 100644 tests/e2e/terminal-perf-diagnostics.ts create mode 100644 tests/e2e/terminal-perf-presentation.ts create mode 100644 tests/e2e/terminal-perf-presentation.unit.test.ts diff --git a/.github/workflows/terminal-perf.yml b/.github/workflows/terminal-perf.yml index 8ad52012c45..6fb186ecea0 100644 --- a/.github/workflows/terminal-perf.yml +++ b/.github/workflows/terminal-perf.yml @@ -55,6 +55,8 @@ jobs: timeout-minutes: 45 env: NODE_OPTIONS: --max-old-space-size=4096 + ORCA_BACKGROUND_LAUNCH: '1' + ORCA_E2E_TERMINAL_PERF_XVFB: '1' ORCA_E2E_FORWARD_APP_LOGS: '1' ORCA_E2E_TERMINAL_PERF_REPORT_PATH: ${{ inputs.report_path || 'test-results/terminal-scale-perf-report.json' }} @@ -91,30 +93,59 @@ jobs: - name: Build Electron app for terminal perf run: npx electron-vite build --mode e2e - - name: Compare hidden and Xvfb-visible windows + - name: Run terminal scale perf report gate env: - ORCA_BACKGROUND_LAUNCH: '1' - ORCA_PERF_DIAGNOSTIC_TRACE: '1' - SKIP_BUILD: '1' - ORCA_E2E_OPENCODE_FRAME_COUNT: '60' - ORCA_E2E_OPENCODE_SCALE_HIDDEN_PRESSURE_PANES: '25' + ORCA_TERMINAL_PERF_GREP: ${{ inputs.grep }} + ORCA_TERMINAL_PERF_FRAME_COUNT: ${{ inputs.frame_count }} + ORCA_TERMINAL_PERF_FRAME_INTERVAL_MS: ${{ inputs.frame_interval_ms }} + ORCA_TERMINAL_PERF_PRESSURE_OUTPUT_CHARS: ${{ inputs.pressure_output_chars }} + ORCA_TERMINAL_PERF_SCALE_PANES: ${{ inputs.scale_panes }} + ORCA_TERMINAL_PERF_SCALE_CROSS_WORKSPACE_PANES: ${{ inputs.scale_cross_workspace_panes }} + ORCA_TERMINAL_PERF_SCALE_PRESSURE_PANES: ${{ inputs.scale_pressure_panes }} + ORCA_TERMINAL_PERF_SCALE_HIDDEN_PRESSURE_PANES: ${{ inputs.scale_hidden_pressure_panes }} run: | - mkdir -p "$RUNNER_TEMP/perf-evidence" - for mode in hidden visible; do - export ORCA_PERF_DIAGNOSTIC_RENDERER=swiftshader - export ORCA_PERF_DIAGNOSTIC_XVFB_VISIBLE=0 - if [ "$mode" = visible ]; then - export ORCA_PERF_DIAGNOSTIC_XVFB_VISIBLE=1 - fi - export ORCA_PERF_DIAGNOSTIC_DIR="$RUNNER_TEMP/perf-evidence/mode-$mode" - mkdir -p "$ORCA_PERF_DIAGNOSTIC_DIR" - xvfb-run --auto-servernum pnpm exec playwright test tests/e2e/artificial-opencode-terminal-load.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1 --grep 'Latin hidden|hidden restore responsive with 25' --repeat-each=5 --reporter=json > "$ORCA_PERF_DIAGNOSTIC_DIR/report.json" 2> "$ORCA_PERF_DIAGNOSTIC_DIR/stderr.log" || echo "mode $mode had a test failure" - done + set -euo pipefail + args=() + if [ -n "${ORCA_TERMINAL_PERF_GREP:-}" ]; then + args+=(--grep "$ORCA_TERMINAL_PERF_GREP") + fi + if [ -n "${ORCA_TERMINAL_PERF_FRAME_COUNT:-}" ]; then + export ORCA_E2E_OPENCODE_FRAME_COUNT="$ORCA_TERMINAL_PERF_FRAME_COUNT" + fi + if [ -n "${ORCA_TERMINAL_PERF_FRAME_INTERVAL_MS:-}" ]; then + export ORCA_E2E_OPENCODE_FRAME_INTERVAL_MS="$ORCA_TERMINAL_PERF_FRAME_INTERVAL_MS" + fi + if [ -n "${ORCA_TERMINAL_PERF_PRESSURE_OUTPUT_CHARS:-}" ]; then + export ORCA_E2E_OPENCODE_PRESSURE_OUTPUT_CHARS="$ORCA_TERMINAL_PERF_PRESSURE_OUTPUT_CHARS" + fi + if [ -n "${ORCA_TERMINAL_PERF_SCALE_PANES:-}" ]; then + export ORCA_E2E_OPENCODE_SCALE_PANES="$ORCA_TERMINAL_PERF_SCALE_PANES" + fi + if [ -n "${ORCA_TERMINAL_PERF_SCALE_CROSS_WORKSPACE_PANES:-}" ]; then + export ORCA_E2E_OPENCODE_SCALE_CROSS_WORKSPACE_PANES="$ORCA_TERMINAL_PERF_SCALE_CROSS_WORKSPACE_PANES" + fi + if [ -n "${ORCA_TERMINAL_PERF_SCALE_PRESSURE_PANES:-}" ]; then + export ORCA_E2E_OPENCODE_SCALE_PRESSURE_PANES="$ORCA_TERMINAL_PERF_SCALE_PRESSURE_PANES" + fi + if [ -n "${ORCA_TERMINAL_PERF_SCALE_HIDDEN_PRESSURE_PANES:-}" ]; then + export ORCA_E2E_OPENCODE_SCALE_HIDDEN_PRESSURE_PANES="$ORCA_TERMINAL_PERF_SCALE_HIDDEN_PRESSURE_PANES" + fi + xvfb-run --auto-servernum env SKIP_BUILD=1 pnpm run test:e2e:terminal-perf:scale:report -- "${args[@]}" - - name: Upload profiles and reports + - name: Upload terminal perf report if: always() uses: actions/upload-artifact@v7 with: - name: terminal-perf-diagnostic - path: ${{ runner.temp }}/perf-evidence + name: terminal-scale-perf-report + path: ${{ env.ORCA_E2E_TERMINAL_PERF_REPORT_PATH }} retention-days: 14 + if-no-files-found: warn + + - name: Upload Playwright traces + if: failure() + uses: actions/upload-artifact@v7 + with: + name: terminal-scale-perf-traces + path: test-results/ + retention-days: 7 + if-no-files-found: ignore diff --git a/docs/reference/terminal-perf-latency-investigation.md b/docs/reference/terminal-perf-latency-investigation.md index a848dc2ff7b..cf9a11610d6 100644 --- a/docs/reference/terminal-perf-latency-investigation.md +++ b/docs/reference/terminal-perf-latency-investigation.md @@ -58,11 +58,31 @@ this is not a one-second xterm parse or proof of a second of CPU consumption. The same throttle is present in Chromium 148.0.7778.218 (Electron 42.3.3) and 146.0.7680.177. This source comparison does not prove identical runtime behavior. +## Visibility control + +[Run 35660847455](https://github.com/stablyai/orca/actions/runs/35660847455) +compared ten hidden-window samples with ten visible-window samples on the same +isolated Xvfb runner, using SwiftShader in both modes. Actual window visibility +was recorded. The background terminal panes remained hidden in both modes. + +| Measurement | Hidden window | Visible window | +| --- | ---: | ---: | +| Undrawn-frame throttle decisions | 1,251 | 0 | +| Largest worst-key latency | 3,062.8 ms | 30.5 ms | +| Largest timer drift | 3,111.8 ms | 67.0 ms | +| Restore range | 213.8–1,862.2 ms (9 completed) | 223.4–734.3 ms (10 completed) | +| Electron tests passed | 9/10 | 10/10 | + +All ten visible-window samples satisfy the existing latency limits. This isolates +the never-presented Linux test window as the trigger for the reproduced native +stalls. It does not establish a newly introduced application-code regression or +prove that every historical outlier had the same cause. + ## Remaining verification -Compare hidden and visible application windows on an isolated Linux CI Xvfb -display, retaining the same hidden-terminal workload and graphics flags. Verify -actual window visibility and native throttling, not only measured restore time. -A supported correction must then pass the full terminal scale report gate with -unchanged latency limits. Keep desktop automation windowless and remove temporary -diagnostic workflow changes before merging a fix. +Run the full terminal scale report gate with the original graphics flags, no +profiling, and unchanged latency limits. The correction presents the benchmark +window only when explicitly enabled on an isolated GitHub Actions Linux display; +ordinary local automation stays windowless. Terminal workloads, hidden-pane gates, +and production launch policy are unchanged. Temporary diagnostic hooks and the +comparison workflow have been removed. diff --git a/tests/e2e/artificial-opencode-hidden-pressure-scenario.ts b/tests/e2e/artificial-opencode-hidden-pressure-scenario.ts index b7395ca7eaf..905e113dc52 100644 --- a/tests/e2e/artificial-opencode-hidden-pressure-scenario.ts +++ b/tests/e2e/artificial-opencode-hidden-pressure-scenario.ts @@ -1,4 +1,3 @@ -import { withTerminalPerfProfile } from './terminal-perf-diagnostics' import type { Page, TestInfo } from '@stablyai/playwright-test' import { expect } from '@stablyai/playwright-test' import { randomUUID } from 'node:crypto' @@ -217,8 +216,10 @@ export async function runHiddenRealPtyPressureScenario< expect(measurement.maxTimerDriftMs).toBeLessThan(MAX_HIDDEN_PRESSURE_TIMER_DRIFT_MS) await deps.releaseTerminalAckGate(orcaPage) - const restoreLatencyMs = await withTerminalPerfProfile(orcaPage, 'restore', runId, () => - measureHiddenOutputRestoreLatency(orcaPage, secondWorktreeId, runId) + const restoreLatencyMs = await measureHiddenOutputRestoreLatency( + orcaPage, + secondWorktreeId, + runId ) testInfo.annotations.push({ type: `opencode-hidden-real-pty-restore${annotationSuffix ?? ''}`, diff --git a/tests/e2e/artificial-opencode-terminal-load.spec.ts b/tests/e2e/artificial-opencode-terminal-load.spec.ts index beec6175b82..60b7efc229a 100644 --- a/tests/e2e/artificial-opencode-terminal-load.spec.ts +++ b/tests/e2e/artificial-opencode-terminal-load.spec.ts @@ -1,7 +1,4 @@ -import { - configureTerminalPerfDiagnostics, - withTerminalPerfProfile -} from './terminal-perf-diagnostics' +import { presentTerminalPerfWindow } from './terminal-perf-presentation' import type { Page, TestInfo } from '@stablyai/playwright-test' import { randomUUID } from 'node:crypto' import { mkdirSync, rmSync, writeFileSync } from 'node:fs' @@ -33,6 +30,10 @@ import { runMainPressureScenario } from './artificial-opencode-main-pressure-sce import { runRendererBackpressureRevisitScenario } from './artificial-opencode-revisit-pressure-scenario' import { startSyntheticOpenCodeInjection } from './artificial-opencode-synthetic-injection' +test.beforeEach(async ({ electronApp }, testInfo) => { + await presentTerminalPerfWindow(electronApp, testInfo) +}) + type TypingMeasurement = { latencies: number[] medianLatencyMs: number @@ -240,24 +241,11 @@ function median(values: number[]): number { return sorted[Math.floor(sorted.length / 2)] ?? 0 } -configureTerminalPerfDiagnostics() - async function measureTypingDuringLoad( page: Page, scriptPath: string, ptyId: string, runId: string -): Promise { - return withTerminalPerfProfile(page, 'typing', runId, () => - measureTypingDuringLoadProfiled(page, scriptPath, ptyId, runId) - ) -} - -async function measureTypingDuringLoadProfiled( - page: Page, - scriptPath: string, - ptyId: string, - runId: string ): Promise { await sendToTerminal(page, ptyId, `node ${JSON.stringify(scriptPath)}\r`) await waitForTerminalOutputForPtyId(page, ptyId, `OPENCODE_TYPING_READY_${runId}`, 10_000) diff --git a/tests/e2e/helpers/electron-launch-args.ts b/tests/e2e/helpers/electron-launch-args.ts index 35b3760281c..6868f48b083 100644 --- a/tests/e2e/helpers/electron-launch-args.ts +++ b/tests/e2e/helpers/electron-launch-args.ts @@ -11,14 +11,7 @@ export function getOrcaElectronLaunchArgs(mainPath: string, headful: boolean): s // Crash tests must not block later launches on AppKit's saved-window recovery dialog. return [...keychainArgs, appPath, '-ApplePersistenceIgnoreState', 'YES'] } - if (process.platform === 'linux' && process.env.ORCA_PERF_DIAGNOSTIC_RENDERER === 'before-c647') { - return ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage', appPath] - } - if ( - (headful || process.env.ORCA_PERF_DIAGNOSTIC_RENDERER === 'swiftshader') && - process.platform === 'linux' && - process.env.CI - ) { + if (headful && process.platform === 'linux' && process.env.CI) { // Hosted runners have no GPU; SwiftShader keeps WebGL assertions from silently skipping. return [ '--use-gl=angle', diff --git a/tests/e2e/terminal-perf-browser-trace.ts b/tests/e2e/terminal-perf-browser-trace.ts deleted file mode 100644 index dac40ecaada..00000000000 --- a/tests/e2e/terminal-perf-browser-trace.ts +++ /dev/null @@ -1,35 +0,0 @@ -import type { Page } from '@stablyai/playwright-test' -import { mkdirSync, writeFileSync } from 'node:fs' -import path from 'node:path' - -export async function withTerminalBrowserTrace( - page: Page, - outputPath: string, - measure: () => Promise -): Promise { - const session = await page.context().newCDPSession(page) - const events: unknown[] = [] - session.on('Tracing.dataCollected', ({ value }) => events.push(...value)) - try { - await session.send('Tracing.start', { - categories: - '-*,devtools.timeline,disabled-by-default-devtools.timeline,blink,blink.user_timing,cc,toplevel,v8,renderer.scheduler,gpu,viz', - transferMode: 'ReportEvents' - }) - try { - await page.evaluate(() => performance.mark('orca-perf-measure-start')) - return await measure() - } finally { - await page.evaluate(() => performance.mark('orca-perf-measure-end')).catch(() => undefined) - const completed = new Promise((resolve) => - session.once('Tracing.tracingComplete', () => resolve()) - ) - await session.send('Tracing.end') - await completed - mkdirSync(path.dirname(outputPath), { recursive: true }) - writeFileSync(outputPath, JSON.stringify({ traceEvents: events })) - } - } finally { - await session.detach() - } -} diff --git a/tests/e2e/terminal-perf-diagnostics.ts b/tests/e2e/terminal-perf-diagnostics.ts deleted file mode 100644 index 6281e4ea67c..00000000000 --- a/tests/e2e/terminal-perf-diagnostics.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { withTerminalBrowserTrace } from './terminal-perf-browser-trace' -import type { Page } from '@stablyai/playwright-test' -import path from 'node:path' -import { test } from './helpers/orca-app' -import { withTypingRendererCpuProfile } from './typing-renderer-cpu-profile' - -export function configureTerminalPerfDiagnostics(): void { - test.beforeEach(async ({ electronApp, orcaPage }, testInfo) => { - if (!process.env.ORCA_PERF_DIAGNOSTIC_DIR) { - return - } - const visibility = await orcaPage.evaluate(() => document.visibilityState) - const unthrottle = process.env.ORCA_PERF_DIAGNOSTIC_UNTHROTTLE === '1' - const visibleOnXvfb = process.env.ORCA_PERF_DIAGNOSTIC_XVFB_VISIBLE === '1' - if (visibleOnXvfb) { - if (process.platform !== 'linux' || !process.env.GITHUB_ACTIONS || !process.env.DISPLAY) { - throw new Error('Visible diagnostic requires an isolated GitHub Actions Xvfb display') - } - await electronApp.evaluate(({ BrowserWindow }) => { - for (const window of BrowserWindow.getAllWindows()) { - window.showInactive() - } - }) - } - const windows = await electronApp.evaluate(({ BrowserWindow }, unthrottle) => { - return BrowserWindow.getAllWindows().map((window) => { - if (unthrottle) { - window.webContents.setBackgroundThrottling(false) - } - return { - visible: window.isVisible(), - throttled: window.webContents.getBackgroundThrottling() - } - }) - }, unthrottle) - testInfo.annotations.push({ - type: 'perf-diagnostic-window', - description: JSON.stringify({ - windows, - visibility, - renderer: process.env.ORCA_PERF_DIAGNOSTIC_RENDERER ?? 'current' - }) - }) - }) -} - -export async function withTerminalPerfProfile( - page: Page, - phase: string, - runId: string, - measure: () => Promise -): Promise { - const directory = process.env.ORCA_PERF_DIAGNOSTIC_DIR - if (!directory) { - return measure() - } - const watcher = await page.evaluateHandle(() => { - const longTasks: { start: number; duration: number }[] = [] - const observer = new PerformanceObserver((list) => { - for (const entry of list.getEntries()) { - longTasks.push({ start: entry.startTime, duration: entry.duration }) - } - }) - observer.observe({ type: 'longtask' }) - const started = performance.now() - return { - stop: () => { - observer.disconnect() - return { - started, - finished: performance.now(), - visibility: document.visibilityState, - longTasks - } - } - } - }) - try { - const result = - process.env.ORCA_PERF_DIAGNOSTIC_TRACE === '1' - ? await withTerminalBrowserTrace( - page, - path.join(directory, `${phase}-${runId}.trace.json`), - measure - ) - : await withTypingRendererCpuProfile( - page, - path.join(directory, `${phase}-${runId}.cpuprofile`), - measure - ) - console.error( - JSON.stringify({ - phase, - runId, - result, - renderer: await watcher.evaluate((watcher) => watcher.stop()) - }) - ) - return result - } finally { - await watcher.evaluate((watcher) => watcher.stop()).catch(() => undefined) - await watcher.dispose() - } -} diff --git a/tests/e2e/terminal-perf-presentation.ts b/tests/e2e/terminal-perf-presentation.ts new file mode 100644 index 00000000000..08dd3b34bbb --- /dev/null +++ b/tests/e2e/terminal-perf-presentation.ts @@ -0,0 +1,35 @@ +import type { ElectronApplication, TestInfo } from '@stablyai/playwright-test' + +export function shouldPresentTerminalPerfWindow( + env: Readonly> = process.env, + platform: string = process.platform +): boolean { + if (env.ORCA_E2E_TERMINAL_PERF_XVFB !== '1') { + return false + } + if (platform !== 'linux' || env.GITHUB_ACTIONS !== 'true' || !env.DISPLAY) { + throw new Error('Terminal perf presentation requires an isolated GitHub Actions Xvfb display') + } + return true +} + +export async function presentTerminalPerfWindow( + electronApp: ElectronApplication, + testInfo: TestInfo +): Promise { + if (!shouldPresentTerminalPerfWindow()) { + return + } + // An unpresented Linux window triggers Chromium's one-second undrawn-frame throttle. + const visible = await electronApp.evaluate(({ BrowserWindow }) => { + const windows = BrowserWindow.getAllWindows() + for (const window of windows) { + window.showInactive() + } + return windows.length > 0 && windows.every((window) => window.isVisible()) + }) + if (!visible) { + throw new Error('Terminal perf window was not presented on the isolated display') + } + testInfo.annotations.push({ type: 'terminal-perf-presentation', description: 'isolated-xvfb' }) +} diff --git a/tests/e2e/terminal-perf-presentation.unit.test.ts b/tests/e2e/terminal-perf-presentation.unit.test.ts new file mode 100644 index 00000000000..749a31ff715 --- /dev/null +++ b/tests/e2e/terminal-perf-presentation.unit.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'vitest' +import { shouldPresentTerminalPerfWindow } from './terminal-perf-presentation' + +describe('terminal perf window presentation', () => { + const isolatedDisplay = { + ORCA_E2E_TERMINAL_PERF_XVFB: '1', + ORCA_BACKGROUND_LAUNCH: '1', + GITHUB_ACTIONS: 'true', + DISPLAY: ':99' + } + + it.each(['darwin', 'linux', 'win32'])('keeps ordinary %s runs hidden', (platform) => { + expect(shouldPresentTerminalPerfWindow({}, platform)).toBe(false) + expect( + shouldPresentTerminalPerfWindow( + { ...isolatedDisplay, ORCA_E2E_TERMINAL_PERF_XVFB: '0' }, + platform + ) + ).toBe(false) + }) + + it('permits the explicit Linux CI display without changing background launch policy', () => { + expect(shouldPresentTerminalPerfWindow(isolatedDisplay, 'linux')).toBe(true) + }) + + it.each(['darwin', 'win32'])('rejects visible diagnostics on %s', (platform) => { + expect(() => shouldPresentTerminalPerfWindow(isolatedDisplay, platform)).toThrow('isolated') + }) + + it.each([ + { ...isolatedDisplay, GITHUB_ACTIONS: undefined }, + { ...isolatedDisplay, GITHUB_ACTIONS: 'false' }, + { ...isolatedDisplay, DISPLAY: undefined }, + { ...isolatedDisplay, DISPLAY: '' } + ])('rejects a missing isolated display: %j', (env) => { + expect(() => shouldPresentTerminalPerfWindow(env, 'linux')).toThrow('isolated') + }) +})