diff --git a/.github/workflows/terminal-perf.yml b/.github/workflows/terminal-perf.yml index 83ada43d2cf..2f3a37524eb 100644 --- a/.github/workflows/terminal-perf.yml +++ b/.github/workflows/terminal-perf.yml @@ -68,25 +68,60 @@ jobs: - name: Install native build tools and xvfb run: sudo apt-get update && sudo apt-get install -y build-essential python3 xvfb zsh + - name: Select dependency preparation + id: install-mode + shell: bash + env: + RUNNER_KIND: ${{ runner.environment }} + JOB_CONTAINER: ${{ job.container.id }} + run: | + node <<'NODE' + const fs = require('node:fs') + const manifest = JSON.parse(fs.readFileSync('package.json', 'utf8')) + const actionPath = '.github/actions/install-node-dependencies/action.yml' + const action = fs.existsSync(actionPath) ? fs.readFileSync(actionPath, 'utf8') : '' + const inputs = action.split(/^inputs:[ \t]*\r?$/m)[1]?.split(/^\S/m)[0] ?? '' + const shared = process.env.RUNNER_KIND === 'github-hosted' && !process.env.JOB_CONTAINER && + process.env.RUNNER_OS === 'Linux' && process.env.RUNNER_ARCH === 'X64' && + manifest.engines?.node === '24' && typeof manifest.packageManager === 'string' && + manifest.packageManager.split('+')[0] === 'pnpm@12.8.1' && + manifest.scripts?.postinstall === 'node config/scripts/rebuild-native-deps.mjs' && + /^ native-runtime:/m.test(inputs) && /^ cache-pnpm-store-lookup-only:/m.test(inputs) && + fs.existsSync('.github/actions/prepare-native-runtime/action.yml') && + fs.existsSync('config/scripts/ensure-native-runtime.mjs') + fs.appendFileSync(process.env.GITHUB_OUTPUT, `shared=${shared}\n`) + NODE + + - name: Prepare current dependencies + if: steps.install-mode.outputs.shared == 'true' + uses: ./.github/actions/install-node-dependencies + with: + native-runtime: electron + cache-electron-package: 'true' + cache-pnpm-store-lookup-only: 'true' + - name: Setup pnpm + if: steps.install-mode.outputs.shared != 'true' uses: pnpm/setup@v2 with: install: false - name: Setup Node.js + if: steps.install-mode.outputs.shared != 'true' uses: actions/setup-node@v6 with: node-version-file: package.json cache: pnpm - # Why: this scheduled/manual workflow uses the same native install path as - # PR and E2E CI, which needs pnpm to bypass its bundled gyp_main.py. + # Historical refs can lack the shared action; retain their original install path. - name: Use external node-gyp to avoid pnpm's bundled copy + if: steps.install-mode.outputs.shared != 'true' run: | npm install -g node-gyp@11.5.0 echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV" - name: Install dependencies + if: steps.install-mode.outputs.shared != 'true' run: pnpm install --frozen-lockfile - name: Build Electron app for terminal perf diff --git a/config/scripts/terminal-perf-dependency-preparation.test.mjs b/config/scripts/terminal-perf-dependency-preparation.test.mjs new file mode 100644 index 00000000000..3f4c5955f7a --- /dev/null +++ b/config/scripts/terminal-perf-dependency-preparation.test.mjs @@ -0,0 +1,116 @@ +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { runInNewContext } from 'node:vm' +import { parse } from 'yaml' +import { expect, it } from 'vitest' +import { runProcessSync } from './script-child-process.mjs' + +const workflow = parse(readFileSync('.github/workflows/terminal-perf.yml', 'utf8')) +const steps = workflow.jobs['terminal-perf'].steps +const selector = steps.find((step) => step.id === 'install-mode') +const script = selector.run.trim().match(/^node <<'NODE'\n([\s\S]*)\nNODE$/)[1] +const supportedAction = readFileSync('.github/actions/install-node-dependencies/action.yml', 'utf8') +const supportedManifest = { + engines: { node: '24' }, + packageManager: 'pnpm@12.8.1', + scripts: { postinstall: 'node config/scripts/rebuild-native-deps.mjs' } +} + +function select(options = {}) { + const directory = mkdtempSync(join(tmpdir(), 'orca-terminal-preparation-')) + const output = join(directory, 'output') + try { + writeFileSync( + join(directory, 'package.json'), + JSON.stringify(options.manifest ?? supportedManifest) + ) + for (const [file, content] of [ + ['.github/actions/install-node-dependencies/action.yml', options.action ?? supportedAction], + ['.github/actions/prepare-native-runtime/action.yml', 'runs: {}'], + ['config/scripts/ensure-native-runtime.mjs', ''] + ]) { + if (options.missing === file) { + continue + } + const path = join(directory, file) + mkdirSync(dirname(path), { recursive: true }) + writeFileSync(path, content) + } + const result = runProcessSync({ + program: process.execPath, + args: ['-e', script], + cwd: directory, + env: { + ...process.env, + GITHUB_OUTPUT: output, + RUNNER_KIND: options.kind ?? 'github-hosted', + JOB_CONTAINER: options.container ?? '', + RUNNER_OS: options.os ?? 'Linux', + RUNNER_ARCH: options.arch ?? 'X64' + } + }) + expect(result.code, result.stderr || result.stdout).toBe(0) + return readFileSync(output, 'utf8').trim() + } finally { + rmSync(directory, { recursive: true, force: true }) + } +} + +it('selects the measured current root profile with the actual installer metadata', () => { + expect(select()).toBe('shared=true') + expect( + select({ manifest: { ...supportedManifest, packageManager: 'pnpm@12.8.1+sha512.fixture' } }) + ).toBe('shared=true') +}) + +it.each([ + ['historical Node', { manifest: { ...supportedManifest, engines: { node: '22' } } }], + ['historical pnpm', { manifest: { ...supportedManifest, packageManager: 'pnpm@10.0.0' } }], + ['unmeasured pnpm', { manifest: { ...supportedManifest, packageManager: 'pnpm@12.8.10' } }], + ['missing toolchain', { manifest: {} }], + [ + 'extra lifecycle work', + { manifest: { ...supportedManifest, scripts: { postinstall: 'generate' } } } + ], + ['self-hosted runner', { kind: 'self-hosted' }], + ['job container', { container: 'container-id' }], + ['another OS', { os: 'Windows' }], + ['another architecture', { arch: 'ARM64' }], + ['missing installer', { missing: '.github/actions/install-node-dependencies/action.yml' }], + ['missing native action', { missing: '.github/actions/prepare-native-runtime/action.yml' }], + ['missing runtime script', { missing: 'config/scripts/ensure-native-runtime.mjs' }], + ['old installer interface', { action: 'inputs:\n native-runtime: {}\nruns: {}\n' }], + [ + 'output-only names', + { action: 'outputs:\n native-runtime: {}\n cache-pnpm-store-lookup-only: {}\n' } + ] +])('retains the original install for %s', (_name, options) => { + expect(select(options)).toBe('shared=false') +}) + +it.each(['true', 'false', ''])('routes mode %s to one complete preparation path', (shared) => { + const enabled = (step) => + runInNewContext(step.if.replaceAll('steps.install-mode.outputs.shared', 'shared'), { shared }) + const current = steps.find((step) => step.name === 'Prepare current dependencies') + const legacy = steps.filter((step) => + [ + 'Setup pnpm', + 'Setup Node.js', + "Use external node-gyp to avoid pnpm's bundled copy", + 'Install dependencies' + ].includes(step.name) + ) + expect(legacy).toHaveLength(4) + expect(enabled(current)).toBe(shared === 'true') + expect(legacy.every((step) => enabled(step) === (shared !== 'true'))).toBe(true) + expect(current.with).toEqual({ + 'native-runtime': 'electron', + 'cache-electron-package': 'true', + 'cache-pnpm-store-lookup-only': 'true' + }) + expect(legacy.at(-1).run).toBe('pnpm install --frozen-lockfile') + expect(steps.find((step) => step.name === 'Run terminal scale perf report gate').run).toContain( + 'pnpm run test:e2e:terminal-perf:scale:report' + ) +}) diff --git a/docs/reference/ci-runner-efficiency.md b/docs/reference/ci-runner-efficiency.md index 312db2f190f..8130b770dd9 100644 --- a/docs/reference/ci-runner-efficiency.md +++ b/docs/reference/ci-runner-efficiency.md @@ -1815,6 +1815,49 @@ received the signal after both builders finished. The qualifying trial requested normal cancellation earlier in the same preparation sequence to account for observed delivery delay; no workload, wait or proof predicate was shortened. +## October 3 Terminal Perf dependency preparation + +The daily/manual Terminal Perf workflow still installed current dependencies through +raw lifecycle scripts and a global node-gyp installation. Its historical `ref` +input also accepts revisions that lack the shared installer, so replacing that +path unconditionally would break older runs. The current-profile path now uses +the existing shared installer with explicit Electron preparation and archive +caching. A guard requires GitHub-hosted Linux x64, Node 24/pnpm 12.8.1, the +native-only root postinstall and the needed local action inputs/files. Other +profiles and historical revisions keep their original frozen install. + +The [hosted comparison](https://github.com/stablyai/orca/actions/runs/37101695800) +ran both preparation paths in each of two Linux x64 jobs, reversing their order. +Legacy/shared preparation took 25.164/16.956 seconds and 27.434/18.032 seconds: +8.208 and 9.402 seconds saved. Both used Node 24.21.0, pnpm 12.8.1 and Electron +43.7.5. Both shared native-module cache lookups missed, so this improvement did +not depend on a warm native build. Electron archive and root pnpm cache lookups +hit. Dependency trees, pnpm data and Electron archives were reset between paths; +compiler headers and external services were not. Bootstrap, resets, validation, +post-job cleanup, queueing and the production guard step are outside those times. +These are preparation measurements, not whole-workflow or billing savings. + +Both paths passed a native-module probe inside the actual Electron executable +with `ELECTRON_RUN_AS_NODE=1`, and built the same Electron-vite e2e application. +The candidate's 18 focused routing/fallback tests, workflow actionlint and changed +code-quality checks passed. Performance tests, budgets and report uploads remain +unchanged. The [existing October 2 run](https://github.com/stablyai/orca/actions/runs/36985792125) +failed the same-workspace 50/100-terminal budgets (46.9/50.2 ms against 25 ms). +This dependency change does not claim to resolve those application regressions. + +The [full candidate integration](https://github.com/stablyai/orca/actions/runs/37104625474) +passed on `df71ad849cd854a232f7063562785563743b641a`: current preparation was +selected, its native cache missed and rebuilt, the app built and all 32 report +annotation rows passed the unchanged budget checker. The downloaded report also +passed the same checker locally. This is integration evidence; it does not +attribute application latency changes to dependency preparation. Subsequent +rebases resolved report documentation and incorporated fixture teardown fixes. +Workflow, installer-action and toolchain content stayed unchanged. Main also +added an import and a Windows-only MSBuild setting to the native-runtime script: +the imported helper has no top-level side effects, and the Linux rebuild branch +is unchanged. Focused tests verify its Linux/macOS no-op behavior. Final-head PR +checks qualify separately. + ## October 3 producer follow-up: automatic selection for the measured profile The first producer rollout in [#24927](https://github.com/stablyai/orca/pull/24927)