Speed up daemon smoke checks and keep compiler slots busy (#28106)

* Remove repeated daemon smoke builds and idle compiler slots

* Test compiler scheduling through its process interface

* Keep preview component below the existing line limit with a type-only split
This commit is contained in:
Neil
2026-10-12 17:20:19 -07:00
committed by GitHub
parent 5e83ca0edd
commit c8996d76a5
11 changed files with 462 additions and 35 deletions
@@ -0,0 +1,42 @@
name: Preflight critical-path comparison
on:
pull_request:
paths:
- .github/workflows/preflight-critical-path-benchmark.yml
- config/scripts/benchmark-ci-preflight-critical-path.mjs
workflow_dispatch:
permissions:
contents: read
concurrency:
group: preflight-critical-path-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
ORCA_BACKGROUND_LAUNCH: '1'
jobs:
compare:
runs-on: ubuntu-24.04-arm
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
comparison: [smoke, sync-spawn, typecheck]
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
fetch-depth: 2
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: node
node-version: '24'
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: 1.4.3
- name: Compare checks and verify their results
run: node config/scripts/benchmark-ci-preflight-critical-path.mjs ${{ matrix.comparison }}
- uses: actions/upload-artifact@v7
if: always()
with:
name: preflight-critical-path-${{ matrix.comparison }}
path: .build/preflight-benchmark/
retention-days: 30
@@ -0,0 +1,183 @@
import { mkdirSync, rmSync, writeFileSync } from 'node:fs'
import { availableParallelism, totalmem } from 'node:os'
import path from 'node:path'
import { fileURLToPath } from 'node:url'
import { runProcessSync } from '@orca/process-host'
import { measurePreflightCommand } from './preflight-benchmark-command.mjs'
import { buildCounterbalancedSchedule } from './counterbalanced-benchmark-schedule.mjs'
import { summarizeBenchmarkSamples } from './benchmark-sample-summary.mjs'
const comparison = process.argv[2]
if (!['smoke', 'sync-spawn', 'typecheck'].includes(comparison) || process.argv.length !== 3) {
throw new Error('Use smoke, sync-spawn or typecheck')
}
const root = fileURLToPath(new URL('../..', import.meta.url))
const output = path.join(root, '.build/preflight-benchmark')
mkdirSync(output, { recursive: true })
const env = { ...process.env, ORCA_BACKGROUND_LAUNCH: '1' }
const report = {
sourceSha: process.env.GITHUB_SHA ?? null,
comparison,
host: {
platform: process.platform,
arch: process.arch,
node: process.version,
memoryBytes: totalmem(),
cores: availableParallelism()
},
method:
'Four counterbalanced pairs on one checkout; baseline comes from its first parent. Smoke clears build output each run; native and download caches are warmed for both arms.',
measurements: []
}
const save = () =>
writeFileSync(
path.join(output, `critical-path-${comparison}.json`),
`${JSON.stringify(report, null, 2)}\n`
)
function baselineSource(file) {
const result = runProcessSync({ program: 'git', args: ['show', `HEAD^1:${file}`], cwd: root })
if (result.code !== 0 || result.outputTruncated) {
throw new Error(`Cannot read baseline ${file}`)
}
return result.stdout
}
async function command(program, args, label) {
const result = await measurePreflightCommand({ program, args, label, root, output, env })
if (result.code !== 0 || result.signal || result.timedOut) {
throw new Error(`${label}: ${JSON.stringify(result)}`)
}
return result
}
const baselineName =
comparison === 'smoke'
? 'runtime-serve-terminal-smoke.mjs'
: comparison === 'typecheck'
? 'run-typecheck-projects-in-parallel.mjs'
: 'check-main-sync-spawn-ratchet.mjs'
const baselineFile = path.join(root, 'config/scripts', `.benchmark-baseline-${baselineName}`)
const runnerFile = path.join(output, 'baseline-smoke-runner.mjs')
writeFileSync(baselineFile, baselineSource(`config/scripts/${baselineName}`))
try {
if (comparison !== 'smoke') {
report.host.bun = (
await command('bun', ['--version'], 'critical-path-bun-version')
).stdout.trim()
if (report.host.bun !== '1.4.3') {
throw new Error('Pin Bun 1.4.3')
}
}
if (comparison === 'sync-spawn') {
const baseline = await import(new URL(`./.benchmark-baseline-${baselineName}`, import.meta.url))
const candidate = await import('./check-main-sync-spawn-ratchet.mjs')
const sources = candidate.collectSyncSpawnSources(root)
let analyzed = 0
for (const [file, contents] of sources) {
const before = baseline.collectSyncLaunchUses(file, contents)
const after = candidate.collectSyncLaunchUses(file, contents)
if (JSON.stringify(before) !== JSON.stringify(after)) {
throw new Error(`Findings changed: ${file}`)
}
analyzed++
}
for (const source of ['import {', 'const broken = ;']) {
for (const implementation of [baseline, candidate]) {
let rejected = false
try {
implementation.collectSyncLaunchUses('src/main/malformed.ts', source)
} catch {
rejected = true
}
if (!rejected) {
throw new Error('Malformed source was accepted')
}
}
}
report.parity = `Identical counts and symbols for all ${analyzed} production files; both reject malformed syntax.`
} else if (comparison === 'smoke') {
const steps = [
[process.execPath, ['config/scripts/ensure-native-runtime.mjs', '--runtime=node']],
['pnpm', ['run', 'build:cli']],
['pnpm', ['run', 'build:orcad']],
[process.execPath, [baselineFile, '--target', 'orcad']]
]
writeFileSync(
runnerFile,
`import { runProcessSync } from '@orca/process-host';
for (const [program, args] of ${JSON.stringify(steps)}) {
const result = runProcessSync({ program, args, cwd: ${JSON.stringify(root)}, stdio: 'inherit', timeoutMs: null });
if (result.code !== 0 || result.signal) process.exit(result.code || 1);
}\n`
)
await command('pnpm', ['run', 'build:orcad'], 'critical-path-smoke-warmup')
report.parity =
'Both clients must pass pairing, worktree, PTY and execution-host CLI round trips.'
} else {
report.parity =
'Same four projects, Bun version, two processes and two threads; both suites must pass.'
}
let findings
for (const [pair, order] of buildCounterbalancedSchedule(4, 'baseline', 'candidate').entries()) {
for (const arm of order) {
let result
if (comparison === 'smoke') {
rmSync(path.join(root, 'out'), { recursive: true, force: true })
result =
arm === 'baseline'
? await command(process.execPath, [runnerFile], `smoke-${pair}-${arm}`)
: await command('pnpm', ['run', 'smoke:orcad-terminal'], `smoke-${pair}-${arm}`)
for (const marker of [
'terminal round trip OK',
'execution host CLI round trip OK',
'[serve-terminal-smoke] PASS'
]) {
if (!result.stdout.includes(marker)) {
throw new Error(`Smoke skipped ${marker}: ${arm}`)
}
}
} else if (comparison === 'typecheck') {
result = await command(
process.execPath,
[
arm === 'baseline' ? baselineFile : `config/scripts/${baselineName}`,
'--compiler=bun',
'--bun-concurrency=2',
'--bun-threads=2'
],
`typecheck-${pair}-${arm}`
)
} else {
result = await command(
'bun',
[arm === 'baseline' ? baselineFile : `config/scripts/${baselineName}`],
`sync-spawn-${pair}-${arm}`
)
if (findings !== undefined && findings !== result.stdout) {
throw new Error(`Ratchet findings changed: ${arm}`)
}
findings = result.stdout
}
report.measurements.push({ pair, arm, ...result })
save()
console.log(JSON.stringify({ comparison, pair, arm, durationMs: result.durationMs }))
}
}
report.summary = Object.fromEntries(
['baseline', 'candidate'].map((arm) => [
arm,
summarizeBenchmarkSamples(
report.measurements
.filter((sample) => sample.arm === arm)
.map((sample) => sample.durationMs)
)
])
)
save()
console.log(JSON.stringify(report.summary))
} finally {
rmSync(baselineFile, { force: true })
rmSync(runnerFile, { force: true })
}
@@ -64,6 +64,12 @@ const GATED_FAMILIES = [
{ modules: ['src/main/win32-utils'], advice: { grantDirAcl: 'use grantDirAclAsync' } }
]
const ADVICE = new Map(GATED_FAMILIES.flatMap((family) => Object.entries(family.advice)))
const GATED_MARKERS = [
...ADVICE.keys(),
...new Set(
GATED_FAMILIES.flatMap((family) => family.modules.map((module) => path.posix.basename(module)))
)
]
// Only spread-free calls with no options or a literal with absent/false `sync` are exempt.
const SYNC_FLAG_SYMBOLS = new Set(['hardenSecurePath'])
const FALLBACK_OPERATORS = new Set([ts.SyntaxKind.QuestionQuestionToken, ts.SyntaxKind.BarBarToken])
@@ -165,10 +171,17 @@ function isValueReference(node) {
* Counted by reference only: not traced through non-const holders or the file's own wrappers.
*/
export function collectSyncLaunchUses(file, contents) {
const source = ts.createSourceFile(file, contents, ts.ScriptTarget.Latest, true)
// Escaped spellings and relative imports inside gated modules still need the full analysis.
const analyze =
contents.includes('\\') ||
GATED_MARKERS.some((marker) => contents.includes(marker) || file.includes(marker))
const source = ts.createSourceFile(file, contents, ts.ScriptTarget.Latest, analyze)
if (source.parseDiagnostics.length > 0) {
throw new Error(`Cannot parse ${file}; the sync-spawn ratchet must not skip it.`)
}
if (!analyze) {
return { count: 0, symbols: [] }
}
const used = new Set()
let count = 0
const namespaces = new Map()
@@ -41,6 +41,19 @@ function captureErrors() {
}
describe('main sync-spawn ratchet', () => {
it('retains escaped imports and API names', () => {
expect(
uses(String.raw`import { \u0065xecSync as run } from 'node:child_\u0070rocess'; run('git')`)
).toEqual({ count: 1, symbols: ['execSync'] })
})
it('retains exports whose module name is inherited from the containing directory', () => {
expect(uses("export * from '.'", 'src/main/wsl/consumer.ts')).toEqual({
count: 4,
symbols: ['getWslHome', 'isWslAvailable', 'listWslDistros', 'wslUncDirectoryExists']
})
})
it.each([
["import { execFileSync } from 'node:child_process'\nexecFileSync('git')", 1, ['execFileSync']],
[
@@ -212,9 +225,12 @@ describe('main sync-spawn ratchet', () => {
expect([...(gatedSymbolsFor('src\\main\\foo\\x.ts', '..\\wsl') ?? [])]).toContain('getWslHome')
})
it('fails closed on source it cannot parse', () => {
expect(() => uses('import {')).toThrow('Cannot parse src/main/consumer.ts')
})
it.each(['import {', 'const broken = ;'])(
'fails closed on source it cannot parse: %s',
(source) => {
expect(() => uses(source)).toThrow('Cannot parse src/main/consumer.ts')
}
)
it('counts only production main and shared source', () => {
const sync = "import { execSync } from 'node:child_process'\nexecSync('a')\nexecSync('b')"
@@ -0,0 +1,80 @@
import { EventEmitter } from 'node:events'
import { beforeEach, expect, it, vi } from 'vitest'
import { runTypecheckProjects, TYPECHECK_PROJECTS } from './run-typecheck-projects-in-parallel.mjs'
const spawn = vi.fn()
let children
let active
let peak
beforeEach(() => {
children = []
active = 0
peak = 0
spawn.mockReset().mockImplementation((spec) => {
const child = new EventEmitter()
children.push({ child, spec })
active++
peak = Math.max(peak, active)
child.once('close', () => {
active--
})
return child
})
})
function start(memoryBytes = 16 * 1024 ** 3) {
return runTypecheckProjects({
compiler: 'bun',
bunConcurrency: 2,
bunThreads: 2,
spawnCompiler: spawn,
memoryBytes,
parallelism: 4
})
}
it('fills a freed compiler slot while the other project is still running', async () => {
const result = start()
expect(children).toHaveLength(2)
children[1].child.emit('close', 0, null)
await vi.waitFor(() => expect(children).toHaveLength(3))
children[2].child.emit('close', 0, null)
await vi.waitFor(() => expect(children).toHaveLength(4))
expect(active).toBe(2)
children[3].child.emit('close', 0, null)
children[0].child.emit('close', 0, null)
expect(await result).toEqual([])
expect(peak).toBe(2)
expect(children.map(({ spec }) => spec.args[spec.args.indexOf('-p') + 1])).toEqual(
TYPECHECK_PROJECTS.map(({ config }) => config)
)
expect(children.every(({ spec }) => spec.args[spec.args.indexOf('--threads') + 1] === '2')).toBe(
true
)
})
it('keeps checking after compiler failures and reports errors in project order', async () => {
const result = start()
children[1].child.emit('close', 1, null)
await vi.waitFor(() => expect(children).toHaveLength(3))
children[0].child.emit('close', null, 'SIGKILL')
await vi.waitFor(() => expect(children).toHaveLength(4))
children[2].child.emit('close', 0, null)
children[3].child.emit('close', 0, null)
expect((await result).map(({ message }) => message)).toEqual([
'bun config/tsconfig.node.json exited with signal SIGKILL',
'bun config/tsconfig.tc.web.json exited with code 1'
])
expect(peak).toBe(2)
})
it('reduces the queue to one slot when two reservations exceed the memory budget', async () => {
const result = start(4 * 1024 ** 3)
expect(children).toHaveLength(1)
for (let index = 0; index < TYPECHECK_PROJECTS.length; index++) {
await vi.waitFor(() => expect(children).toHaveLength(index + 1))
children[index].child.emit('close', 0, null)
}
expect(await result).toEqual([])
expect(peak).toBe(1)
})
@@ -101,7 +101,7 @@ export function typecheckBatches(projects, { compiler, bunConcurrency, memoryByt
)
}
function checkProject(config, { compiler, bunThreads, root }) {
function checkProject(config, { compiler, bunThreads, root, spawnCompiler }) {
return new Promise((resolve, reject) => {
const installedCompiler = resolveProjectCompilerInfo(config, root)
// Bun 1.4.3 targets TypeScript 7.0.2; compiler upgrades require a new parity comparison.
@@ -110,7 +110,7 @@ function checkProject(config, { compiler, bunThreads, root }) {
`Bun CI compiler targets TypeScript 7.0.2, but ${config} resolves ${installedCompiler.version}`
)
}
const child = spawnProcess({
const child = spawnCompiler({
program: compiler === 'bun' ? 'bun' : process.execPath,
args:
compiler === 'bun'
@@ -138,15 +138,37 @@ export async function runTypecheckProjects({
bunThreads = 4,
root = repoRoot,
projects = TYPECHECK_PROJECTS,
spawnCompiler = spawnProcess,
memoryBytes = totalmem(),
parallelism = availableParallelism()
} = {}) {
const batches = typecheckBatches(projects, { compiler, bunConcurrency, memoryBytes, parallelism })
if (compiler === 'bun') {
// Equal Bun reservations let a freed slot start the next project without waiting for peers.
const pending = batches.flat()
const failures = []
let next = 0
await Promise.all(
Array.from({ length: Math.max(0, ...batches.map((batch) => batch.length)) }, async () => {
while (next < pending.length) {
const index = next++
try {
await checkProject(pending[index].config, { compiler, bunThreads, root, spawnCompiler })
} catch (reason) {
failures.push({ index, reason })
}
}
})
)
return failures.sort((left, right) => left.index - right.index).map(({ reason }) => reason)
}
// Every batch runs even after one fails, so a single broken project still reports the rest.
const failures = []
for (const batch of batches) {
const results = await Promise.allSettled(
batch.map((project) => checkProject(project.config, { compiler, bunThreads, root }))
batch.map((project) =>
checkProject(project.config, { compiler, bunThreads, root, spawnCompiler })
)
)
for (const result of results) {
if (result.status === 'rejected') {
+31 -11
View File
@@ -28,7 +28,10 @@ import { dirname, join, resolve } from 'node:path'
import { pathToFileURL } from 'node:url'
import { randomBytes } from 'node:crypto'
import process from 'node:process'
import { ORCAD_SERVER_ENTRY_FILENAME } from '../../src/shared/orcad-artifacts.ts'
import {
ORCAD_CLI_ENTRY_FILENAME,
ORCAD_SERVER_ENTRY_FILENAME
} from '../../src/shared/orcad-artifacts.ts'
import { packagedNodeRuntimePath } from './build-orcad-node.mjs'
import { currentTarget } from './server-build-target.mjs'
@@ -52,11 +55,21 @@ function fail(message) {
}
/**
* Prefer the CLI built from this checkout over whatever `orca` is on PATH: it is the
* version under test, and a CI runner has no installed Orca app to fall back on.
* Orcad uses its packaged CLI and Node; Electron prefers the checkout's desktop CLI.
*/
function resolveCli() {
const built = join(projectDir, 'out', 'cli', 'index.js')
export function resolveCli(target, root = projectDir) {
if (target === 'orcad') {
const directory = join(root, 'out', 'orcad')
const built = join(directory, ORCAD_CLI_ENTRY_FILENAME)
if (!existsSync(built)) {
throw new Error(`Missing packaged Orcad CLI at ${built}; run pnpm build:orcad`)
}
return {
command: packagedNodeRuntimePath(directory, currentTarget()),
prefix: [built]
}
}
const built = join(root, 'out', 'cli', 'index.js')
return existsSync(built)
? { command: process.execPath, prefix: [built] }
: { command: 'orca', prefix: [] }
@@ -64,7 +77,7 @@ function resolveCli() {
/** The `orca` CLI, driven with an explicit pairing code so it targets this server only. */
function orca(pairingCode, args) {
const cli = resolveCli()
const cli = resolveCli(smokeTarget())
const result = spawnSync(
cli.command,
[...cli.prefix, ...args, '--pairing-code', pairingCode, '--json'],
@@ -170,12 +183,17 @@ async function waitForNonce(pairingCode, terminalHandle, nonce) {
* contract as the Electron server, proven by the same code rather than a parallel test
* that could drift into asserting less.
*/
function resolveLaunch(userDataDir) {
function smokeTarget() {
// Why a flag and not just an env var: package scripts have to set this on Windows too,
// and `FOO=bar cmd` is not portable there.
const flagIndex = process.argv.indexOf('--target')
const target =
flagIndex !== -1 ? process.argv[flagIndex + 1] : (process.env.ORCA_SMOKE_TARGET ?? 'electron')
return flagIndex !== -1
? process.argv[flagIndex + 1]
: (process.env.ORCA_SMOKE_TARGET ?? 'electron')
}
function resolveLaunch(userDataDir) {
const target = smokeTarget()
if (target === 'orcad') {
return {
label: `orcad (${ORCAD_ENTRY})`,
@@ -372,7 +390,7 @@ async function main() {
} finally {
// Why before SIGTERM: worktree removal is a server operation, so it needs the server.
if (seeded?.worktreeId && pairing) {
const cleanupCli = resolveCli()
const cleanupCli = resolveCli(smokeTarget())
const removed = spawnSync(
cleanupCli.command,
[
@@ -428,4 +446,6 @@ async function main() {
}
}
await main()
if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
await main()
}
@@ -0,0 +1,48 @@
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path'
import { afterEach, expect, it } from 'vitest'
import { resolveCli } from './runtime-serve-terminal-smoke.mjs'
import { packagedNodeRuntimePath } from './build-orcad-node.mjs'
import { currentTarget } from './server-build-target.mjs'
import { ORCAD_CLI_ENTRY_FILENAME } from '../../src/shared/orcad-artifacts.ts'
const roots = []
afterEach(() => {
for (const root of roots.splice(0)) {
rmSync(root, { recursive: true, force: true })
}
})
function checkout() {
const root = mkdtempSync(join(tmpdir(), 'orca-smoke-cli-'))
roots.push(root)
const emitted = join(root, 'out', 'cli', 'index.js')
mkdirSync(dirname(emitted), { recursive: true })
writeFileSync(emitted, '')
return root
}
it('drives Orcad with its packaged CLI and pinned Node even when a desktop CLI exists', () => {
const root = checkout()
const directory = join(root, 'out', 'orcad')
const bundled = join(directory, ORCAD_CLI_ENTRY_FILENAME)
mkdirSync(dirname(bundled), { recursive: true })
writeFileSync(bundled, '')
expect(resolveCli('orcad', root)).toEqual({
command: packagedNodeRuntimePath(directory, currentTarget()),
prefix: [bundled]
})
})
it('fails if the packaged CLI is missing instead of using a stale desktop CLI', () => {
expect(() => resolveCli('orcad', checkout())).toThrow('Missing packaged Orcad CLI')
})
it('retains the built desktop CLI for the Electron smoke', () => {
const root = checkout()
expect(resolveCli('electron', root)).toEqual({
command: process.execPath,
prefix: [join(root, 'out', 'cli', 'index.js')]
})
})
+1 -1
View File
@@ -48,7 +48,7 @@
"build:orcad": "pnpm run build:packages && node config/scripts/build-orcad-node.mjs",
"build:orcad-template": "pnpm run build:packages && node config/scripts/build-orcad-template.mjs",
"build:orcad-prebuilds": "pnpm run build:packages && node config/scripts/build-orcad-prebuilds.mjs",
"smoke:orcad-terminal": "node config/scripts/ensure-native-runtime.mjs --runtime=node && pnpm run build:cli && pnpm run build:orcad && node config/scripts/runtime-serve-terminal-smoke.mjs --target orcad",
"smoke:orcad-terminal": "node config/scripts/ensure-native-runtime.mjs --runtime=node && pnpm run build:orcad && node config/scripts/runtime-serve-terminal-smoke.mjs --target orcad",
"smoke:serve-terminal": "node config/scripts/runtime-serve-terminal-smoke.mjs",
"check:feature-wall-assets": "node config/scripts/check-feature-wall-assets.mjs",
"generate:rpc-params-catalog": "node config/scripts/generate-rpc-params-catalog.mjs",
@@ -4,7 +4,7 @@ import type {
DocPreviewFileFailure,
DocPreviewFileFailureReason
} from '../../../../../shared/doc-preview-scheme'
import type { BrowserPageConversionOrigin } from '../../../../../shared/browser-workspace-types'
import type { HtmlDocPreviewProps } from './html-doc-preview-props'
import {
advanceAcrossBrowserPageConversion,
returnAcrossBrowserPageConversion
@@ -52,21 +52,7 @@ export function HtmlDocPreview({
externalSshTargetId = null,
convertedFrom = null,
convertedTo = null
}: {
previewId: string
filePath: string
relativePath: string
worktreeId: string
/** Whether this preview is the surface the reader is in, and so may hold the keyboard. */
holdsGuestFocus?: boolean
isActive?: boolean
runtimeEnvironmentId?: string | null
externalSshTargetId?: string | null
/** Set when the address bar converted this page; Back returns across it once guest history runs out. */
convertedFrom?: BrowserPageConversionOrigin | null
/** Set when Back returned across a conversion to this page; Forward re-crosses it. */
convertedTo?: BrowserPageConversionOrigin | null
}): React.JSX.Element {
}: HtmlDocPreviewProps): React.JSX.Element {
const containerRef = useRef<HTMLDivElement | null>(null)
const webviewRef = useRef<Electron.WebviewTag | null>(null)
const reloadRef = useRef<(() => void) | null>(null)
@@ -0,0 +1,17 @@
import type { BrowserPageConversionOrigin } from '../../../../../shared/browser-workspace-types'
export type HtmlDocPreviewProps = {
previewId: string
filePath: string
relativePath: string
worktreeId: string
/** Whether this preview is the surface the reader is in, and so may hold the keyboard. */
holdsGuestFocus?: boolean
isActive?: boolean
runtimeEnvironmentId?: string | null
externalSshTargetId?: string | null
/** Set when the address bar converted this page; Back returns across it once guest history runs out. */
convertedFrom?: BrowserPageConversionOrigin | null
/** Set when Back returned across a conversion to this page; Forward re-crosses it. */
convertedTo?: BrowserPageConversionOrigin | null
}