perf(checks): use Bun for end-to-end typechecking (#31046)

config/scripts/check-e2e-typecheck-ratchet.mjs

@@ -1,8 +1,8 @@
-import { spawnSync } from 'node:child_process'
 import fs from 'node:fs'
 import path from 'node:path'
 import process from 'node:process'
 import { pathToFileURL } from 'node:url'
+import { runE2eTypecheck } from './run-e2e-typecheck.mjs'
 import {
   diffCounts,
   parseCountBaseline,
@@ -15,7 +15,6 @@ import {
 // lane runs it. Counts per file may only go down.
 
 const BASELINE_PATH = 'config/e2e-typecheck-baseline.txt'
-const TSCONFIG = 'config/tsconfig.e2e.json'
 const HEADER = [
   '# Type errors per file under tests/ (tsc -p config/tsconfig.e2e.json).',
   '# This is a RATCHET: counts may only go DOWN. Fix the errors in a file you touch.',
@@ -37,21 +36,11 @@ export function countTestTypeErrors(tscOutput) {
   return counts
 }
 
-function runTsc(root) {
-  const result = spawnSync(
-    process.execPath,
-    ['node_modules/typescript/bin/tsc', '--noEmit', '--pretty', 'false', '-p', TSCONFIG],
-    { cwd: root, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }
-  )
-  const output = `${result.stdout ?? ''}${result.stderr ?? ''}`
-  // tsc exits 2 when it reports errors; anything else without diagnostics means it never ran.
-  if (result.error || (result.status !== 0 && !/error TS\d+:/.test(output))) {
-    throw new Error(`tsc -p ${TSCONFIG} failed to run:\n${result.error ?? output}`)
-  }
-  return output
-}
-
-export function main(root = process.cwd(), arg = undefined, tscOutput = runTsc(root)) {
+export function main(
+  root = process.cwd(),
+  arg = undefined,
+  tscOutput = runE2eTypecheck({ root }).output
+) {
   const current = countTestTypeErrors(tscOutput)
   if (arg === '--prune') {
     const total = pruneCountBaseline(root, BASELINE_PATH, HEADER, current)

config/scripts/run-e2e-typecheck.mjs

@@ -0,0 +1,58 @@
+import { availableParallelism } from 'node:os'
+import { runProcessSync } from '@orca/process-host'
+import { resolvePinnedBunExecutable } from './pinned-bun-executable.mjs'
+import { describeProcessFailure } from './process-failure-message.mjs'
+import { isDirectInvocation } from './script-entry-detection.mjs'
+import {
+  resolveTypecheckCommand,
+  selectTypecheckCompiler
+} from './run-typecheck-projects-in-parallel.mjs'
+
+const CONFIG = 'config/tsconfig.e2e.json'
+
+export function runE2eTypecheck({
+  root = process.cwd(),
+  compiler = 'bun',
+  execute = runProcessSync,
+  resolveBunExecutable = (directory) => resolvePinnedBunExecutable(directory, runProcessSync)
+} = {}) {
+  const selection = selectTypecheckCompiler(compiler, { root, resolveBunExecutable })
+  const command = resolveTypecheckCommand(CONFIG, {
+    ...selection,
+    root,
+    bunThreads: availableParallelism()
+  })
+  const result = execute({
+    ...command,
+    args: [...command.args, ...(compiler === 'native' ? ['--pretty', 'false'] : [])],
+    timeoutMs: null,
+    maxOutputBytes: 64 * 1024 * 1024
+  })
+  const output = `${result.stdout ?? ''}${result.stderr ?? ''}`
+  const diagnosticExit = result.code === 1 || (compiler === 'native' && result.code === 2)
+  // Incomplete diagnostics must never look like fixes when pruning the baseline.
+  if (
+    result.signal ||
+    result.timedOut ||
+    result.outputTruncated ||
+    (result.code !== 0 && (!diagnosticExit || !/error TS\d+:/.test(output)))
+  ) {
+    throw new Error(`${compiler} ${CONFIG} failed to run: ${describeProcessFailure(result)}`)
+  }
+  return { code: result.code, output }
+}
+
+if (isDirectInvocation(import.meta.url, process.argv[1])) {
+  try {
+    const args = process.argv.slice(2)
+    if (args.length > 1 || (args.length === 1 && args[0] !== '--compiler=native')) {
+      throw new Error('Usage: run-e2e-typecheck.mjs [--compiler=native]')
+    }
+    const result = runE2eTypecheck({ compiler: args.length ? 'native' : 'bun' })
+    process.stdout.write(result.output)
+    process.exitCode = result.code
+  } catch (error) {
+    console.error(error.message)
+    process.exitCode = 1
+  }
+}

config/scripts/run-e2e-typecheck.test.mjs

@@ -0,0 +1,119 @@
+import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'
+import { availableParallelism, tmpdir } from 'node:os'
+import { join, resolve } from 'node:path'
+import { runProcessSync } from '@orca/process-host'
+import { afterEach, expect, it, vi } from 'vitest'
+import { countTestTypeErrors } from './check-e2e-typecheck-ratchet.mjs'
+import { resolvePinnedBunExecutable } from './pinned-bun-executable.mjs'
+import { runE2eTypecheck } from './run-e2e-typecheck.mjs'
+
+const ROOT = process.cwd()
+const resolveBunExecutable = () => resolvePinnedBunExecutable(ROOT, runProcessSync)
+const directories = []
+const diagnostic = 'tests/entry.ts(1,1): error TS2322: Type error.\n'
+
+afterEach(() => {
+  for (const directory of directories.splice(0)) {
+    rmSync(directory, { recursive: true, force: true })
+  }
+})
+
+function fixture() {
+  const root = mkdtempSync(join(tmpdir(), 'orca-e2e-check-'))
+  directories.push(root)
+  mkdirSync(join(root, 'config'))
+  mkdirSync(join(root, 'tests'))
+  symlinkSync(resolve('node_modules'), join(root, 'node_modules'), 'junction')
+  writeFileSync(join(root, 'package.json'), '{"type":"commonjs"}')
+  writeFileSync(
+    join(root, 'config', 'tsconfig.e2e.json'),
+    JSON.stringify({
+      compilerOptions: {
+        module: 'Node20',
+        moduleResolution: 'Node16',
+        target: 'ES2022',
+        strict: true,
+        types: []
+      },
+      include: ['../tests/**/*.ts']
+    })
+  )
+  writeFileSync(join(root, 'tests', 'exports.ts'), 'export const value = 1\n')
+  writeFileSync(
+    join(root, 'tests', 'entry.ts'),
+    "import { missing } from './exports'\nexport const value: number = 'wrong'\n"
+  )
+  return root
+}
+
+it('captures all diagnostics using the pinned Bun and all available CPU cores', () => {
+  const execute = vi.fn(() => ({ code: 1, stdout: diagnostic, stderr: 'summary' }))
+  expect(runE2eTypecheck({ execute, resolveBunExecutable: () => 'pinned-bun' })).toEqual({
+    code: 1,
+    output: `${diagnostic}summary`
+  })
+  expect(execute).toHaveBeenCalledWith({
+    program: 'pinned-bun',
+    args: [
+      'check',
+      '-p',
+      'config/tsconfig.e2e.json',
+      '--threads',
+      String(availableParallelism()),
+      '--no-pretty',
+      '--all'
+    ],
+    cwd: ROOT,
+    timeoutMs: null,
+    maxOutputBytes: 64 * 1024 * 1024
+  })
+})
+
+it.each([
+  { code: 1, stdout: '', stderr: 'failed to start' },
+  { code: null, stdout: diagnostic, signal: 'SIGKILL' },
+  { code: 0, stdout: diagnostic, timedOut: true },
+  { code: 1, stdout: diagnostic, outputTruncated: true },
+  { code: 99, stdout: diagnostic },
+  { code: null, stdout: diagnostic }
+])('rejects failed or incomplete checks before the ratchet can prune errors: %j', (result) => {
+  expect(() =>
+    runE2eTypecheck({
+      execute: () => result,
+      resolveBunExecutable: () => 'pinned-bun'
+    })
+  ).toThrow('failed to run')
+})
+
+it('does not run when Bun is missing or the project compiler version has drifted', () => {
+  const execute = vi.fn()
+  expect(() => runE2eTypecheck({ execute, resolveBunExecutable: () => null })).toThrow('missing')
+  const root = fixture()
+  mkdirSync(join(root, 'config', 'node_modules', 'typescript'), { recursive: true })
+  writeFileSync(
+    join(root, 'config', 'node_modules', 'typescript', 'package.json'),
+    JSON.stringify({ version: '0.0.0', bin: { tsc: './bin/tsc' } })
+  )
+  expect(() =>
+    runE2eTypecheck({
+      root,
+      execute,
+      resolveBunExecutable: () => 'pinned-bun'
+    })
+  ).toThrow('resolves 0.0.0')
+  expect(execute).not.toHaveBeenCalled()
+})
+
+it('keeps real missing-export and type-error diagnostics identical to TypeScript', () => {
+  const root = fixture()
+  const native = runE2eTypecheck({ root, compiler: 'native' })
+  const bun = runE2eTypecheck({ root, resolveBunExecutable })
+  const locations = (output) =>
+    [...output.matchAll(/^.+\(\d+,\d+\): error TS\d+:/gm)].map(([line]) => line).sort()
+  expect(locations(bun.output)).toEqual(locations(native.output))
+  expect(locations(bun.output)).toHaveLength(2)
+  expect(countTestTypeErrors(bun.output)).toEqual(new Map([['tests/entry.ts', 2]]))
+  writeFileSync(join(root, 'tests', 'entry.ts'), "export { value } from './exports'\n")
+  expect(runE2eTypecheck({ root, resolveBunExecutable }).code).toBe(0)
+  expect(runE2eTypecheck({ root, compiler: 'native' }).code).toBe(0)
+})

config/scripts/run-typecheck-projects-in-parallel.mjs

@@ -138,21 +138,27 @@ export function typecheckBatches(projects, { compiler, bunConcurrency, memoryByt
   )
 }
 
+export function resolveTypecheckCommand(config, { compiler, bunExecutable, bunThreads, root }) {
+  const installedCompiler = resolveProjectCompilerInfo(config, root)
+  if (compiler === 'bun' && installedCompiler.version !== BUN_TYPESCRIPT_VERSION) {
+    throw new Error(
+      `Bun CI compiler targets TypeScript ${BUN_TYPESCRIPT_VERSION}, but ${config} resolves ${installedCompiler.version}`
+    )
+  }
+  return {
+    program: compiler === 'bun' ? bunExecutable : process.execPath,
+    args:
+      compiler === 'bun'
+        ? ['check', '-p', config, '--threads', String(bunThreads), '--no-pretty', '--all']
+        : [installedCompiler.program, '--noEmit', '-p', config],
+    cwd: root
+  }
+}
+
 function checkProject(config, { compiler, bunExecutable, bunThreads, root, spawnCompiler }) {
   return new Promise((resolve, reject) => {
-    const installedCompiler = resolveProjectCompilerInfo(config, root)
-    if (compiler === 'bun' && installedCompiler.version !== BUN_TYPESCRIPT_VERSION) {
-      throw new Error(
-        `Bun CI compiler targets TypeScript ${BUN_TYPESCRIPT_VERSION}, but ${config} resolves ${installedCompiler.version}`
-      )
-    }
     const child = spawnCompiler({
-      program: compiler === 'bun' ? bunExecutable : process.execPath,
-      args:
-        compiler === 'bun'
-          ? ['check', '-p', config, '--threads', String(bunThreads), '--no-pretty', '--all']
-          : [installedCompiler.program, '--noEmit', '-p', config],
-      cwd: root,
+      ...resolveTypecheckCommand(config, { compiler, bunExecutable, bunThreads, root }),
       stdio: 'inherit'
     })
     child.on('error', reject)

package.json

@@ -74,7 +74,8 @@
     "typecheck:node": "pnpm run build:packages && node config/scripts/run-typecheck-projects-in-parallel.mjs --project=config/tsconfig.node.json",
     "typecheck:cli": "pnpm run build:packages && node config/scripts/run-typecheck-projects-in-parallel.mjs --project=config/tsconfig.tc.cli.json",
     "typecheck:web": "pnpm run build:packages && node config/scripts/run-typecheck-projects-in-parallel.mjs --project=config/tsconfig.tc.web.json",
-    "typecheck:e2e": "pnpm run build:packages && node node_modules/typescript/bin/tsc --noEmit -p config/tsconfig.e2e.json",
+    "typecheck:e2e": "pnpm run build:packages && node config/scripts/run-e2e-typecheck.mjs",
+    "typecheck:tsc:e2e": "pnpm run build:packages && node config/scripts/run-e2e-typecheck.mjs --compiler=native",
     "typecheck": "pnpm run build:packages && node config/scripts/run-typecheck-projects-in-parallel.mjs",
     "typecheck:ci": "pnpm run build:packages && node config/scripts/run-typecheck-projects-in-parallel.mjs --compiler=bun",
     "typecheck:tsc:node": "pnpm run build:packages && node node_modules/typescript/bin/tsc --noEmit -p config/tsconfig.node.json --composite false",