fix(build): admit typecheck projects by memory, not core count (#22074)

The typecheck job ran all four tsc projects at once whenever the machine had more than one core. Two of them are expensive -- tsconfig.node.json peaks at 6.3 GB of tsc heap and tsconfig.tc.web.json at 5.6 GB, measured with --extendedDiagnostics -- so together they reach ~14.5 GB on a 16 GB runner. Past that the runner agent is killed mid-check, and the job reports "The runner has received a shutdown signal" with an orphaned tsc, not a type error. Attempt-1 typecheck failures were 0 across Sept 11-17 and then 5-26% per day from Sept 18, with no change to the scheduler in that window. What moved was the codebase: src/main grew 23% and src/renderer 8.5% between Sept 1 and Sept 21, which is what pushed the pair over the line. Projects now carry their measured peak heap and are admitted heaviest-first while the batch fits both a memory budget and the core count, so the two expensive projects never share a runner. On a 16 GB / 4-core runner that plans node+cli+mobile-web (10 GiB) then web (6 GiB), measured at 8.6 GB peak instead of 14.5 GB. A roomy machine still runs all four together, so local typecheck is unchanged. A project larger than the whole budget still runs alone rather than producing an empty batch. The runner body moves behind the standard direct-invocation guard so the admission planner can be imported and tested without spawning tsc.
2026-09-22 00:02:31 +00:00 · 2026-09-21 14:59:31 -07:00
parent 297cfe0cf3
commit 93e4407180
2 changed files with 149 additions and 26 deletions
@@ -1,21 +1,61 @@
 import { spawn } from 'node:child_process'
-import { availableParallelism } from 'node:os'
-import { fileURLToPath } from 'node:url'
+import { availableParallelism, totalmem } from 'node:os'
+import { fileURLToPath, pathToFileURL } from 'node:url'

-// These projects overlap heavily in src/shared but have no build dependency on
-// each other, so tsc can check them concurrently instead of in a `&&` chain.
-const projects = [
-  'tsconfig.node.json',
-  'tsconfig.tc.cli.json',
-  'tsconfig.tc.web.json',
-  'tsconfig.mobile-web.json'
+const BYTES_PER_GIB = 1024 ** 3
+
+// Peak heap per project, read from `tsc --extendedDiagnostics` and rounded up. node and
+// web are the expensive pair: run together they exceed a 16 GB CI runner, and an
+// out-of-memory runner is killed mid-check, so the job reports a lost runner instead of a
+// type error. Admission is therefore by memory, not by core count alone.
+export const TYPECHECK_PROJECTS = [
+  { config: 'tsconfig.node.json', heapGib: 7 },
+  { config: 'tsconfig.tc.web.json', heapGib: 6 },
+  { config: 'tsconfig.tc.cli.json', heapGib: 2 },
+  { config: 'tsconfig.mobile-web.json', heapGib: 1 }
 ]
+
+// The OS, node itself, and the runner agent need their share; the rest is what tsc may hold.
+export function admissibleHeapGib(totalBytes) {
+  return Math.max(1, (totalBytes / BYTES_PER_GIB) * 0.75)
+}
+
+/**
+ * Heaviest first, admitting another project only while it fits both the memory budget and
+ * the core count. A project larger than the whole budget still runs, alone, so a small
+ * machine makes progress rather than producing an empty batch forever.
+ */
+export function planTypecheckBatches(projects, { budgetGib, parallelism }) {
+  const pending = [...projects].sort((left, right) => right.heapGib - left.heapGib)
+  const batches = []
+
+  while (pending.length > 0) {
+    const batch = []
+    let claimed = 0
+
+    for (let index = 0; index < pending.length;) {
+      const project = pending[index]
+      const admit =
+        batch.length === 0 || (batch.length < parallelism && claimed + project.heapGib <= budgetGib)
+
+      if (admit) {
+        batch.push(project)
+        claimed += project.heapGib
+        pending.splice(index, 1)
+      } else {
+        index += 1
+      }
+    }
+
+    batches.push(batch)
+  }
+
+  return batches
+}
+
 const repoRoot = fileURLToPath(new URL('../..', import.meta.url))
 const tsc = fileURLToPath(new URL('../../node_modules/typescript/bin/tsc', import.meta.url))

-// Why serialize on a single-core runner: three tsc processes there thrash rather than overlap.
-const concurrent = availableParallelism() > 1
-
 function checkProject(project) {
  return new Promise((resolve, reject) => {
    const child = spawn(process.execPath, [tsc, '--noEmit', '-p', `config/${project}`], {
@@ -36,23 +76,32 @@ function checkProject(project) {
  })
 }

-let failures = []
-if (concurrent) {
-  const results = await Promise.allSettled(projects.map(checkProject))
-  failures = results.filter((result) => result.status === 'rejected').map((result) => result.reason)
-} else {
-  for (const project of projects) {
-    try {
-      await checkProject(project)
-    } catch (error) {
-      failures.push(error)
+async function runTypecheckProjects() {
+  const batches = planTypecheckBatches(TYPECHECK_PROJECTS, {
+    budgetGib: admissibleHeapGib(totalmem()),
+    parallelism: availableParallelism()
+  })
+
+  // Every batch runs even after one fails, so a single broken project still reports the rest.
+  const failures = []
+  for (const batch of batches) {
+    const results = await Promise.allSettled(batch.map((project) => checkProject(project.config)))
+    for (const result of results) {
+      if (result.status === 'rejected') {
+        failures.push(result.reason)
+      }
    }
  }
+
+  return failures
 }

-if (failures.length > 0) {
-  for (const failure of failures) {
-    console.error(failure.message ?? failure)
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  const failures = await runTypecheckProjects()
+  if (failures.length > 0) {
+    for (const failure of failures) {
+      console.error(failure.message ?? failure)
+    }
+    process.exit(1)
  }
-  process.exit(1)
 }
@@ -0,0 +1,74 @@
+import { describe, expect, it } from 'vitest'
+import {
+  TYPECHECK_PROJECTS,
+  admissibleHeapGib,
+  planTypecheckBatches
+} from './run-typecheck-projects-in-parallel.mjs'
+
+const CI_RUNNER = { totalBytes: 16 * 1024 ** 3, parallelism: 4 }
+const DEV_LAPTOP = { totalBytes: 64 * 1024 ** 3, parallelism: 18 }
+
+function planFor({ totalBytes, parallelism }, projects = TYPECHECK_PROJECTS) {
+  return planTypecheckBatches(projects, {
+    budgetGib: admissibleHeapGib(totalBytes),
+    parallelism
+  })
+}
+
+function batchOf(batches, config) {
+  return batches.findIndex((batch) => batch.some((project) => project.config === config))
+}
+
+describe('typecheck project admission', () => {
+  it('keeps the two expensive projects off the same CI runner', () => {
+    const batches = planFor(CI_RUNNER)
+    expect(batchOf(batches, 'tsconfig.node.json')).not.toBe(
+      batchOf(batches, 'tsconfig.tc.web.json')
+    )
+  })
+
+  it('holds every CI batch inside the memory budget', () => {
+    const budget = admissibleHeapGib(CI_RUNNER.totalBytes)
+    for (const batch of planFor(CI_RUNNER)) {
+      const claimed = batch.reduce((total, project) => total + project.heapGib, 0)
+      expect(claimed).toBeLessThanOrEqual(budget)
+    }
+  })
+
+  it('still fills a CI batch with the cheap projects rather than serializing everything', () => {
+    // Why: strict serialization measured ~60% slower than pairing the cheap work alongside.
+    expect(planFor(CI_RUNNER).length).toBeLessThan(TYPECHECK_PROJECTS.length)
+  })
+
+  it('leaves a roomy machine fully parallel', () => {
+    expect(planFor(DEV_LAPTOP)).toHaveLength(1)
+  })
+
+  it('serializes on a single core', () => {
+    const batches = planFor({ totalBytes: 64 * 1024 ** 3, parallelism: 1 })
+    expect(batches).toHaveLength(TYPECHECK_PROJECTS.length)
+    expect(batches.every((batch) => batch.length === 1)).toBe(true)
+  })
+
+  it('runs a project that alone exceeds the budget instead of stalling', () => {
+    const batches = planTypecheckBatches([{ config: 'huge.json', heapGib: 512 }], {
+      budgetGib: admissibleHeapGib(2 * 1024 ** 3),
+      parallelism: 4
+    })
+    expect(batches).toEqual([[{ config: 'huge.json', heapGib: 512 }]])
+  })
+
+  it('schedules every project exactly once', () => {
+    const scheduled = planFor(CI_RUNNER)
+      .flat()
+      .map((project) => project.config)
+      .sort()
+    expect(scheduled).toEqual(TYPECHECK_PROJECTS.map((project) => project.config).sort())
+  })
+
+  it('never lets one project outgrow a CI runner on its own', () => {
+    // A single project over budget runs anyway, so this is the ceiling the pool cannot rescue.
+    const heaviest = Math.max(...TYPECHECK_PROJECTS.map((project) => project.heapGib))
+    expect(heaviest).toBeLessThanOrEqual(admissibleHeapGib(CI_RUNNER.totalBytes))
+  })
+})