ci: restore main's Vitest transform cache in PR unit shards (#34512)

* ci: restore main's Vitest transform cache in PR unit shards Main produces per-shard transform caches on a schedule; PR shards restore the newest one for the same config/lockfile/runtime policy. A Vitest cache key generator folds each literal specifier's resolution candidates into the per-module key so newly added files that shadow a resolution invalidate the importer, and bails on glob-expanded dynamic imports. * ci: list the resolution key plugin in the node tsconfig project * ci: accept Vitest's CACHEDIR.TAG when sealing the unit module cache Vitest 5.0.5 tags its cache root, which made every seal fail with 'invalid cache entry name' so no producer cache was ever saved. * ci: reseed the unit module cache when any cache-key input lands on main Co-Authored-By: Claude <noreply@anthropic.com> --------- Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com>
2026-10-30 16:47:34 +00:00 · 2026-10-27 20:04:56 -07:00
parent 6bb0c14a98
commit 8b64ec281a
11 changed files with 273 additions and 35 deletions
@@ -888,6 +888,8 @@ jobs:
      node_versions: '["24"]'
      runner: blacksmith-8vcpu-ubuntu-2404
      phase_diagnostics: true
+      # Restore main's transform cache; the `ci:cold-unit-cache` label or ORCA_UNIT_MODULE_CACHE=off forces cold.
+      module_cache: ${{ (vars.ORCA_UNIT_MODULE_CACHE == 'off' || contains(github.event.pull_request.labels.*.name, 'ci:cold-unit-cache')) && 'off' || 'warm' }}
      shards: ${{ needs.preflight.outputs.shards }}
      run_shards: ${{ needs.code_paths.outputs.mobile_ci_tools != 'true' }}
      run_relay_integration: ${{ needs.code_paths.outputs.relay_integration != 'false' || needs.code_paths.outputs.orcad_browser == 'true' }}
@@ -0,0 +1,50 @@
+name: Unit transform cache producer
+
+# Cold full unit run on main that publishes per-shard Vitest transform caches for PRs.
+on:
+  schedule:
+    - cron: '17 */6 * * *'
+  workflow_dispatch:
+  push:
+    branches: [main]
+    # Cache-key inputs (see ci-unit-module-cache.mjs) change the key; reseed right away.
+    paths:
+      - '.github/workflows/unit-module-cache.yml'
+      - '.github/workflows/unit-tests.yml'
+      - 'config/vitest.config.ts'
+      - 'config/scripts/vitest*'
+      - 'config/scripts/ci-unit-module-cache*.mjs'
+      - 'config/patches/**'
+      - 'config/scripts/workspace-source-exports.mjs'
+      - 'config/dark-build-switches.cjs'
+      - 'config/pnpmfile.cjs'
+      - '.npmrc'
+      - '**/package.json'
+      - '**/tsconfig*.json'
+      - 'pnpm-lock.yaml'
+      - 'pnpm-workspace.yaml'
+      - 'config/.bun-version'
+
+concurrency:
+  group: unit-module-cache-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  unit_plan:
+    if: github.ref == 'refs/heads/main'
+    uses: ./.github/workflows/unit-plan.yml
+
+  test:
+    needs: unit_plan
+    uses: ./.github/workflows/unit-tests.yml
+    with:
+      # Match the PR unit matrix so the cache key's runtime and shard fields line up.
+      node_versions: '["24"]'
+      runner: blacksmith-8vcpu-ubuntu-2404
+      shards: ${{ needs.unit_plan.outputs.shards }}
+      module_cache: cold
+      artifact_prefix: cache-producer-unit
+      run_relay_integration: false
@@ -35,6 +35,8 @@ jobs:
        env:
          ORCA_UNIT_SELECTION_MODE: ${{ inputs.selection_mode }}
          ORCA_PULL_REQUEST_DRAFT: ${{ github.event.pull_request.draft }}
+          # Same count as PR preflight so transform cache shard keys line up.
+          ORCA_UNIT_FULL_SHARD_COUNT: ${{ vars.ORCA_UNIT_FULL_SHARD_COUNT || '5' }}
        run: node config/scripts/ci-unit-plan.mjs
      - uses: actions/upload-artifact@v7
        continue-on-error: true
@@ -32,7 +32,7 @@ on:
        type: string

      module_cache:
-        description: Transform cache policy (off, cold producer, or warm exact restore).
+        description: Transform cache policy (off, cold producer, or warm restore).
        default: 'off'
        type: string

@@ -110,19 +110,22 @@ jobs:
          ORCA_UNIT_CACHE_NAMESPACE: ${{ inputs.module_cache_namespace }}
        run: node config/scripts/ci-unit-module-cache.mjs identity

-      - name: Restore exact unit module cache
+      # The key pins config, plugins, lockfile and runtimes; the newest producer for it wins.
+      # Source edits are keyed per module by Vitest plus vitest-module-cache-resolution-key.ts.
+      - name: Restore unit module cache
        id: module-cache-restore
        if: inputs.module_cache == 'warm' && steps.module-cache.outputs.key != ''
        continue-on-error: true
        uses: actions/cache/restore@v5
        with:
          path: ${{ steps.module-cache.outputs.path }}
-          key: ${{ steps.module-cache.outputs.key }}
+          key: ${{ steps.module-cache.outputs.key }}-${{ github.sha }}
+          restore-keys: ${{ steps.module-cache.outputs.key }}-

      - name: Validate restored unit module cache or use cold transforms
        if: steps.module-cache.outputs.key != ''
        env:
-          ORCA_UNIT_CACHE_RESTORE_HIT: ${{ steps.module-cache-restore.outputs.cache-hit }}
+          ORCA_UNIT_CACHE_RESTORE_HIT: ${{ steps.module-cache-restore.outputs.cache-matched-key != '' }}
        run: node config/scripts/ci-unit-module-cache.mjs prepare

      - uses: actions/download-artifact@v8
@@ -155,11 +158,11 @@ jobs:

      - name: Seal unit module cache
        id: module-cache-seal
+        # Only cold main/seeded producers publish; PRs consume and never seed each other.
        if: >-
          success() && steps.module-cache.outputs.key != '' &&
-          steps.module-cache-restore.outputs.cache-hit != 'true' &&
-          (github.event_name == 'pull_request' ||
-          github.ref == 'refs/heads/main' && github.event_name != 'pull_request' ||
+          inputs.module_cache == 'cold' &&
+          (github.ref == 'refs/heads/main' && github.event_name != 'pull_request' ||
          inputs.module_cache_seed && github.event_name == 'workflow_dispatch')
        run: node config/scripts/ci-unit-module-cache.mjs seal

@@ -169,7 +172,7 @@ jobs:
        uses: actions/cache/save@v5
        with:
          path: ${{ steps.module-cache.outputs.path }}
-          key: ${{ steps.module-cache.outputs.key }}
+          key: ${{ steps.module-cache.outputs.key }}-${{ github.run_id }}-${{ github.run_attempt }}

      - name: Upload unit shard assignment
        if: always()
@@ -11,6 +11,8 @@ export const MODULE_CACHE_LIMITS = Object.freeze({
  entryBytes: 32 * 2 ** 20
 })
 const METADATA = '_metadata.json'
+// Vitest 5.0.5+ marks its cache root with a fixed-content CACHEDIR.TAG.
+const CACHEDIR_TAG = 'CACHEDIR.TAG'
 const MODULE_NAME = /^[a-f0-9]{40}$/
 const isRecord = (value) => value !== null && typeof value === 'object' && !Array.isArray(value)
 const fail = (message) => {
@@ -76,7 +78,12 @@ export async function inventoryModuleCache({ path, lockfileHash }) {
      while (next < names.length) {
        const index = next++
        const name = names[index]
-        if (name !== METADATA && name !== MODULE_CACHE_MANIFEST && !MODULE_NAME.test(name)) {
+        if (
+          name !== METADATA &&
+          name !== MODULE_CACHE_MANIFEST &&
+          name !== CACHEDIR_TAG &&
+          !MODULE_NAME.test(name)
+        ) {
          fail(`invalid cache entry name: ${name}`)
        }
        const buffer = await readRegularFile(join(path, name))
@@ -96,6 +103,10 @@ export async function inventoryModuleCache({ path, lockfileHash }) {
          if (!isRecord(metadata) || metadata.lockfileHash !== lockfileHash) {
            fail('cache lockfile hash mismatch')
          }
+        } else if (name === CACHEDIR_TAG) {
+          if (!buffer.toString('utf8').startsWith('Signature: 8a477f597d28d172789f06886806bc55')) {
+            fail('invalid cache directory tag')
+          }
        } else {
          checkModuleTrailer(buffer)
        }
@@ -52,7 +52,8 @@ export async function moduleCachePolicyHash(root = ROOT) {
  return digestInputs(root, [
    ...configInputs,
    ...workspacePackageManifests(root),
-    ...globSync(['config/patches/**', '**/tsconfig*.json'], {
+    // tsconfig and package.json (type/imports) steer transforms but are not in Vitest's per-module key.
+    ...globSync(['config/patches/**', '**/tsconfig*.json', '**/package.json'], {
      cwd: root,
      exclude: ['**/node_modules/**', '**/.git/**', '**/dist/**', '**/out/**', 'mobile/vendor/**']
    }).filter((file) => statSync(join(root, file)).isFile()),
@@ -62,6 +63,7 @@ export async function moduleCachePolicyHash(root = ROOT) {
    '.npmrc',
    'config/pnpmfile.cjs',
    'config/.bun-version',
+    'config/dark-build-switches.cjs',
    'config/scripts/ci-unit-module-cache.mjs',
    'config/scripts/ci-unit-module-cache-contents.mjs',
    'config/scripts/ci-unit-module-cache-trailer.mjs'
@@ -109,7 +111,7 @@ export function moduleCacheIdentity({
  const digest = createHash('sha256').update(JSON.stringify(identity)).digest('hex')
  return {
    ...identity,
-    key: `unit-module-v1-${platform}-${arch}-n${versions.node}-b${versions.bun}-v${versions.vitest}-s${shard.index}of${shard.count}-${digest}`,
+    key: `unit-module-v2-${platform}-${arch}-n${versions.node}-b${versions.bun}-v${versions.vitest}-s${shard.index}of${shard.count}-${digest}`,
    path: join(resolve(cacheRoot), 'orca-vitest-module-cache')
  }
 }
@@ -117,7 +117,7 @@ describe('persistent unit module cache boundary', () => {
  })
 })

-it('uses an optional exact restore after install and saves successful scoped producers', () => {
+it('restores the newest same-policy producer and saves only cold main or seeded producers', () => {
  const workflow = parse(readFileSync('.github/workflows/unit-tests.yml', 'utf8'))
  const steps = workflow.jobs.test.steps
  const dependencies = steps.findIndex(
@@ -134,33 +134,30 @@ it('uses an optional exact restore after install and saves successful scoped pro
  expect(prepare).toBeGreaterThan(restore)
  expect(tests).toBeGreaterThan(prepare)
  expect(steps[restore].uses).toBe('actions/cache/restore@v5')
-  expect(steps[restore].with['restore-keys']).toBeUndefined()
+  expect(steps[restore].with['restore-keys']).toBe('${{ steps.module-cache.outputs.key }}-')
  expect(steps[restore]['continue-on-error']).toBe(true)
-  const guard = seal.if
-    .replaceAll('steps.module-cache.outputs.key', 'steps["module-cache"].outputs.key')
-    .replaceAll(
-      'steps.module-cache-restore.outputs.cache-hit',
-      'steps["module-cache-restore"].outputs["cache-hit"]'
-    )
+  const guard = seal.if.replaceAll(
+    'steps.module-cache.outputs.key',
+    'steps["module-cache"].outputs.key'
+  )
  const producer = {
-    event: 'pull_request',
-    ref: 'refs/pull/1/merge',
+    event: 'schedule',
+    ref: 'refs/heads/main',
    success: true,
    key: 'scoped',
-    hit: '',
+    mode: 'cold',
    seed: false
  }
  for (const [name, update, expected] of [
-    ['successful PR cache miss', {}, true],
-    ['successful main producer', { event: 'push', ref: 'refs/heads/main' }, true],
+    ['successful main producer', {}, true],
+    ['PR', { event: 'pull_request', ref: 'refs/pull/1/merge' }, false],
+    ['warm main consumer', { mode: 'warm' }, false],
    [
      'successful manually seeded branch',
      { event: 'workflow_dispatch', ref: 'refs/heads/experiment', seed: true },
      true
    ],
-    ['failed PR', { success: false }, false],
    ['missing identity', { key: '' }, false],
-    ['exact restored hit', { hit: 'true' }, false],
    ['ordinary branch push', { event: 'push', ref: 'refs/heads/experiment' }, false],
    ['unseeded manual branch', { event: 'workflow_dispatch', ref: 'refs/heads/experiment' }, false],
    [
@@ -174,16 +171,15 @@ it('uses an optional exact restore after install and saves successful scoped pro
    const admitted = runInNewContext(guard, {
      github: { event_name: control.event, ref: control.ref },
      steps: {
-        'module-cache': { outputs: { key: control.key } },
-        'module-cache-restore': { outputs: { 'cache-hit': control.hit } }
+        'module-cache': { outputs: { key: control.key } }
      },
-      inputs: { module_cache_seed: control.seed },
+      inputs: { module_cache: control.mode, module_cache_seed: control.seed },
      success: () => control.success
    })
    expect(admitted, name).toBe(expected)
  }
  expect(save.if).toBe("steps.module-cache-seal.outputs.ready == 'true'")
-  expect(save.with.key).toBe(steps[restore].with.key)
+  expect(save.with.key.startsWith(steps[restore].with['restore-keys'])).toBe(true)
  expect(workflow.on.workflow_call.inputs.module_cache.default).toBe('off')
  expect(workflow.on.workflow_call.inputs.run_relay_integration.default).toBe(true)
 })
@@ -0,0 +1,57 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, expect, it } from 'vitest'
+import {
+  createResolutionFingerprint,
+  hasExpandedImport
+} from './vitest-module-cache-resolution-key'
+
+const roots: string[] = []
+afterEach(() => {
+  for (const root of roots.splice(0)) {
+    rmSync(root, { recursive: true, force: true })
+  }
+})
+
+function fixture(): string {
+  const root = mkdtempSync(join(tmpdir(), 'orca-resolution-key-'))
+  roots.push(root)
+  mkdirSync(join(root, 'src/lib/foo'), { recursive: true })
+  writeFileSync(join(root, 'src/lib/foo/index.ts'), '')
+  return root
+}
+
+const source = "import a from './lib/foo'\nimport b from '@/lib/foo?raw'"
+const aliases = (root: string) => [{ find: '@', replacement: join(root, 'src') }]
+
+it.each([
+  ['a same-stem sibling shadows a directory index', 'src/lib/foo.ts'],
+  ['a higher-precedence extension appears', 'src/lib/foo.mts'],
+  ['another index candidate appears', 'src/lib/foo/index.mts'],
+  ['the directory gains a package.json', 'src/lib/foo/package.json']
+])('changes the key when %s', (_, added) => {
+  const root = fixture()
+  const before = createResolutionFingerprint(root, aliases(root))(join(root, 'src/a.ts'), source)
+  writeFileSync(join(root, added), '{}')
+  const after = createResolutionFingerprint(root, aliases(root))(join(root, 'src/a.ts'), source)
+  expect(after).not.toBe(before)
+})
+
+it('keeps the key when an unrelated sibling appears', () => {
+  const root = fixture()
+  const before = createResolutionFingerprint(root, aliases(root))(join(root, 'src/a.ts'), source)
+  writeFileSync(join(root, 'src/lib/bar.ts'), '')
+  const after = createResolutionFingerprint(root, aliases(root))(join(root, 'src/a.ts'), source)
+  expect(after).toBe(before)
+})
+
+it.each([
+  ['import(`./locales/${name}.ts`)', true],
+  ["import('./locales/' + name + '.ts')", true],
+  ['import.meta.glob("./*.ts")', true],
+  ["import('./locales/en.ts')", false],
+  ['import(specifier)', false]
+])('treats %s as directory-dependent: %s', (code, expected) => {
+  expect(hasExpandedImport(code)).toBe(expected)
+})
@@ -0,0 +1,113 @@
+import { readdirSync, readFileSync } from 'node:fs'
+import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'
+import type { Alias, Plugin } from 'vite'
+
+// Why: Vitest keys a cached transform by its own source, so a new sibling such as
+// `foo.ts` beside `foo/index.ts` would keep serving the old resolution. Fold every
+// candidate a literal specifier could resolve to into the key instead.
+const LITERAL = /(?=(['"`])([^'"`\n]*)\1)/g
+const DYNAMIC_IMPORT = /\bimport\s*\(\s*/g
+const PLAIN_LITERAL_ARGUMENT = /^(['"`])[^'"`\n$]*\1\s*[,)]/
+
+// Vite expands template/concatenated dynamic imports into globs over directory contents.
+export function hasExpandedImport(source: string): boolean {
+  if (/\bimport\.meta\.glob\b/.test(source)) {
+    return true
+  }
+  for (const match of source.matchAll(DYNAMIC_IMPORT)) {
+    const rest = source.slice(match.index + match[0].length)
+    if (/^['"`]/.test(rest) && !PLAIN_LITERAL_ARGUMENT.test(rest)) {
+      return true
+    }
+  }
+  return false
+}
+
+function applyAlias(specifier: string, aliases: readonly Alias[]): string | undefined {
+  for (const { find, replacement } of aliases) {
+    if (typeof find === 'string') {
+      if (specifier === find || specifier.startsWith(`${find}/`)) {
+        return replacement + specifier.slice(find.length)
+      }
+    } else if (find.test(specifier)) {
+      return specifier.replace(find, replacement)
+    }
+  }
+  return undefined
+}
+
+export function createResolutionFingerprint(root: string, aliases: readonly Alias[]) {
+  const listings = new Map<string, readonly string[]>()
+  const list = (dir: string): readonly string[] => {
+    let names = listings.get(dir)
+    if (!names) {
+      try {
+        // Directories carry a trailing slash so a file/directory swap also changes the key.
+        names = readdirSync(dir, { withFileTypes: true })
+          .map((entry) => (entry.isDirectory() ? `${entry.name}/` : entry.name))
+          .sort()
+      } catch {
+        names = []
+      }
+      listings.set(dir, names)
+    }
+    return names
+  }
+  // Outside-root strings (e.g. '/tmp') are not module specifiers here and their listings churn.
+  const insideRoot = (path: string) => {
+    const rel = relative(root, path)
+    return rel !== '' && rel !== '..' && !rel.startsWith(`..${sep}`) && !isAbsolute(rel)
+  }
+  // Every file Vite could pick for `base`: same-stem siblings, plus index/package.json if a directory.
+  const candidates = (base: string): string => {
+    const dir = dirname(base)
+    const stem = basename(base).split('.')[0]
+    const parts: string[] = []
+    for (const name of list(dir).filter((entry) => entry.startsWith(stem))) {
+      parts.push(name)
+      if (name.endsWith('/')) {
+        const path = join(dir, name)
+        const inner = list(path)
+        const entries = inner.filter((entry) => /^(?:index\.|package\.json$)/.test(entry))
+        parts.push(`[${entries.join(',')}]`)
+        if (inner.includes('package.json')) {
+          parts.push(readFileSync(join(path, 'package.json'), 'utf8'))
+        }
+      }
+    }
+    return `${relative(root, base)}:${parts.join(',')}`
+  }
+  return (id: string, source: string): string => {
+    const importerDir = dirname(id.split('?')[0])
+    const seen = new Set<string>()
+    for (const [, , literal] of source.matchAll(LITERAL)) {
+      const specifier = literal.split(/[?#]/)[0]
+      let target: string | undefined
+      if (specifier === '.' || specifier === '..' || /^\.{1,2}\//.test(specifier)) {
+        target = resolve(importerDir, specifier)
+      } else if (specifier.startsWith('/')) {
+        target = specifier.startsWith(root) ? specifier : join(root, specifier)
+      } else {
+        const aliased = applyAlias(specifier, aliases)
+        target = aliased && isAbsolute(aliased) ? aliased : undefined
+      }
+      if (target && insideRoot(target)) {
+        seen.add(target)
+      }
+    }
+    return [...seen].sort().map(candidates).join('\n')
+  }
+}
+
+export function moduleCacheResolutionKey(): Plugin {
+  return {
+    name: 'module-cache-resolution-key',
+    configureVitest({ project, defineCacheKeyGenerator }) {
+      const config = project.vite.config
+      const fingerprint = createResolutionFingerprint(config.root, config.resolve.alias)
+      defineCacheKeyGenerator(({ id, sourceCode }) =>
+        hasExpandedImport(sourceCode) ? false : fingerprint(id, sourceCode)
+      )
+    }
+  }
+}
@@ -13,6 +13,7 @@
    "./scripts/happy-dom-mutation-observer-retention.ts",
    "./scripts/happy-dom-animation-cancel.ts",
    "./scripts/vitest-node-runtime-pool.ts",
+    "./scripts/vitest-module-cache-resolution-key.ts",
    "../src/main/**/*",
    "../src/renderer/src/store/slices/worktrees/create/worktree-create-payload.ts",
    "../src/renderer/src/components/native-chat/native-chat-resolution-receipt.ts",
@@ -10,11 +10,13 @@ import { getNodeRuntimeInclude } from './scripts/vitest-node-runtime-files.mjs'
 import { nodeRuntimePool } from './scripts/vitest-node-runtime-pool'
 import { readRunFilters } from './scripts/vitest-run-filters.mjs'
 import { createWorkspaceSourceResolver } from './scripts/workspace-source-exports.mjs'
+import { moduleCacheResolutionKey } from './scripts/vitest-module-cache-resolution-key'

 const nodeRuntimeInclude = process.versions.bun
  ? getNodeRuntimeInclude({ filters: readRunFilters() })
  : []
 const balancedShards = process.env.ORCA_BALANCE_UNIT_SHARDS === '1'
+const moduleCachePath = process.env.ORCA_UNIT_MODULE_CACHE_PATH
 const measurementFile = 'src/main/foreign-sqlite-readers/foreign-sqlite-reader-event-loop.test.ts'
 const paletteMeasurementFile =
  'src/renderer/src/lib/palette-match/palette-match-performance.test.ts'
@@ -45,6 +47,7 @@ const transforms = {
        return { code: result.code, map: result.map }
      }
    } satisfies Plugin,
+    ...(moduleCachePath ? [moduleCacheResolutionKey()] : []),
    {
      // Unit tests have no asset server; retain real catalog parsing through fetch(data:).
      name: 'locale-catalog-test-assets',
@@ -83,11 +86,9 @@ const testOptions = {
  environment: 'node',
  clearMocks: false,
  ...(balancedShards ? { silent: 'passed-only' as const } : {}),
-  // Persistent transforms can miss source additions; only explicit cache experiments opt in.
-  fsModuleCache: Boolean(process.env.ORCA_UNIT_MODULE_CACHE_PATH),
-  ...(process.env.ORCA_UNIT_MODULE_CACHE_PATH
-    ? { fsModuleCachePath: process.env.ORCA_UNIT_MODULE_CACHE_PATH }
-    : {}),
+  // Persistent transforms are CI-keyed (ci-unit-module-cache.mjs); local runs stay cold.
+  fsModuleCache: Boolean(moduleCachePath),
+  ...(moduleCachePath ? { fsModuleCachePath: moduleCachePath } : {}),
  env: { ORCA_VITEST_RUNTIME: 'node' },
  server: { deps: { inline: ['zod'] } },
  // Node's storage globals and V8 retention checks require the existing child flags.