diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index da43aa890a5..d76f7063a51 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -888,6 +888,8 @@ jobs: node_versions: '["24"]' runner: blacksmith-8vcpu-ubuntu-2404 phase_diagnostics: true + # Restore main's transform cache; the `ci:cold-unit-cache` label or ORCA_UNIT_MODULE_CACHE=off forces cold. + module_cache: ${{ (vars.ORCA_UNIT_MODULE_CACHE == 'off' || contains(github.event.pull_request.labels.*.name, 'ci:cold-unit-cache')) && 'off' || 'warm' }} shards: ${{ needs.preflight.outputs.shards }} run_shards: ${{ needs.code_paths.outputs.mobile_ci_tools != 'true' }} run_relay_integration: ${{ needs.code_paths.outputs.relay_integration != 'false' || needs.code_paths.outputs.orcad_browser == 'true' }} diff --git a/.github/workflows/unit-module-cache.yml b/.github/workflows/unit-module-cache.yml new file mode 100644 index 00000000000..ff2683b8289 --- /dev/null +++ b/.github/workflows/unit-module-cache.yml @@ -0,0 +1,50 @@ +name: Unit transform cache producer + +# Cold full unit run on main that publishes per-shard Vitest transform caches for PRs. +on: + schedule: + - cron: '17 */6 * * *' + workflow_dispatch: + push: + branches: [main] + # Cache-key inputs (see ci-unit-module-cache.mjs) change the key; reseed right away. + paths: + - '.github/workflows/unit-module-cache.yml' + - '.github/workflows/unit-tests.yml' + - 'config/vitest.config.ts' + - 'config/scripts/vitest*' + - 'config/scripts/ci-unit-module-cache*.mjs' + - 'config/patches/**' + - 'config/scripts/workspace-source-exports.mjs' + - 'config/dark-build-switches.cjs' + - 'config/pnpmfile.cjs' + - '.npmrc' + - '**/package.json' + - '**/tsconfig*.json' + - 'pnpm-lock.yaml' + - 'pnpm-workspace.yaml' + - 'config/.bun-version' + +concurrency: + group: unit-module-cache-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + unit_plan: + if: github.ref == 'refs/heads/main' + uses: ./.github/workflows/unit-plan.yml + + test: + needs: unit_plan + uses: ./.github/workflows/unit-tests.yml + with: + # Match the PR unit matrix so the cache key's runtime and shard fields line up. + node_versions: '["24"]' + runner: blacksmith-8vcpu-ubuntu-2404 + shards: ${{ needs.unit_plan.outputs.shards }} + module_cache: cold + artifact_prefix: cache-producer-unit + run_relay_integration: false diff --git a/.github/workflows/unit-plan.yml b/.github/workflows/unit-plan.yml index 0080a184bfb..30c7cf10715 100644 --- a/.github/workflows/unit-plan.yml +++ b/.github/workflows/unit-plan.yml @@ -35,6 +35,8 @@ jobs: env: ORCA_UNIT_SELECTION_MODE: ${{ inputs.selection_mode }} ORCA_PULL_REQUEST_DRAFT: ${{ github.event.pull_request.draft }} + # Same count as PR preflight so transform cache shard keys line up. + ORCA_UNIT_FULL_SHARD_COUNT: ${{ vars.ORCA_UNIT_FULL_SHARD_COUNT || '5' }} run: node config/scripts/ci-unit-plan.mjs - uses: actions/upload-artifact@v7 continue-on-error: true diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 4e85f2d41de..e2121b3775d 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -32,7 +32,7 @@ on: type: string module_cache: - description: Transform cache policy (off, cold producer, or warm exact restore). + description: Transform cache policy (off, cold producer, or warm restore). default: 'off' type: string @@ -110,19 +110,22 @@ jobs: ORCA_UNIT_CACHE_NAMESPACE: ${{ inputs.module_cache_namespace }} run: node config/scripts/ci-unit-module-cache.mjs identity - - name: Restore exact unit module cache + # The key pins config, plugins, lockfile and runtimes; the newest producer for it wins. + # Source edits are keyed per module by Vitest plus vitest-module-cache-resolution-key.ts. + - name: Restore unit module cache id: module-cache-restore if: inputs.module_cache == 'warm' && steps.module-cache.outputs.key != '' continue-on-error: true uses: actions/cache/restore@v5 with: path: ${{ steps.module-cache.outputs.path }} - key: ${{ steps.module-cache.outputs.key }} + key: ${{ steps.module-cache.outputs.key }}-${{ github.sha }} + restore-keys: ${{ steps.module-cache.outputs.key }}- - name: Validate restored unit module cache or use cold transforms if: steps.module-cache.outputs.key != '' env: - ORCA_UNIT_CACHE_RESTORE_HIT: ${{ steps.module-cache-restore.outputs.cache-hit }} + ORCA_UNIT_CACHE_RESTORE_HIT: ${{ steps.module-cache-restore.outputs.cache-matched-key != '' }} run: node config/scripts/ci-unit-module-cache.mjs prepare - uses: actions/download-artifact@v8 @@ -155,11 +158,11 @@ jobs: - name: Seal unit module cache id: module-cache-seal + # Only cold main/seeded producers publish; PRs consume and never seed each other. if: >- success() && steps.module-cache.outputs.key != '' && - steps.module-cache-restore.outputs.cache-hit != 'true' && - (github.event_name == 'pull_request' || - github.ref == 'refs/heads/main' && github.event_name != 'pull_request' || + inputs.module_cache == 'cold' && + (github.ref == 'refs/heads/main' && github.event_name != 'pull_request' || inputs.module_cache_seed && github.event_name == 'workflow_dispatch') run: node config/scripts/ci-unit-module-cache.mjs seal @@ -169,7 +172,7 @@ jobs: uses: actions/cache/save@v5 with: path: ${{ steps.module-cache.outputs.path }} - key: ${{ steps.module-cache.outputs.key }} + key: ${{ steps.module-cache.outputs.key }}-${{ github.run_id }}-${{ github.run_attempt }} - name: Upload unit shard assignment if: always() diff --git a/config/scripts/ci-unit-module-cache-contents.mjs b/config/scripts/ci-unit-module-cache-contents.mjs index f0f1ffa9a75..559e3487a42 100644 --- a/config/scripts/ci-unit-module-cache-contents.mjs +++ b/config/scripts/ci-unit-module-cache-contents.mjs @@ -11,6 +11,8 @@ export const MODULE_CACHE_LIMITS = Object.freeze({ entryBytes: 32 * 2 ** 20 }) const METADATA = '_metadata.json' +// Vitest 5.0.5+ marks its cache root with a fixed-content CACHEDIR.TAG. +const CACHEDIR_TAG = 'CACHEDIR.TAG' const MODULE_NAME = /^[a-f0-9]{40}$/ const isRecord = (value) => value !== null && typeof value === 'object' && !Array.isArray(value) const fail = (message) => { @@ -76,7 +78,12 @@ export async function inventoryModuleCache({ path, lockfileHash }) { while (next < names.length) { const index = next++ const name = names[index] - if (name !== METADATA && name !== MODULE_CACHE_MANIFEST && !MODULE_NAME.test(name)) { + if ( + name !== METADATA && + name !== MODULE_CACHE_MANIFEST && + name !== CACHEDIR_TAG && + !MODULE_NAME.test(name) + ) { fail(`invalid cache entry name: ${name}`) } const buffer = await readRegularFile(join(path, name)) @@ -96,6 +103,10 @@ export async function inventoryModuleCache({ path, lockfileHash }) { if (!isRecord(metadata) || metadata.lockfileHash !== lockfileHash) { fail('cache lockfile hash mismatch') } + } else if (name === CACHEDIR_TAG) { + if (!buffer.toString('utf8').startsWith('Signature: 8a477f597d28d172789f06886806bc55')) { + fail('invalid cache directory tag') + } } else { checkModuleTrailer(buffer) } diff --git a/config/scripts/ci-unit-module-cache.mjs b/config/scripts/ci-unit-module-cache.mjs index 30d448aa9f5..c7478b4f017 100644 --- a/config/scripts/ci-unit-module-cache.mjs +++ b/config/scripts/ci-unit-module-cache.mjs @@ -52,7 +52,8 @@ export async function moduleCachePolicyHash(root = ROOT) { return digestInputs(root, [ ...configInputs, ...workspacePackageManifests(root), - ...globSync(['config/patches/**', '**/tsconfig*.json'], { + // tsconfig and package.json (type/imports) steer transforms but are not in Vitest's per-module key. + ...globSync(['config/patches/**', '**/tsconfig*.json', '**/package.json'], { cwd: root, exclude: ['**/node_modules/**', '**/.git/**', '**/dist/**', '**/out/**', 'mobile/vendor/**'] }).filter((file) => statSync(join(root, file)).isFile()), @@ -62,6 +63,7 @@ export async function moduleCachePolicyHash(root = ROOT) { '.npmrc', 'config/pnpmfile.cjs', 'config/.bun-version', + 'config/dark-build-switches.cjs', 'config/scripts/ci-unit-module-cache.mjs', 'config/scripts/ci-unit-module-cache-contents.mjs', 'config/scripts/ci-unit-module-cache-trailer.mjs' @@ -109,7 +111,7 @@ export function moduleCacheIdentity({ const digest = createHash('sha256').update(JSON.stringify(identity)).digest('hex') return { ...identity, - key: `unit-module-v1-${platform}-${arch}-n${versions.node}-b${versions.bun}-v${versions.vitest}-s${shard.index}of${shard.count}-${digest}`, + key: `unit-module-v2-${platform}-${arch}-n${versions.node}-b${versions.bun}-v${versions.vitest}-s${shard.index}of${shard.count}-${digest}`, path: join(resolve(cacheRoot), 'orca-vitest-module-cache') } } diff --git a/config/scripts/ci-unit-module-cache.test.mjs b/config/scripts/ci-unit-module-cache.test.mjs index af496f3ba7a..b56801fe072 100644 --- a/config/scripts/ci-unit-module-cache.test.mjs +++ b/config/scripts/ci-unit-module-cache.test.mjs @@ -117,7 +117,7 @@ describe('persistent unit module cache boundary', () => { }) }) -it('uses an optional exact restore after install and saves successful scoped producers', () => { +it('restores the newest same-policy producer and saves only cold main or seeded producers', () => { const workflow = parse(readFileSync('.github/workflows/unit-tests.yml', 'utf8')) const steps = workflow.jobs.test.steps const dependencies = steps.findIndex( @@ -134,33 +134,30 @@ it('uses an optional exact restore after install and saves successful scoped pro expect(prepare).toBeGreaterThan(restore) expect(tests).toBeGreaterThan(prepare) expect(steps[restore].uses).toBe('actions/cache/restore@v5') - expect(steps[restore].with['restore-keys']).toBeUndefined() + expect(steps[restore].with['restore-keys']).toBe('${{ steps.module-cache.outputs.key }}-') expect(steps[restore]['continue-on-error']).toBe(true) - const guard = seal.if - .replaceAll('steps.module-cache.outputs.key', 'steps["module-cache"].outputs.key') - .replaceAll( - 'steps.module-cache-restore.outputs.cache-hit', - 'steps["module-cache-restore"].outputs["cache-hit"]' - ) + const guard = seal.if.replaceAll( + 'steps.module-cache.outputs.key', + 'steps["module-cache"].outputs.key' + ) const producer = { - event: 'pull_request', - ref: 'refs/pull/1/merge', + event: 'schedule', + ref: 'refs/heads/main', success: true, key: 'scoped', - hit: '', + mode: 'cold', seed: false } for (const [name, update, expected] of [ - ['successful PR cache miss', {}, true], - ['successful main producer', { event: 'push', ref: 'refs/heads/main' }, true], + ['successful main producer', {}, true], + ['PR', { event: 'pull_request', ref: 'refs/pull/1/merge' }, false], + ['warm main consumer', { mode: 'warm' }, false], [ 'successful manually seeded branch', { event: 'workflow_dispatch', ref: 'refs/heads/experiment', seed: true }, true ], - ['failed PR', { success: false }, false], ['missing identity', { key: '' }, false], - ['exact restored hit', { hit: 'true' }, false], ['ordinary branch push', { event: 'push', ref: 'refs/heads/experiment' }, false], ['unseeded manual branch', { event: 'workflow_dispatch', ref: 'refs/heads/experiment' }, false], [ @@ -174,16 +171,15 @@ it('uses an optional exact restore after install and saves successful scoped pro const admitted = runInNewContext(guard, { github: { event_name: control.event, ref: control.ref }, steps: { - 'module-cache': { outputs: { key: control.key } }, - 'module-cache-restore': { outputs: { 'cache-hit': control.hit } } + 'module-cache': { outputs: { key: control.key } } }, - inputs: { module_cache_seed: control.seed }, + inputs: { module_cache: control.mode, module_cache_seed: control.seed }, success: () => control.success }) expect(admitted, name).toBe(expected) } expect(save.if).toBe("steps.module-cache-seal.outputs.ready == 'true'") - expect(save.with.key).toBe(steps[restore].with.key) + expect(save.with.key.startsWith(steps[restore].with['restore-keys'])).toBe(true) expect(workflow.on.workflow_call.inputs.module_cache.default).toBe('off') expect(workflow.on.workflow_call.inputs.run_relay_integration.default).toBe(true) }) diff --git a/config/scripts/vitest-module-cache-resolution-key.test.ts b/config/scripts/vitest-module-cache-resolution-key.test.ts new file mode 100644 index 00000000000..4130dff53fb --- /dev/null +++ b/config/scripts/vitest-module-cache-resolution-key.test.ts @@ -0,0 +1,57 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, expect, it } from 'vitest' +import { + createResolutionFingerprint, + hasExpandedImport +} from './vitest-module-cache-resolution-key' + +const roots: string[] = [] +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +function fixture(): string { + const root = mkdtempSync(join(tmpdir(), 'orca-resolution-key-')) + roots.push(root) + mkdirSync(join(root, 'src/lib/foo'), { recursive: true }) + writeFileSync(join(root, 'src/lib/foo/index.ts'), '') + return root +} + +const source = "import a from './lib/foo'\nimport b from '@/lib/foo?raw'" +const aliases = (root: string) => [{ find: '@', replacement: join(root, 'src') }] + +it.each([ + ['a same-stem sibling shadows a directory index', 'src/lib/foo.ts'], + ['a higher-precedence extension appears', 'src/lib/foo.mts'], + ['another index candidate appears', 'src/lib/foo/index.mts'], + ['the directory gains a package.json', 'src/lib/foo/package.json'] +])('changes the key when %s', (_, added) => { + const root = fixture() + const before = createResolutionFingerprint(root, aliases(root))(join(root, 'src/a.ts'), source) + writeFileSync(join(root, added), '{}') + const after = createResolutionFingerprint(root, aliases(root))(join(root, 'src/a.ts'), source) + expect(after).not.toBe(before) +}) + +it('keeps the key when an unrelated sibling appears', () => { + const root = fixture() + const before = createResolutionFingerprint(root, aliases(root))(join(root, 'src/a.ts'), source) + writeFileSync(join(root, 'src/lib/bar.ts'), '') + const after = createResolutionFingerprint(root, aliases(root))(join(root, 'src/a.ts'), source) + expect(after).toBe(before) +}) + +it.each([ + ['import(`./locales/${name}.ts`)', true], + ["import('./locales/' + name + '.ts')", true], + ['import.meta.glob("./*.ts")', true], + ["import('./locales/en.ts')", false], + ['import(specifier)', false] +])('treats %s as directory-dependent: %s', (code, expected) => { + expect(hasExpandedImport(code)).toBe(expected) +}) diff --git a/config/scripts/vitest-module-cache-resolution-key.ts b/config/scripts/vitest-module-cache-resolution-key.ts new file mode 100644 index 00000000000..9520cc74271 --- /dev/null +++ b/config/scripts/vitest-module-cache-resolution-key.ts @@ -0,0 +1,113 @@ +import { readdirSync, readFileSync } from 'node:fs' +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path' +import type { Alias, Plugin } from 'vite' + +// Why: Vitest keys a cached transform by its own source, so a new sibling such as +// `foo.ts` beside `foo/index.ts` would keep serving the old resolution. Fold every +// candidate a literal specifier could resolve to into the key instead. +const LITERAL = /(?=(['"`])([^'"`\n]*)\1)/g +const DYNAMIC_IMPORT = /\bimport\s*\(\s*/g +const PLAIN_LITERAL_ARGUMENT = /^(['"`])[^'"`\n$]*\1\s*[,)]/ + +// Vite expands template/concatenated dynamic imports into globs over directory contents. +export function hasExpandedImport(source: string): boolean { + if (/\bimport\.meta\.glob\b/.test(source)) { + return true + } + for (const match of source.matchAll(DYNAMIC_IMPORT)) { + const rest = source.slice(match.index + match[0].length) + if (/^['"`]/.test(rest) && !PLAIN_LITERAL_ARGUMENT.test(rest)) { + return true + } + } + return false +} + +function applyAlias(specifier: string, aliases: readonly Alias[]): string | undefined { + for (const { find, replacement } of aliases) { + if (typeof find === 'string') { + if (specifier === find || specifier.startsWith(`${find}/`)) { + return replacement + specifier.slice(find.length) + } + } else if (find.test(specifier)) { + return specifier.replace(find, replacement) + } + } + return undefined +} + +export function createResolutionFingerprint(root: string, aliases: readonly Alias[]) { + const listings = new Map() + const list = (dir: string): readonly string[] => { + let names = listings.get(dir) + if (!names) { + try { + // Directories carry a trailing slash so a file/directory swap also changes the key. + names = readdirSync(dir, { withFileTypes: true }) + .map((entry) => (entry.isDirectory() ? `${entry.name}/` : entry.name)) + .sort() + } catch { + names = [] + } + listings.set(dir, names) + } + return names + } + // Outside-root strings (e.g. '/tmp') are not module specifiers here and their listings churn. + const insideRoot = (path: string) => { + const rel = relative(root, path) + return rel !== '' && rel !== '..' && !rel.startsWith(`..${sep}`) && !isAbsolute(rel) + } + // Every file Vite could pick for `base`: same-stem siblings, plus index/package.json if a directory. + const candidates = (base: string): string => { + const dir = dirname(base) + const stem = basename(base).split('.')[0] + const parts: string[] = [] + for (const name of list(dir).filter((entry) => entry.startsWith(stem))) { + parts.push(name) + if (name.endsWith('/')) { + const path = join(dir, name) + const inner = list(path) + const entries = inner.filter((entry) => /^(?:index\.|package\.json$)/.test(entry)) + parts.push(`[${entries.join(',')}]`) + if (inner.includes('package.json')) { + parts.push(readFileSync(join(path, 'package.json'), 'utf8')) + } + } + } + return `${relative(root, base)}:${parts.join(',')}` + } + return (id: string, source: string): string => { + const importerDir = dirname(id.split('?')[0]) + const seen = new Set() + for (const [, , literal] of source.matchAll(LITERAL)) { + const specifier = literal.split(/[?#]/)[0] + let target: string | undefined + if (specifier === '.' || specifier === '..' || /^\.{1,2}\//.test(specifier)) { + target = resolve(importerDir, specifier) + } else if (specifier.startsWith('/')) { + target = specifier.startsWith(root) ? specifier : join(root, specifier) + } else { + const aliased = applyAlias(specifier, aliases) + target = aliased && isAbsolute(aliased) ? aliased : undefined + } + if (target && insideRoot(target)) { + seen.add(target) + } + } + return [...seen].sort().map(candidates).join('\n') + } +} + +export function moduleCacheResolutionKey(): Plugin { + return { + name: 'module-cache-resolution-key', + configureVitest({ project, defineCacheKeyGenerator }) { + const config = project.vite.config + const fingerprint = createResolutionFingerprint(config.root, config.resolve.alias) + defineCacheKeyGenerator(({ id, sourceCode }) => + hasExpandedImport(sourceCode) ? false : fingerprint(id, sourceCode) + ) + } + } +} diff --git a/config/tsconfig.node.json b/config/tsconfig.node.json index bc9bf87d0d9..061c6f89f81 100644 --- a/config/tsconfig.node.json +++ b/config/tsconfig.node.json @@ -13,6 +13,7 @@ "./scripts/happy-dom-mutation-observer-retention.ts", "./scripts/happy-dom-animation-cancel.ts", "./scripts/vitest-node-runtime-pool.ts", + "./scripts/vitest-module-cache-resolution-key.ts", "../src/main/**/*", "../src/renderer/src/store/slices/worktrees/create/worktree-create-payload.ts", "../src/renderer/src/components/native-chat/native-chat-resolution-receipt.ts", diff --git a/config/vitest.config.ts b/config/vitest.config.ts index b5c3dc6d508..5e38c0ee338 100644 --- a/config/vitest.config.ts +++ b/config/vitest.config.ts @@ -10,11 +10,13 @@ import { getNodeRuntimeInclude } from './scripts/vitest-node-runtime-files.mjs' import { nodeRuntimePool } from './scripts/vitest-node-runtime-pool' import { readRunFilters } from './scripts/vitest-run-filters.mjs' import { createWorkspaceSourceResolver } from './scripts/workspace-source-exports.mjs' +import { moduleCacheResolutionKey } from './scripts/vitest-module-cache-resolution-key' const nodeRuntimeInclude = process.versions.bun ? getNodeRuntimeInclude({ filters: readRunFilters() }) : [] const balancedShards = process.env.ORCA_BALANCE_UNIT_SHARDS === '1' +const moduleCachePath = process.env.ORCA_UNIT_MODULE_CACHE_PATH const measurementFile = 'src/main/foreign-sqlite-readers/foreign-sqlite-reader-event-loop.test.ts' const paletteMeasurementFile = 'src/renderer/src/lib/palette-match/palette-match-performance.test.ts' @@ -45,6 +47,7 @@ const transforms = { return { code: result.code, map: result.map } } } satisfies Plugin, + ...(moduleCachePath ? [moduleCacheResolutionKey()] : []), { // Unit tests have no asset server; retain real catalog parsing through fetch(data:). name: 'locale-catalog-test-assets', @@ -83,11 +86,9 @@ const testOptions = { environment: 'node', clearMocks: false, ...(balancedShards ? { silent: 'passed-only' as const } : {}), - // Persistent transforms can miss source additions; only explicit cache experiments opt in. - fsModuleCache: Boolean(process.env.ORCA_UNIT_MODULE_CACHE_PATH), - ...(process.env.ORCA_UNIT_MODULE_CACHE_PATH - ? { fsModuleCachePath: process.env.ORCA_UNIT_MODULE_CACHE_PATH } - : {}), + // Persistent transforms are CI-keyed (ci-unit-module-cache.mjs); local runs stay cold. + fsModuleCache: Boolean(moduleCachePath), + ...(moduleCachePath ? { fsModuleCachePath: moduleCachePath } : {}), env: { ORCA_VITEST_RUNTIME: 'node' }, server: { deps: { inline: ['zod'] } }, // Node's storage globals and V8 retention checks require the existing child flags.