diff --git a/config/scripts/reclaim-electron-dists.mjs b/config/scripts/reclaim-electron-dists.mjs index 2276a5f862d..03e7b539d54 100644 --- a/config/scripts/reclaim-electron-dists.mjs +++ b/config/scripts/reclaim-electron-dists.mjs @@ -6,6 +6,8 @@ import { execFileSync } from 'node:child_process' import { randomUUID } from 'node:crypto' import { + accessSync, + constants, existsSync, readFileSync, readdirSync, @@ -131,6 +133,18 @@ function main() { continue } + try { + // Elevated POSIX users can pass W_OK even when the install is deliberately read-only. + if (process.platform !== 'win32' && (statSync(electronPackageDir).mode & 0o222) === 0) { + throw new Error('Electron package directory has no write permission bits') + } + accessSync(electronPackageDir, constants.W_OK) + } catch { + console.log(`skip ${worktree} (Electron package directory is not writable)`) + skipped += 1 + continue + } + const size = measure(distPath) if (!apply) { console.log(`would share ${worktree} ${(size / 1024 ** 3).toFixed(2)} GiB (${version})`) @@ -145,6 +159,9 @@ function main() { recordAdoptedSharedElectronDist(entry, writeFileSync) console.log(`seeded ${worktree} -> ${entry.entryPath}`) converted += 1 + } else { + console.warn(`skip ${worktree} (shared Electron cache could not be prepared)`) + skipped += 1 } continue } @@ -153,6 +170,9 @@ function main() { reclaimed += size converted += 1 console.log(`shared ${worktree} ${(size / 1024 ** 3).toFixed(2)} GiB logical file data`) + } else { + console.warn(`skip ${worktree} (shared Electron copy could not be prepared)`) + skipped += 1 } } catch (error) { // A worktree that fails is left exactly as it was; it still has its own working dist. diff --git a/config/scripts/reclaim-electron-dists.test.ts b/config/scripts/reclaim-electron-dists.test.ts new file mode 100644 index 00000000000..b1b3e08e094 --- /dev/null +++ b/config/scripts/reclaim-electron-dists.test.ts @@ -0,0 +1,86 @@ +import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { afterEach, expect, it } from 'vitest' +import { runProcess } from '@orca/process-host' +import { getElectronPlatformPath } from './electron-platform-path.mjs' + +const roots: string[] = [] +afterEach(async () => { + for (const root of roots.splice(0)) { + await rm(root, { recursive: true, force: true }) + } +}) + +async function fixture() { + const root = await mkdtemp(join(tmpdir(), 'orca-reclaim-electron-')) + roots.push(root) + const result = await runProcess({ + program: 'git', + args: ['init', '--quiet'], + cwd: root, + timeoutMs: 10_000, + terminationBarrier: true + }) + expect(result.code).toBe(0) + const pkg = join(root, 'node_modules', 'electron') + const executable = join(pkg, 'dist', getElectronPlatformPath(process.platform)) + await mkdir(join(executable, '..'), { recursive: true }) + await writeFile(join(pkg, 'package.json'), JSON.stringify({ version: '99.1.0' })) + await writeFile(join(pkg, 'dist', 'version'), '99.1.0\n') + await writeFile(executable, 'fake electron executable\n') + const run = (apply = false) => + runProcess({ + program: process.env.ORCA_TEST_NODE_EXECUTABLE ?? process.execPath, + args: [ + resolve(import.meta.dirname, 'reclaim-electron-dists.mjs'), + '--repo', + root, + ...(apply ? ['--apply'] : []) + ], + cwd: root, + env: { ...process.env, CI: '', ORCA_BACKGROUND_LAUNCH: '1' }, + timeoutMs: 20_000, + terminationBarrier: true + }) + return { root, pkg, executable, run } +} + +it.skipIf(process.platform === 'win32')( + 'reports a read-only package as protected in preview and apply', + async () => { + const { pkg, executable, run } = await fixture() + await chmod(pkg, 0o555) + try { + for (const apply of [false, true]) { + const result = await run(apply) + expect(result.code, result.stderr).toBe(0) + expect(result.stdout).toContain('Electron package directory is not writable') + expect(result.stdout).toContain('0 worktree(s)') + expect(result.stdout).toContain('skipped 1') + expect(await readFile(executable, 'utf8')).toBe('fake electron executable\n') + await expect(readFile(join(pkg, '.orca-shared-dist'))).rejects.toMatchObject({ + code: 'ENOENT' + }) + } + } finally { + await chmod(pkg, 0o755) + } + } +) + +it('shares a writable install and preserves its executable bytes', async () => { + const { pkg, executable, run } = await fixture() + const preview = await run() + expect(preview.code, preview.stderr).toBe(0) + expect(preview.stdout).toContain('Would share 1 worktree(s)') + const applied = await run(true) + expect(applied.code, applied.stderr).toBe(0) + expect(applied.stdout).toContain('Shared 1 worktree(s)') + expect(await readFile(executable, 'utf8')).toBe('fake electron executable\n') + expect(await readFile(join(pkg, '.orca-shared-dist'), 'utf8')).toBe( + `99.1.0-${process.platform}-${process.arch}` + ) + const repeated = await run() + expect(repeated.stdout).toContain('Would share 0 worktree(s)') +}) diff --git a/config/scripts/reclaim-release-test-caches.mjs b/config/scripts/reclaim-release-test-caches.mjs new file mode 100644 index 00000000000..55f90a1b149 --- /dev/null +++ b/config/scripts/reclaim-release-test-caches.mjs @@ -0,0 +1,307 @@ +#!/usr/bin/env node +import { lstat, readFile, readdir, realpath, rm } from 'node:fs/promises' +import { join, resolve } from 'node:path' +import { runProcess } from '@orca/process-host' +import { lock } from 'proper-lockfile' +import { decodeGitCQuotedPath } from '../../src/shared/git-cquoted-path.ts' +import { CHECKOUT_FORMAT } from '../../tests/e2e/cross-version-wire/release-checkout-publication.ts' +import { withReleaseSourceFormatLease } from '../../tests/e2e/cross-version-wire/release-source-retention.ts' + +const COMMIT = '(?:[\\da-f]{40}|[\\da-f]{64})' +const LOCAL_ENTRY = new RegExp(`^(${COMMIT})-format-([1-9]\\d*)$`) +const SHARED_ENTRY = new RegExp(`^(${COMMIT})$`) +const DAY_MS = 24 * 60 * 60 * 1000 + +async function directory(path) { + try { + const info = await lstat(path) + return info.isDirectory() && !info.isSymbolicLink() ? info : null + } catch (error) { + if (error.code === 'ENOENT') { + return null + } + throw error + } +} + +async function descend(root, parts) { + let path = root + for (const part of parts) { + path = join(path, part) + if (!(await directory(path))) { + return null + } + } + return path +} + +async function matchingStamp(path, commit, format) { + try { + const marker = join(path, 'checkout-stamp.json') + const info = await lstat(marker) + if (!info.isFile() || info.isSymbolicLink() || info.size > 4096) { + return false + } + const value = JSON.parse(await readFile(marker, 'utf8')) + return value?.commit === commit && value?.format === format + } catch (error) { + if (error.code === 'ENOENT' || error instanceof SyntaxError) { + return false + } + throw error + } +} + +async function logicalBytes(root) { + let bytes = 0 + const pending = [root] + while (pending.length) { + const path = pending.pop() + for (const entry of await readdir(path, { withFileTypes: true })) { + const child = join(path, entry.name) + const info = await lstat(child) + if (info.isDirectory() && !info.isSymbolicLink()) { + pending.push(child) + } else if (info.isFile() && !info.isSymbolicLink()) { + bytes += info.size + } + } + } + return bytes +} + +async function git(repoRoot, args) { + const result = await runProcess({ + program: 'git', + args, + cwd: repoRoot, + timeoutMs: 30_000, + maxOutputBytes: 4 * 1024 * 1024, + terminationBarrier: true + }) + if (result.code !== 0 || result.timedOut) { + throw new Error('Could not read the local Git repository') + } + return result.stdout +} + +async function repositoryTargets(repoRoot) { + const common = (await git(repoRoot, ['rev-parse', '--git-common-dir'])).replace( + process.platform === 'win32' ? /\r?\n$/ : /\n$/, + '' + ) + if (!common || common.includes('\n')) { + throw new Error('Unrecognised Git common directory') + } + const output = await git(repoRoot, [ + '-c', + 'core.quotePath=false', + 'worktree', + 'list', + '--porcelain' + ]) + const worktrees = [] + for (const block of output.split(/\r?\n\r?\n/)) { + const lines = block.split(/\r?\n/).filter(Boolean) + if (lines.length === 0) { + continue + } + if ( + lines.some( + (line) => + !/^(worktree |HEAD |branch |bare$|detached$|locked(?: |$)|prunable(?: |$))/.test(line) + ) + ) { + throw new Error('Ambiguous worktree listing; refusing cleanup') + } + const field = lines.find((line) => line.startsWith('worktree ')) + if (field && !lines.includes('bare')) { + worktrees.push(decodeGitCQuotedPath(field.slice(9))) + } + } + return { commonDirectory: await realpath(resolve(repoRoot, common)), worktrees } +} + +export async function reclaimReleaseTestCaches({ + repoRoot = process.cwd(), + apply = false, + testsStopped = false, + minAgeDays = 30, + nowMs = Date.now(), + targets, + report = console.log +} = {}) { + if (!Number.isFinite(minAgeDays) || minAgeDays < 0) { + throw new Error('--min-age-days requires a non-negative number') + } + if (apply && !testsStopped) { + throw new Error( + 'Stop cross-version tests in this repository, then pass --tests-stopped with --apply' + ) + } + const { commonDirectory, worktrees } = targets ?? (await repositoryTargets(repoRoot)) + const cutoff = nowMs - minAgeDays * DAY_MS + const result = { entries: 0, logicalBytes: 0, protected: 0 } + + async function processEntry(path, commit, format, recheckParents, verifyLease = () => {}) { + const info = await directory(path) + if (!info || info.mtimeMs > cutoff || !(await matchingStamp(path, commit, format))) { + return + } + let release + try { + if (apply) { + release = await lock(path, { realpath: false, stale: 60_000, retries: 0 }) + } + } catch (error) { + if (error.code !== 'ELOCKED') { + throw error + } + result.protected += 1 + report(`skip locked cache ${JSON.stringify(path)}`) + return + } + try { + const bytes = await logicalBytes(path) + if (apply) { + verifyLease() + if (!(await recheckParents())) { + return + } + const current = await directory(path) + if ( + !current || + current.dev !== info.dev || + current.ino !== info.ino || + current.mtimeMs > cutoff || + !(await matchingStamp(path, commit, format)) + ) { + return + } + verifyLease() + await rm(path, { recursive: true, force: true }) + } + result.entries += 1 + result.logicalBytes += bytes + report( + `${apply ? 'removed' : 'would remove'} ${JSON.stringify(path)} ${(bytes / 1024 ** 2).toFixed(1)} MiB logical` + ) + } finally { + await release?.() + } + } + + for (const worktree of new Set(worktrees)) { + const cache = await descend(worktree, ['tests', 'e2e', '.cross-version-checkouts']) + if (!cache) { + continue + } + for (const label of await readdir(cache, { withFileTypes: true })) { + if (!label.isDirectory() || label.isSymbolicLink()) { + continue + } + const labelRoot = join(cache, label.name) + for (const entry of await readdir(labelRoot, { withFileTypes: true })) { + const match = LOCAL_ENTRY.exec(entry.name) + if (!entry.isDirectory() || !match || Number(match[2]) > CHECKOUT_FORMAT) { + continue + } + await processEntry(join(labelRoot, entry.name), match[1], Number(match[2]), async () => + Boolean(await descend(worktree, ['tests', 'e2e', '.cross-version-checkouts', label.name])) + ) + } + } + } + + const shared = await descend(commonDirectory, ['orca-cache', 'release-source']) + if (shared) { + for (const entry of await readdir(shared, { withFileTypes: true })) { + const match = /^format-([1-9]\d*)$/.exec(entry.name) + if (!entry.isDirectory() || !match || Number(match[1]) > CHECKOUT_FORMAT) { + continue + } + if (!(await directory(join(shared, entry.name)))) { + continue + } + const format = Number(match[1]) + const visit = async (formatRoot, verifyLease) => { + for (const seed of await readdir(formatRoot, { withFileTypes: true })) { + const key = SHARED_ENTRY.exec(seed.name) + if (!seed.isDirectory() || !key) { + continue + } + await processEntry( + join(formatRoot, seed.name), + key[1], + format, + async () => + Boolean(await descend(commonDirectory, ['orca-cache', 'release-source', entry.name])), + verifyLease + ) + } + } + if (!apply) { + await visit(join(shared, entry.name), () => {}) + } else { + try { + await withReleaseSourceFormatLease(shared, format, visit, { + realpath: false, + stale: 60_000, + update: 10_000, + retries: { + retries: 0, + factor: 1, + minTimeout: 250, + maxTimeout: 250, + randomize: false + } + }) + } catch (error) { + if (error.code !== 'ELOCKED') { + throw error + } + result.protected += 1 + report(`skip locked shared format ${entry.name}`) + } + } + } + } + report( + `${apply ? 'Removed' : 'Would remove'} ${result.entries} cache(s); ${(result.logicalBytes / 1024 ** 3).toFixed(2)} GiB logical file data. Shared blocks affect actual savings.` + ) + return result +} + +if (process.argv[1] && resolve(process.argv[1]) === resolve(import.meta.filename)) { + const options = {} + for (let index = 2; index < process.argv.length; index += 1) { + const argument = process.argv[index] + if (argument === '--help') { + console.log( + 'Usage: pnpm run reclaim:worktree-space [--repo DIRECTORY] [--release-cache-min-age-days 30] [--apply --tests-stopped]' + ) + process.exit(0) + } + if (argument === '--apply') { + options.apply = true + } else if (argument === '--tests-stopped') { + options.testsStopped = true + } else if (argument === '--repo' || argument === '--min-age-days') { + const value = process.argv[++index] + if (!value || value.startsWith('--')) { + throw new Error(`${argument} requires a value`) + } + if (argument === '--repo') { + options.repoRoot = resolve(value) + } else { + options.minAgeDays = Number(value) + } + } else { + throw new Error(`Unknown argument: ${argument}`) + } + } + await reclaimReleaseTestCaches(options).catch((error) => { + console.error(error.message) + process.exitCode = 1 + }) +} diff --git a/config/scripts/reclaim-release-test-caches.test.ts b/config/scripts/reclaim-release-test-caches.test.ts new file mode 100644 index 00000000000..5715229ed5f --- /dev/null +++ b/config/scripts/reclaim-release-test-caches.test.ts @@ -0,0 +1,140 @@ +import { lstat, mkdir, mkdtemp, readFile, rm, symlink, utimes, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, expect, it } from 'vitest' +import { lock } from 'proper-lockfile' +import { reclaimReleaseTestCaches } from './reclaim-release-test-caches.mjs' +import { withReleaseSourceLease } from '../../tests/e2e/cross-version-wire/release-source-retention' +import { CHECKOUT_FORMAT } from '../../tests/e2e/cross-version-wire/release-checkout-publication' + +const roots: string[] = [] +const commit = 'c'.repeat(40) +const nowMs = Date.now() +const day = 24 * 60 * 60 * 1000 + +afterEach(async () => { + for (const root of roots.splice(0)) { + await rm(root, { recursive: true, force: true }) + } +}) + +async function fixture() { + const root = await mkdtemp(join(tmpdir(), 'orca-release-cache-cleanup-')) + roots.push(root) + const worktree = join(root, 'worktree') + const commonDirectory = join(root, 'git') + const cache = join(worktree, 'tests', 'e2e', '.cross-version-checkouts') + const shared = join(commonDirectory, 'orca-cache', 'release-source') + await mkdir(cache, { recursive: true }) + await mkdir(shared, { recursive: true }) + const targets = { commonDirectory, worktrees: [worktree, worktree] } + return { root, worktree, cache, shared, targets, nowMs, report: () => {} } +} + +async function stamped(path: string, format = CHECKOUT_FORMAT, ageDays = 60) { + await mkdir(path, { recursive: true }) + await writeFile(join(path, 'checkout-stamp.json'), JSON.stringify({ commit, format })) + await writeFile(join(path, 'source.ts'), 'export const historical = true\n') + const time = new Date(nowMs - ageDays * day) + await utimes(path, time, time) + return path +} + +it('previews old stamped views and seeds without changing them', async () => { + const options = await fixture() + const local = await stamped(join(options.cache, 'v-old', `${commit}-format-4`), 4) + const seed = await stamped(join(options.shared, `format-${CHECKOUT_FORMAT}`, commit)) + const result = await reclaimReleaseTestCaches(options) + expect(result.entries).toBe(2) + expect(result.logicalBytes).toBeGreaterThan(0) + expect((await lstat(local)).isDirectory()).toBe(true) + expect((await lstat(seed)).isDirectory()).toBe(true) + await expect(lstat(`${local}.lock`)).rejects.toMatchObject({ code: 'ENOENT' }) +}) + +it('requires stopped-test acknowledgement before deleting anything', async () => { + const options = await fixture() + const local = await stamped(join(options.cache, 'v-old', `${commit}-format-4`), 4) + await expect(reclaimReleaseTestCaches({ ...options, apply: true })).rejects.toThrow( + '--tests-stopped' + ) + expect((await lstat(local)).isDirectory()).toBe(true) +}) + +it('removes old generated trees and retains recent, future, malformed and mutable trees', async () => { + const options = await fixture() + const old = await stamped(join(options.cache, 'v-old', `${commit}-format-4`), 4) + const seed = await stamped(join(options.shared, `format-${CHECKOUT_FORMAT}`, commit)) + const recent = await stamped( + join(options.cache, 'v-new', `${commit}-format-${CHECKOUT_FORMAT}`), + CHECKOUT_FORMAT, + 1 + ) + const future = await stamped( + join(options.shared, `format-${CHECKOUT_FORMAT + 1}`, commit), + CHECKOUT_FORMAT + 1 + ) + const malformed = await stamped(join(options.cache, 'invalid', `${commit}-format-4`), 5) + const mutable = join(options.cache, 'working-tree', 'src', 'current.ts') + await mkdir(join(mutable, '..'), { recursive: true }) + await writeFile(mutable, 'mutable working tree copy') + const result = await reclaimReleaseTestCaches({ ...options, apply: true, testsStopped: true }) + expect(result.entries).toBe(2) + for (const path of [old, seed]) { + await expect(lstat(path)).rejects.toMatchObject({ code: 'ENOENT' }) + } + for (const path of [recent, future, malformed]) { + expect((await lstat(path)).isDirectory()).toBe(true) + } + expect(await readFile(mutable, 'utf8')).toBe('mutable working tree copy') +}) + +it('does not follow cache, label, entry or shared-format symlinks', async () => { + const options = await fixture() + const outside = await stamped(join(options.root, 'outside', `${commit}-format-4`), 4) + const second = join(options.root, 'linked-worktree') + await mkdir(join(second, 'tests', 'e2e'), { recursive: true }) + await symlink( + join(outside, '..'), + join(second, 'tests', 'e2e', '.cross-version-checkouts'), + 'junction' + ) + options.targets.worktrees.push(second) + await symlink(join(outside, '..'), join(options.cache, 'linked-label'), 'junction') + await mkdir(join(options.cache, 'linked-entry')) + await symlink(outside, join(options.cache, 'linked-entry', `${commit}-format-4`), 'junction') + await symlink(join(outside, '..'), join(options.shared, 'format-4'), 'junction') + const result = await reclaimReleaseTestCaches({ + ...options, + apply: true, + testsStopped: true, + minAgeDays: 0 + }) + expect(result.entries).toBe(0) + expect(await readFile(join(outside, 'source.ts'), 'utf8')).toContain('historical') +}) + +it('skips publication locks and a shared source copy lease', async () => { + const options = await fixture() + const local = await stamped(join(options.cache, 'v-old', `${commit}-format-4`), 4) + const seed = await stamped(join(options.shared, `format-${CHECKOUT_FORMAT}`, commit)) + const release = await lock(local, { realpath: false }) + try { + await withReleaseSourceLease(options.shared, async () => { + const result = await reclaimReleaseTestCaches({ ...options, apply: true, testsStopped: true }) + expect(result.entries).toBe(0) + expect(result.protected).toBe(2) + }) + } finally { + await release() + } + expect((await lstat(local)).isDirectory()).toBe(true) + expect((await lstat(seed)).isDirectory()).toBe(true) +}) + +it.each([-1, Number.NaN, Number.POSITIVE_INFINITY])( + 'rejects invalid minimum age %s', + async (age) => { + await expect(reclaimReleaseTestCaches({ minAgeDays: age })).rejects.toThrow('--min-age-days') + } +) diff --git a/config/scripts/reclaim-worktree-space.mjs b/config/scripts/reclaim-worktree-space.mjs index 561a5ea4623..e99b791845a 100644 --- a/config/scripts/reclaim-worktree-space.mjs +++ b/config/scripts/reclaim-worktree-space.mjs @@ -7,10 +7,19 @@ import { runProcess } from '@orca/process-host' export async function reclaimWorktreeSpace({ repoRoot = process.cwd(), apply = false, + automatic = false, execute = runProcess, acquireLock = lock, - commonDirectory + commonDirectory, + testsStopped = false, + releaseCacheMinAgeDays = 30 } = {}) { + if (apply && !automatic && !testsStopped) { + throw new Error('Stop cross-version tests, then pass --tests-stopped with --apply') + } + if (!Number.isFinite(releaseCacheMinAgeDays) || releaseCacheMinAgeDays < 0) { + throw new Error('--release-cache-min-age-days requires a non-negative number') + } if (!commonDirectory) { const result = await runProcess({ program: 'git', @@ -37,11 +46,21 @@ export async function reclaimWorktreeSpace({ let cleanupError try { const scripts = ['reclaim-electron-dists.mjs', 'reclaim-dev-electron-bundles.mjs'] + if (!automatic) { + scripts.unshift('reclaim-release-test-caches.mjs') + } for (const script of scripts) { const args = [resolve(import.meta.dirname, script), '--repo', repoRoot] if (apply) { args.push('--apply') } + if (script === 'reclaim-release-test-caches.mjs') { + args.unshift('--import', import.meta.resolve('tsx')) + args.push('--min-age-days', String(releaseCacheMinAgeDays)) + if (testsStopped) { + args.push('--tests-stopped') + } + } const result = await execute({ program: process.execPath, args, @@ -76,18 +95,21 @@ export async function reclaimWorktreeSpace({ if (process.argv[1] && resolve(process.argv[1]) === resolve(import.meta.filename)) { if (process.argv.includes('--help')) { - console.log('Usage: pnpm run reclaim:worktree-space [--repo DIRECTORY] [--apply]') console.log( - 'Previews Electron distribution sharing and idle dev bundle removal; --apply performs them.' + 'Usage: pnpm run reclaim:worktree-space [--repo DIRECTORY] [--release-cache-min-age-days 30] [--apply --tests-stopped]' + ) + console.log( + 'Previews historical test cache cleanup, Electron sharing, and idle dev bundle removal.' ) process.exit(0) } const repoIndex = process.argv.indexOf('--repo') + const ageIndex = process.argv.indexOf('--release-cache-min-age-days') for (let index = 2; index < process.argv.length; index += 1) { const argument = process.argv[index] - if (argument === '--repo') { + if (argument === '--repo' || argument === '--release-cache-min-age-days') { index += 1 - } else if (argument !== '--apply') { + } else if (argument !== '--apply' && argument !== '--tests-stopped') { throw new Error(`Unknown argument: ${argument}`) } } @@ -97,9 +119,17 @@ if (process.argv[1] && resolve(process.argv[1]) === resolve(import.meta.filename ) { throw new Error('--repo requires a directory') } + if ( + ageIndex !== -1 && + (!process.argv[ageIndex + 1] || process.argv[ageIndex + 1].startsWith('--')) + ) { + throw new Error('--release-cache-min-age-days requires a number') + } await reclaimWorktreeSpace({ repoRoot: repoIndex !== -1 ? resolve(process.argv[repoIndex + 1]) : process.cwd(), - apply: process.argv.includes('--apply') + apply: process.argv.includes('--apply'), + testsStopped: process.argv.includes('--tests-stopped'), + releaseCacheMinAgeDays: ageIndex === -1 ? 30 : Number(process.argv[ageIndex + 1]) }).catch((error) => { console.error(error.message) process.exitCode = 1 diff --git a/config/scripts/reclaim-worktree-space.test.mjs b/config/scripts/reclaim-worktree-space.test.mjs index f449408d204..0b5967a0b41 100644 --- a/config/scripts/reclaim-worktree-space.test.mjs +++ b/config/scripts/reclaim-worktree-space.test.mjs @@ -1,8 +1,9 @@ -import { mkdtemp, rm } from 'node:fs/promises' +import { mkdir, mkdtemp, readFile, rm, utimes, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' -import { join } from 'node:path' +import { join, resolve } from 'node:path' import { afterEach, expect, it, vi } from 'vitest' import { reclaimWorktreeSpace } from './reclaim-worktree-space.mjs' +import { runProcess } from '@orca/process-host' const roots = [] afterEach(async () => { @@ -17,26 +18,114 @@ async function fixture() { return { commonDirectory: root, repoRoot: root, execute } } -it('previews both existing cleanup operations without applying either', async () => { +it('previews historical test caches and both Electron cleanup operations', async () => { const options = await fixture() await reclaimWorktreeSpace(options) - expect(options.execute).toHaveBeenCalledTimes(2) + expect(options.execute).toHaveBeenCalledTimes(3) expect(options.execute.mock.calls.flatMap(([spec]) => spec.args)).not.toContain('--apply') }) -it('forwards apply and repository selection to both cleanup operations', async () => { - const options = { ...(await fixture()), apply: true } +it('forwards apply and repository selection to all cleanup operations', async () => { + const options = { ...(await fixture()), apply: true, testsStopped: true } await reclaimWorktreeSpace(options) - expect(options.execute).toHaveBeenCalledTimes(2) + expect(options.execute).toHaveBeenCalledTimes(3) for (const [spec] of options.execute.mock.calls) { expect(spec.args).toContain('--apply') expect(spec.args).toContain(options.repoRoot) expect(spec.terminationBarrier).toBe(true) } + const cache = options.execute.mock.calls[0][0] + expect(cache.args).toContain('--tests-stopped') + expect(cache.args).toContain('--import') + for (const [spec] of options.execute.mock.calls.slice(1)) { + expect(spec.args).not.toContain('--tests-stopped') + } +}) + +it('rejects manual apply before any operation unless cross-version tests are stopped', async () => { + const options = { ...(await fixture()), apply: true } + await expect(reclaimWorktreeSpace(options)).rejects.toThrow('--tests-stopped') + expect(options.execute).not.toHaveBeenCalled() +}) + +it('leaves historical caches out of automatic invocations', async () => { + const options = { ...(await fixture()), automatic: true, apply: true } + await reclaimWorktreeSpace(options) + expect(options.execute.mock.calls.flatMap(([spec]) => spec.args).join(' ')).not.toContain( + 'reclaim-release-test-caches' + ) +}) + +it('forwards an age override only to historical test cache cleanup', async () => { + const options = { ...(await fixture()), releaseCacheMinAgeDays: 7 } + await reclaimWorktreeSpace(options) + expect(options.execute.mock.calls[0][0].args).toEqual( + expect.arrayContaining(['--min-age-days', '7']) + ) + for (const [spec] of options.execute.mock.calls.slice(1)) { + expect(spec.args).not.toContain('--min-age-days') + } +}) + +it('rejects an invalid cache age before starting cleanup', async () => { + const options = { ...(await fixture()), releaseCacheMinAgeDays: -1 } + await expect(reclaimWorktreeSpace(options)).rejects.toThrow('--release-cache-min-age-days') + expect(options.execute).not.toHaveBeenCalled() +}) + +it('runs the combined CLI from outside a disposable repository and requires acknowledgement', async () => { + const options = await fixture() + const init = await runProcess({ + program: 'git', + args: ['init', '--quiet'], + cwd: options.repoRoot, + timeoutMs: 10_000, + terminationBarrier: true + }) + expect(init.code).toBe(0) + const commit = 'd'.repeat(40) + const cache = join( + options.repoRoot, + 'tests', + 'e2e', + '.cross-version-checkouts', + 'old', + `${commit}-format-4` + ) + await mkdir(cache, { recursive: true }) + const marker = join(cache, 'checkout-stamp.json') + await writeFile(marker, JSON.stringify({ commit, format: 4 })) + await utimes(cache, new Date(0), new Date(0)) + const run = (flags = []) => + runProcess({ + program: process.env.ORCA_TEST_NODE_EXECUTABLE ?? process.execPath, + args: [ + resolve(import.meta.dirname, 'reclaim-worktree-space.mjs'), + '--repo', + options.repoRoot, + ...flags + ], + cwd: tmpdir(), + env: { ...process.env, ORCA_BACKGROUND_LAUNCH: '1' }, + timeoutMs: 20_000, + terminationBarrier: true + }) + const preview = await run() + expect(preview.code, preview.stderr).toBe(0) + expect(preview.stdout).toContain('Would remove 1 cache(s)') + expect(await readFile(marker, 'utf8')).toContain(commit) + const blocked = await run(['--apply']) + expect(blocked.code).not.toBe(0) + expect(blocked.stderr).toContain('--tests-stopped') + expect(await readFile(marker, 'utf8')).toContain(commit) + const applied = await run(['--apply', '--tests-stopped']) + expect(applied.code, applied.stderr).toBe(0) + expect(applied.stdout).toContain('Removed 1 cache(s)') + await expect(readFile(marker)).rejects.toMatchObject({ code: 'ENOENT' }) }) it('rejects competing sweeps and releases the lock after a failed operation', async () => { - const options = { ...(await fixture()), apply: true } + const options = { ...(await fixture()), apply: true, testsStopped: true } let release const blocked = new Promise((resolve) => { release = resolve @@ -58,7 +147,7 @@ it('rejects competing sweeps and releases the lock after a failed operation', as await rejected options.execute.mockResolvedValue({ code: 0, timedOut: false }) await reclaimWorktreeSpace(options) - expect(options.execute).toHaveBeenCalledTimes(3) + expect(options.execute).toHaveBeenCalledTimes(4) }) it('preserves the cleanup failure and reports a concurrent lock release failure separately', async () => { @@ -85,5 +174,5 @@ it('fails if cleanup succeeds but the repository lock cannot be released', async await expect(reclaimWorktreeSpace({ ...options, acquireLock: async () => release })).rejects.toBe( failure ) - expect(options.execute).toHaveBeenCalledTimes(2) + expect(options.execute).toHaveBeenCalledTimes(3) }) diff --git a/config/scripts/space-sharing-copy.mjs b/config/scripts/space-sharing-copy.mjs index 01e9c8ac5ef..d678a95ac7a 100644 --- a/config/scripts/space-sharing-copy.mjs +++ b/config/scripts/space-sharing-copy.mjs @@ -10,7 +10,8 @@ import { statSync, symlinkSync } from 'node:fs' -import { join } from 'node:path' +import { join, relative } from 'node:path' +import * as filesystem from 'node:fs/promises' // -c asks for clonefile(2). -P keeps Electron.framework's relative symlinks as symlinks; resolving // them breaks Chromium's bundle lookup. @@ -173,3 +174,83 @@ function copyTreeVerbatim(sourcePath, destinationPath) { verbatimSymlinks: true }) } + +export async function shareTreeAsync(sourcePath, destinationPath, options = {}) { + const platform = options.platform ?? process.platform + const errors = [] + for (const mechanism of getShareMechanisms(platform)) { + try { + await (options[mechanism] ?? asyncShareMechanisms[mechanism])(sourcePath, destinationPath) + return mechanism + } catch (error) { + errors.push(error) + await filesystem.rm(destinationPath, { recursive: true, force: true }) + } + } + throw new AggregateError(errors, `Could not share storage for ${destinationPath}`) +} + +async function runSharingCopy(program, args) { + const { runProcess } = await import('@orca/process-host') + const result = await runProcess({ program, args, timeoutMs: 45_000, terminationBarrier: true }) + if (result.code !== 0 || result.timedOut) { + throw new Error(`${program} could not share source storage`) + } +} + +const asyncShareMechanisms = { + clone: (source, destination) => + runSharingCopy('/bin/cp', [...MACOS_CLONE_ARGS, source, destination]), + reflink: (source, destination) => + runSharingCopy('cp', [...LINUX_REFLINK_ARGS, source, destination]), + hardlink: async (source, destination) => { + await walkTreeAsync(source, async (directory, entries) => { + const target = join(destination, relative(source, directory)) + await filesystem.mkdir(target, { recursive: true }) + return entries.map((entry) => async () => { + const from = join(directory, entry.name) + const to = join(target, entry.name) + if (entry.isSymbolicLink()) { + await filesystem.symlink(await filesystem.readlink(from), to) + } else if (!entry.isDirectory()) { + await filesystem.link(from, to) + } + }) + }) + } +} + +async function walkTreeAsync(root, visitDirectory) { + const directories = [root] + while (directories.length > 0) { + const directory = directories.pop() + const entries = await filesystem.readdir(directory, { withFileTypes: true }) + for (const entry of entries) { + if (entry.isDirectory()) { + directories.push(join(directory, entry.name)) + } + } + const operations = await visitDirectory(directory, entries) + for (let offset = 0; offset < operations.length; offset += 32) { + const settled = await Promise.allSettled( + operations.slice(offset, offset + 32).map((run) => run()) + ) + for (const result of settled) { + if (result.status === 'rejected') { + throw result.reason + } + } + } + } +} + +export async function makeTreeReadOnlyAsync(root) { + await walkTreeAsync(root, (directory, entries) => + entries + .filter((entry) => !entry.isDirectory() && !entry.isSymbolicLink()) + .map((entry) => async () => { + const file = join(directory, entry.name) + await filesystem.chmod(file, (await filesystem.stat(file)).mode & ~0o222) + }) + ) +} diff --git a/config/scripts/space-sharing-copy.test.ts b/config/scripts/space-sharing-copy.test.ts index 351f44b286e..7d6eb32b5f6 100644 --- a/config/scripts/space-sharing-copy.test.ts +++ b/config/scripts/space-sharing-copy.test.ts @@ -19,12 +19,66 @@ import { copyPrivateTree, hardlinkTree, makeTreeReadOnly, + makeTreeReadOnlyAsync, makeTreeWritable, - shareTree + shareTree, + shareTreeAsync } from './space-sharing-copy.mjs' const roots: string[] = [] +describe('asynchronous immutable sharing', () => { + it('preserves read-only source and relative symlinks on the execution host', async () => { + const { root, source } = makeTree() + await makeTreeReadOnlyAsync(source) + const destination = path.join(root, 'async-copy') + await shareTreeAsync(source, destination) + expect(readFileSync(path.join(destination, 'nested', 'file'), 'utf8')).toBe('contents') + expect(readlinkSync(path.join(destination, 'relative-link'))).toBe(path.join('nested', 'file')) + expect(statSync(path.join(destination, 'nested', 'file')).mode & 0o222).toBe(0) + }) + + it('removes partial copies before its Linux hardlink fallback', async () => { + const { root, source } = makeTree() + const destination = path.join(root, 'async-copy') + const reflink = async (_source: string, target: string) => { + mkdirSync(target) + writeFileSync(path.join(target, 'partial'), 'partial') + throw new Error('unsupported reflink') + } + expect(await shareTreeAsync(source, destination, { platform: 'linux', reflink })).toBe( + 'hardlink' + ) + expect(existsSync(path.join(destination, 'partial'))).toBe(false) + expect(statSync(path.join(destination, 'nested', 'file')).ino).toBe( + statSync(path.join(source, 'nested', 'file')).ino + ) + expect(readlinkSync(path.join(destination, 'relative-link'))).toBe(path.join('nested', 'file')) + }) + + it('keeps the event loop available while a sharing process is pending', async () => { + const { root, source } = makeTree() + let finish!: () => void + const pending = new Promise((resolve) => { + finish = resolve + }) + let completed = false + const sharing = shareTreeAsync(source, path.join(root, 'async-copy'), { + platform: 'darwin', + clone: async () => { + await pending + } + }).then(() => { + completed = true + }) + await new Promise((resolve) => setImmediate(resolve)) + expect(completed).toBe(false) + finish() + await sharing + expect(completed).toBe(true) + }) +}) + afterEach(() => { while (roots.length > 0) { rmSync(roots.pop()!, { recursive: true, force: true }) diff --git a/config/tsconfig.e2e.json b/config/tsconfig.e2e.json index 1304f50f963..426c7340c9e 100644 --- a/config/tsconfig.e2e.json +++ b/config/tsconfig.e2e.json @@ -16,6 +16,7 @@ ], "exclude": ["../tests/e2e/.cross-version-checkouts/**"], "compilerOptions": { + "allowImportingTsExtensions": true, // Why: specs run browser code inside page.evaluate, so the DOM lib has to be visible here // even though the harness itself is Node. "lib": ["ESNext", "DOM", "DOM.Iterable"], diff --git a/docs/readme/release-source-sharing.md b/docs/readme/release-source-sharing.md new file mode 100644 index 00000000000..4ccf3449ef4 --- /dev/null +++ b/docs/readme/release-source-sharing.md @@ -0,0 +1,107 @@ +# Shared historical source for cross-version tests (P2) + +Cross-version tests extract historical releases under each worktree's +`tests/e2e/.cross-version-checkouts`. Previously, branches testing the same release each +retained a full copy. Local tests now prepare one read-only source seed and share its file +data with each worktree's independent import directory. + +## How it works + +Seeds live under `/orca-cache/release-source/format-8/`. +The resolved commit and extraction format identify the source, including its rewritten imports. +The common Git directory lets sibling worktrees reuse the same seed without scanning them. +Existing local checkouts remain in place; the new format creates fresh import directories. + +Missing-package stand-ins depend on the current branch's dependencies, so each worktree +creates those locally. Mutable copies of current-branch modules also remain local. +The existing lock, staging directory, and rename transaction publishes complete seeds and +import directories. Invalid stamps, missing protocol source, or invalid package metadata +cause rebuilding or ordinary local extraction. + +macOS uses APFS clones. Linux tries reflinks, then read-only hardlinks. Copies preserve the +seed's read-only permissions; these historical files are never edited by the harness. +Windows uses independent copies because removing read-only hardlinks can change permissions +on surviving links. Unsupported sharing falls back to a private copy. If the optional shared +cache fails, the harness extracts locally and continues the test. + +The current format retains at most eight seeds, ordered by recent use. A shared lock covers +publication, copying, and eviction, so cleanup cannot delete a source while another test +copies it. Copying and permission changes are asynchronous so the lock heartbeat keeps running. +Interrupted staging directories are removed on the next use. Retention only touches the +current format; older implementations may still be using other formats. Eviction leaves +already-created import directories usable. + +## Controls and scope + +Sharing is enabled for ordinary local cross-version tests. Set +`ORCA_DISABLE_SHARED_RELEASE_CACHE=1` to keep ordinary local extraction. CI and explicit +`cacheRoot` options remain isolated unless a caller explicitly supplies `sharedCacheRoot`. + +This only changes test tooling. It adds no development-server task or background worktree +scan. Sharing does not migrate or delete existing local caches. Tests running on a remote host use that +host's repository and filesystem; no client filesystem or remote protocol is involved. +Folder contexts without a Git repository cannot provide a historical release checkout. + +## Cleaning up older historical caches + +Preview cleanup across this repository's registered local worktrees and shared source cache: + +```sh +pnpm run reclaim:worktree-space +``` + +The default selects stamped release checkouts older than 30 days, including previous extraction +formats. To apply the preview, first stop cross-version tests in all this repository's worktrees: + +```sh +pnpm run reclaim:worktree-space --apply --tests-stopped +``` + +Use `--release-cache-min-age-days 7` for a shorter retention window, or `--release-cache-min-age-days 0` to select all +recognised completed release checkouts. `--repo DIRECTORY` selects another local repository. +Each run rescans and rechecks eligibility; it does not apply a saved preview. + +This is manual because older harnesses do not hold a lease while importing source. Locks prevent +deletion during publication or shared-source copying, but do not prove that a completed checkout +is unused. Local checkout age uses directory modification time, which ordinary reuse does not +refresh; shared seed timestamps reflect recent copying. `--tests-stopped` acknowledges that the +caller has stopped consumers; it does not detect them. + +The script checks matching commit/format stamps, skips symlinked cache paths and future formats, +and leaves mutable `working-tree` copies, source, dependencies, and worktree registrations alone. +It reports logical bytes rather than promised disk savings: another clone may retain the same +blocks. Deleted historical source rebuilds on the next test run. Run the command on the execution +host for SSH repositories. The same manual command also performs P1's Electron maintenance; +automatic dev-bundle sweeps exclude historical test cache cleanup. + +## Measurement and validation + +A disposable macOS/APFS run against `v1.4.190` compared ordinary extraction with two separate +local caches using one shared seed. Each import tree contained 8,175 files. APFS private file +allocation was measured after all three views existed, without following symlinks. + +| Method | Elapsed time | Private file data per import directory | +| -------------------------------- | ------------ | -------------------------------------- | +| Ordinary extraction | 4.64 s | 68.79 MiB | +| Shared source, creating the seed | 22.06 s | 44 KiB | +| Shared source, reusing the seed | 1.33 s | 44 KiB | + +Shared blocks still occupy approximately one source copy. Private bytes exclude those blocks +and filesystem metadata, so they are not the total footprint. The cold seed costs extra work; +one timing sample on a busy machine does not establish a speed guarantee. Deleting the seed +left the other import directory readable. No existing worktrees were cleaned up. + +Tests cover concurrent publication, lock heartbeats and eviction, independent branch stand-ins, +corrupt seeds, fallback, the opt-out, and preserved read-only files. Run them with: + +```sh +ORCA_BACKGROUND_LAUNCH=1 pnpm test tests/e2e/cross-version-wire/release-source-cache.unit.test.ts tests/e2e/cross-version-wire/release-source-retention.unit.test.ts tests/e2e/cross-version-wire/release-checkout.unit.test.ts tests/e2e/cross-version-wire/release-checkout-tree.unit.test.ts tests/e2e/cross-version-wire/release-missing-packages.unit.test.ts config/scripts/space-sharing-copy.test.ts +``` + +## P3 remains an experiment + +P3 shares the tracked source of new Orca worktrees. It requires independent writable files, +Git indexes, and matching checkout settings, and has more compatibility risk than P2's +immutable test source. The APFS experiment demonstrated substantial storage savings but +variable creation time and stays out of this change. Its code, measurements, and design notes +are preserved on `nwparker/tracked-source-sharing-experiment`; [the draft PR is closed](https://github.com/stablyai/orca/pull/27618). diff --git a/docs/readme/worktree-storage-maintenance.md b/docs/readme/worktree-storage-maintenance.md index 98e9b98f027..536a5b2a31d 100644 --- a/docs/readme/worktree-storage-maintenance.md +++ b/docs/readme/worktree-storage-maintenance.md @@ -6,18 +6,27 @@ Run from the repository to preview maintenance across its locally registered Git pnpm run reclaim:worktree-space ``` -Apply the previewed operations with: +After stopping cross-version tests in all this repository's worktrees, apply cleanup with: ```sh -pnpm run reclaim:worktree-space --apply +pnpm run reclaim:worktree-space --apply --tests-stopped ``` -The command reuses the existing Electron distribution migration and dev bundle cleanup scripts. +The command reuses the historical test cache, Electron distribution, and dev bundle cleanup scripts. It shares complete Electron distributions through the repository cache and removes idle -`out/electron-dev` bundles, which development rebuilds on demand. It leaves source files, +`out/electron-dev` bundles, which development rebuilds on demand. It also removes stamped historical +test checkouts and shared seeds older than 30 days; tests regenerate them when needed. It leaves source files, dependency installs, worktree registrations, and other build output alone. Reported sizes are logical file data; APFS clones and hardlinks mean actual disk savings can be smaller. +Use `--release-cache-min-age-days 7` to shorten historical-cache retention, or `0` to select all +recognised completed historical checkouts. This age option applies only to test caches. Local +checkout age uses directory modification time, which ordinary reuse does not refresh. The +`--tests-stopped` flag acknowledges stopped tests; older harnesses cannot prove a completed +checkout is unused. Missing acknowledgement rejects manual apply before any cleanup begins. +Cache cleanup skips symlinks, malformed stamps, future formats, and publication/copy locks. +See [release source sharing](./release-source-sharing.md) for the cache and locking details. + Use `--repo DIRECTORY` to choose another local repository. Git's worktree list determines scope, including registered worktrees outside Orca's workspace directory. Folder workspaces without Git have no repository-wide sweep. These commands run on the local host: they do not inspect or delete @@ -27,8 +36,11 @@ anything on SSH hosts. Run maintenance on the execution host when maintaining re Healthy Electron installs join the shared distribution cache during native dependency setup, without waiting for an Electron upgrade. The manual command migrates older installs immediately. +Read-only Electron package directories are retained and reported as skipped. Cache publication +or copying failures are also reported as skipped rather than silently omitted. -Cleanup runs only when explicitly requested. Starting `pnpm dev` does not trigger a sweep. +Historical test cache cleanup runs only in this explicit manual command. Automatic invocations +exclude it because tests may still be importing source. The repository lock prevents competing cleanup commands. Dev bundle removal checks active processes and recent unfinished builds, then rechecks eligibility immediately before deletion. An unreadable process table prevents removal. diff --git a/tests/e2e/cross-version-wire/release-checkout-publication.ts b/tests/e2e/cross-version-wire/release-checkout-publication.ts new file mode 100644 index 00000000000..05e662c6a2f --- /dev/null +++ b/tests/e2e/cross-version-wire/release-checkout-publication.ts @@ -0,0 +1,175 @@ +import { constants } from 'node:fs' +import { access, mkdtemp, mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { lock } from 'proper-lockfile' +import { scavengeReleaseCheckoutStaging } from './release-checkout-tree.ts' + +export const CHECKOUT_FORMAT = 8 + +export type CheckoutLockOptions = { + realpath: false + stale: number + update: number + retries: { + retries: number + factor: number + minTimeout: number + maxTimeout: number + randomize: boolean + } +} + +type CheckoutLockRelease = () => Promise +type AcquireCheckoutLock = ( + root: string, + options: CheckoutLockOptions +) => Promise + +export type CheckoutLifecycleContext = { + root: string + stagingPrefix: string +} + +export type CheckoutStagingContext = CheckoutLifecycleContext & { + staging: string +} + +export type MaterializeReleaseCheckoutTestHooks = { + lockOptions?: CheckoutLockOptions + acquireLock?: AcquireCheckoutLock + onLockAttempt?: (context: CheckoutLifecycleContext) => void + onLockAcquired?: (context: CheckoutLifecycleContext) => void | Promise + onStagingCreated?: (context: CheckoutStagingContext) => void | Promise + populateStaging?: (context: CheckoutStagingContext) => Promise +} + +const DEFAULT_LOCK_OPTIONS: CheckoutLockOptions = { + realpath: false, + stale: 60_000, + update: 10_000, + retries: { retries: 480, factor: 1, minTimeout: 250, maxTimeout: 250, randomize: true } +} + +type CheckoutStamp = { commit: string; format: number } + +async function readStamp(root: string): Promise { + try { + const value: unknown = JSON.parse(await readFile(join(root, 'checkout-stamp.json'), 'utf8')) + return typeof value === 'object' && + value !== null && + 'commit' in value && + typeof value.commit === 'string' && + 'format' in value && + typeof value.format === 'number' + ? { commit: value.commit, format: value.format } + : null + } catch { + return null + } +} + +async function checkoutMatches(root: string, commit: string): Promise { + const stamp = await readStamp(root) + if (stamp?.commit !== commit || stamp.format !== CHECKOUT_FORMAT) { + return false + } + try { + await assertCheckoutWireSurface(root, commit) + return true + } catch { + return false + } +} + +async function assertCheckoutWireSurface(root: string, ref: string): Promise { + try { + await access(join(root, 'src', 'shared', 'terminal-stream-protocol.ts'), constants.F_OK) + } catch { + throw new Error( + `Cross-version checkout for ${ref} is missing the terminal stream protocol; ` + + 'the wire surface moved and the harness needs updating.' + ) + } +} + +export async function publishReleaseCheckout({ + root, + ref, + commit, + populateStaging, + prepareForPublication, + hooks +}: { + root: string + ref: string + commit: string + populateStaging: (staging: string) => Promise + prepareForPublication?: (staging: string) => Promise + hooks?: MaterializeReleaseCheckoutTestHooks +}): Promise { + if (await checkoutMatches(root, commit)) { + return + } + + const stagingPrefix = `.staging-${commit}-format-${CHECKOUT_FORMAT}-` + const lifecycleContext = { root, stagingPrefix } + const lockOptions = hooks?.lockOptions ?? DEFAULT_LOCK_OPTIONS + const acquireLock: AcquireCheckoutLock = + hooks?.acquireLock ?? ((target, value) => lock(target, value)) + await mkdir(dirname(root), { recursive: true }) + let releaseLock: CheckoutLockRelease | undefined + try { + const acquiring = acquireLock(root, lockOptions) + try { + hooks?.onLockAttempt?.(lifecycleContext) + } catch (error) { + await acquiring.then( + async (release) => release(), + () => undefined + ) + throw error + } + releaseLock = await acquiring + } catch (error) { + if (await checkoutMatches(root, commit)) { + return + } + throw new Error(`Cross-version harness could not lock ${ref} (${commit}): ${String(error)}`) + } + + let staging: string | undefined + try { + await hooks?.onLockAcquired?.(lifecycleContext) + if (await checkoutMatches(root, commit)) { + return + } + await scavengeReleaseCheckoutStaging(dirname(root), stagingPrefix) + staging = await mkdtemp(join(dirname(root), stagingPrefix)) + const stagingContext = { ...lifecycleContext, staging } + await hooks?.onStagingCreated?.(stagingContext) + await (hooks?.populateStaging + ? hooks.populateStaging(stagingContext) + : populateStaging(staging)) + await assertCheckoutWireSurface(staging, ref) + await writeFile( + join(staging, 'checkout-stamp.json'), + `${JSON.stringify({ commit, format: CHECKOUT_FORMAT } satisfies CheckoutStamp, null, 2)}\n` + ) + await prepareForPublication?.(staging) + await rm(root, { recursive: true, force: true }) + await rename(staging, root) + staging = undefined + } catch (error) { + if (await checkoutMatches(root, commit)) { + return + } + throw new Error(`Cross-version harness failed to extract ${ref} (${commit}): ${String(error)}`) + } finally { + if (staging) { + await rm(staging, { recursive: true, force: true }) + } + await releaseLock() + } + + await assertCheckoutWireSurface(root, ref) +} diff --git a/tests/e2e/cross-version-wire/release-checkout-tree.ts b/tests/e2e/cross-version-wire/release-checkout-tree.ts index 24085ede7c4..fdfee479d93 100644 --- a/tests/e2e/cross-version-wire/release-checkout-tree.ts +++ b/tests/e2e/cross-version-wire/release-checkout-tree.ts @@ -240,8 +240,9 @@ async function releaseWorkspaceArchivePaths( export async function extractReleaseCheckoutTree( repoRoot: string, staging: string, - commit: string -): Promise { + commit: string, + sourceOnly = false +): Promise { const archive = join(staging, '.release-checkout.tar') const deadline = Date.now() + CHECKOUT_PROCESS_TIMEOUT_MS const workspacePaths = await releaseWorkspaceArchivePaths(repoRoot, commit, deadline) @@ -273,12 +274,15 @@ export async function extractReleaseCheckoutTree( } const packageImports: PackageImports = new Map() await prepareExtractedTree(staging, workspacePaths, packageImports) - await installMissingPackageStandIns( - staging, - commit, - packageImports, - JSON.parse(await readFile(join(repoRoot, 'package.json'), 'utf8')) - ) + if (!sourceOnly) { + await installMissingPackageStandIns( + staging, + commit, + packageImports, + JSON.parse(await readFile(join(repoRoot, 'package.json'), 'utf8')) + ) + } + return packageImports } export async function scavengeReleaseCheckoutStaging( diff --git a/tests/e2e/cross-version-wire/release-checkout.ts b/tests/e2e/cross-version-wire/release-checkout.ts index 2267b74817c..e3ea64469b0 100644 --- a/tests/e2e/cross-version-wire/release-checkout.ts +++ b/tests/e2e/cross-version-wire/release-checkout.ts @@ -1,23 +1,28 @@ import { execFileSync } from 'node:child_process' -import { constants } from 'node:fs' -import { access, copyFile, mkdtemp, mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises' +import { copyFile, mkdir, rename } from 'node:fs/promises' import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path' -import { lock } from 'proper-lockfile' -import { - extractReleaseCheckoutTree, - scavengeReleaseCheckoutStaging -} from './release-checkout-tree.ts' import { selectLatestStableReleaseTag } from '../../../config/scripts/stable-release-tags.mjs' +import { + populateReleaseCheckoutSource, + resolveSharedReleaseCacheRoot +} from './release-source-cache.ts' +import { + CHECKOUT_FORMAT, + publishReleaseCheckout, + type MaterializeReleaseCheckoutTestHooks +} from './release-checkout-publication.ts' +export type { + CheckoutLockOptions, + CheckoutLifecycleContext, + CheckoutStagingContext, + MaterializeReleaseCheckoutTestHooks +} from './release-checkout-publication.ts' export { selectLatestStableReleaseTag } export const REPO_ROOT = resolve(import.meta.dirname, '..', '..', '..') const DEFAULT_CACHE_ROOT = join(REPO_ROOT, 'tests', 'e2e', '.cross-version-checkouts') -// Released sources still import @streamparser/json; retain its pinned test-only dependency. -// Bump when extraction or the alias rewrite changes so cached trees are rebuilt. -const CHECKOUT_FORMAT = 6 - const BASELINE_REF_ENV = 'ORCA_CROSS_VERSION_BASELINE_REF' export type ReleaseCheckout = { @@ -33,54 +38,11 @@ export type ReleaseCheckout = { export type MaterializeReleaseCheckoutOptions = { cacheRoot?: string + sharedCacheRoot?: string /** Test-only lifecycle seams; production callers must use the defaults. */ testHooks?: MaterializeReleaseCheckoutTestHooks } -export type CheckoutLockOptions = { - realpath: false - stale: number - update: number - retries: { - retries: number - factor: number - minTimeout: number - maxTimeout: number - randomize: boolean - } -} - -type CheckoutLockRelease = () => Promise -type AcquireCheckoutLock = ( - root: string, - options: CheckoutLockOptions -) => Promise - -export type CheckoutLifecycleContext = { - root: string - stagingPrefix: string -} - -export type CheckoutStagingContext = CheckoutLifecycleContext & { - staging: string -} - -export type MaterializeReleaseCheckoutTestHooks = { - lockOptions?: CheckoutLockOptions - acquireLock?: AcquireCheckoutLock - onLockAttempt?: (context: CheckoutLifecycleContext) => void - onLockAcquired?: (context: CheckoutLifecycleContext) => void | Promise - onStagingCreated?: (context: CheckoutStagingContext) => void | Promise - populateStaging?: (context: CheckoutStagingContext) => Promise -} - -const DEFAULT_LOCK_OPTIONS: CheckoutLockOptions = { - realpath: false, - stale: 60_000, - update: 10_000, - retries: { retries: 480, factor: 1, minTimeout: 250, maxTimeout: 250, randomize: true } -} - function git(args: string[]): string { return execFileSync('git', args, { cwd: REPO_ROOT, @@ -132,32 +94,6 @@ function resolveCommit(ref: string): string { } } -type CheckoutStamp = { commit: string; format: number } - -async function readStamp(root: string): Promise { - try { - return JSON.parse(await readFile(join(root, 'checkout-stamp.json'), 'utf8')) as CheckoutStamp - } catch { - return null - } -} - -async function checkoutMatches(root: string, commit: string): Promise { - const stamp = await readStamp(root) - return stamp?.commit === commit && stamp.format === CHECKOUT_FORMAT -} - -async function assertCheckoutWireSurface(root: string, ref: string): Promise { - try { - await access(join(root, 'src', 'shared', 'terminal-stream-protocol.ts'), constants.F_OK) - } catch { - throw new Error( - `Cross-version checkout for ${ref} is missing the terminal stream protocol; ` + - 'the wire surface moved and the harness needs updating.' - ) - } -} - function checkoutModulePath(root: string, rootRelativePath: string): string { const fromRoot = rootRelativePath.replace(/^[/\\]+/, '') const absolute = resolve(root, fromRoot) @@ -225,70 +161,17 @@ export async function materializeReleaseCheckout( const label = ref.replace(/[^A-Za-z0-9._-]/g, '_') const cacheRoot = options.cacheRoot ?? DEFAULT_CACHE_ROOT const root = join(cacheRoot, label, `${commit}-format-${CHECKOUT_FORMAT}`) - if (await checkoutMatches(root, commit)) { - return { ref, commit, label, root } - } - - const stagingPrefix = `.staging-${commit}-format-${CHECKOUT_FORMAT}-` - const lifecycleContext = { root, stagingPrefix } - const hooks = options.testHooks - const lockOptions = hooks?.lockOptions ?? DEFAULT_LOCK_OPTIONS - const acquireLock: AcquireCheckoutLock = - hooks?.acquireLock ?? ((target, value) => lock(target, value)) - await mkdir(dirname(root), { recursive: true }) - let releaseLock: CheckoutLockRelease | undefined - try { - const acquiring = acquireLock(root, lockOptions) - try { - hooks?.onLockAttempt?.(lifecycleContext) - } catch (error) { - await acquiring.then( - async (release) => release(), - () => undefined - ) - throw error + await publishReleaseCheckout({ + root, + ref, + commit, + hooks: options.testHooks, + populateStaging: async (staging) => { + const sharedCacheRoot = + options.sharedCacheRoot ?? + (await resolveSharedReleaseCacheRoot(REPO_ROOT, options.cacheRoot)) + await populateReleaseCheckoutSource(REPO_ROOT, staging, commit, sharedCacheRoot) } - releaseLock = await acquiring - } catch (error) { - if (await checkoutMatches(root, commit)) { - return { ref, commit, label, root } - } - throw new Error(`Cross-version harness could not lock ${ref} (${commit}): ${String(error)}`) - } - - let staging: string | undefined - try { - await hooks?.onLockAcquired?.(lifecycleContext) - if (await checkoutMatches(root, commit)) { - return { ref, commit, label, root } - } - await scavengeReleaseCheckoutStaging(dirname(root), stagingPrefix) - staging = await mkdtemp(join(dirname(root), stagingPrefix)) - const stagingContext = { ...lifecycleContext, staging } - await hooks?.onStagingCreated?.(stagingContext) - await (hooks?.populateStaging - ? hooks.populateStaging(stagingContext) - : extractReleaseCheckoutTree(REPO_ROOT, staging, commit)) - await assertCheckoutWireSurface(staging, ref) - await writeFile( - join(staging, 'checkout-stamp.json'), - `${JSON.stringify({ commit, format: CHECKOUT_FORMAT } satisfies CheckoutStamp, null, 2)}\n` - ) - await rm(root, { recursive: true, force: true }) - await rename(staging, root) - staging = undefined - } catch (error) { - if (await checkoutMatches(root, commit)) { - return { ref, commit, label, root } - } - throw new Error(`Cross-version harness failed to extract ${ref} (${commit}): ${String(error)}`) - } finally { - if (staging) { - await rm(staging, { recursive: true, force: true }) - } - await releaseLock() - } - - await assertCheckoutWireSurface(root, ref) + }) return { ref, commit, label, root } } diff --git a/tests/e2e/cross-version-wire/release-source-cache.ts b/tests/e2e/cross-version-wire/release-source-cache.ts new file mode 100644 index 00000000000..3c99fdaa585 --- /dev/null +++ b/tests/e2e/cross-version-wire/release-source-cache.ts @@ -0,0 +1,169 @@ +import { cp, mkdir, readFile, readdir, rename, rm, utimes, writeFile } from 'node:fs/promises' +import { join, resolve } from 'node:path' +import { + makeTreeReadOnlyAsync, + shareTreeAsync +} from '../../../config/scripts/space-sharing-copy.mjs' +import { extractReleaseCheckoutTree } from './release-checkout-tree.ts' +import { installMissingPackageStandIns, type PackageImports } from './release-missing-packages.ts' +import { publishReleaseCheckout } from './release-checkout-publication.ts' +import { pruneSharedReleaseSources, withReleaseSourceLease } from './release-source-retention.ts' + +const IMPORTS_FILENAME = 'release-package-imports.json' + +export async function populateReleaseCheckoutSource( + repoRoot: string, + staging: string, + commit: string, + sharedCacheRoot: string | undefined +): Promise { + if (sharedCacheRoot) { + try { + await populateFromSharedReleaseSource(repoRoot, staging, commit, sharedCacheRoot) + return + } catch (error) { + console.warn( + `[cross-version] Shared release source unavailable; extracting locally: ${String(error)}` + ) + await rm(staging, { recursive: true, force: true }) + await mkdir(staging, { recursive: true }) + } + } + await extractReleaseCheckoutTree(repoRoot, staging, commit) +} + +function readPackageImports(value: unknown): PackageImports { + if (!Array.isArray(value)) { + throw new Error('Invalid cached release package imports') + } + const result: PackageImports = new Map() + for (const entry of value) { + if ( + !Array.isArray(entry) || + entry.length !== 2 || + typeof entry[0] !== 'string' || + !Array.isArray(entry[1]) + ) { + throw new Error('Invalid cached release package imports') + } + const names = entry[1].filter((name): name is string => typeof name === 'string') + if (names.length !== entry[1].length) { + throw new Error('Invalid cached release import names') + } + result.set(entry[0], new Set(names)) + } + return result +} + +export async function populateFromSharedReleaseSource( + repoRoot: string, + staging: string, + commit: string, + sharedCacheRoot: string, + { extract = extractReleaseCheckoutTree, copy = copySharedReleaseSource } = {} +): Promise { + await withReleaseSourceLease(sharedCacheRoot, async (formatRoot, verifyLease) => { + const seed = join(formatRoot, commit) + try { + await pruneSharedReleaseSources(formatRoot, commit, verifyLease) + await publishReleaseCheckout({ + root: seed, + ref: commit, + commit, + populateStaging: async (seedStaging) => { + const imports = await extract(repoRoot, seedStaging, commit, true) + await writeFile( + join(seedStaging, IMPORTS_FILENAME), + JSON.stringify([...imports].map(([specifier, names]) => [specifier, [...names]])) + ) + }, + prepareForPublication: async (seedStaging) => { + verifyLease() + await makeTreeReadOnlyAsync(seedStaging) + verifyLease() + } + }) + const imports = readPackageImports( + JSON.parse(await readFile(join(seed, IMPORTS_FILENAME), 'utf8')) + ) + const copied = join(staging, '.release-source') + verifyLease() + await copy(seed, copied) + verifyLease() + for (const entry of await readdir(copied)) { + if (entry !== 'checkout-stamp.json' && entry !== IMPORTS_FILENAME) { + await rename(join(copied, entry), join(staging, entry)) + } + } + await rm(copied, { recursive: true, force: true }) + const now = new Date() + await utimes(seed, now, now) + await installMissingPackageStandIns( + staging, + commit, + imports, + JSON.parse(await readFile(join(repoRoot, 'package.json'), 'utf8')) + ) + } catch (error) { + verifyLease() + await rm(seed, { recursive: true, force: true }).catch(() => {}) + throw error + } + }) +} + +export async function copySharedReleaseSource( + source: string, + destination: string, + platform: NodeJS.Platform = process.platform +): Promise { + try { + await shareTreeAsync( + source, + destination, + platform === 'win32' + ? { + platform, + hardlink: async () => { + throw new Error('Windows removal changes hardlink permissions') + } + } + : { platform } + ) + } catch { + await cp(source, destination, { recursive: true, dereference: false, verbatimSymlinks: true }) + } +} + +export async function resolveSharedReleaseCacheRoot( + repoRoot: string, + isolatedCacheRoot?: string +): Promise { + if ( + isolatedCacheRoot || + process.env.CI || + process.env.ORCA_DISABLE_SHARED_RELEASE_CACHE === '1' + ) { + return undefined + } + try { + const { runProcess } = await import('@orca/process-host') + const result = await runProcess({ + program: 'git', + args: ['rev-parse', '--git-common-dir'], + cwd: repoRoot, + timeoutMs: 10_000, + terminationBarrier: true + }) + if (result.code !== 0 || result.timedOut) { + return undefined + } + const common = result.stdout.replace(process.platform === 'win32' ? /\r?\n$/ : /\n$/, '') + if (!common || common.includes('\n')) { + return undefined + } + return join(resolve(repoRoot, common), 'orca-cache', 'release-source') + } catch { + return undefined + } +} diff --git a/tests/e2e/cross-version-wire/release-source-cache.unit.test.ts b/tests/e2e/cross-version-wire/release-source-cache.unit.test.ts new file mode 100644 index 00000000000..e88880ffff1 --- /dev/null +++ b/tests/e2e/cross-version-wire/release-source-cache.unit.test.ts @@ -0,0 +1,202 @@ +import { chmod, mkdtemp, mkdir, readFile, rm, writeFile, readdir, stat } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, expect, it, vi } from 'vitest' +import { runProcess } from '@orca/process-host' +import { materializeReleaseCheckout } from './release-checkout' +import { + copySharedReleaseSource, + populateFromSharedReleaseSource, + resolveSharedReleaseCacheRoot +} from './release-source-cache' +import { CHECKOUT_FORMAT } from './release-checkout-publication' + +const roots: string[] = [] +afterEach(async () => { + vi.unstubAllEnvs() + for (const root of roots.splice(0)) { + await rm(root, { recursive: true, force: true }) + } +}) + +it('keeps Windows copies independent and read-only after seed removal', async () => { + const root = await scratch() + const seed = join(root, 'seed') + const copied = join(root, 'copy') + await mkdir(seed) + await writeFile(join(seed, 'source.ts'), 'released bytes\n') + await chmod(join(seed, 'source.ts'), 0o444) + await copySharedReleaseSource(seed, copied, 'win32') + expect((await stat(join(copied, 'source.ts'))).ino).not.toBe( + (await stat(join(seed, 'source.ts'))).ino + ) + await rm(seed, { recursive: true, force: true }) + expect(await readFile(join(copied, 'source.ts'), 'utf8')).toBe('released bytes\n') + expect((await stat(join(copied, 'source.ts'))).mode & 0o222).toBe(0) +}) + +it('keeps CI, isolated roots and the environment opt-out on local extraction', async () => { + vi.stubEnv('CI', '') + vi.stubEnv('ORCA_DISABLE_SHARED_RELEASE_CACHE', '') + expect( + await resolveSharedReleaseCacheRoot(process.cwd(), '/explicit-isolated-root') + ).toBeUndefined() + vi.stubEnv('CI', '1') + expect(await resolveSharedReleaseCacheRoot(process.cwd())).toBeUndefined() + vi.stubEnv('CI', '') + vi.stubEnv('ORCA_DISABLE_SHARED_RELEASE_CACHE', '1') + expect(await resolveSharedReleaseCacheRoot(process.cwd())).toBeUndefined() + vi.stubEnv('ORCA_DISABLE_SHARED_RELEASE_CACHE', '') + const folder = await scratch() + expect(await resolveSharedReleaseCacheRoot(folder)).toBeUndefined() +}) + +it('resolves the default cache inside the execution repository common Git directory', async () => { + vi.stubEnv('CI', '') + vi.stubEnv('ORCA_DISABLE_SHARED_RELEASE_CACHE', '') + const repo = await scratch() + const result = await runProcess({ + program: 'git', + args: ['init', '--quiet'], + cwd: repo, + timeoutMs: 10_000, + terminationBarrier: true + }) + expect(result.code).toBe(0) + expect(await resolveSharedReleaseCacheRoot(repo)).toBe( + join(repo, '.git', 'orca-cache', 'release-source') + ) +}) + +it.each(['missing wire source', 'invalid imports metadata'])( + 'repairs a shared seed with %s', + async (damage) => { + const root = await scratch() + const shared = join(root, 'shared') + const commit = 'b'.repeat(40) + const seed = join(shared, `format-${CHECKOUT_FORMAT}`, commit) + await writeFile(join(root, 'package.json'), '{}') + const extract = vi.fn(async (_repo: string, staging: string) => { + await mkdir(join(staging, 'src', 'shared'), { recursive: true }) + await writeFile(join(staging, 'src', 'shared', 'terminal-stream-protocol.ts'), 'export {}\n') + return new Map>() + }) + const populate = async (name: string): Promise => { + const staging = join(root, name) + await mkdir(staging) + await populateFromSharedReleaseSource(root, staging, commit, shared, { extract }) + } + await populate('first') + if (damage === 'missing wire source') { + await rm(join(seed, 'src', 'shared', 'terminal-stream-protocol.ts')) + } else { + const metadata = join(seed, 'release-package-imports.json') + await chmod(metadata, 0o644) + await writeFile(metadata, '[false]') + await expect(populate('damaged')).rejects.toThrow('Invalid cached release package imports') + await expect(stat(seed)).rejects.toThrow() + } + await populate('repaired') + expect(extract).toHaveBeenCalledTimes(2) + expect( + await readFile(join(root, 'repaired', 'src', 'shared', 'terminal-stream-protocol.ts'), 'utf8') + ).toBe('export {}\n') + } +) + +async function scratch(): Promise { + const root = await mkdtemp(join(tmpdir(), 'orca-release-source-sharing-')) + roots.push(root) + return root +} + +it('shares one published release source between separate local caches', async () => { + const root = await scratch() + const sharedCacheRoot = join(root, 'shared') + const [a, b] = await Promise.all([ + materializeReleaseCheckout('v1.4.190', { cacheRoot: join(root, 'a'), sharedCacheRoot }), + materializeReleaseCheckout('v1.4.190', { cacheRoot: join(root, 'b'), sharedCacheRoot }) + ]) + expect(a.root).not.toBe(b.root) + expect(a.commit).toBe(b.commit) + expect(await readdir(sharedCacheRoot)).toEqual([`format-${CHECKOUT_FORMAT}`]) + const source = join('src', 'shared', 'terminal-stream-protocol.ts') + expect(await readFile(join(a.root, source), 'utf8')).toBe( + await readFile(join(b.root, source), 'utf8') + ) + const seed = join(sharedCacheRoot, `format-${CHECKOUT_FORMAT}`, a.commit) + expect(await readFile(join(seed, source), 'utf8')).toBe( + await readFile(join(a.root, source), 'utf8') + ) + expect((await stat(join(b.root, source))).mode & 0o222).toBe(0) + await rm(a.root, { recursive: true }) + expect(await readFile(join(b.root, source), 'utf8')).toBe( + await readFile(join(seed, source), 'utf8') + ) +}, 120_000) + +it('extracts locally when the optional shared cache cannot be created', async () => { + const root = await scratch() + const sharedCacheRoot = join(root, 'blocked-cache') + await writeFile(sharedCacheRoot, 'occupied by a file') + const warning = vi.spyOn(console, 'warn').mockImplementation(() => {}) + try { + const checkout = await materializeReleaseCheckout('v1.4.190', { + cacheRoot: join(root, 'local'), + sharedCacheRoot + }) + expect( + await readFile(join(checkout.root, 'src', 'shared', 'terminal-stream-protocol.ts'), 'utf8') + ).toContain('export') + expect(warning).toHaveBeenCalledTimes(1) + expect(await readFile(sharedCacheRoot, 'utf8')).toBe('occupied by a file') + } finally { + warning.mockRestore() + } +}, 120_000) + +it('creates branch-dependent package stand-ins locally without changing the shared source', async () => { + const root = await scratch() + const shared = join(root, 'shared') + const commit = 'a'.repeat(40) + const seed = join(shared, `format-${CHECKOUT_FORMAT}`, commit) + const extract = vi.fn(async (_repo: string, staging: string) => { + await mkdir(join(staging, 'src', 'shared'), { recursive: true }) + await writeFile(join(staging, 'src', 'released.ts'), 'export const released = true\n') + await writeFile( + join(staging, 'src', 'shared', 'terminal-stream-protocol.ts'), + 'export const protocol = true\n' + ) + return new Map([['missing-release-package', new Set(['oldExport'])]]) + }) + for (const declared of [false, true]) { + const repo = join(root, declared ? 'declared' : 'removed') + const staging = join(repo, 'staging') + await mkdir(staging, { recursive: true }) + await writeFile( + join(repo, 'package.json'), + JSON.stringify({ + dependencies: declared ? { 'missing-release-package': '1.0.0' } : {} + }) + ) + await populateFromSharedReleaseSource(repo, staging, commit, shared, { extract }) + const stub = readFile( + join(staging, 'node_modules', 'missing-release-package', 'stand-in-0.js'), + 'utf8' + ) + if (declared) { + await expect(stub).rejects.toThrow() + } else { + expect(await stub).toContain('oldExport') + } + await expect(readFile(join(staging, 'checkout-stamp.json'))).rejects.toThrow() + } + await expect( + readFile(join(seed, 'node_modules', 'missing-release-package', 'stand-in-0.js')) + ).rejects.toThrow() + expect(JSON.parse(await readFile(join(seed, 'checkout-stamp.json'), 'utf8'))).toEqual({ + commit, + format: CHECKOUT_FORMAT + }) + expect(extract).toHaveBeenCalledTimes(1) +}) diff --git a/tests/e2e/cross-version-wire/release-source-retention.ts b/tests/e2e/cross-version-wire/release-source-retention.ts new file mode 100644 index 00000000000..e5267637d8d --- /dev/null +++ b/tests/e2e/cross-version-wire/release-source-retention.ts @@ -0,0 +1,78 @@ +import { mkdir, readdir, rm, stat } from 'node:fs/promises' +import { join } from 'node:path' +import { lock } from 'proper-lockfile' +import { CHECKOUT_FORMAT, type CheckoutLockOptions } from './release-checkout-publication.ts' + +const COMMIT_DIRECTORY = /^(?:[\da-f]{40}|[\da-f]{64})$/ +export const MAX_SHARED_RELEASE_SEEDS = 8 +const ORPHAN_STAGING = new RegExp( + `^\\.staging-(?:[\\da-f]{40}|[\\da-f]{64})-format-${CHECKOUT_FORMAT}-[\\da-zA-Z]{6}$` +) +const LEASE_OPTIONS: CheckoutLockOptions = { + realpath: false, + stale: 60_000, + update: 10_000, + retries: { retries: 480, factor: 1, minTimeout: 250, maxTimeout: 250, randomize: true } +} + +export async function withReleaseSourceLease( + cacheRoot: string, + apply: (formatRoot: string, verifyLease: () => void) => Promise, + lockOptions = LEASE_OPTIONS +): Promise { + return withReleaseSourceFormatLease(cacheRoot, CHECKOUT_FORMAT, apply, lockOptions) +} + +export async function withReleaseSourceFormatLease( + cacheRoot: string, + format: number, + apply: (formatRoot: string, verifyLease: () => void) => Promise, + lockOptions = LEASE_OPTIONS +): Promise { + const formatRoot = join(cacheRoot, `format-${format}`) + await mkdir(formatRoot, { recursive: true }) + let compromised: Error | undefined + const release = await lock(join(formatRoot, 'maintenance'), { + ...lockOptions, + onCompromised: (error) => { + compromised = error + } + }) + const verifyLease = (): void => { + if (compromised) { + throw compromised + } + } + try { + verifyLease() + const result = await apply(formatRoot, verifyLease) + verifyLease() + return result + } finally { + await release() + } +} + +export async function pruneSharedReleaseSources( + formatRoot: string, + keepCommit: string, + verifyLease: () => void, + maxSeeds = MAX_SHARED_RELEASE_SEEDS +): Promise { + const seeds: { name: string; modified: number }[] = [] + for (const entry of await readdir(formatRoot, { withFileTypes: true })) { + if (entry.isDirectory() && COMMIT_DIRECTORY.test(entry.name)) { + seeds.push({ name: entry.name, modified: (await stat(join(formatRoot, entry.name))).mtimeMs }) + } else if (entry.isDirectory() && ORPHAN_STAGING.test(entry.name)) { + verifyLease() + await rm(join(formatRoot, entry.name), { recursive: true, force: true }) + } + } + const others = seeds + .filter((seed) => seed.name !== keepCommit) + .sort((a, b) => b.modified - a.modified || a.name.localeCompare(b.name)) + for (const seed of others.slice(maxSeeds - 1)) { + verifyLease() + await rm(join(formatRoot, seed.name), { recursive: true, force: true }) + } +} diff --git a/tests/e2e/cross-version-wire/release-source-retention.unit.test.ts b/tests/e2e/cross-version-wire/release-source-retention.unit.test.ts new file mode 100644 index 00000000000..4c686be4739 --- /dev/null +++ b/tests/e2e/cross-version-wire/release-source-retention.unit.test.ts @@ -0,0 +1,133 @@ +import { + chmod, + mkdir, + mkdtemp, + readFile, + readdir, + rm, + stat, + utimes, + writeFile +} from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, expect, it, vi } from 'vitest' +import { CHECKOUT_FORMAT, type CheckoutLockOptions } from './release-checkout-publication' +import { + MAX_SHARED_RELEASE_SEEDS, + pruneSharedReleaseSources, + withReleaseSourceLease +} from './release-source-retention' +import { copySharedReleaseSource } from './release-source-cache' + +const roots: string[] = [] +async function scratch() { + const root = await mkdtemp(join(tmpdir(), 'orca-release-source-retention-')) + roots.push(root) + return root +} +afterEach(async () => { + for (const root of roots.splice(0)) { + await rm(root, { recursive: true, force: true }) + } +}) + +it('caps seeds by recent use, preserves imported copies, and scopes cleanup to its format', async () => { + const root = await scratch() + const oldFormat = join(root, 'format-old', 'old-seed') + await mkdir(oldFormat, { recursive: true }) + await withReleaseSourceLease(root, async (formatRoot, verify) => { + const commits: string[] = [] + for (let index = 0; index < MAX_SHARED_RELEASE_SEEDS + 2; index += 1) { + const commit = index.toString(16).padStart(40, '0') + commits.push(commit) + const seed = join(formatRoot, commit) + await mkdir(seed) + await writeFile(join(seed, 'source.ts'), 'export const release = true\n') + await chmod(join(seed, 'source.ts'), 0o444) + await utimes(seed, index + 1, index + 1) + } + const imported = join(root, 'already-imported') + await copySharedReleaseSource(join(formatRoot, commits[1]), imported) + const orphan = `.staging-${commits[1]}-format-${CHECKOUT_FORMAT}-abc123` + await mkdir(join(formatRoot, orphan)) + await mkdir(join(formatRoot, '.staging-unrelated-live-owner')) + await pruneSharedReleaseSources(formatRoot, commits[0], verify) + const retained = (await readdir(formatRoot)).filter((name) => /^[\da-f]{40}$/.test(name)) + expect(retained).toHaveLength(MAX_SHARED_RELEASE_SEEDS) + expect(retained).toContain(commits[0]) + expect(retained).not.toContain(commits[1]) + expect(retained).not.toContain(commits[2]) + expect(await readFile(join(imported, 'source.ts'), 'utf8')).toBe( + 'export const release = true\n' + ) + expect((await stat(join(imported, 'source.ts'))).mode & 0o222).toBe(0) + await expect(stat(join(formatRoot, orphan))).rejects.toMatchObject({ code: 'ENOENT' }) + expect((await stat(join(formatRoot, '.staging-unrelated-live-owner'))).isDirectory()).toBe(true) + }) + expect((await stat(oldFormat)).isDirectory()).toBe(true) +}) + +it('keeps a copy lease alive beyond its stale interval and blocks competing cleanup', async () => { + const root = await scratch() + const options: CheckoutLockOptions = { + realpath: false, + stale: 2500, + update: 1000, + retries: { retries: 200, factor: 1, minTimeout: 50, maxTimeout: 50, randomize: false } + } + let releaseCopy!: () => void + let entered!: () => void + const copying = new Promise((resolve) => { + releaseCopy = resolve + }) + const enteredLease = new Promise((resolve) => { + entered = resolve + }) + const first = withReleaseSourceLease( + root, + async () => { + entered() + await copying + }, + options + ) + await enteredLease + const rivalEntered = vi.fn() + const rival = withReleaseSourceLease( + root, + async () => { + rivalEntered() + }, + options + ) + try { + const start = Date.now() + const mtimes = new Set() + await vi.waitFor( + async () => { + mtimes.add( + (await stat(join(root, `format-${CHECKOUT_FORMAT}`, 'maintenance.lock'))).mtimeMs + ) + expect(Date.now() - start).toBeGreaterThan(options.stale + 250) + expect(mtimes.size).toBeGreaterThanOrEqual(3) + }, + { timeout: 10_000, interval: 100 } + ) + expect(rivalEntered).not.toHaveBeenCalled() + } finally { + releaseCopy() + await Promise.allSettled([first, rival]) + } + expect(rivalEntered).toHaveBeenCalledOnce() +}, 15_000) + +it('releases the copy lease after an operation fails', async () => { + const root = await scratch() + await expect( + withReleaseSourceLease(root, async () => { + throw new Error('copy failed') + }) + ).rejects.toThrow('copy failed') + await expect(withReleaseSourceLease(root, async () => 'recovered')).resolves.toBe('recovered') +})