diff --git a/config/scripts/dev-electron-bundle-cache.mjs b/config/scripts/dev-electron-bundle-cache.mjs index 8732661c874..244d1c964c9 100644 --- a/config/scripts/dev-electron-bundle-cache.mjs +++ b/config/scripts/dev-electron-bundle-cache.mjs @@ -1,3 +1,25 @@ +import { execFileSync } from 'node:child_process' + +/** Written once a bundle is fully built; its absence is what marks a build still in flight. */ +export const DEV_BUNDLE_MARKER_FILENAME = 'orca-dev-electron-app.json' + +export function getDevBundleProcessTable(execFile = execFileSync) { + // Not pgrep: macOS pgrep has no -a (a Linux procps extension) and silently prints bare PIDs, + // which reads as "nothing is running" and deletes a live bundle. -ww keeps the command column + // from being truncated. The raw text is searched directly; see isDevBundleInUse for why it is + // deliberately not parsed into paths. + try { + return execFile('/bin/ps', ['-Awwo', 'command='], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + timeout: 5000 + }) + } catch { + // Treating a failure as "nothing live" would risk deleting a running bundle, so skip pruning. + return null + } +} + // Why this module exists: `out/electron-dev` accumulates one ~270MB copy of Electron.app per // (branch title x Electron version x bundle layout). The runner only ever clears the directory it is // about to rebuild, so siblings from renamed branches and past upgrades are never reclaimed -- diff --git a/config/scripts/reclaim-dev-electron-bundles.mjs b/config/scripts/reclaim-dev-electron-bundles.mjs new file mode 100644 index 00000000000..c2d8a9fd886 --- /dev/null +++ b/config/scripts/reclaim-dev-electron-bundles.mjs @@ -0,0 +1,131 @@ +#!/usr/bin/env node + +// Removes idle `out/electron-dev` bundles across every worktree of a repository. +// +// The dev runner already prunes these, but only within the worktree it is starting and only when +// that worktree holds more than one bundle -- and a worktree almost always holds exactly one. So +// nothing ever reclaims a bundle belonging to a worktree you are not currently running, and one +// ~275MB copy per branch accumulates indefinitely. +// +// Bundles are pure build output: `pnpm dev` rebuilds one on demand, and since the Electron dist is +// now shared, rebuilding is cheap. + +import { execFileSync } from 'node:child_process' +import { existsSync, readdirSync, rmSync, statSync } from 'node:fs' +import path from 'node:path' +import { + DEV_BUNDLE_MARKER_FILENAME, + getDevBundleProcessTable, + selectStaleDevBundleDirs +} from './dev-electron-bundle-cache.mjs' + +const apply = process.argv.includes('--apply') +const repoRoot = process.argv.includes('--repo') + ? path.resolve(process.argv[process.argv.indexOf('--repo') + 1]) + : process.cwd() + +function listWorktrees(root) { + const raw = execFileSync('git', ['-C', root, 'worktree', 'list', '--porcelain'], { + encoding: 'utf8' + }) + return raw + .split('\n') + .filter((line) => line.startsWith('worktree ')) + .map((line) => line.slice('worktree '.length).trim()) +} + +function measure(targetPath) { + let total = 0 + let entries + try { + entries = readdirSync(targetPath, { withFileTypes: true }) + } catch { + return 0 + } + for (const entry of entries) { + const entryPath = path.join(targetPath, entry.name) + if (entry.isDirectory()) { + total += measure(entryPath) + } else if (!entry.isSymbolicLink()) { + total += statSync(entryPath, { throwIfNoEntry: false })?.size ?? 0 + } + } + return total +} + +export function collectDevBundles(worktree) { + const root = path.join(worktree, 'out', 'electron-dev') + try { + return readdirSync(root, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => { + const dir = path.join(root, entry.name) + return { + dir, + hasMarker: existsSync(path.join(dir, DEV_BUNDLE_MARKER_FILENAME)), + mtimeMs: statSync(dir, { throwIfNoEntry: false })?.mtimeMs ?? 0 + } + }) + } catch { + return [] + } +} + +function main() { + // The patched dev bundle is only built on macOS; elsewhere the dev app runs from dist directly. + if (process.platform !== 'darwin') { + console.log('No dev Electron bundles on this platform; nothing to reclaim.') + return + } + + const processTable = getDevBundleProcessTable() + if (processTable === null) { + // Same rule the dev runner uses: no process table means we cannot prove a bundle is idle. + console.error('Could not read the process table; refusing to guess which bundles are idle.') + process.exitCode = 1 + return + } + + const bundles = listWorktrees(repoRoot).flatMap((worktree) => collectDevBundles(worktree)) + // currentDir is null on purpose: unlike the dev runner, this sweep is not about to launch anything, + // so the only thing protecting a bundle is a live process or an in-flight build. + const stale = selectStaleDevBundleDirs({ + bundles, + currentDir: null, + processTable, + nowMs: Date.now() + }) + + let reclaimed = 0 + let removed = 0 + for (const dir of stale) { + const size = measure(dir) + if (!apply) { + console.log(`would remove ${dir} ${(size / 1024 ** 3).toFixed(2)} GiB`) + reclaimed += size + removed += 1 + continue + } + try { + rmSync(dir, { recursive: true, force: true }) + reclaimed += size + removed += 1 + console.log(`removed ${dir} ${(size / 1024 ** 3).toFixed(2)} GiB`) + } catch (error) { + console.warn(`skip ${dir} (${error instanceof Error ? error.message : String(error)})`) + } + } + + const inUse = bundles.length - stale.length + console.log( + `\n${apply ? 'Removed' : 'Would remove'} ${removed} bundle(s); ` + + `${apply ? 'reclaimed' : 'reclaimable'} ~${(reclaimed / 1024 ** 3).toFixed(2)} GiB` + + `${inUse > 0 ? `; left ${inUse} in use or still building` : ''}` + + `${apply ? '' : '\nRe-run with --apply to do it.'}` + ) +} + +// Guarded so importing this module for tests does not sweep the whole repository. +if (process.argv[1] && path.resolve(process.argv[1]) === path.resolve(import.meta.filename)) { + main() +} diff --git a/config/scripts/reclaim-dev-electron-bundles.test.ts b/config/scripts/reclaim-dev-electron-bundles.test.ts new file mode 100644 index 00000000000..071d0f3aaba --- /dev/null +++ b/config/scripts/reclaim-dev-electron-bundles.test.ts @@ -0,0 +1,97 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + DEV_BUNDLE_MARKER_FILENAME, + getDevBundleProcessTable, + selectStaleDevBundleDirs +} from './dev-electron-bundle-cache.mjs' +import { collectDevBundles } from './reclaim-dev-electron-bundles.mjs' + +const roots: string[] = [] + +afterEach(() => { + while (roots.length > 0) { + rmSync(roots.pop()!, { recursive: true, force: true }) + } +}) + +function makeWorktree(bundles: { name: string; marker: boolean }[]): string { + const worktree = mkdtempSync(path.join(tmpdir(), 'orca-dev-bundles-')) + roots.push(worktree) + for (const bundle of bundles) { + const dir = path.join(worktree, 'out', 'electron-dev', bundle.name) + mkdirSync(dir, { recursive: true }) + if (bundle.marker) { + writeFileSync(path.join(dir, DEV_BUNDLE_MARKER_FILENAME), '{}') + } + } + return worktree +} + +describe('collectDevBundles', () => { + it('reports each bundle and whether its build finished', () => { + const worktree = makeWorktree([ + { name: 'aaaa', marker: true }, + { name: 'bbbb', marker: false } + ]) + const bundles = collectDevBundles(worktree).sort((a, b) => a.dir.localeCompare(b.dir)) + expect(bundles).toHaveLength(2) + expect(bundles[0].hasMarker).toBe(true) + expect(bundles[1].hasMarker).toBe(false) + expect(bundles[0].mtimeMs).toBeGreaterThan(0) + }) + + it('returns nothing for a worktree that has never run the dev app', () => { + const worktree = mkdtempSync(path.join(tmpdir(), 'orca-dev-bundles-')) + roots.push(worktree) + expect(collectDevBundles(worktree)).toEqual([]) + }) +}) + +describe('sweeping across worktrees', () => { + it('spares a bundle a live process is running from, and takes the idle ones', () => { + const worktree = makeWorktree([ + { name: 'live', marker: true }, + { name: 'idle', marker: true } + ]) + const bundles = collectDevBundles(worktree) + const live = bundles.find((bundle) => bundle.dir.endsWith('live'))! + // Why currentDir is null here: unlike the dev runner, the sweep is not about to launch + // anything, so only a live process or an in-flight build may protect a bundle. + const stale = selectStaleDevBundleDirs({ + bundles, + currentDir: null, + processTable: `/usr/bin/foo ${live.dir}/Orca.app/Contents/MacOS/Electron`, + nowMs: Date.now() + }) + expect(stale).toEqual([bundles.find((bundle) => bundle.dir.endsWith('idle'))!.dir]) + }) + + it('spares a build still in flight, which has no marker yet', () => { + const worktree = makeWorktree([{ name: 'building', marker: false }]) + const stale = selectStaleDevBundleDirs({ + bundles: collectDevBundles(worktree), + currentDir: null, + processTable: '', + nowMs: Date.now() + }) + expect(stale).toEqual([]) + }) +}) + +describe('getDevBundleProcessTable', () => { + it('returns null rather than an empty table when ps fails', () => { + expect( + getDevBundleProcessTable(() => { + throw new Error('ps unavailable') + }) + ).toBeNull() + }) + + it('reads the real process table on this host', () => { + const table = getDevBundleProcessTable() + expect(typeof table === 'string' || table === null).toBe(true) + }) +}) diff --git a/config/scripts/reclaim-electron-dists.mjs b/config/scripts/reclaim-electron-dists.mjs index 4d4fb43c995..c3a0fb5b42f 100644 --- a/config/scripts/reclaim-electron-dists.mjs +++ b/config/scripts/reclaim-electron-dists.mjs @@ -81,89 +81,96 @@ function adoptInto(distPath, entry, identity) { return true } -let reclaimed = 0 -let converted = 0 -let skipped = 0 +function main() { + let reclaimed = 0 + let converted = 0 + let skipped = 0 -for (const worktree of listWorktrees(repoRoot)) { - const electronPackageDir = path.join(worktree, 'node_modules', 'electron') - const distPath = path.join(electronPackageDir, 'dist') - if (!existsSync(path.join(electronPackageDir, 'package.json')) || !existsSync(distPath)) { - continue - } - if (statSync(distPath, { throwIfNoEntry: false })?.isDirectory() !== true) { - continue - } - - let version - try { - version = JSON.parse( - readFileSync(path.join(electronPackageDir, 'package.json'), 'utf8') - ).version - } catch { - continue - } - const targetPlatform = process.platform - const targetArch = process.arch - let platformPath - try { - platformPath = getElectronPlatformPath(targetPlatform) - } catch { - continue - } - if (!isUsableElectronDist(distPath, version, platformPath)) { - console.log(`skip ${worktree} (dist is not a complete Electron ${version})`) - skipped += 1 - continue - } - - const entry = resolveSharedElectronDistEntry({ - repoRoot: worktree, - electronPackageDir, - version, - targetPlatform, - targetArch - }) - if (entry === null) { - continue - } - if (hasAdoptedSharedElectronDist(entry)) { - continue - } - - const size = measure(distPath) - if (!apply) { - console.log(`would share ${worktree} ${(size / 1024 ** 3).toFixed(2)} GiB (${version})`) - reclaimed += size - converted += 1 - continue - } - - try { - if (!existsSync(entry.entryPath)) { - if (publishSharedElectronDist(distPath, entry, { version, platformPath })) { - recordAdoptedSharedElectronDist(entry, writeFileSync) - console.log(`seeded ${worktree} -> ${entry.entryPath}`) - converted += 1 - } + for (const worktree of listWorktrees(repoRoot)) { + const electronPackageDir = path.join(worktree, 'node_modules', 'electron') + const distPath = path.join(electronPackageDir, 'dist') + if (!existsSync(path.join(electronPackageDir, 'package.json')) || !existsSync(distPath)) { continue } - if (adoptInto(distPath, entry, { version, platformPath })) { - recordAdoptedSharedElectronDist(entry, writeFileSync) + if (statSync(distPath, { throwIfNoEntry: false })?.isDirectory() !== true) { + continue + } + + let version + try { + version = JSON.parse( + readFileSync(path.join(electronPackageDir, 'package.json'), 'utf8') + ).version + } catch { + continue + } + const targetPlatform = process.platform + const targetArch = process.arch + let platformPath + try { + platformPath = getElectronPlatformPath(targetPlatform) + } catch { + continue + } + if (!isUsableElectronDist(distPath, version, platformPath)) { + console.log(`skip ${worktree} (dist is not a complete Electron ${version})`) + skipped += 1 + continue + } + + const entry = resolveSharedElectronDistEntry({ + repoRoot: worktree, + electronPackageDir, + version, + targetPlatform, + targetArch + }) + if (entry === null) { + continue + } + if (hasAdoptedSharedElectronDist(entry)) { + continue + } + + const size = measure(distPath) + if (!apply) { + console.log(`would share ${worktree} ${(size / 1024 ** 3).toFixed(2)} GiB (${version})`) reclaimed += size converted += 1 - console.log(`shared ${worktree} reclaimed ${(size / 1024 ** 3).toFixed(2)} GiB`) + continue + } + + try { + if (!existsSync(entry.entryPath)) { + if (publishSharedElectronDist(distPath, entry, { version, platformPath })) { + recordAdoptedSharedElectronDist(entry, writeFileSync) + console.log(`seeded ${worktree} -> ${entry.entryPath}`) + converted += 1 + } + continue + } + if (adoptInto(distPath, entry, { version, platformPath })) { + recordAdoptedSharedElectronDist(entry, writeFileSync) + reclaimed += size + converted += 1 + console.log(`shared ${worktree} reclaimed ${(size / 1024 ** 3).toFixed(2)} GiB`) + } + } catch (error) { + // A worktree that fails is left exactly as it was; it still has its own working dist. + console.warn(`skip ${worktree} (${error instanceof Error ? error.message : String(error)})`) + skipped += 1 } - } catch (error) { - // A worktree that fails is left exactly as it was; it still has its own working dist. - console.warn(`skip ${worktree} (${error instanceof Error ? error.message : String(error)})`) - skipped += 1 } + + console.log( + `\n${apply ? 'Shared' : 'Would share'} ${converted} worktree(s); ` + + `${apply ? 'reclaimed' : 'reclaimable'} ~${(reclaimed / 1024 ** 3).toFixed(2)} GiB` + + `${skipped > 0 ? `; skipped ${skipped}` : ''}` + + `${apply ? '' : '\nRe-run with --apply to do it.'}` + ) } -console.log( - `\n${apply ? 'Shared' : 'Would share'} ${converted} worktree(s); ` + - `${apply ? 'reclaimed' : 'reclaimable'} ~${(reclaimed / 1024 ** 3).toFixed(2)} GiB` + - `${skipped > 0 ? `; skipped ${skipped}` : ''}` + - `${apply ? '' : '\nRe-run with --apply to do it.'}` -) +// Guarded so importing this module for tests does not sweep the whole repository. +if (process.argv[1] && path.resolve(process.argv[1]) === path.resolve(import.meta.filename)) { + main() +} diff --git a/config/scripts/run-electron-vite-dev.mjs b/config/scripts/run-electron-vite-dev.mjs index b7914943701..dfb0a0aceb7 100644 --- a/config/scripts/run-electron-vite-dev.mjs +++ b/config/scripts/run-electron-vite-dev.mjs @@ -17,7 +17,12 @@ import { createRequire } from 'node:module' import path from 'node:path' import { prepareDevCliTerminalWrappers } from './dev-cli-terminal-wrapper.mjs' -import { isDevBundleInUse, selectStaleDevBundleDirs } from './dev-electron-bundle-cache.mjs' +import { + DEV_BUNDLE_MARKER_FILENAME, + getDevBundleProcessTable, + isDevBundleInUse, + selectStaleDevBundleDirs +} from './dev-electron-bundle-cache.mjs' import { copyPrivateTree } from './space-sharing-copy.mjs' import { DEV_BUNDLE_ID, @@ -117,23 +122,6 @@ function sanitizeMacAppBundleName(value) { ) } -function getDevBundleProcessTable() { - // Not pgrep: macOS pgrep has no -a (a Linux procps extension) and silently prints bare PIDs, - // which reads as "nothing is running" and deletes a live bundle. -ww keeps the command column - // from being truncated. The raw text is searched directly; see dev-electron-bundle-cache.mjs - // for why it is deliberately not parsed into paths. - try { - return execFileSync('/bin/ps', ['-Awwo', 'command='], { - encoding: 'utf8', - stdio: ['ignore', 'pipe', 'ignore'], - timeout: 5000 - }) - } catch { - // Treating a failure as "nothing live" would risk deleting a running bundle, so skip pruning. - return null - } -} - function pruneStaleDevBundles(distDir) { const root = path.dirname(distDir) let bundles @@ -144,7 +132,7 @@ function pruneStaleDevBundles(distDir) { const dir = path.join(root, entry.name) return { dir, - hasMarker: existsSync(path.join(dir, 'orca-dev-electron-app.json')), + hasMarker: existsSync(path.join(dir, DEV_BUNDLE_MARKER_FILENAME)), mtimeMs: getMtimeMs(dir) } }) @@ -204,7 +192,7 @@ function prepareMacDevElectronApp() { // and it sits outside the code signature, so varying it does not disturb the cdhash. const appBundleName = `${sanitizeMacAppBundleName(title)}.app` const appPath = path.join(distDir, appBundleName) - const markerPath = path.join(distDir, 'orca-dev-electron-app.json') + const markerPath = path.join(distDir, DEV_BUNDLE_MARKER_FILENAME) // Why: one stable id for every dev instance. Per-instance ids registered a // new macOS Notification Settings entry for each branch × Electron version, // piling up "Orca: " rows forever and breaking the notification diff --git a/package.json b/package.json index 04128b67922..4ddc4e8fdb1 100644 --- a/package.json +++ b/package.json @@ -87,6 +87,7 @@ "postinstall": "node config/scripts/rebuild-native-deps.mjs", "rebuild:electron": "node config/scripts/rebuild-native-deps.mjs", "reclaim:electron-dists": "node config/scripts/reclaim-electron-dists.mjs", + "reclaim:dev-bundles": "node config/scripts/reclaim-dev-electron-bundles.mjs", "rebuild:node": "pnpm rebuild node-pty", "build:unpack": "pnpm run build && pnpm run ensure:electron-runtime && electron-builder --config config/electron-builder.config.cjs --dir", "build:win": "pnpm run build:desktop && pnpm run ensure:electron-runtime && electron-builder --config config/electron-builder.config.cjs --win",