mirror of
https://github.com/stablyai/orca.git
synced 2026-09-21 16:02:20 +00:00
* perf(dev): clone one Electron dist per repo instead of per worktree Every worktree extracted its own ~295MB node_modules/electron/dist, measured at 69GB across 241 worktrees on one machine. Extract once per repository into <git-common-dir>/orca-cache/electron, then APFS-clone it into each worktree: copy-on-write, so the second worktree allocates ~0 bytes and still gets a real, private, writable directory. Hangs off install-electron-package-binary.mjs, inside the transaction it already uses to swap dist. Every cache path returns a boolean and false means "install normally", so non-APFS, cross-volume, corrupt entry, no Git, folder workspace and CI all keep today's behavior. No symlinks, no lifecycle changes. out/electron-dev's per-branch Electron.app copy clones too, via the same helper. Refs #13709 * perf(dev): share the Electron dist on Linux and Windows too Extends the shared dist cache beyond macOS APFS. Three mechanisms, strongest isolation first: macOS APFS cp -c private copy-on-write Linux btrfs cp --reflink private copy-on-write ext4 / NTFS hardlink + 0555 shared inodes, forced read-only Reflinks cover btrfs/XFS/bcachefs/ZFS but not ext4, and Windows block cloning is ReFS-only, so most Linux and effectively all Windows developers need hardlinks to get any saving at all. Extracted dist is 327MB on linux-x64 and 374MB on win32-x64, both larger than macOS. Hardlinks share inodes, so a write through one worktree would rewrite every sibling and the cache. Nothing in this repo writes inside dist -- every mutation replaces the directory via rename -- but Electron's own install.js extracts over an existing dist with O_TRUNC, and is reachable through `pnpm rebuild electron`. Publishing the entry read-only turns that from silent cross-worktree corruption into EPERM. Directories stay writable so the install transaction's renames and unlinks still work. out/electron-dev's per-branch Electron.app is patched and codesigned after it is copied, so it uses copyPrivateTree, which never hardlinks. Refs #13709 * test(dev): keep shared-dist tests honest across ext4 and NTFS Verified on real hardware: Ubuntu 24.04/ext4 (no reflink support, so the hardlink tier is the only thing that helps there) and Windows/NTFS. Three tests faked platform: 'darwin' while invoking the real mechanism, so they failed on Linux where /bin/cp -c does not exist. Mechanism selection is now asserted with injected stubs; real filesystem behavior is asserted against whatever the host actually supports. Windows maps chmod onto the read-only attribute alone, so a directory never reports 0o755 and a read-only file reports 0o444. Mode-bit assertions that encoded POSIX semantics are now behavioral (the tree stays removable), and the executable-bit assertion is POSIX-only -- confirmed on NTFS that a read-only hardlinked .exe still runs. * fix(dev): stop a losing publisher from discarding a good cache entry Greptile caught a TOCTOU in the shared Electron dist cache. Quarantining an invalid entry happened before sharing the replacement tree, which takes seconds -- long enough for a sibling worktree to publish a good entry that this one would then rename away. If the follow-up publish also failed, the cache was left empty and every worktree re-downloaded. Stage first, then re-validate immediately before the destructive rename, so an entry that became good during the share is kept. On a failed swap, restore the quarantined entry instead of leaving no entry at all: a stale entry still beats an empty cache, because the next publisher re-validates and replaces it. An entry that cannot be validated is never displaced, matching the pre-staging rule. Also covers the Electron upgrade path end to end: a version bump gets its own cache entry and leaves the previous one for worktrees still on the old branch. * feat(dev): add a script to share existing worktrees' Electron dists An install only shares when Electron is (re)installed, and rebuild-native-deps returns early when the package is already usable -- so a worktree that already has a working dist never reaches the sharing path and keeps its own copy until the next Electron upgrade. pnpm reclaim:electron-dists reports what it would share; --apply does it. Each worktree is converted behind a rename, so an interrupted run leaves a working dist either way, and any worktree that fails is left untouched. Measured on one machine: 677 worktrees, ~195 GiB reclaimable. * fix(dev): keep the reclaim script's error formatting type-safe
190 lines
6.7 KiB
JavaScript
190 lines
6.7 KiB
JavaScript
import { execFileSync } from 'node:child_process'
|
|
import { randomUUID } from 'node:crypto'
|
|
import { existsSync, lstatSync, mkdirSync, readFileSync, renameSync, rmSync } from 'node:fs'
|
|
import path from 'node:path'
|
|
import { makeTreeReadOnly, shareTree } from './space-sharing-copy.mjs'
|
|
|
|
const IDENTITY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/
|
|
// Sibling of path.txt, never inside dist: an install replaces dist wholesale.
|
|
const MARKER_FILENAME = '.orca-shared-dist'
|
|
|
|
/**
|
|
* Where sibling worktrees of one repository keep their shared extracted Electron.
|
|
*
|
|
* Null means "install normally" -- every caller treats a missing entry as "do what you did before".
|
|
*/
|
|
export function resolveSharedElectronDistEntry(options) {
|
|
const { repoRoot, version, targetPlatform, targetArch } = options
|
|
const env = options.env ?? process.env
|
|
// Packaging jobs get a fresh checkout per run, so a cache only adds a failure mode.
|
|
if (env.CI === '1' || env.CI === 'true') {
|
|
return null
|
|
}
|
|
if (![version, targetPlatform, targetArch].every((part) => IDENTITY_PATTERN.test(part ?? ''))) {
|
|
return null
|
|
}
|
|
let gitCommonDir
|
|
try {
|
|
gitCommonDir = resolveGitCommonDir(repoRoot, options.execFile ?? execFileSync)
|
|
} catch {
|
|
return null // Folder workspace, or no Git on PATH.
|
|
}
|
|
const cacheRoot = path.join(gitCommonDir, 'orca-cache', 'electron')
|
|
return {
|
|
cacheRoot,
|
|
entryPath: path.join(cacheRoot, `${version}-${targetPlatform}-${targetArch}`),
|
|
markerPath: path.join(options.electronPackageDir, MARKER_FILENAME)
|
|
}
|
|
}
|
|
|
|
export function resolveGitCommonDir(repoRoot, execFile = execFileSync) {
|
|
const rawPath = execFile('git', ['-C', repoRoot, 'rev-parse', '--git-common-dir'], {
|
|
encoding: 'utf8',
|
|
stdio: ['ignore', 'pipe', 'ignore']
|
|
}).trim()
|
|
if (!rawPath) {
|
|
throw new Error('Git returned an empty common directory')
|
|
}
|
|
return path.resolve(repoRoot, rawPath)
|
|
}
|
|
|
|
/** True once this worktree's dist is already a clone of the current cache entry. */
|
|
export function hasAdoptedSharedElectronDist(entry) {
|
|
try {
|
|
return readFileSync(entry.markerPath, 'utf8') === path.basename(entry.entryPath)
|
|
} catch {
|
|
return false
|
|
}
|
|
}
|
|
|
|
export function recordAdoptedSharedElectronDist(entry, write) {
|
|
try {
|
|
write(entry.markerPath, path.basename(entry.entryPath))
|
|
} catch {
|
|
// The marker is only an optimization: a missing one costs one extra clone.
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Share a validated cache entry into `stagePath`. Deliberately never falls back to a byte copy --
|
|
* with no storage to share the caller is better off with its normal install, which this must not
|
|
* slow down.
|
|
*/
|
|
export function shareElectronDistFromCache(entry, stagePath, options) {
|
|
const { version, platformPath } = options
|
|
if (!isUsableElectronDist(entry.entryPath, version, platformPath)) {
|
|
return false
|
|
}
|
|
try {
|
|
;(options.share ?? shareTree)(entry.entryPath, stagePath)
|
|
} catch {
|
|
return false
|
|
}
|
|
return isUsableElectronDist(stagePath, version, platformPath)
|
|
}
|
|
|
|
/**
|
|
* Publish this worktree's dist as the shared entry, best effort.
|
|
*
|
|
* No lock: staging names are unique and `rename` onto a populated directory fails with ENOTEMPTY,
|
|
* so a concurrent publisher either wins the rename or cleans up its own staging tree. Neither can
|
|
* observe a half-written entry, and a usable entry already in place is never overwritten.
|
|
*/
|
|
export function publishSharedElectronDist(distPath, entry, options = {}) {
|
|
const { version, platformPath } = options
|
|
const uuid = options.uuid ?? randomUUID
|
|
const canValidate = Boolean(version) && Boolean(platformPath)
|
|
// Without an identity to check against, "unusable" is unknowable -- never discard on a guess.
|
|
if (existsSync(entry.entryPath) && (!canValidate || isUsable(entry, version, platformPath))) {
|
|
return false
|
|
}
|
|
|
|
const stagePath = `${entry.entryPath}.staging-${process.pid}-${uuid()}`
|
|
try {
|
|
mkdirSync(entry.cacheRoot, { recursive: true })
|
|
;(options.share ?? shareTree)(distPath, stagePath)
|
|
// Before publishing, not after: an entry is visible the instant the rename lands, and under
|
|
// hardlink sharing this is the only thing standing between a stray write and every worktree.
|
|
;(options.protect ?? makeTreeReadOnly)(stagePath)
|
|
} catch {
|
|
rmSync(stagePath, { recursive: true, force: true })
|
|
return false
|
|
}
|
|
|
|
return swapInElectronDistEntry(entry, stagePath, {
|
|
canValidate,
|
|
version,
|
|
platformPath,
|
|
uuid,
|
|
rename: options.rename ?? renameSync
|
|
})
|
|
}
|
|
|
|
/**
|
|
* Replace the entry with a staged tree, re-checking first.
|
|
*
|
|
* Sharing the tree above takes seconds, and a sibling worktree can publish a perfectly good entry
|
|
* in that time. Re-validating here, immediately before the destructive rename, keeps us from
|
|
* discarding that entry -- and restoring the quarantine on a failed swap keeps a losing publisher
|
|
* from leaving the cache empty.
|
|
*/
|
|
function swapInElectronDistEntry(entry, stagePath, options) {
|
|
const { canValidate, version, platformPath, uuid, rename } = options
|
|
let quarantinePath = null
|
|
if (existsSync(entry.entryPath)) {
|
|
// Same rule as before staging: an entry we cannot judge, or one that is good, is never
|
|
// displaced. Both mean another worktree got there first, so keep theirs.
|
|
if (!canValidate || isUsable(entry, version, platformPath)) {
|
|
rmSync(stagePath, { recursive: true, force: true })
|
|
return false
|
|
}
|
|
quarantinePath = `${entry.entryPath}.unusable-${process.pid}-${uuid()}`
|
|
try {
|
|
renameSync(entry.entryPath, quarantinePath)
|
|
} catch {
|
|
rmSync(stagePath, { recursive: true, force: true })
|
|
return false // Another worktree is already replacing it.
|
|
}
|
|
}
|
|
|
|
try {
|
|
rename(stagePath, entry.entryPath)
|
|
} catch {
|
|
rmSync(stagePath, { recursive: true, force: true })
|
|
if (quarantinePath !== null) {
|
|
// Put it back rather than leave no entry at all; a bad entry still beats an empty cache,
|
|
// because the next publisher re-validates and replaces it.
|
|
try {
|
|
renameSync(quarantinePath, entry.entryPath)
|
|
return false
|
|
} catch {
|
|
rmSync(quarantinePath, { recursive: true, force: true })
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
if (quarantinePath !== null) {
|
|
rmSync(quarantinePath, { recursive: true, force: true })
|
|
}
|
|
return true
|
|
}
|
|
|
|
function isUsable(entry, version, platformPath) {
|
|
return isUsableElectronDist(entry.entryPath, version, platformPath)
|
|
}
|
|
|
|
export function isUsableElectronDist(distPath, version, platformPath) {
|
|
try {
|
|
if (!lstatSync(distPath).isDirectory()) {
|
|
return false
|
|
}
|
|
const installedVersion = readFileSync(path.join(distPath, 'version'), 'utf8')
|
|
.trim()
|
|
.replace(/^v/, '')
|
|
return installedVersion === version && existsSync(path.join(distPath, platformPath))
|
|
} catch {
|
|
return false
|
|
}
|
|
}
|