mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 00:02:31 +00:00
* perf(dev): clone one Electron dist per repo instead of per worktree Every worktree extracted its own ~295MB node_modules/electron/dist, measured at 69GB across 241 worktrees on one machine. Extract once per repository into <git-common-dir>/orca-cache/electron, then APFS-clone it into each worktree: copy-on-write, so the second worktree allocates ~0 bytes and still gets a real, private, writable directory. Hangs off install-electron-package-binary.mjs, inside the transaction it already uses to swap dist. Every cache path returns a boolean and false means "install normally", so non-APFS, cross-volume, corrupt entry, no Git, folder workspace and CI all keep today's behavior. No symlinks, no lifecycle changes. out/electron-dev's per-branch Electron.app copy clones too, via the same helper. Refs #13709 * perf(dev): share the Electron dist on Linux and Windows too Extends the shared dist cache beyond macOS APFS. Three mechanisms, strongest isolation first: macOS APFS cp -c private copy-on-write Linux btrfs cp --reflink private copy-on-write ext4 / NTFS hardlink + 0555 shared inodes, forced read-only Reflinks cover btrfs/XFS/bcachefs/ZFS but not ext4, and Windows block cloning is ReFS-only, so most Linux and effectively all Windows developers need hardlinks to get any saving at all. Extracted dist is 327MB on linux-x64 and 374MB on win32-x64, both larger than macOS. Hardlinks share inodes, so a write through one worktree would rewrite every sibling and the cache. Nothing in this repo writes inside dist -- every mutation replaces the directory via rename -- but Electron's own install.js extracts over an existing dist with O_TRUNC, and is reachable through `pnpm rebuild electron`. Publishing the entry read-only turns that from silent cross-worktree corruption into EPERM. Directories stay writable so the install transaction's renames and unlinks still work. out/electron-dev's per-branch Electron.app is patched and codesigned after it is copied, so it uses copyPrivateTree, which never hardlinks. Refs #13709 * test(dev): keep shared-dist tests honest across ext4 and NTFS Verified on real hardware: Ubuntu 24.04/ext4 (no reflink support, so the hardlink tier is the only thing that helps there) and Windows/NTFS. Three tests faked platform: 'darwin' while invoking the real mechanism, so they failed on Linux where /bin/cp -c does not exist. Mechanism selection is now asserted with injected stubs; real filesystem behavior is asserted against whatever the host actually supports. Windows maps chmod onto the read-only attribute alone, so a directory never reports 0o755 and a read-only file reports 0o444. Mode-bit assertions that encoded POSIX semantics are now behavioral (the tree stays removable), and the executable-bit assertion is POSIX-only -- confirmed on NTFS that a read-only hardlinked .exe still runs. * fix(dev): stop a losing publisher from discarding a good cache entry Greptile caught a TOCTOU in the shared Electron dist cache. Quarantining an invalid entry happened before sharing the replacement tree, which takes seconds -- long enough for a sibling worktree to publish a good entry that this one would then rename away. If the follow-up publish also failed, the cache was left empty and every worktree re-downloaded. Stage first, then re-validate immediately before the destructive rename, so an entry that became good during the share is kept. On a failed swap, restore the quarantined entry instead of leaving no entry at all: a stale entry still beats an empty cache, because the next publisher re-validates and replaces it. An entry that cannot be validated is never displaced, matching the pre-staging rule. Also covers the Electron upgrade path end to end: a version bump gets its own cache entry and leaves the previous one for worktrees still on the old branch. * feat(dev): add a script to share existing worktrees' Electron dists An install only shares when Electron is (re)installed, and rebuild-native-deps returns early when the package is already usable -- so a worktree that already has a working dist never reaches the sharing path and keeps its own copy until the next Electron upgrade. pnpm reclaim:electron-dists reports what it would share; --apply does it. Each worktree is converted behind a rename, so an interrupted run leaves a working dist either way, and any worktree that fails is left untouched. Measured on one machine: 677 worktrees, ~195 GiB reclaimable. * fix(dev): keep the reclaim script's error formatting type-safe
211 lines
7.1 KiB
JavaScript
211 lines
7.1 KiB
JavaScript
import { execFileSync, spawnSync } from 'node:child_process'
|
|
import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'
|
|
import { tmpdir } from 'node:os'
|
|
import { join } from 'node:path'
|
|
import { fileURLToPath } from 'node:url'
|
|
import { copyScriptWithLocalModules } from './script-module-dependencies.mjs'
|
|
|
|
const sourceScriptPath = fileURLToPath(
|
|
new URL('./install-electron-package-binary.mjs', import.meta.url)
|
|
)
|
|
/** Matches the fake package version and the platform/arch runInstallScript installs for. */
|
|
export const sharedEntryName = '41.5.0-linux-x64'
|
|
export const sharedEntryNameFor = (version) => `${version}-linux-x64`
|
|
|
|
export function mkTempProject() {
|
|
const projectDir = mkdtempSync(join(tmpdir(), 'orca-install-electron-'))
|
|
copyScriptWithLocalModules(sourceScriptPath, join(projectDir, 'config', 'scripts'))
|
|
return projectDir
|
|
}
|
|
|
|
export function runInstallScript(projectDir, extraEnv = {}) {
|
|
return spawnSync(process.execPath, ['config/scripts/install-electron-package-binary.mjs'], {
|
|
cwd: projectDir,
|
|
encoding: 'utf8',
|
|
env: {
|
|
...process.env,
|
|
ELECTRON_CACHE: undefined,
|
|
ORCA_ELECTRON_PACKAGE_CACHE_ROOT: undefined,
|
|
npm_config_platform: 'linux',
|
|
npm_config_arch: 'x64',
|
|
ORCA_ELECTRON_PACKAGE_EXTRACTOR: join(projectDir, 'fake-extractor.cjs'),
|
|
...extraEnv
|
|
}
|
|
})
|
|
}
|
|
|
|
export function writeFakeElectronPackage(
|
|
projectDir,
|
|
{ lazyRequireMarker = null, version = '41.5.0' } = {}
|
|
) {
|
|
const electronDir = join(projectDir, 'node_modules', 'electron')
|
|
mkdirSync(electronDir, { recursive: true })
|
|
writeFileSync(join(electronDir, 'package.json'), JSON.stringify({ name: 'electron', version }))
|
|
writeFileSync(join(electronDir, 'checksums.json'), '{}')
|
|
writeFileSync(
|
|
join(electronDir, 'index.js'),
|
|
`
|
|
const fs = require('node:fs')
|
|
const path = require('node:path')
|
|
${lazyRequireMarker ? `fs.writeFileSync(${JSON.stringify(lazyRequireMarker)}, 'required')` : ''}
|
|
const pathFile = path.join(__dirname, 'path.txt')
|
|
if (!fs.existsSync(pathFile)) {
|
|
throw new Error('Electron failed to install correctly, please delete node_modules/electron and try installing again')
|
|
}
|
|
module.exports = path.join(__dirname, 'dist', fs.readFileSync(pathFile, 'utf8'))
|
|
`
|
|
)
|
|
}
|
|
|
|
export function writeFakeElectronDist(
|
|
projectDir,
|
|
{ version = 'v41.5.0', executableContents = '', pathContents } = {}
|
|
) {
|
|
const electronDir = join(projectDir, 'node_modules', 'electron')
|
|
mkdirSync(join(electronDir, 'dist'), { recursive: true })
|
|
writeFileSync(join(electronDir, 'dist/version'), version)
|
|
writeFileSync(join(electronDir, 'dist/electron'), executableContents)
|
|
if (pathContents !== undefined) {
|
|
writeFileSync(join(electronDir, 'path.txt'), pathContents)
|
|
}
|
|
}
|
|
|
|
export function writeFakeElectronGet(
|
|
projectDir,
|
|
{
|
|
downloadNeverSettles = false,
|
|
downloadFailures = 0,
|
|
downloadErrorCode = 'ECONNRESET',
|
|
downloadHttpStatus = null
|
|
} = {}
|
|
) {
|
|
const getDir = join(projectDir, 'node_modules', 'electron', 'node_modules', '@electron', 'get')
|
|
mkdirSync(getDir, { recursive: true })
|
|
writeFileSync(
|
|
join(getDir, 'index.js'),
|
|
`
|
|
const { mkdirSync, writeFileSync, appendFileSync } = require('node:fs')
|
|
const { join } = require('node:path')
|
|
let downloadAttempt = 0
|
|
exports.downloadArtifact = async function downloadArtifact(details) {
|
|
downloadAttempt += 1
|
|
appendFileSync(
|
|
'electron-get.log',
|
|
'cacheRoot=' + details.cacheRoot + ' platform=' + details.platform + ' arch=' + details.arch + ' force=' + details.force + '\\n'
|
|
)
|
|
if (${JSON.stringify(downloadNeverSettles)}) {
|
|
return new Promise(() => {})
|
|
}
|
|
if (downloadAttempt <= ${JSON.stringify(downloadFailures)}) {
|
|
if (${JSON.stringify(downloadHttpStatus)} != null) {
|
|
const error = new Error('Response code ' + ${JSON.stringify(downloadHttpStatus)})
|
|
error.response = { status: ${JSON.stringify(downloadHttpStatus)} }
|
|
throw error
|
|
}
|
|
const cause = Object.assign(new Error('download failed'), {
|
|
code: ${JSON.stringify(downloadErrorCode)}
|
|
})
|
|
throw Object.assign(new TypeError('fetch failed'), { cause })
|
|
}
|
|
mkdirSync(details.cacheRoot, { recursive: true })
|
|
const artifactPath = join(details.cacheRoot, 'electron.zip')
|
|
writeFileSync(artifactPath, 'fake zip')
|
|
return artifactPath
|
|
}
|
|
`
|
|
)
|
|
}
|
|
|
|
export function writeFakeExtractor(projectDir, { createExecutable, version = '41.5.0' }) {
|
|
writeFileSync(
|
|
join(projectDir, 'fake-extractor.cjs'),
|
|
`
|
|
const { appendFileSync, mkdirSync, symlinkSync, writeFileSync } = require('node:fs')
|
|
const { join } = require('node:path')
|
|
const extractDir = process.argv[3]
|
|
appendFileSync(join(__dirname, 'fake-extractor.log'), extractDir + '\\n')
|
|
mkdirSync(join(extractDir, 'locales'), { recursive: true })
|
|
if (${JSON.stringify(createExecutable)}) {
|
|
writeFileSync(join(extractDir, 'electron'), '')
|
|
writeFileSync(join(extractDir, 'electron.exe'), '')
|
|
writeFileSync(join(extractDir, 'electron.d.ts'), 'replacement types')
|
|
writeFileSync(join(extractDir, 'version'), ${JSON.stringify(`v${version}`)})
|
|
if (process.platform !== 'win32') {
|
|
symlinkSync('version', join(extractDir, 'version-link'))
|
|
}
|
|
}
|
|
`
|
|
)
|
|
}
|
|
|
|
export function writeTypeDefPublishFailurePreload(projectDir) {
|
|
const preloadPath = join(projectDir, 'type-def-publish-failure.cjs')
|
|
writeFileSync(
|
|
preloadPath,
|
|
`
|
|
const fs = require('node:fs')
|
|
const { syncBuiltinESMExports } = require('node:module')
|
|
const { basename, dirname } = require('node:path')
|
|
const renameSync = fs.renameSync
|
|
fs.renameSync = (source, target) => {
|
|
if (basename(source) === 'electron.d.ts' && basename(dirname(source)) === 'dist') {
|
|
const error = new Error('injected Electron type definition publish failure')
|
|
error.code = 'EACCES'
|
|
throw error
|
|
}
|
|
return renameSync(source, target)
|
|
}
|
|
syncBuiltinESMExports()
|
|
`
|
|
)
|
|
return preloadPath
|
|
}
|
|
|
|
export function initGitRepo(projectDir) {
|
|
runGit(projectDir, ['init', '--quiet', '--initial-branch=main'])
|
|
runGit(projectDir, ['config', 'user.email', 'orca-test@example.com'])
|
|
runGit(projectDir, ['config', 'user.name', 'Orca Test'])
|
|
runGit(projectDir, ['commit', '--quiet', '--allow-empty', '-m', 'init'])
|
|
}
|
|
|
|
export function addSiblingWorktree(projectDir, siblingDir) {
|
|
runGit(projectDir, ['worktree', 'add', '--quiet', '-b', 'sibling', siblingDir])
|
|
copyScriptWithLocalModules(sourceScriptPath, join(siblingDir, 'config', 'scripts'))
|
|
return siblingDir
|
|
}
|
|
|
|
function runGit(projectDir, args) {
|
|
execFileSync('git', ['-C', projectDir, ...args], { stdio: 'ignore' })
|
|
}
|
|
|
|
export function sharedCacheRoot(repoDir) {
|
|
return join(repoDir, '.git', 'orca-cache', 'electron')
|
|
}
|
|
|
|
export function readSharedDistMarker(projectDir) {
|
|
try {
|
|
return readFileSync(join(projectDir, 'node_modules/electron/.orca-shared-dist'), 'utf8')
|
|
} catch {
|
|
return null
|
|
}
|
|
}
|
|
|
|
export function readExtractorCallCount(projectDir) {
|
|
try {
|
|
return readFileSync(join(projectDir, 'fake-extractor.log'), 'utf8').trim().split('\n').length
|
|
} catch {
|
|
return 0
|
|
}
|
|
}
|
|
|
|
export function writeNonDarwinPlatformPreload(projectDir) {
|
|
const preloadPath = join(projectDir, 'non-darwin-platform.cjs')
|
|
writeFileSync(
|
|
preloadPath,
|
|
`
|
|
Object.defineProperty(process, 'platform', { value: 'linux', configurable: true })
|
|
`
|
|
)
|
|
return preloadPath
|
|
}
|