mirror of
https://github.com/stablyai/orca.git
synced 2026-09-21 16:02:20 +00:00
perf(dev): share one Electron dist per repo instead of per worktree (#17664)
* perf(dev): clone one Electron dist per repo instead of per worktree Every worktree extracted its own ~295MB node_modules/electron/dist, measured at 69GB across 241 worktrees on one machine. Extract once per repository into <git-common-dir>/orca-cache/electron, then APFS-clone it into each worktree: copy-on-write, so the second worktree allocates ~0 bytes and still gets a real, private, writable directory. Hangs off install-electron-package-binary.mjs, inside the transaction it already uses to swap dist. Every cache path returns a boolean and false means "install normally", so non-APFS, cross-volume, corrupt entry, no Git, folder workspace and CI all keep today's behavior. No symlinks, no lifecycle changes. out/electron-dev's per-branch Electron.app copy clones too, via the same helper. Refs #13709 * perf(dev): share the Electron dist on Linux and Windows too Extends the shared dist cache beyond macOS APFS. Three mechanisms, strongest isolation first: macOS APFS cp -c private copy-on-write Linux btrfs cp --reflink private copy-on-write ext4 / NTFS hardlink + 0555 shared inodes, forced read-only Reflinks cover btrfs/XFS/bcachefs/ZFS but not ext4, and Windows block cloning is ReFS-only, so most Linux and effectively all Windows developers need hardlinks to get any saving at all. Extracted dist is 327MB on linux-x64 and 374MB on win32-x64, both larger than macOS. Hardlinks share inodes, so a write through one worktree would rewrite every sibling and the cache. Nothing in this repo writes inside dist -- every mutation replaces the directory via rename -- but Electron's own install.js extracts over an existing dist with O_TRUNC, and is reachable through `pnpm rebuild electron`. Publishing the entry read-only turns that from silent cross-worktree corruption into EPERM. Directories stay writable so the install transaction's renames and unlinks still work. out/electron-dev's per-branch Electron.app is patched and codesigned after it is copied, so it uses copyPrivateTree, which never hardlinks. Refs #13709 * test(dev): keep shared-dist tests honest across ext4 and NTFS Verified on real hardware: Ubuntu 24.04/ext4 (no reflink support, so the hardlink tier is the only thing that helps there) and Windows/NTFS. Three tests faked platform: 'darwin' while invoking the real mechanism, so they failed on Linux where /bin/cp -c does not exist. Mechanism selection is now asserted with injected stubs; real filesystem behavior is asserted against whatever the host actually supports. Windows maps chmod onto the read-only attribute alone, so a directory never reports 0o755 and a read-only file reports 0o444. Mode-bit assertions that encoded POSIX semantics are now behavioral (the tree stays removable), and the executable-bit assertion is POSIX-only -- confirmed on NTFS that a read-only hardlinked .exe still runs. * fix(dev): stop a losing publisher from discarding a good cache entry Greptile caught a TOCTOU in the shared Electron dist cache. Quarantining an invalid entry happened before sharing the replacement tree, which takes seconds -- long enough for a sibling worktree to publish a good entry that this one would then rename away. If the follow-up publish also failed, the cache was left empty and every worktree re-downloaded. Stage first, then re-validate immediately before the destructive rename, so an entry that became good during the share is kept. On a failed swap, restore the quarantined entry instead of leaving no entry at all: a stale entry still beats an empty cache, because the next publisher re-validates and replaces it. An entry that cannot be validated is never displaced, matching the pre-staging rule. Also covers the Electron upgrade path end to end: a version bump gets its own cache entry and leaves the previous one for worktrees still on the old branch. * feat(dev): add a script to share existing worktrees' Electron dists An install only shares when Electron is (re)installed, and rebuild-native-deps returns early when the package is already usable -- so a worktree that already has a working dist never reaches the sharing path and keeps its own copy until the next Electron upgrade. pnpm reclaim:electron-dists reports what it would share; --apply does it. Each worktree is converted behind a rename, so an interrupted run leaves a working dist either way, and any worktree that fails is left untouched. Measured on one machine: 677 worktrees, ~195 GiB reclaimable. * fix(dev): keep the reclaim script's error formatting type-safe
This commit is contained in:
@@ -0,0 +1,16 @@
|
||||
/** Where the Electron executable sits inside a `dist` tree, relative to it. */
|
||||
export function getElectronPlatformPath(targetPlatform) {
|
||||
switch (targetPlatform) {
|
||||
case 'mas':
|
||||
case 'darwin':
|
||||
return 'Electron.app/Contents/MacOS/Electron'
|
||||
case 'freebsd':
|
||||
case 'openbsd':
|
||||
case 'linux':
|
||||
return 'electron'
|
||||
case 'win32':
|
||||
return 'electron.exe'
|
||||
default:
|
||||
throw new Error(`Electron builds are not available on platform: ${targetPlatform}`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
import { execFileSync, spawnSync } from 'node:child_process'
|
||||
import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { copyScriptWithLocalModules } from './script-module-dependencies.mjs'
|
||||
|
||||
const sourceScriptPath = fileURLToPath(
|
||||
new URL('./install-electron-package-binary.mjs', import.meta.url)
|
||||
)
|
||||
/** Matches the fake package version and the platform/arch runInstallScript installs for. */
|
||||
export const sharedEntryName = '41.5.0-linux-x64'
|
||||
export const sharedEntryNameFor = (version) => `${version}-linux-x64`
|
||||
|
||||
export function mkTempProject() {
|
||||
const projectDir = mkdtempSync(join(tmpdir(), 'orca-install-electron-'))
|
||||
copyScriptWithLocalModules(sourceScriptPath, join(projectDir, 'config', 'scripts'))
|
||||
return projectDir
|
||||
}
|
||||
|
||||
export function runInstallScript(projectDir, extraEnv = {}) {
|
||||
return spawnSync(process.execPath, ['config/scripts/install-electron-package-binary.mjs'], {
|
||||
cwd: projectDir,
|
||||
encoding: 'utf8',
|
||||
env: {
|
||||
...process.env,
|
||||
ELECTRON_CACHE: undefined,
|
||||
ORCA_ELECTRON_PACKAGE_CACHE_ROOT: undefined,
|
||||
npm_config_platform: 'linux',
|
||||
npm_config_arch: 'x64',
|
||||
ORCA_ELECTRON_PACKAGE_EXTRACTOR: join(projectDir, 'fake-extractor.cjs'),
|
||||
...extraEnv
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
export function writeFakeElectronPackage(
|
||||
projectDir,
|
||||
{ lazyRequireMarker = null, version = '41.5.0' } = {}
|
||||
) {
|
||||
const electronDir = join(projectDir, 'node_modules', 'electron')
|
||||
mkdirSync(electronDir, { recursive: true })
|
||||
writeFileSync(join(electronDir, 'package.json'), JSON.stringify({ name: 'electron', version }))
|
||||
writeFileSync(join(electronDir, 'checksums.json'), '{}')
|
||||
writeFileSync(
|
||||
join(electronDir, 'index.js'),
|
||||
`
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
${lazyRequireMarker ? `fs.writeFileSync(${JSON.stringify(lazyRequireMarker)}, 'required')` : ''}
|
||||
const pathFile = path.join(__dirname, 'path.txt')
|
||||
if (!fs.existsSync(pathFile)) {
|
||||
throw new Error('Electron failed to install correctly, please delete node_modules/electron and try installing again')
|
||||
}
|
||||
module.exports = path.join(__dirname, 'dist', fs.readFileSync(pathFile, 'utf8'))
|
||||
`
|
||||
)
|
||||
}
|
||||
|
||||
export function writeFakeElectronDist(
|
||||
projectDir,
|
||||
{ version = 'v41.5.0', executableContents = '', pathContents } = {}
|
||||
) {
|
||||
const electronDir = join(projectDir, 'node_modules', 'electron')
|
||||
mkdirSync(join(electronDir, 'dist'), { recursive: true })
|
||||
writeFileSync(join(electronDir, 'dist/version'), version)
|
||||
writeFileSync(join(electronDir, 'dist/electron'), executableContents)
|
||||
if (pathContents !== undefined) {
|
||||
writeFileSync(join(electronDir, 'path.txt'), pathContents)
|
||||
}
|
||||
}
|
||||
|
||||
export function writeFakeElectronGet(
|
||||
projectDir,
|
||||
{
|
||||
downloadNeverSettles = false,
|
||||
downloadFailures = 0,
|
||||
downloadErrorCode = 'ECONNRESET',
|
||||
downloadHttpStatus = null
|
||||
} = {}
|
||||
) {
|
||||
const getDir = join(projectDir, 'node_modules', 'electron', 'node_modules', '@electron', 'get')
|
||||
mkdirSync(getDir, { recursive: true })
|
||||
writeFileSync(
|
||||
join(getDir, 'index.js'),
|
||||
`
|
||||
const { mkdirSync, writeFileSync, appendFileSync } = require('node:fs')
|
||||
const { join } = require('node:path')
|
||||
let downloadAttempt = 0
|
||||
exports.downloadArtifact = async function downloadArtifact(details) {
|
||||
downloadAttempt += 1
|
||||
appendFileSync(
|
||||
'electron-get.log',
|
||||
'cacheRoot=' + details.cacheRoot + ' platform=' + details.platform + ' arch=' + details.arch + ' force=' + details.force + '\\n'
|
||||
)
|
||||
if (${JSON.stringify(downloadNeverSettles)}) {
|
||||
return new Promise(() => {})
|
||||
}
|
||||
if (downloadAttempt <= ${JSON.stringify(downloadFailures)}) {
|
||||
if (${JSON.stringify(downloadHttpStatus)} != null) {
|
||||
const error = new Error('Response code ' + ${JSON.stringify(downloadHttpStatus)})
|
||||
error.response = { status: ${JSON.stringify(downloadHttpStatus)} }
|
||||
throw error
|
||||
}
|
||||
const cause = Object.assign(new Error('download failed'), {
|
||||
code: ${JSON.stringify(downloadErrorCode)}
|
||||
})
|
||||
throw Object.assign(new TypeError('fetch failed'), { cause })
|
||||
}
|
||||
mkdirSync(details.cacheRoot, { recursive: true })
|
||||
const artifactPath = join(details.cacheRoot, 'electron.zip')
|
||||
writeFileSync(artifactPath, 'fake zip')
|
||||
return artifactPath
|
||||
}
|
||||
`
|
||||
)
|
||||
}
|
||||
|
||||
export function writeFakeExtractor(projectDir, { createExecutable, version = '41.5.0' }) {
|
||||
writeFileSync(
|
||||
join(projectDir, 'fake-extractor.cjs'),
|
||||
`
|
||||
const { appendFileSync, mkdirSync, symlinkSync, writeFileSync } = require('node:fs')
|
||||
const { join } = require('node:path')
|
||||
const extractDir = process.argv[3]
|
||||
appendFileSync(join(__dirname, 'fake-extractor.log'), extractDir + '\\n')
|
||||
mkdirSync(join(extractDir, 'locales'), { recursive: true })
|
||||
if (${JSON.stringify(createExecutable)}) {
|
||||
writeFileSync(join(extractDir, 'electron'), '')
|
||||
writeFileSync(join(extractDir, 'electron.exe'), '')
|
||||
writeFileSync(join(extractDir, 'electron.d.ts'), 'replacement types')
|
||||
writeFileSync(join(extractDir, 'version'), ${JSON.stringify(`v${version}`)})
|
||||
if (process.platform !== 'win32') {
|
||||
symlinkSync('version', join(extractDir, 'version-link'))
|
||||
}
|
||||
}
|
||||
`
|
||||
)
|
||||
}
|
||||
|
||||
export function writeTypeDefPublishFailurePreload(projectDir) {
|
||||
const preloadPath = join(projectDir, 'type-def-publish-failure.cjs')
|
||||
writeFileSync(
|
||||
preloadPath,
|
||||
`
|
||||
const fs = require('node:fs')
|
||||
const { syncBuiltinESMExports } = require('node:module')
|
||||
const { basename, dirname } = require('node:path')
|
||||
const renameSync = fs.renameSync
|
||||
fs.renameSync = (source, target) => {
|
||||
if (basename(source) === 'electron.d.ts' && basename(dirname(source)) === 'dist') {
|
||||
const error = new Error('injected Electron type definition publish failure')
|
||||
error.code = 'EACCES'
|
||||
throw error
|
||||
}
|
||||
return renameSync(source, target)
|
||||
}
|
||||
syncBuiltinESMExports()
|
||||
`
|
||||
)
|
||||
return preloadPath
|
||||
}
|
||||
|
||||
export function initGitRepo(projectDir) {
|
||||
runGit(projectDir, ['init', '--quiet', '--initial-branch=main'])
|
||||
runGit(projectDir, ['config', 'user.email', 'orca-test@example.com'])
|
||||
runGit(projectDir, ['config', 'user.name', 'Orca Test'])
|
||||
runGit(projectDir, ['commit', '--quiet', '--allow-empty', '-m', 'init'])
|
||||
}
|
||||
|
||||
export function addSiblingWorktree(projectDir, siblingDir) {
|
||||
runGit(projectDir, ['worktree', 'add', '--quiet', '-b', 'sibling', siblingDir])
|
||||
copyScriptWithLocalModules(sourceScriptPath, join(siblingDir, 'config', 'scripts'))
|
||||
return siblingDir
|
||||
}
|
||||
|
||||
function runGit(projectDir, args) {
|
||||
execFileSync('git', ['-C', projectDir, ...args], { stdio: 'ignore' })
|
||||
}
|
||||
|
||||
export function sharedCacheRoot(repoDir) {
|
||||
return join(repoDir, '.git', 'orca-cache', 'electron')
|
||||
}
|
||||
|
||||
export function readSharedDistMarker(projectDir) {
|
||||
try {
|
||||
return readFileSync(join(projectDir, 'node_modules/electron/.orca-shared-dist'), 'utf8')
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
export function readExtractorCallCount(projectDir) {
|
||||
try {
|
||||
return readFileSync(join(projectDir, 'fake-extractor.log'), 'utf8').trim().split('\n').length
|
||||
} catch {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
export function writeNonDarwinPlatformPreload(projectDir) {
|
||||
const preloadPath = join(projectDir, 'non-darwin-platform.cjs')
|
||||
writeFileSync(
|
||||
preloadPath,
|
||||
`
|
||||
Object.defineProperty(process, 'platform', { value: 'linux', configurable: true })
|
||||
`
|
||||
)
|
||||
return preloadPath
|
||||
}
|
||||
@@ -15,6 +15,14 @@ import { spawnSync } from 'node:child_process'
|
||||
import { createRequire } from 'node:module'
|
||||
import { platform as osPlatform, tmpdir } from 'node:os'
|
||||
import { join, resolve } from 'node:path'
|
||||
import { getElectronPlatformPath } from './electron-platform-path.mjs'
|
||||
import {
|
||||
shareElectronDistFromCache,
|
||||
hasAdoptedSharedElectronDist,
|
||||
publishSharedElectronDist,
|
||||
recordAdoptedSharedElectronDist,
|
||||
resolveSharedElectronDistEntry
|
||||
} from './shared-electron-dist-cache.mjs'
|
||||
|
||||
const projectDir = resolve(import.meta.dirname, '../..')
|
||||
const electronPackageDir = resolve(projectDir, 'node_modules/electron')
|
||||
@@ -54,7 +62,18 @@ try {
|
||||
|
||||
async function main() {
|
||||
repairElectronPathFile()
|
||||
const sharedEntry = resolveSharedElectronDistEntry({
|
||||
repoRoot: projectDir,
|
||||
electronPackageDir,
|
||||
version: electronVersion,
|
||||
targetPlatform,
|
||||
targetArch
|
||||
})
|
||||
|
||||
if (electronPackageIsUsable()) {
|
||||
if (sharedEntry !== null && !hasAdoptedSharedElectronDist(sharedEntry)) {
|
||||
shareExistingElectronDist(sharedEntry)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -62,7 +81,7 @@ async function main() {
|
||||
// Node. Install only Electron's npm package binary here; do not run the full
|
||||
// Electron native-module rebuild path, which would undo the Node ABI rebuild.
|
||||
console.log('[electron-package] Electron package binary is missing; running Electron install.')
|
||||
await installElectronPackageBinary()
|
||||
await installElectronPackageBinary(sharedEntry)
|
||||
|
||||
repairElectronPathFile()
|
||||
|
||||
@@ -122,8 +141,11 @@ function repairElectronPathFile() {
|
||||
}
|
||||
}
|
||||
|
||||
async function installElectronPackageBinary() {
|
||||
async function installElectronPackageBinary(sharedEntry) {
|
||||
const electronDistDir = resolve(electronPackageDir, 'dist')
|
||||
if (sharedEntry !== null && adoptSharedElectronDist(sharedEntry, electronDistDir)) {
|
||||
return
|
||||
}
|
||||
const tempDir = mkdtempSync(resolve(tmpdir(), 'orca-electron-'))
|
||||
const persistentCacheRoot =
|
||||
process.env.ORCA_ELECTRON_PACKAGE_CACHE_ROOT || process.env.ELECTRON_CACHE || null
|
||||
@@ -158,11 +180,73 @@ async function installElectronPackageBinary() {
|
||||
}
|
||||
|
||||
moveExtractedElectronDist(extractDir, electronDistDir)
|
||||
if (sharedEntry !== null) {
|
||||
publishElectronDistForSiblingWorktrees(sharedEntry, electronDistDir)
|
||||
}
|
||||
} finally {
|
||||
rmSync(tempDir, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Point this worktree's dist at the copy its siblings already share, so the ~295MB tree costs one
|
||||
* allocation per repository instead of one per worktree.
|
||||
*
|
||||
* Staged inside node_modules/electron on purpose: clonefile only shares blocks within a volume, and
|
||||
* staging elsewhere would silently downgrade the publish rename to a cross-device byte copy.
|
||||
*/
|
||||
function adoptSharedElectronDist(sharedEntry, electronDistDir) {
|
||||
const stageRoot = mkdtempSync(resolve(electronPackageDir, '.dist-clone-'))
|
||||
try {
|
||||
const stagePath = join(stageRoot, 'dist')
|
||||
if (
|
||||
!shareElectronDistFromCache(sharedEntry, stagePath, {
|
||||
version: electronVersion,
|
||||
platformPath
|
||||
})
|
||||
) {
|
||||
return false
|
||||
}
|
||||
moveExtractedElectronDist(stagePath, electronDistDir)
|
||||
recordAdoptedSharedElectronDist(sharedEntry, writeFileSync)
|
||||
console.log(
|
||||
`[electron-package] Shared Electron ${electronVersion} from ${sharedEntry.entryPath}`
|
||||
)
|
||||
return true
|
||||
} catch (error) {
|
||||
// The download path below is always a correct fallback, so sharing never fails an install.
|
||||
console.warn(`[electron-package] Shared Electron dist unavailable: ${formatShareError(error)}`)
|
||||
return false
|
||||
} finally {
|
||||
rmSync(stageRoot, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
/** An already-installed dist joins the cache: clone from it if it exists, seed it otherwise. */
|
||||
function shareExistingElectronDist(sharedEntry) {
|
||||
const electronDistDir = resolve(electronPackageDir, 'dist')
|
||||
if (!adoptSharedElectronDist(sharedEntry, electronDistDir)) {
|
||||
publishElectronDistForSiblingWorktrees(sharedEntry, electronDistDir)
|
||||
}
|
||||
}
|
||||
|
||||
function publishElectronDistForSiblingWorktrees(sharedEntry, electronDistDir) {
|
||||
const published = publishSharedElectronDist(electronDistDir, sharedEntry, {
|
||||
version: electronVersion,
|
||||
platformPath
|
||||
})
|
||||
if (published) {
|
||||
console.log(
|
||||
`[electron-package] Published Electron ${electronVersion} to ${sharedEntry.entryPath}`
|
||||
)
|
||||
recordAdoptedSharedElectronDist(sharedEntry, writeFileSync)
|
||||
}
|
||||
}
|
||||
|
||||
function formatShareError(error) {
|
||||
return error instanceof Error ? error.message : String(error)
|
||||
}
|
||||
|
||||
async function downloadElectronArtifactWithRetry(downloadOptions, { cacheRootIsPersistent }) {
|
||||
const retryDelays = getDownloadRetryDelays()
|
||||
|
||||
@@ -438,19 +522,3 @@ function getElectronTargetPlatform() {
|
||||
function getElectronTargetArch() {
|
||||
return process.env.ELECTRON_INSTALL_ARCH || process.env.npm_config_arch || process.arch
|
||||
}
|
||||
|
||||
function getElectronPlatformPath(targetPlatform) {
|
||||
switch (targetPlatform) {
|
||||
case 'mas':
|
||||
case 'darwin':
|
||||
return 'Electron.app/Contents/MacOS/Electron'
|
||||
case 'freebsd':
|
||||
case 'openbsd':
|
||||
case 'linux':
|
||||
return 'electron'
|
||||
case 'win32':
|
||||
return 'electron.exe'
|
||||
default:
|
||||
throw new Error(`Electron builds are not available on platform: ${targetPlatform}`)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,22 +1,32 @@
|
||||
import {
|
||||
copyFileSync,
|
||||
existsSync,
|
||||
lstatSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
writeFileSync
|
||||
} from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
const sourceScriptPath = fileURLToPath(
|
||||
new URL('./install-electron-package-binary.mjs', import.meta.url)
|
||||
)
|
||||
import {
|
||||
addSiblingWorktree,
|
||||
initGitRepo,
|
||||
mkTempProject,
|
||||
readExtractorCallCount,
|
||||
readSharedDistMarker,
|
||||
runInstallScript,
|
||||
sharedCacheRoot,
|
||||
sharedEntryName,
|
||||
sharedEntryNameFor,
|
||||
writeFakeElectronDist,
|
||||
writeFakeElectronGet,
|
||||
writeFakeElectronPackage,
|
||||
writeFakeExtractor,
|
||||
writeNonDarwinPlatformPreload,
|
||||
writeTypeDefPublishFailurePreload
|
||||
} from './install-electron-package-binary-test-fixtures.mjs'
|
||||
|
||||
describe('install-electron-package-binary', () => {
|
||||
it('installs Electron from an isolated cache and repairs path.txt', () => {
|
||||
@@ -403,6 +413,199 @@ describe('install-electron-package-binary', () => {
|
||||
}
|
||||
})
|
||||
|
||||
// The shared cache is macOS-only: it exists to avoid a second copy via APFS clonefile.
|
||||
it('publishes a shared Electron dist entry after a fresh download', () => {
|
||||
const projectDir = mkTempProject()
|
||||
|
||||
try {
|
||||
initGitRepo(projectDir)
|
||||
writeFakeElectronPackage(projectDir)
|
||||
writeFakeElectronGet(projectDir)
|
||||
writeFakeExtractor(projectDir, { createExecutable: true })
|
||||
|
||||
const result = runInstallScript(projectDir, { CI: '' })
|
||||
const entryPath = join(sharedCacheRoot(projectDir), sharedEntryName)
|
||||
|
||||
expect(result.status, result.stderr).toBe(0)
|
||||
expect(lstatSync(entryPath).isDirectory()).toBe(true)
|
||||
expect(lstatSync(entryPath).isSymbolicLink()).toBe(false)
|
||||
expect(readFileSync(join(entryPath, 'version'), 'utf8')).toBe('v41.5.0')
|
||||
expect(existsSync(join(entryPath, 'electron'))).toBe(true)
|
||||
expect(readSharedDistMarker(projectDir)).toBe(sharedEntryName)
|
||||
expect(result.stdout).toMatch(/Published Electron 41\.5\.0 to .*41\.5\.0-linux-x64$/m)
|
||||
} finally {
|
||||
rmSync(projectDir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('shares the Electron dist into a sibling worktree without downloading', () => {
|
||||
const projectDir = mkTempProject()
|
||||
const siblingDir = `${projectDir}-sibling`
|
||||
|
||||
try {
|
||||
initGitRepo(projectDir)
|
||||
writeFakeElectronPackage(projectDir)
|
||||
writeFakeElectronGet(projectDir)
|
||||
writeFakeExtractor(projectDir, { createExecutable: true })
|
||||
expect(runInstallScript(projectDir, { CI: '' }).status).toBe(0)
|
||||
|
||||
addSiblingWorktree(projectDir, siblingDir)
|
||||
writeFakeElectronPackage(siblingDir)
|
||||
writeFakeElectronGet(siblingDir)
|
||||
writeFakeExtractor(siblingDir, { createExecutable: true })
|
||||
|
||||
const result = runInstallScript(siblingDir, { CI: '' })
|
||||
const siblingDistDir = join(siblingDir, 'node_modules/electron/dist')
|
||||
|
||||
expect(result.status, result.stderr).toBe(0)
|
||||
expect(readExtractorCallCount(siblingDir)).toBe(0)
|
||||
expect(existsSync(join(siblingDir, 'electron-get.log'))).toBe(false)
|
||||
expect(lstatSync(siblingDistDir).isDirectory()).toBe(true)
|
||||
expect(lstatSync(siblingDistDir).isSymbolicLink()).toBe(false)
|
||||
expect(readFileSync(join(siblingDistDir, 'version'), 'utf8')).toBe('v41.5.0')
|
||||
expect(existsSync(join(siblingDistDir, 'electron'))).toBe(true)
|
||||
expect(readFileSync(join(siblingDir, 'node_modules/electron/path.txt'), 'utf8')).toBe(
|
||||
'electron'
|
||||
)
|
||||
expect(readSharedDistMarker(siblingDir)).toBe(sharedEntryName)
|
||||
expect(result.stdout).toContain('Shared Electron 41.5.0 from')
|
||||
} finally {
|
||||
rmSync(siblingDir, { recursive: true, force: true })
|
||||
rmSync(projectDir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('publishes an already installed Electron dist that predates the shared cache', () => {
|
||||
const projectDir = mkTempProject()
|
||||
|
||||
try {
|
||||
initGitRepo(projectDir)
|
||||
writeFakeElectronPackage(projectDir)
|
||||
writeFakeElectronGet(projectDir)
|
||||
writeFakeExtractor(projectDir, { createExecutable: true })
|
||||
writeFakeElectronDist(projectDir, {
|
||||
executableContents: 'existing executable',
|
||||
pathContents: 'electron'
|
||||
})
|
||||
|
||||
const result = runInstallScript(projectDir, { CI: '' })
|
||||
const entryPath = join(sharedCacheRoot(projectDir), sharedEntryName)
|
||||
const distDir = join(projectDir, 'node_modules/electron/dist')
|
||||
|
||||
expect(result.status, result.stderr).toBe(0)
|
||||
expect(readExtractorCallCount(projectDir)).toBe(0)
|
||||
expect(existsSync(join(projectDir, 'electron-get.log'))).toBe(false)
|
||||
expect(readFileSync(join(entryPath, 'version'), 'utf8')).toBe('v41.5.0')
|
||||
expect(readFileSync(join(entryPath, 'electron'), 'utf8')).toBe('existing executable')
|
||||
expect(readSharedDistMarker(projectDir)).toBe(sharedEntryName)
|
||||
expect(readFileSync(join(distDir, 'electron'), 'utf8')).toBe('existing executable')
|
||||
expect(readFileSync(join(distDir, 'version'), 'utf8')).toBe('v41.5.0')
|
||||
} finally {
|
||||
rmSync(projectDir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('replaces a corrupt shared Electron dist entry instead of re-downloading forever', () => {
|
||||
const projectDir = mkTempProject()
|
||||
|
||||
try {
|
||||
initGitRepo(projectDir)
|
||||
writeFakeElectronPackage(projectDir)
|
||||
writeFakeElectronGet(projectDir)
|
||||
writeFakeExtractor(projectDir, { createExecutable: true })
|
||||
const entryPath = join(sharedCacheRoot(projectDir), sharedEntryName)
|
||||
mkdirSync(entryPath, { recursive: true })
|
||||
writeFileSync(join(entryPath, 'version'), 'v40.0.0')
|
||||
writeFileSync(join(entryPath, 'electron'), 'stale executable')
|
||||
|
||||
const result = runInstallScript(projectDir, { CI: '' })
|
||||
const distDir = join(projectDir, 'node_modules/electron/dist')
|
||||
|
||||
expect(result.status, result.stderr).toBe(0)
|
||||
expect(result.stderr).not.toContain('Failed to install Electron package binary')
|
||||
expect(readExtractorCallCount(projectDir)).toBe(1)
|
||||
expect(readFileSync(join(distDir, 'version'), 'utf8')).toBe('v41.5.0')
|
||||
expect(readFileSync(join(projectDir, 'node_modules/electron/path.txt'), 'utf8')).toBe(
|
||||
'electron'
|
||||
)
|
||||
// Why not just fall back: an entry left corrupt makes every sibling worktree download again.
|
||||
expect(readFileSync(join(entryPath, 'version'), 'utf8')).toBe('v41.5.0')
|
||||
expect(readSharedDistMarker(projectDir)).toBe(sharedEntryName)
|
||||
expect(readdirSync(sharedCacheRoot(projectDir))).toEqual([sharedEntryName])
|
||||
} finally {
|
||||
rmSync(projectDir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('hardlinks the shared Electron dist on a host without copy-on-write', () => {
|
||||
const projectDir = mkTempProject()
|
||||
|
||||
try {
|
||||
initGitRepo(projectDir)
|
||||
writeFakeElectronPackage(projectDir)
|
||||
writeFakeElectronGet(projectDir)
|
||||
writeFakeExtractor(projectDir, { createExecutable: true })
|
||||
const preloadPath = writeNonDarwinPlatformPreload(projectDir)
|
||||
const nonDarwinEnv = {
|
||||
CI: '',
|
||||
NODE_OPTIONS: [process.env.NODE_OPTIONS, `--require=${preloadPath}`]
|
||||
.filter(Boolean)
|
||||
.join(' ')
|
||||
}
|
||||
|
||||
const result = runInstallScript(projectDir, nonDarwinEnv)
|
||||
const entryPath = join(sharedCacheRoot(projectDir), sharedEntryName)
|
||||
const distDir = join(projectDir, 'node_modules/electron/dist')
|
||||
|
||||
expect(result.status, result.stderr).toBe(0)
|
||||
expect(readFileSync(join(distDir, 'version'), 'utf8')).toBe('v41.5.0')
|
||||
expect(readFileSync(join(entryPath, 'version'), 'utf8')).toBe('v41.5.0')
|
||||
// Why read-only: these are the same inodes, so an extract over dist would otherwise rewrite
|
||||
// the cache and every sibling worktree at once.
|
||||
expect(statSync(join(entryPath, 'electron')).mode & 0o222).toBe(0)
|
||||
expect(statSync(join(entryPath, 'electron')).ino).toBe(
|
||||
statSync(join(distDir, 'electron')).ino
|
||||
)
|
||||
} finally {
|
||||
rmSync(projectDir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('gives an Electron upgrade its own cache entry and leaves the old one for other branches', () => {
|
||||
const projectDir = mkTempProject()
|
||||
|
||||
try {
|
||||
initGitRepo(projectDir)
|
||||
writeFakeElectronPackage(projectDir)
|
||||
writeFakeElectronGet(projectDir)
|
||||
writeFakeExtractor(projectDir, { createExecutable: true })
|
||||
expect(runInstallScript(projectDir, { CI: '' }).status).toBe(0)
|
||||
expect(readSharedDistMarker(projectDir)).toBe(sharedEntryNameFor('41.5.0'))
|
||||
|
||||
// Upgrade the pinned Electron, exactly as a branch bumping the dependency would.
|
||||
writeFakeElectronPackage(projectDir, { version: '42.0.0' })
|
||||
writeFakeExtractor(projectDir, { createExecutable: true, version: '42.0.0' })
|
||||
const upgraded = runInstallScript(projectDir, { CI: '' })
|
||||
const cacheRoot = sharedCacheRoot(projectDir)
|
||||
|
||||
expect(upgraded.status, upgraded.stderr).toBe(0)
|
||||
expect(readFileSync(join(projectDir, 'node_modules/electron/dist/version'), 'utf8')).toBe(
|
||||
'v42.0.0'
|
||||
)
|
||||
expect(readSharedDistMarker(projectDir)).toBe(sharedEntryNameFor('42.0.0'))
|
||||
// Why the old entry stays: sibling worktrees on the previous branch still share it.
|
||||
expect(readdirSync(cacheRoot).sort()).toEqual([
|
||||
sharedEntryNameFor('41.5.0'),
|
||||
sharedEntryNameFor('42.0.0')
|
||||
])
|
||||
expect(readFileSync(join(cacheRoot, sharedEntryNameFor('41.5.0'), 'version'), 'utf8')).toBe(
|
||||
'v41.5.0'
|
||||
)
|
||||
} finally {
|
||||
rmSync(projectDir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('does not exit successfully when Electron download never settles', () => {
|
||||
const projectDir = mkTempProject()
|
||||
|
||||
@@ -421,155 +624,3 @@ describe('install-electron-package-binary', () => {
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
function mkTempProject() {
|
||||
const projectDir = mkdtempSync(join(tmpdir(), 'orca-install-electron-'))
|
||||
mkdirSync(join(projectDir, 'config', 'scripts'), { recursive: true })
|
||||
copyFileSync(
|
||||
sourceScriptPath,
|
||||
join(projectDir, 'config', 'scripts', 'install-electron-package-binary.mjs')
|
||||
)
|
||||
return projectDir
|
||||
}
|
||||
|
||||
function runInstallScript(projectDir, extraEnv = {}) {
|
||||
return spawnSync(process.execPath, ['config/scripts/install-electron-package-binary.mjs'], {
|
||||
cwd: projectDir,
|
||||
encoding: 'utf8',
|
||||
env: {
|
||||
...process.env,
|
||||
ELECTRON_CACHE: undefined,
|
||||
ORCA_ELECTRON_PACKAGE_CACHE_ROOT: undefined,
|
||||
npm_config_platform: 'linux',
|
||||
npm_config_arch: 'x64',
|
||||
ORCA_ELECTRON_PACKAGE_EXTRACTOR: join(projectDir, 'fake-extractor.cjs'),
|
||||
...extraEnv
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function writeFakeElectronPackage(projectDir, { lazyRequireMarker = null } = {}) {
|
||||
const electronDir = join(projectDir, 'node_modules', 'electron')
|
||||
mkdirSync(electronDir, { recursive: true })
|
||||
writeFileSync(
|
||||
join(electronDir, 'package.json'),
|
||||
JSON.stringify({ name: 'electron', version: '41.5.0' })
|
||||
)
|
||||
writeFileSync(join(electronDir, 'checksums.json'), '{}')
|
||||
writeFileSync(
|
||||
join(electronDir, 'index.js'),
|
||||
`
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
${lazyRequireMarker ? `fs.writeFileSync(${JSON.stringify(lazyRequireMarker)}, 'required')` : ''}
|
||||
const pathFile = path.join(__dirname, 'path.txt')
|
||||
if (!fs.existsSync(pathFile)) {
|
||||
throw new Error('Electron failed to install correctly, please delete node_modules/electron and try installing again')
|
||||
}
|
||||
module.exports = path.join(__dirname, 'dist', fs.readFileSync(pathFile, 'utf8'))
|
||||
`
|
||||
)
|
||||
}
|
||||
|
||||
function writeFakeElectronDist(
|
||||
projectDir,
|
||||
{ version = 'v41.5.0', executableContents = '', pathContents } = {}
|
||||
) {
|
||||
const electronDir = join(projectDir, 'node_modules', 'electron')
|
||||
mkdirSync(join(electronDir, 'dist'), { recursive: true })
|
||||
writeFileSync(join(electronDir, 'dist/version'), version)
|
||||
writeFileSync(join(electronDir, 'dist/electron'), executableContents)
|
||||
if (pathContents !== undefined) {
|
||||
writeFileSync(join(electronDir, 'path.txt'), pathContents)
|
||||
}
|
||||
}
|
||||
|
||||
function writeFakeElectronGet(
|
||||
projectDir,
|
||||
{
|
||||
downloadNeverSettles = false,
|
||||
downloadFailures = 0,
|
||||
downloadErrorCode = 'ECONNRESET',
|
||||
downloadHttpStatus = null
|
||||
} = {}
|
||||
) {
|
||||
const getDir = join(projectDir, 'node_modules', 'electron', 'node_modules', '@electron', 'get')
|
||||
mkdirSync(getDir, { recursive: true })
|
||||
writeFileSync(
|
||||
join(getDir, 'index.js'),
|
||||
`
|
||||
const { mkdirSync, writeFileSync, appendFileSync } = require('node:fs')
|
||||
const { join } = require('node:path')
|
||||
let downloadAttempt = 0
|
||||
exports.downloadArtifact = async function downloadArtifact(details) {
|
||||
downloadAttempt += 1
|
||||
appendFileSync(
|
||||
'electron-get.log',
|
||||
'cacheRoot=' + details.cacheRoot + ' platform=' + details.platform + ' arch=' + details.arch + ' force=' + details.force + '\\n'
|
||||
)
|
||||
if (${JSON.stringify(downloadNeverSettles)}) {
|
||||
return new Promise(() => {})
|
||||
}
|
||||
if (downloadAttempt <= ${JSON.stringify(downloadFailures)}) {
|
||||
if (${JSON.stringify(downloadHttpStatus)} != null) {
|
||||
const error = new Error('Response code ' + ${JSON.stringify(downloadHttpStatus)})
|
||||
error.response = { status: ${JSON.stringify(downloadHttpStatus)} }
|
||||
throw error
|
||||
}
|
||||
const cause = Object.assign(new Error('download failed'), {
|
||||
code: ${JSON.stringify(downloadErrorCode)}
|
||||
})
|
||||
throw Object.assign(new TypeError('fetch failed'), { cause })
|
||||
}
|
||||
mkdirSync(details.cacheRoot, { recursive: true })
|
||||
const artifactPath = join(details.cacheRoot, 'electron.zip')
|
||||
writeFileSync(artifactPath, 'fake zip')
|
||||
return artifactPath
|
||||
}
|
||||
`
|
||||
)
|
||||
}
|
||||
|
||||
function writeFakeExtractor(projectDir, { createExecutable }) {
|
||||
writeFileSync(
|
||||
join(projectDir, 'fake-extractor.cjs'),
|
||||
`
|
||||
const { mkdirSync, symlinkSync, writeFileSync } = require('node:fs')
|
||||
const { join } = require('node:path')
|
||||
const extractDir = process.argv[3]
|
||||
mkdirSync(join(extractDir, 'locales'), { recursive: true })
|
||||
if (${JSON.stringify(createExecutable)}) {
|
||||
writeFileSync(join(extractDir, 'electron'), '')
|
||||
writeFileSync(join(extractDir, 'electron.exe'), '')
|
||||
writeFileSync(join(extractDir, 'electron.d.ts'), 'replacement types')
|
||||
writeFileSync(join(extractDir, 'version'), 'v41.5.0')
|
||||
if (process.platform !== 'win32') {
|
||||
symlinkSync('version', join(extractDir, 'version-link'))
|
||||
}
|
||||
}
|
||||
`
|
||||
)
|
||||
}
|
||||
|
||||
function writeTypeDefPublishFailurePreload(projectDir) {
|
||||
const preloadPath = join(projectDir, 'type-def-publish-failure.cjs')
|
||||
writeFileSync(
|
||||
preloadPath,
|
||||
`
|
||||
const fs = require('node:fs')
|
||||
const { syncBuiltinESMExports } = require('node:module')
|
||||
const { basename, dirname } = require('node:path')
|
||||
const renameSync = fs.renameSync
|
||||
fs.renameSync = (source, target) => {
|
||||
if (basename(source) === 'electron.d.ts' && basename(dirname(source)) === 'dist') {
|
||||
const error = new Error('injected Electron type definition publish failure')
|
||||
error.code = 'EACCES'
|
||||
throw error
|
||||
}
|
||||
return renameSync(source, target)
|
||||
}
|
||||
syncBuiltinESMExports()
|
||||
`
|
||||
)
|
||||
return preloadPath
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ import { chmodSync, copyFileSync, mkdirSync, mkdtempSync, writeFileSync } from '
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { copyScriptWithLocalModules } from './script-module-dependencies.mjs'
|
||||
|
||||
const sourceScriptPath = fileURLToPath(new URL('./rebuild-native-deps.mjs', import.meta.url))
|
||||
const sourceInstallScriptPath = fileURLToPath(
|
||||
@@ -19,10 +20,7 @@ export function mkTempProject() {
|
||||
const projectDir = mkdtempSync(join(tmpdir(), 'orca-rebuild-native-deps-'))
|
||||
mkdirSync(join(projectDir, 'config', 'scripts'), { recursive: true })
|
||||
copyFileSync(sourceScriptPath, join(projectDir, 'config', 'scripts', 'rebuild-native-deps.mjs'))
|
||||
copyFileSync(
|
||||
sourceInstallScriptPath,
|
||||
join(projectDir, 'config', 'scripts', 'install-electron-package-binary.mjs')
|
||||
)
|
||||
copyScriptWithLocalModules(sourceInstallScriptPath, join(projectDir, 'config', 'scripts'))
|
||||
copyFileSync(
|
||||
sourceNodePtyJobOwnershipPath,
|
||||
join(projectDir, 'config', 'scripts', 'node-pty-job-ownership.cjs')
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
// Converts worktrees that already have their own Electron dist over to the shared cache.
|
||||
// A normal install only shares when Electron is (re)installed, and an existing healthy worktree
|
||||
// never reaches that path -- so without this, sharing only arrives at the next Electron upgrade.
|
||||
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { existsSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import {
|
||||
hasAdoptedSharedElectronDist,
|
||||
isUsableElectronDist,
|
||||
publishSharedElectronDist,
|
||||
recordAdoptedSharedElectronDist,
|
||||
resolveSharedElectronDistEntry,
|
||||
shareElectronDistFromCache
|
||||
} from './shared-electron-dist-cache.mjs'
|
||||
import { getElectronPlatformPath } from './electron-platform-path.mjs'
|
||||
|
||||
const apply = process.argv.includes('--apply')
|
||||
const repoRoot = process.argv.includes('--repo')
|
||||
? path.resolve(process.argv[process.argv.indexOf('--repo') + 1])
|
||||
: process.cwd()
|
||||
|
||||
function listWorktrees(root) {
|
||||
const raw = execFileSync('git', ['-C', root, 'worktree', 'list', '--porcelain'], {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
return raw
|
||||
.split('\n')
|
||||
.filter((line) => line.startsWith('worktree '))
|
||||
.map((line) => line.slice('worktree '.length).trim())
|
||||
}
|
||||
|
||||
function measure(distPath) {
|
||||
try {
|
||||
return (
|
||||
Number(execFileSync('du', ['-sk', distPath], { encoding: 'utf8' }).split(/\s+/)[0]) * 1024
|
||||
)
|
||||
} catch {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
/** Swap in a shared copy behind a rename, so an interrupted run never leaves a partial dist. */
|
||||
function adoptInto(distPath, entry, identity) {
|
||||
const stagePath = `${distPath}.reclaim-${process.pid}-${randomUUID()}`
|
||||
if (!shareElectronDistFromCache(entry, stagePath, identity)) {
|
||||
rmSync(stagePath, { recursive: true, force: true })
|
||||
return false
|
||||
}
|
||||
const previousPath = `${distPath}.previous-${process.pid}-${randomUUID()}`
|
||||
renameSync(distPath, previousPath)
|
||||
try {
|
||||
renameSync(stagePath, distPath)
|
||||
} catch (error) {
|
||||
renameSync(previousPath, distPath)
|
||||
rmSync(stagePath, { recursive: true, force: true })
|
||||
throw error
|
||||
}
|
||||
rmSync(previousPath, { recursive: true, force: true })
|
||||
return true
|
||||
}
|
||||
|
||||
let reclaimed = 0
|
||||
let converted = 0
|
||||
let skipped = 0
|
||||
|
||||
for (const worktree of listWorktrees(repoRoot)) {
|
||||
const electronPackageDir = path.join(worktree, 'node_modules', 'electron')
|
||||
const distPath = path.join(electronPackageDir, 'dist')
|
||||
if (!existsSync(path.join(electronPackageDir, 'package.json')) || !existsSync(distPath)) {
|
||||
continue
|
||||
}
|
||||
if (statSync(distPath, { throwIfNoEntry: false })?.isDirectory() !== true) {
|
||||
continue
|
||||
}
|
||||
|
||||
let version
|
||||
try {
|
||||
version = JSON.parse(
|
||||
readFileSync(path.join(electronPackageDir, 'package.json'), 'utf8')
|
||||
).version
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
const targetPlatform = process.platform
|
||||
const targetArch = process.arch
|
||||
let platformPath
|
||||
try {
|
||||
platformPath = getElectronPlatformPath(targetPlatform)
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
if (!isUsableElectronDist(distPath, version, platformPath)) {
|
||||
console.log(`skip ${worktree} (dist is not a complete Electron ${version})`)
|
||||
skipped += 1
|
||||
continue
|
||||
}
|
||||
|
||||
const entry = resolveSharedElectronDistEntry({
|
||||
repoRoot: worktree,
|
||||
electronPackageDir,
|
||||
version,
|
||||
targetPlatform,
|
||||
targetArch
|
||||
})
|
||||
if (entry === null) {
|
||||
continue
|
||||
}
|
||||
if (hasAdoptedSharedElectronDist(entry)) {
|
||||
continue
|
||||
}
|
||||
|
||||
const size = measure(distPath)
|
||||
if (!apply) {
|
||||
console.log(`would share ${worktree} ${(size / 1024 ** 3).toFixed(2)} GiB (${version})`)
|
||||
reclaimed += size
|
||||
converted += 1
|
||||
continue
|
||||
}
|
||||
|
||||
try {
|
||||
if (!existsSync(entry.entryPath)) {
|
||||
if (publishSharedElectronDist(distPath, entry, { version, platformPath })) {
|
||||
recordAdoptedSharedElectronDist(entry, writeFileSync)
|
||||
console.log(`seeded ${worktree} -> ${entry.entryPath}`)
|
||||
converted += 1
|
||||
}
|
||||
continue
|
||||
}
|
||||
if (adoptInto(distPath, entry, { version, platformPath })) {
|
||||
recordAdoptedSharedElectronDist(entry, writeFileSync)
|
||||
reclaimed += size
|
||||
converted += 1
|
||||
console.log(`shared ${worktree} reclaimed ${(size / 1024 ** 3).toFixed(2)} GiB`)
|
||||
}
|
||||
} catch (error) {
|
||||
// A worktree that fails is left exactly as it was; it still has its own working dist.
|
||||
console.warn(`skip ${worktree} (${error instanceof Error ? error.message : String(error)})`)
|
||||
skipped += 1
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`\n${apply ? 'Shared' : 'Would share'} ${converted} worktree(s); ` +
|
||||
`${apply ? 'reclaimed' : 'reclaimable'} ~${(reclaimed / 1024 ** 3).toFixed(2)} GiB` +
|
||||
`${skipped > 0 ? `; skipped ${skipped}` : ''}` +
|
||||
`${apply ? '' : '\nRe-run with --apply to do it.'}`
|
||||
)
|
||||
@@ -1,7 +1,6 @@
|
||||
import { execFileSync, spawn } from 'node:child_process'
|
||||
import { createHash } from 'node:crypto'
|
||||
import {
|
||||
cpSync,
|
||||
existsSync,
|
||||
lstatSync,
|
||||
mkdirSync,
|
||||
@@ -16,8 +15,10 @@ import {
|
||||
import net from 'node:net'
|
||||
import { createRequire } from 'node:module'
|
||||
import path from 'node:path'
|
||||
|
||||
import { prepareDevCliTerminalWrappers } from './dev-cli-terminal-wrapper.mjs'
|
||||
import { isDevBundleInUse, selectStaleDevBundleDirs } from './dev-electron-bundle-cache.mjs'
|
||||
import { copyPrivateTree } from './space-sharing-copy.mjs'
|
||||
import {
|
||||
DEV_BUNDLE_ID,
|
||||
getDevBundlePlistPatches,
|
||||
@@ -298,9 +299,9 @@ function prepareMacDevElectronApp() {
|
||||
|
||||
rmSync(distDir, { recursive: true, force: true })
|
||||
mkdirSync(distDir, { recursive: true })
|
||||
// Why: Electron.framework uses relative symlinks for its bundle resources;
|
||||
// resolving them to pnpm-store absolutes breaks Chromium's bundle lookup.
|
||||
cpSync(sourceAppPath, appPath, { recursive: true, verbatimSymlinks: true })
|
||||
// Why clone-first: this ~280MB copy is made per branch title x Electron version, and only the
|
||||
// plist/helper/codesign bytes patched below ever diverge from the source.
|
||||
copyPrivateTree(sourceAppPath, appPath)
|
||||
restoreElectronFrameworkSymlinks(appPath)
|
||||
|
||||
const plistPath = path.join(appPath, 'Contents', 'Info.plist')
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import { copyFileSync, mkdirSync, readFileSync } from 'node:fs'
|
||||
import { basename, dirname, join } from 'node:path'
|
||||
|
||||
/**
|
||||
* Copy a script and every co-located module it imports into a fixture's `config/scripts`.
|
||||
*
|
||||
* Walked rather than listed: a module the script needs but the fixture never copied fails every
|
||||
* test in the suite with a module-resolution error that looks nothing like the defect it hides.
|
||||
*/
|
||||
export function copyScriptWithLocalModules(sourceScriptPath, destinationScriptsDir) {
|
||||
mkdirSync(destinationScriptsDir, { recursive: true })
|
||||
for (const modulePath of collectScriptModules(sourceScriptPath)) {
|
||||
copyFileSync(modulePath, join(destinationScriptsDir, basename(modulePath)))
|
||||
}
|
||||
}
|
||||
|
||||
function collectScriptModules(scriptPath, seen = new Set()) {
|
||||
if (seen.has(scriptPath)) {
|
||||
return seen
|
||||
}
|
||||
seen.add(scriptPath)
|
||||
for (const [, specifier] of readFileSync(scriptPath, 'utf8').matchAll(/from '(\.\/[^']+)'/g)) {
|
||||
collectScriptModules(join(dirname(scriptPath), specifier), seen)
|
||||
}
|
||||
return seen
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { existsSync, lstatSync, mkdirSync, readFileSync, renameSync, rmSync } from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import { makeTreeReadOnly, shareTree } from './space-sharing-copy.mjs'
|
||||
|
||||
const IDENTITY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/
|
||||
// Sibling of path.txt, never inside dist: an install replaces dist wholesale.
|
||||
const MARKER_FILENAME = '.orca-shared-dist'
|
||||
|
||||
/**
|
||||
* Where sibling worktrees of one repository keep their shared extracted Electron.
|
||||
*
|
||||
* Null means "install normally" -- every caller treats a missing entry as "do what you did before".
|
||||
*/
|
||||
export function resolveSharedElectronDistEntry(options) {
|
||||
const { repoRoot, version, targetPlatform, targetArch } = options
|
||||
const env = options.env ?? process.env
|
||||
// Packaging jobs get a fresh checkout per run, so a cache only adds a failure mode.
|
||||
if (env.CI === '1' || env.CI === 'true') {
|
||||
return null
|
||||
}
|
||||
if (![version, targetPlatform, targetArch].every((part) => IDENTITY_PATTERN.test(part ?? ''))) {
|
||||
return null
|
||||
}
|
||||
let gitCommonDir
|
||||
try {
|
||||
gitCommonDir = resolveGitCommonDir(repoRoot, options.execFile ?? execFileSync)
|
||||
} catch {
|
||||
return null // Folder workspace, or no Git on PATH.
|
||||
}
|
||||
const cacheRoot = path.join(gitCommonDir, 'orca-cache', 'electron')
|
||||
return {
|
||||
cacheRoot,
|
||||
entryPath: path.join(cacheRoot, `${version}-${targetPlatform}-${targetArch}`),
|
||||
markerPath: path.join(options.electronPackageDir, MARKER_FILENAME)
|
||||
}
|
||||
}
|
||||
|
||||
export function resolveGitCommonDir(repoRoot, execFile = execFileSync) {
|
||||
const rawPath = execFile('git', ['-C', repoRoot, 'rev-parse', '--git-common-dir'], {
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'ignore']
|
||||
}).trim()
|
||||
if (!rawPath) {
|
||||
throw new Error('Git returned an empty common directory')
|
||||
}
|
||||
return path.resolve(repoRoot, rawPath)
|
||||
}
|
||||
|
||||
/** True once this worktree's dist is already a clone of the current cache entry. */
|
||||
export function hasAdoptedSharedElectronDist(entry) {
|
||||
try {
|
||||
return readFileSync(entry.markerPath, 'utf8') === path.basename(entry.entryPath)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
export function recordAdoptedSharedElectronDist(entry, write) {
|
||||
try {
|
||||
write(entry.markerPath, path.basename(entry.entryPath))
|
||||
} catch {
|
||||
// The marker is only an optimization: a missing one costs one extra clone.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Share a validated cache entry into `stagePath`. Deliberately never falls back to a byte copy --
|
||||
* with no storage to share the caller is better off with its normal install, which this must not
|
||||
* slow down.
|
||||
*/
|
||||
export function shareElectronDistFromCache(entry, stagePath, options) {
|
||||
const { version, platformPath } = options
|
||||
if (!isUsableElectronDist(entry.entryPath, version, platformPath)) {
|
||||
return false
|
||||
}
|
||||
try {
|
||||
;(options.share ?? shareTree)(entry.entryPath, stagePath)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
return isUsableElectronDist(stagePath, version, platformPath)
|
||||
}
|
||||
|
||||
/**
|
||||
* Publish this worktree's dist as the shared entry, best effort.
|
||||
*
|
||||
* No lock: staging names are unique and `rename` onto a populated directory fails with ENOTEMPTY,
|
||||
* so a concurrent publisher either wins the rename or cleans up its own staging tree. Neither can
|
||||
* observe a half-written entry, and a usable entry already in place is never overwritten.
|
||||
*/
|
||||
export function publishSharedElectronDist(distPath, entry, options = {}) {
|
||||
const { version, platformPath } = options
|
||||
const uuid = options.uuid ?? randomUUID
|
||||
const canValidate = Boolean(version) && Boolean(platformPath)
|
||||
// Without an identity to check against, "unusable" is unknowable -- never discard on a guess.
|
||||
if (existsSync(entry.entryPath) && (!canValidate || isUsable(entry, version, platformPath))) {
|
||||
return false
|
||||
}
|
||||
|
||||
const stagePath = `${entry.entryPath}.staging-${process.pid}-${uuid()}`
|
||||
try {
|
||||
mkdirSync(entry.cacheRoot, { recursive: true })
|
||||
;(options.share ?? shareTree)(distPath, stagePath)
|
||||
// Before publishing, not after: an entry is visible the instant the rename lands, and under
|
||||
// hardlink sharing this is the only thing standing between a stray write and every worktree.
|
||||
;(options.protect ?? makeTreeReadOnly)(stagePath)
|
||||
} catch {
|
||||
rmSync(stagePath, { recursive: true, force: true })
|
||||
return false
|
||||
}
|
||||
|
||||
return swapInElectronDistEntry(entry, stagePath, {
|
||||
canValidate,
|
||||
version,
|
||||
platformPath,
|
||||
uuid,
|
||||
rename: options.rename ?? renameSync
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace the entry with a staged tree, re-checking first.
|
||||
*
|
||||
* Sharing the tree above takes seconds, and a sibling worktree can publish a perfectly good entry
|
||||
* in that time. Re-validating here, immediately before the destructive rename, keeps us from
|
||||
* discarding that entry -- and restoring the quarantine on a failed swap keeps a losing publisher
|
||||
* from leaving the cache empty.
|
||||
*/
|
||||
function swapInElectronDistEntry(entry, stagePath, options) {
|
||||
const { canValidate, version, platformPath, uuid, rename } = options
|
||||
let quarantinePath = null
|
||||
if (existsSync(entry.entryPath)) {
|
||||
// Same rule as before staging: an entry we cannot judge, or one that is good, is never
|
||||
// displaced. Both mean another worktree got there first, so keep theirs.
|
||||
if (!canValidate || isUsable(entry, version, platformPath)) {
|
||||
rmSync(stagePath, { recursive: true, force: true })
|
||||
return false
|
||||
}
|
||||
quarantinePath = `${entry.entryPath}.unusable-${process.pid}-${uuid()}`
|
||||
try {
|
||||
renameSync(entry.entryPath, quarantinePath)
|
||||
} catch {
|
||||
rmSync(stagePath, { recursive: true, force: true })
|
||||
return false // Another worktree is already replacing it.
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
rename(stagePath, entry.entryPath)
|
||||
} catch {
|
||||
rmSync(stagePath, { recursive: true, force: true })
|
||||
if (quarantinePath !== null) {
|
||||
// Put it back rather than leave no entry at all; a bad entry still beats an empty cache,
|
||||
// because the next publisher re-validates and replaces it.
|
||||
try {
|
||||
renameSync(quarantinePath, entry.entryPath)
|
||||
return false
|
||||
} catch {
|
||||
rmSync(quarantinePath, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
if (quarantinePath !== null) {
|
||||
rmSync(quarantinePath, { recursive: true, force: true })
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
function isUsable(entry, version, platformPath) {
|
||||
return isUsableElectronDist(entry.entryPath, version, platformPath)
|
||||
}
|
||||
|
||||
export function isUsableElectronDist(distPath, version, platformPath) {
|
||||
try {
|
||||
if (!lstatSync(distPath).isDirectory()) {
|
||||
return false
|
||||
}
|
||||
const installedVersion = readFileSync(path.join(distPath, 'version'), 'utf8')
|
||||
.trim()
|
||||
.replace(/^v/, '')
|
||||
return installedVersion === version && existsSync(path.join(distPath, platformPath))
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,358 @@
|
||||
import type { execFileSync } from 'node:child_process'
|
||||
import {
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readdirSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
symlinkSync,
|
||||
writeFileSync
|
||||
} from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
shareElectronDistFromCache,
|
||||
hasAdoptedSharedElectronDist,
|
||||
isUsableElectronDist,
|
||||
publishSharedElectronDist,
|
||||
recordAdoptedSharedElectronDist,
|
||||
resolveSharedElectronDistEntry
|
||||
} from './shared-electron-dist-cache.mjs'
|
||||
import { makeTreeReadOnly } from './space-sharing-copy.mjs'
|
||||
|
||||
const VERSION = '43.4.1'
|
||||
const PLATFORM_PATH = path.join('Electron.app', 'Contents', 'MacOS', 'Electron')
|
||||
const identity = { version: VERSION, platformPath: PLATFORM_PATH }
|
||||
const roots: string[] = []
|
||||
|
||||
afterEach(() => {
|
||||
while (roots.length > 0) {
|
||||
rmSync(roots.pop()!, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
function makeRoot(): string {
|
||||
const root = mkdtempSync(path.join(tmpdir(), 'orca-shared-electron-'))
|
||||
roots.push(root)
|
||||
return root
|
||||
}
|
||||
|
||||
function writeDist(distPath: string, version = VERSION): string {
|
||||
mkdirSync(path.join(distPath, path.dirname(PLATFORM_PATH)), { recursive: true })
|
||||
writeFileSync(path.join(distPath, 'version'), `v${version}\n`)
|
||||
writeFileSync(path.join(distPath, PLATFORM_PATH), 'electron')
|
||||
return distPath
|
||||
}
|
||||
|
||||
function makeEntry(root: string, entryName = `${VERSION}-darwin-arm64`) {
|
||||
const cacheRoot = path.join(root, 'cache')
|
||||
return {
|
||||
cacheRoot,
|
||||
entryPath: path.join(cacheRoot, entryName),
|
||||
markerPath: path.join(root, '.orca-shared-dist')
|
||||
}
|
||||
}
|
||||
|
||||
const baseOptions = {
|
||||
repoRoot: '/repo',
|
||||
electronPackageDir: '/repo/node_modules/electron',
|
||||
version: VERSION,
|
||||
targetPlatform: 'darwin',
|
||||
targetArch: 'arm64',
|
||||
hostPlatform: 'darwin' as const,
|
||||
env: {} as NodeJS.ProcessEnv,
|
||||
execFile: (() => '/repo/.git\n') as unknown as typeof execFileSync
|
||||
}
|
||||
|
||||
describe('resolveSharedElectronDistEntry', () => {
|
||||
it('keys the entry by version, platform, and arch under the git common dir', () => {
|
||||
const entry = resolveSharedElectronDistEntry(baseOptions)
|
||||
expect(entry?.cacheRoot).toBe(path.join('/repo/.git', 'orca-cache', 'electron'))
|
||||
expect(entry?.entryPath).toBe(
|
||||
path.join('/repo/.git', 'orca-cache', 'electron', '43.4.1-darwin-arm64')
|
||||
)
|
||||
expect(entry?.markerPath).toBe(path.join('/repo/node_modules/electron', '.orca-shared-dist'))
|
||||
})
|
||||
|
||||
it('offers an entry on every platform a worktree is developed on', () => {
|
||||
for (const hostPlatform of ['darwin', 'linux', 'win32']) {
|
||||
expect(resolveSharedElectronDistEntry({ ...baseOptions, hostPlatform })).not.toBeNull()
|
||||
}
|
||||
})
|
||||
|
||||
it('declines on CI, where every job gets a fresh checkout', () => {
|
||||
expect(resolveSharedElectronDistEntry({ ...baseOptions, env: { CI: '1' } })).toBeNull()
|
||||
expect(resolveSharedElectronDistEntry({ ...baseOptions, env: { CI: 'true' } })).toBeNull()
|
||||
expect(resolveSharedElectronDistEntry({ ...baseOptions, env: { CI: 'false' } })).not.toBeNull()
|
||||
})
|
||||
|
||||
it('declines outside a Git worktree so folder workspaces install normally', () => {
|
||||
const execFile = (() => {
|
||||
throw new Error('not a git repository')
|
||||
}) as unknown as typeof execFileSync
|
||||
expect(resolveSharedElectronDistEntry({ ...baseOptions, execFile })).toBeNull()
|
||||
})
|
||||
|
||||
it('declines an identity that would not be a single safe path segment', () => {
|
||||
expect(resolveSharedElectronDistEntry({ ...baseOptions, targetArch: '../escape' })).toBeNull()
|
||||
expect(resolveSharedElectronDistEntry({ ...baseOptions, targetPlatform: 'dar/win' })).toBeNull()
|
||||
expect(resolveSharedElectronDistEntry({ ...baseOptions, version: '' })).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('isUsableElectronDist', () => {
|
||||
it('accepts a complete dist and tolerates the leading v in the version file', () => {
|
||||
const root = makeRoot()
|
||||
expect(isUsableElectronDist(writeDist(path.join(root, 'dist')), VERSION, PLATFORM_PATH)).toBe(
|
||||
true
|
||||
)
|
||||
})
|
||||
|
||||
it('rejects a version mismatch, a missing executable, and a missing directory', () => {
|
||||
const root = makeRoot()
|
||||
expect(
|
||||
isUsableElectronDist(writeDist(path.join(root, 'a'), '40.0.0'), VERSION, PLATFORM_PATH)
|
||||
).toBe(false)
|
||||
const partial = path.join(root, 'b')
|
||||
mkdirSync(partial, { recursive: true })
|
||||
writeFileSync(path.join(partial, 'version'), `v${VERSION}`)
|
||||
expect(isUsableElectronDist(partial, VERSION, PLATFORM_PATH)).toBe(false)
|
||||
expect(isUsableElectronDist(path.join(root, 'missing'), VERSION, PLATFORM_PATH)).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects a symlink so a redirected entry is never treated as cache content', () => {
|
||||
const root = makeRoot()
|
||||
writeDist(path.join(root, 'real'))
|
||||
symlinkSync(path.join(root, 'real'), path.join(root, 'link'), 'dir')
|
||||
expect(isUsableElectronDist(path.join(root, 'link'), VERSION, PLATFORM_PATH)).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('publishSharedElectronDist', () => {
|
||||
it('publishes through a staging directory and an atomic rename', () => {
|
||||
const root = makeRoot()
|
||||
const entry = makeEntry(root)
|
||||
const dist = writeDist(path.join(root, 'dist'))
|
||||
const share = vi.fn((source: string, destination: string) => {
|
||||
expect(path.basename(destination)).toMatch(/^43\.4\.1-darwin-arm64\.staging-/)
|
||||
writeDist(destination)
|
||||
expect(source).toBe(dist)
|
||||
})
|
||||
expect(publishSharedElectronDist(dist, entry, { share, ...identity })).toBe(true)
|
||||
expect(isUsableElectronDist(entry.entryPath, VERSION, PLATFORM_PATH)).toBe(true)
|
||||
expect(readdirSync(entry.cacheRoot)).toEqual([path.basename(entry.entryPath)])
|
||||
})
|
||||
|
||||
it('publishes the entry read-only, before it is reachable under its final name', () => {
|
||||
const root = makeRoot()
|
||||
const entry = makeEntry(root)
|
||||
const dist = writeDist(path.join(root, 'dist'))
|
||||
const protectedPaths: string[] = []
|
||||
const share = (source: string, destination: string) => {
|
||||
writeDist(destination)
|
||||
expect(source).toBe(dist)
|
||||
}
|
||||
const protect = (target: string) => {
|
||||
// Why order matters: a reader can clone the entry the instant the rename lands.
|
||||
expect(existsSync(entry.entryPath)).toBe(false)
|
||||
protectedPaths.push(target)
|
||||
makeTreeReadOnly(target)
|
||||
}
|
||||
expect(publishSharedElectronDist(dist, entry, { share, protect, ...identity })).toBe(true)
|
||||
expect(protectedPaths).toHaveLength(1)
|
||||
expect(statSync(path.join(entry.entryPath, 'version')).mode & 0o222).toBe(0)
|
||||
// Entry directories stay removable, which is what the install transaction actually needs.
|
||||
expect(() => rmSync(entry.entryPath, { recursive: true })).not.toThrow()
|
||||
})
|
||||
|
||||
it('never overwrites an entry another worktree already published', () => {
|
||||
const root = makeRoot()
|
||||
const entry = makeEntry(root)
|
||||
mkdirSync(entry.cacheRoot, { recursive: true })
|
||||
writeDist(entry.entryPath)
|
||||
writeFileSync(path.join(entry.entryPath, 'marker'), 'first-writer')
|
||||
const share = vi.fn()
|
||||
expect(publishSharedElectronDist(writeDist(path.join(root, 'dist')), entry, { share })).toBe(
|
||||
false
|
||||
)
|
||||
expect(share).not.toHaveBeenCalled()
|
||||
expect(existsSync(path.join(entry.entryPath, 'marker'))).toBe(true)
|
||||
})
|
||||
|
||||
it('loses a publish race without clobbering the winner or leaking staging', () => {
|
||||
const root = makeRoot()
|
||||
const entry = makeEntry(root)
|
||||
const share = (_source: string, destination: string) => {
|
||||
writeDist(destination)
|
||||
// The winner lands between our existence check and our rename.
|
||||
writeDist(entry.entryPath)
|
||||
writeFileSync(path.join(entry.entryPath, 'marker'), 'winner')
|
||||
}
|
||||
expect(publishSharedElectronDist(writeDist(path.join(root, 'dist')), entry, { share })).toBe(
|
||||
false
|
||||
)
|
||||
expect(existsSync(path.join(entry.entryPath, 'marker'))).toBe(true)
|
||||
expect(readdirSync(entry.cacheRoot)).toEqual([path.basename(entry.entryPath)])
|
||||
})
|
||||
|
||||
it('keeps a good entry a sibling published while this one was still sharing', () => {
|
||||
const root = makeRoot()
|
||||
const entry = makeEntry(root)
|
||||
mkdirSync(entry.cacheRoot, { recursive: true })
|
||||
writeDist(entry.entryPath, '40.0.0') // Unusable, so this worktree intends to replace it.
|
||||
const share = (_source: string, destination: string) => {
|
||||
writeDist(destination)
|
||||
// A sibling replaces the bad entry with a good one while this share is still running.
|
||||
rmSync(entry.entryPath, { recursive: true, force: true })
|
||||
writeDist(entry.entryPath)
|
||||
writeFileSync(path.join(entry.entryPath, 'marker'), 'sibling')
|
||||
}
|
||||
expect(
|
||||
publishSharedElectronDist(writeDist(path.join(root, 'dist')), entry, { share, ...identity })
|
||||
).toBe(false)
|
||||
expect(existsSync(path.join(entry.entryPath, 'marker'))).toBe(true)
|
||||
expect(readdirSync(entry.cacheRoot)).toEqual([path.basename(entry.entryPath)])
|
||||
})
|
||||
|
||||
it('restores the quarantined entry rather than leaving the cache empty', () => {
|
||||
const root = makeRoot()
|
||||
const entry = makeEntry(root)
|
||||
mkdirSync(entry.cacheRoot, { recursive: true })
|
||||
writeDist(entry.entryPath, '40.0.0')
|
||||
writeFileSync(path.join(entry.entryPath, 'marker'), 'stale')
|
||||
const share = (_source: string, destination: string) => writeDist(destination)
|
||||
// The swap itself fails; a bad entry still beats no entry, since the next publisher replaces it.
|
||||
const failingRename = () => {
|
||||
throw new Error('rename failed')
|
||||
}
|
||||
expect(
|
||||
publishSharedElectronDist(writeDist(path.join(root, 'dist')), entry, {
|
||||
share,
|
||||
rename: failingRename,
|
||||
...identity
|
||||
})
|
||||
).toBe(false)
|
||||
expect(existsSync(path.join(entry.entryPath, 'marker'))).toBe(true)
|
||||
expect(readdirSync(entry.cacheRoot)).toEqual([path.basename(entry.entryPath)])
|
||||
})
|
||||
|
||||
it('replaces an entry that fails validation instead of stranding every worktree', () => {
|
||||
const root = makeRoot()
|
||||
const entry = makeEntry(root)
|
||||
mkdirSync(entry.cacheRoot, { recursive: true })
|
||||
writeDist(entry.entryPath, '40.0.0')
|
||||
const share = (_source: string, destination: string) => writeDist(destination)
|
||||
expect(
|
||||
publishSharedElectronDist(writeDist(path.join(root, 'dist')), entry, { share, ...identity })
|
||||
).toBe(true)
|
||||
expect(isUsableElectronDist(entry.entryPath, VERSION, PLATFORM_PATH)).toBe(true)
|
||||
expect(readdirSync(entry.cacheRoot)).toEqual([path.basename(entry.entryPath)])
|
||||
})
|
||||
|
||||
it('never discards an entry it was given no identity to check', () => {
|
||||
const root = makeRoot()
|
||||
const entry = makeEntry(root)
|
||||
mkdirSync(entry.cacheRoot, { recursive: true })
|
||||
writeDist(entry.entryPath, '40.0.0')
|
||||
const share = vi.fn()
|
||||
expect(publishSharedElectronDist(writeDist(path.join(root, 'dist')), entry, { share })).toBe(
|
||||
false
|
||||
)
|
||||
expect(share).not.toHaveBeenCalled()
|
||||
expect(existsSync(path.join(entry.entryPath, 'version'))).toBe(true)
|
||||
})
|
||||
|
||||
it('leaves no entry and no staging tree when sharing fails', () => {
|
||||
const root = makeRoot()
|
||||
const entry = makeEntry(root)
|
||||
const share = (_source: string, destination: string) => {
|
||||
writeDist(destination)
|
||||
throw new Error('no shareable storage')
|
||||
}
|
||||
expect(publishSharedElectronDist(writeDist(path.join(root, 'dist')), entry, { share })).toBe(
|
||||
false
|
||||
)
|
||||
expect(existsSync(entry.entryPath)).toBe(false)
|
||||
expect(readdirSync(entry.cacheRoot)).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('shareElectronDistFromCache', () => {
|
||||
it('shares a validated entry with real filesystem semantics', () => {
|
||||
const root = makeRoot()
|
||||
const entry = makeEntry(root)
|
||||
mkdirSync(entry.cacheRoot, { recursive: true })
|
||||
writeDist(entry.entryPath)
|
||||
const stagePath = path.join(root, 'stage')
|
||||
expect(
|
||||
shareElectronDistFromCache(entry, stagePath, {
|
||||
version: VERSION,
|
||||
platformPath: PLATFORM_PATH
|
||||
})
|
||||
).toBe(true)
|
||||
expect(isUsableElectronDist(stagePath, VERSION, PLATFORM_PATH)).toBe(true)
|
||||
})
|
||||
|
||||
it('refuses an entry that fails validation instead of installing it', () => {
|
||||
const root = makeRoot()
|
||||
const entry = makeEntry(root)
|
||||
mkdirSync(entry.cacheRoot, { recursive: true })
|
||||
writeDist(entry.entryPath, '40.0.0')
|
||||
const stagePath = path.join(root, 'stage')
|
||||
expect(
|
||||
shareElectronDistFromCache(entry, stagePath, {
|
||||
version: VERSION,
|
||||
platformPath: PLATFORM_PATH
|
||||
})
|
||||
).toBe(false)
|
||||
expect(existsSync(stagePath)).toBe(false)
|
||||
})
|
||||
|
||||
it('reports failure rather than falling back to a full copy', () => {
|
||||
const root = makeRoot()
|
||||
const entry = makeEntry(root)
|
||||
mkdirSync(entry.cacheRoot, { recursive: true })
|
||||
writeDist(entry.entryPath)
|
||||
// Injected rather than provoked: what counts as an unshareable destination differs per
|
||||
// mechanism, and a byte-copy fallback here would defeat the point of the cache.
|
||||
const stagePath = path.join(root, 'stage')
|
||||
const share = () => {
|
||||
throw new Error('no shareable storage')
|
||||
}
|
||||
expect(
|
||||
shareElectronDistFromCache(entry, stagePath, {
|
||||
version: VERSION,
|
||||
platformPath: PLATFORM_PATH,
|
||||
share
|
||||
})
|
||||
).toBe(false)
|
||||
expect(existsSync(stagePath)).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('shared dist marker', () => {
|
||||
it('reports adoption only for the entry the marker names', () => {
|
||||
const root = makeRoot()
|
||||
const entry = makeEntry(root)
|
||||
expect(hasAdoptedSharedElectronDist(entry)).toBe(false)
|
||||
recordAdoptedSharedElectronDist(entry, writeFileSync)
|
||||
expect(hasAdoptedSharedElectronDist(entry)).toBe(true)
|
||||
expect(
|
||||
hasAdoptedSharedElectronDist({
|
||||
...entry,
|
||||
entryPath: path.join(entry.cacheRoot, '44.0.0-darwin-arm64')
|
||||
})
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('swallows a marker write failure, which only costs one extra share', () => {
|
||||
const entry = makeEntry(makeRoot())
|
||||
expect(() =>
|
||||
recordAdoptedSharedElectronDist(entry, () => {
|
||||
throw new Error('read-only node_modules')
|
||||
})
|
||||
).not.toThrow()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,143 @@
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import {
|
||||
chmodSync,
|
||||
cpSync,
|
||||
linkSync,
|
||||
mkdirSync,
|
||||
readdirSync,
|
||||
readlinkSync,
|
||||
rmSync,
|
||||
symlinkSync
|
||||
} from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
|
||||
// -c asks for clonefile(2). -P keeps Electron.framework's relative symlinks as symlinks; resolving
|
||||
// them breaks Chromium's bundle lookup.
|
||||
export const MACOS_CLONE_ARGS = Object.freeze(['-c', '-R', '-P'])
|
||||
// -a implies -d (no symlink following) and preserves mode. --reflink=always fails loudly on a
|
||||
// filesystem without reflinks rather than silently writing a second full copy.
|
||||
export const LINUX_REFLINK_ARGS = Object.freeze(['--reflink=always', '-a'])
|
||||
|
||||
/**
|
||||
* Copy a directory tree so the destination costs no new storage.
|
||||
*
|
||||
* Three mechanisms, strongest isolation first. Clone and reflink are copy-on-write, so the
|
||||
* destination is genuinely private. Hardlinks are not: the two trees share inodes, and a write
|
||||
* through either mutates both. That is only sound for a tree nothing writes to, which is why
|
||||
* `makeTreeReadOnly` exists and why the caller must apply it.
|
||||
*
|
||||
* Throws when no mechanism is available, so a caller can fall back to installing normally rather
|
||||
* than silently paying for a second full copy.
|
||||
*/
|
||||
export function shareTree(sourcePath, destinationPath, options = {}) {
|
||||
const platform = options.platform ?? process.platform
|
||||
const errors = []
|
||||
for (const mechanism of getShareMechanisms(platform)) {
|
||||
try {
|
||||
;(options[mechanism] ?? shareMechanisms[mechanism])(sourcePath, destinationPath)
|
||||
return mechanism
|
||||
} catch (error) {
|
||||
errors.push(error)
|
||||
// A mechanism can fail part-way through a tree; the next one needs a clean destination.
|
||||
rmSync(destinationPath, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
throw new AggregateError(errors, `Could not share storage for ${destinationPath}`)
|
||||
}
|
||||
|
||||
function getShareMechanisms(platform) {
|
||||
switch (platform) {
|
||||
case 'darwin':
|
||||
// APFS only. HFS+ has no clonefile, and hardlinking a 585-entry bundle buys little.
|
||||
return ['clone']
|
||||
case 'linux':
|
||||
// reflink covers btrfs/XFS/bcachefs/ZFS; ext4 has none, which is most developers.
|
||||
return ['reflink', 'hardlink']
|
||||
case 'win32':
|
||||
// Block cloning is ReFS-only, so NTFS gets hardlinks or nothing.
|
||||
return ['hardlink']
|
||||
default:
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
const shareMechanisms = {
|
||||
clone: (sourcePath, destinationPath) =>
|
||||
execFileSync('/bin/cp', [...MACOS_CLONE_ARGS, sourcePath, destinationPath], {
|
||||
stdio: 'ignore'
|
||||
}),
|
||||
reflink: (sourcePath, destinationPath) =>
|
||||
execFileSync('cp', [...LINUX_REFLINK_ARGS, sourcePath, destinationPath], { stdio: 'ignore' }),
|
||||
hardlink: hardlinkTree
|
||||
}
|
||||
|
||||
export function hardlinkTree(sourcePath, destinationPath) {
|
||||
mkdirSync(destinationPath, { recursive: true })
|
||||
for (const entry of readdirSync(sourcePath, { withFileTypes: true })) {
|
||||
const from = join(sourcePath, entry.name)
|
||||
const to = join(destinationPath, entry.name)
|
||||
if (entry.isDirectory()) {
|
||||
hardlinkTree(from, to)
|
||||
} else if (entry.isSymbolicLink()) {
|
||||
symlinkSync(readlinkSync(from), to)
|
||||
} else {
|
||||
linkSync(from, to)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop write permission across a tree.
|
||||
*
|
||||
* This is what makes hardlink sharing safe: Electron's own install.js extracts over an existing
|
||||
* dist with O_TRUNC, which through a hardlink would rewrite every sibling worktree and the cache at
|
||||
* once. Read-only turns that into EPERM. Directories stay writable because unlink needs a writable
|
||||
* parent, not a writable file, so the install transaction's renames still work.
|
||||
*/
|
||||
export function makeTreeReadOnly(targetPath, chmod = chmodSync) {
|
||||
for (const entry of readdirSync(targetPath, { withFileTypes: true })) {
|
||||
const entryPath = join(targetPath, entry.name)
|
||||
if (entry.isDirectory()) {
|
||||
makeTreeReadOnly(entryPath, chmod)
|
||||
} else if (!entry.isSymbolicLink()) {
|
||||
chmod(entryPath, 0o555)
|
||||
}
|
||||
}
|
||||
chmod(targetPath, 0o755)
|
||||
}
|
||||
|
||||
/**
|
||||
* Share storage when possible, otherwise copy the bytes.
|
||||
*
|
||||
* Never hardlinks: this is for trees the caller goes on to patch, where shared inodes would write
|
||||
* through into the source.
|
||||
*/
|
||||
export function copyPrivateTree(sourcePath, destinationPath, options = {}) {
|
||||
const platform = options.platform ?? process.platform
|
||||
const copy = options.copy ?? copyTreeVerbatim
|
||||
const privateMechanisms = new Set(['clone', 'reflink'])
|
||||
if (getShareMechanisms(platform).some((mechanism) => privateMechanisms.has(mechanism))) {
|
||||
try {
|
||||
const mechanism = shareTree(sourcePath, destinationPath, {
|
||||
...options,
|
||||
hardlink: () => {
|
||||
throw new Error('hardlinks would not be private')
|
||||
}
|
||||
})
|
||||
return { mechanism, copyError: null }
|
||||
} catch (copyError) {
|
||||
copy(sourcePath, destinationPath)
|
||||
return { mechanism: null, copyError }
|
||||
}
|
||||
}
|
||||
copy(sourcePath, destinationPath)
|
||||
return { mechanism: null, copyError: null }
|
||||
}
|
||||
|
||||
function copyTreeVerbatim(sourcePath, destinationPath) {
|
||||
cpSync(sourcePath, destinationPath, {
|
||||
recursive: true,
|
||||
dereference: false,
|
||||
verbatimSymlinks: true
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
import {
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
readlinkSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
symlinkSync,
|
||||
writeFileSync
|
||||
} from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
LINUX_REFLINK_ARGS,
|
||||
MACOS_CLONE_ARGS,
|
||||
copyPrivateTree,
|
||||
hardlinkTree,
|
||||
makeTreeReadOnly,
|
||||
shareTree
|
||||
} from './space-sharing-copy.mjs'
|
||||
|
||||
const roots: string[] = []
|
||||
|
||||
afterEach(() => {
|
||||
while (roots.length > 0) {
|
||||
rmSync(roots.pop()!, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
function makeTree(): { root: string; source: string } {
|
||||
const root = mkdtempSync(path.join(tmpdir(), 'orca-share-'))
|
||||
roots.push(root)
|
||||
const source = path.join(root, 'source')
|
||||
mkdirSync(path.join(source, 'nested'), { recursive: true })
|
||||
writeFileSync(path.join(source, 'nested', 'file'), 'contents')
|
||||
symlinkSync(path.join('nested', 'file'), path.join(source, 'relative-link'))
|
||||
return { root, source }
|
||||
}
|
||||
|
||||
describe('shareTree', () => {
|
||||
// Mechanism selection is asserted with stubs, because the real mechanisms only exist on the host
|
||||
// that owns them: /bin/cp -c is macOS-only and `cp --reflink` is GNU-only.
|
||||
it('prefers the strongest isolation each platform offers', () => {
|
||||
const stub = () =>
|
||||
vi.fn((_source: string, target: string) => mkdirSync(target, { recursive: true }))
|
||||
const stubs = { clone: stub(), reflink: stub(), hardlink: stub() }
|
||||
const { root, source } = makeTree()
|
||||
expect(shareTree(source, path.join(root, 'a'), { platform: 'darwin', ...stubs })).toBe('clone')
|
||||
expect(shareTree(source, path.join(root, 'b'), { platform: 'linux', ...stubs })).toBe('reflink')
|
||||
expect(shareTree(source, path.join(root, 'c'), { platform: 'win32', ...stubs })).toBe(
|
||||
'hardlink'
|
||||
)
|
||||
})
|
||||
|
||||
it('keeps relative symlinks unresolved on whatever this host supports', () => {
|
||||
const { root, source } = makeTree()
|
||||
const destination = path.join(root, 'shared')
|
||||
expect(shareTree(source, destination)).toBeTruthy()
|
||||
expect(readFileSync(path.join(destination, 'nested', 'file'), 'utf8')).toBe('contents')
|
||||
expect(readlinkSync(path.join(destination, 'relative-link'))).toBe(path.join('nested', 'file'))
|
||||
})
|
||||
|
||||
it('falls from reflink to hardlink on Linux, where ext4 has no reflinks', () => {
|
||||
const { root, source } = makeTree()
|
||||
const destination = path.join(root, 'shared')
|
||||
const reflink = vi.fn(() => {
|
||||
throw new Error('failed to clone: Invalid cross-device link')
|
||||
})
|
||||
expect(shareTree(source, destination, { platform: 'linux', reflink })).toBe('hardlink')
|
||||
expect(reflink).toHaveBeenCalledOnce()
|
||||
expect(statSync(path.join(destination, 'nested', 'file')).ino).toBe(
|
||||
statSync(path.join(source, 'nested', 'file')).ino
|
||||
)
|
||||
})
|
||||
|
||||
it('hardlinks on Windows, the only mechanism NTFS offers', () => {
|
||||
const { root, source } = makeTree()
|
||||
expect(shareTree(source, path.join(root, 'shared'), { platform: 'win32' })).toBe('hardlink')
|
||||
})
|
||||
|
||||
it('clears a part-way tree before trying the next mechanism', () => {
|
||||
const { root, source } = makeTree()
|
||||
const destination = path.join(root, 'shared')
|
||||
const reflink = (_source: string, target: string) => {
|
||||
mkdirSync(target, { recursive: true })
|
||||
writeFileSync(path.join(target, 'half-written'), 'partial')
|
||||
throw new Error('reflink failed midway')
|
||||
}
|
||||
expect(shareTree(source, destination, { platform: 'linux', reflink })).toBe('hardlink')
|
||||
expect(existsSync(path.join(destination, 'half-written'))).toBe(false)
|
||||
})
|
||||
|
||||
it('throws rather than silently paying for a second full copy', () => {
|
||||
const { root, source } = makeTree()
|
||||
expect(() => shareTree(source, path.join(root, 'shared'), { platform: 'freebsd' })).toThrow(
|
||||
/Could not share storage/
|
||||
)
|
||||
})
|
||||
|
||||
it('fails loudly instead of degrading, on both copy-out mechanisms', () => {
|
||||
expect(MACOS_CLONE_ARGS).toContain('-P')
|
||||
expect(LINUX_REFLINK_ARGS).toContain('--reflink=always')
|
||||
})
|
||||
})
|
||||
|
||||
describe('hardlinkTree', () => {
|
||||
it('shares inodes for files but recreates symlinks as their own entries', () => {
|
||||
const { root, source } = makeTree()
|
||||
const destination = path.join(root, 'linked')
|
||||
hardlinkTree(source, destination)
|
||||
expect(statSync(path.join(destination, 'nested', 'file')).ino).toBe(
|
||||
statSync(path.join(source, 'nested', 'file')).ino
|
||||
)
|
||||
expect(readlinkSync(path.join(destination, 'relative-link'))).toBe(path.join('nested', 'file'))
|
||||
})
|
||||
|
||||
it('propagates a write through the shared inode, which is why callers must protect it', () => {
|
||||
const { root, source } = makeTree()
|
||||
const destination = path.join(root, 'linked')
|
||||
hardlinkTree(source, destination)
|
||||
writeFileSync(path.join(destination, 'nested', 'file'), 'mutated')
|
||||
expect(readFileSync(path.join(source, 'nested', 'file'), 'utf8')).toBe('mutated')
|
||||
})
|
||||
})
|
||||
|
||||
describe('makeTreeReadOnly', () => {
|
||||
it('drops write permission on files while leaving directories traversable and unlinkable', () => {
|
||||
const { source } = makeTree()
|
||||
makeTreeReadOnly(source)
|
||||
expect(statSync(path.join(source, 'nested', 'file')).mode & 0o222).toBe(0)
|
||||
// Asserted as behavior, not mode bits: Windows maps chmod onto the read-only attribute alone,
|
||||
// so a directory there never reports 0o755. What has to hold everywhere is that the install
|
||||
// transaction can still rename dist aside and remove it.
|
||||
expect(() => rmSync(path.join(source, 'nested'), { recursive: true })).not.toThrow()
|
||||
})
|
||||
|
||||
it('turns an extract-over-dist write into an error instead of silent shared corruption', () => {
|
||||
const { root, source } = makeTree()
|
||||
const destination = path.join(root, 'linked')
|
||||
hardlinkTree(source, destination)
|
||||
makeTreeReadOnly(destination)
|
||||
expect(() => writeFileSync(path.join(destination, 'nested', 'file'), 'mutated')).toThrow()
|
||||
expect(readFileSync(path.join(source, 'nested', 'file'), 'utf8')).toBe('contents')
|
||||
})
|
||||
|
||||
it.runIf(process.platform !== 'win32')(
|
||||
'keeps the executable bit, which Electron needs to launch',
|
||||
() => {
|
||||
const { source } = makeTree()
|
||||
const executable = path.join(source, 'electron')
|
||||
writeFileSync(executable, 'binary', { mode: 0o755 })
|
||||
makeTreeReadOnly(source)
|
||||
// Verified on real ext4: 0o555. Windows has no execute bit -- the read-only attribute does
|
||||
// not gate execution there, confirmed by running a read-only hardlinked .exe on NTFS.
|
||||
expect(statSync(executable).mode & 0o111).toBe(0o111)
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
describe('copyPrivateTree', () => {
|
||||
it('never hardlinks, because the caller patches what it gets back', () => {
|
||||
const { root, source } = makeTree()
|
||||
const destination = path.join(root, 'private')
|
||||
const hardlink = vi.fn()
|
||||
const result = copyPrivateTree(source, destination, { platform: 'linux', hardlink })
|
||||
expect(hardlink).not.toHaveBeenCalled()
|
||||
expect(statSync(path.join(destination, 'nested', 'file')).ino).not.toBe(
|
||||
statSync(path.join(source, 'nested', 'file')).ino
|
||||
)
|
||||
expect(result.mechanism === 'reflink' || result.mechanism === null).toBe(true)
|
||||
})
|
||||
|
||||
it('copies bytes on a platform with no private mechanism at all', () => {
|
||||
const { root, source } = makeTree()
|
||||
const destination = path.join(root, 'private')
|
||||
const hardlink = vi.fn()
|
||||
expect(copyPrivateTree(source, destination, { platform: 'win32', hardlink })).toEqual({
|
||||
mechanism: null,
|
||||
copyError: null
|
||||
})
|
||||
expect(hardlink).not.toHaveBeenCalled()
|
||||
expect(readFileSync(path.join(destination, 'nested', 'file'), 'utf8')).toBe('contents')
|
||||
expect(readlinkSync(path.join(destination, 'relative-link'))).toBe(path.join('nested', 'file'))
|
||||
})
|
||||
|
||||
it('reports the private mechanism it used', () => {
|
||||
const { root, source } = makeTree()
|
||||
const clone = vi.fn((_source: string, target: string) => mkdirSync(target, { recursive: true }))
|
||||
expect(
|
||||
copyPrivateTree(source, path.join(root, 'private'), { platform: 'darwin', clone })
|
||||
).toEqual({
|
||||
mechanism: 'clone',
|
||||
copyError: null
|
||||
})
|
||||
})
|
||||
|
||||
it('falls back to a byte copy when the private mechanism fails', () => {
|
||||
const { root, source } = makeTree()
|
||||
const destination = path.join(root, 'private')
|
||||
const clone = () => {
|
||||
throw new Error('clonefile unsupported')
|
||||
}
|
||||
const result = copyPrivateTree(source, destination, { platform: 'darwin', clone })
|
||||
expect(result.mechanism).toBeNull()
|
||||
expect(result.copyError).toBeInstanceOf(Error)
|
||||
expect(readFileSync(path.join(destination, 'nested', 'file'), 'utf8')).toBe('contents')
|
||||
})
|
||||
})
|
||||
@@ -86,6 +86,7 @@
|
||||
"build:release:parallel": "pnpm run build:relay && pnpm run build:native && pnpm run verify:computer-native && pnpm run build:cli && pnpm run build:electron-vite:parallel && pnpm run verify:built-skills-cli && pnpm run build:web-from-renderer",
|
||||
"postinstall": "node config/scripts/rebuild-native-deps.mjs",
|
||||
"rebuild:electron": "node config/scripts/rebuild-native-deps.mjs",
|
||||
"reclaim:electron-dists": "node config/scripts/reclaim-electron-dists.mjs",
|
||||
"rebuild:node": "pnpm rebuild node-pty",
|
||||
"build:unpack": "pnpm run build && pnpm run ensure:electron-runtime && electron-builder --config config/electron-builder.config.cjs --dir",
|
||||
"build:win": "pnpm run build:desktop && pnpm run ensure:electron-runtime && electron-builder --config config/electron-builder.config.cjs --win",
|
||||
|
||||
Reference in New Issue
Block a user