diff --git a/config/packaged-native-build-entries.cjs b/config/packaged-native-build-entries.cjs new file mode 100644 index 00000000000..f0c8301e35c --- /dev/null +++ b/config/packaged-native-build-entries.cjs @@ -0,0 +1,40 @@ +const { existsSync, readdirSync, renameSync, rmSync } = require('node:fs') +const { join } = require('node:path') +const { + NATIVE_BUILD_PACKAGES, + isNativeBuildEntryName, + nativeBuildEntryName +} = require('./scripts/native-build-entry.cjs') + +/** + * Ship a dev install's per-ABI native build as build/Release. + * + * A package dir shared across worktrees (pnpm's global virtual store) holds Orca's source build + * in build/ rather than build/Release; see scripts/native-build-entry.cjs. Packaged + * consumers (daemon relocation, the macOS terminal host, the job-ownership gate) all read + * build/Release, so the packaged copy gets exactly the layout a CI install produces. + */ +function adoptPackagedNativeBuildEntries(resourcesDir, electronPlatformName, architecture) { + for (const moduleName of Object.keys(NATIVE_BUILD_PACKAGES)) { + const buildDir = join(resourcesDir, 'node_modules', ...moduleName.split('/'), 'build') + if (!existsSync(buildDir)) { + continue + } + const entryDir = join( + buildDir, + nativeBuildEntryName(moduleName, { platform: electronPlatformName, arch: architecture }) + ) + if (existsSync(entryDir)) { + rmSync(join(buildDir, 'Release'), { recursive: true, force: true }) + renameSync(entryDir, join(buildDir, 'Release')) + } + // Other arches, plus any staging or quarantine leftovers, are dead weight in this slice. + for (const name of readdirSync(buildDir)) { + if (isNativeBuildEntryName(name)) { + rmSync(join(buildDir, name), { recursive: true, force: true }) + } + } + } +} + +module.exports = { adoptPackagedNativeBuildEntries } diff --git a/config/packaged-runtime-node-modules.cjs b/config/packaged-runtime-node-modules.cjs index 63556ad3b6d..62ed2ff47f5 100644 --- a/config/packaged-runtime-node-modules.cjs +++ b/config/packaged-runtime-node-modules.cjs @@ -10,6 +10,11 @@ const { const { dirname, join, resolve } = require('node:path') const { builtinModules, createRequire } = require('node:module') const { PE_MACHINE, readPeMachine } = require('./scripts/windows-pe-machine.cjs') +const { adoptPackagedNativeBuildEntries } = require('./packaged-native-build-entries.cjs') +const { + NATIVE_BUILD_PACKAGES, + TRANSIENT_NATIVE_BUILD_ENTRY_FILTER +} = require('./scripts/native-build-entry.cjs') const projectDir = resolve(__dirname, '..') const requireFromProject = createRequire(join(projectDir, 'package.json')) @@ -223,7 +228,9 @@ function createPackagedRuntimeNodeModuleResources(electronPlatformName = process return collectPackagedRuntimePackages(electronPlatformName).map(([packageName, packageDir]) => ({ from: packageDir, to: join('node_modules', ...packageName.split('/')), - ...(packageName === '@orca/process-host' ? { filter: ['package.json', 'dist/**/*'] } : {}) + ...(packageName === '@orca/process-host' ? { filter: ['package.json', 'dist/**/*'] } : {}), + // Why: another worktree can rename these away mid-copy, failing the walk with ENOENT. + ...(NATIVE_BUILD_PACKAGES[packageName] ? { filter: TRANSIENT_NATIVE_BUILD_ENTRY_FILTER } : {}) })) } @@ -658,6 +665,8 @@ function assertPackagedNativeVariantsInstalled(electronPlatformName, electronArc function prunePackagedRuntimeNodeModules(resourcesDir, electronPlatformName, electronArch) { const architecture = normalizeElectronArchitecture(electronArch) + // Before the node-pty prune, which judges build/Release. + adoptPackagedNativeBuildEntries(resourcesDir, electronPlatformName, architecture) prunePackagedNodePty(resourcesDir, electronPlatformName, architecture) prunePackagedParcelWatcher(resourcesDir, electronPlatformName, architecture) // Why before the filename walk: zod/src is deleted wholesale, so walking it first is wasted work. diff --git a/config/patches/@vscode__windows-process-tree@0.8.0.patch b/config/patches/@vscode__windows-process-tree@0.8.0.patch index bc50a8cfa63..ac0defcee9f 100644 --- a/config/patches/@vscode__windows-process-tree@0.8.0.patch +++ b/config/patches/@vscode__windows-process-tree@0.8.0.patch @@ -32,13 +32,17 @@ index 11a5e71..1b2e74b 100644 "/guard:cf", "/sdl", diff --git a/lib/index.js b/lib/index.js -index e586cd6..0ee62c3 100644 +index 9747a7402600cd252859144d32580ed45c8c93f7..69cf7fe6d14e52b199ac9ab869567be5d9bb708c 100644 --- a/lib/index.js +++ b/lib/index.js -@@ -7,11 +7,14 @@ +@@ -6,12 +6,17 @@ + Object.defineProperty(exports, "__esModule", { value: true }); exports.getAllProcesses = exports.getProcessTree = exports.getProcessCpuUsage = exports.getProcessList = exports.filterProcessList = exports.buildProcessTree = exports.ProcessDataFlag = void 0; const util_1 = require("util"); - const native = process.platform === 'win32' ? require('../build/Release/windows_process_tree.node') : undefined; +-const native = process.platform === 'win32' ? require('../build/Release/windows_process_tree.node') : undefined; ++// Orca's per-ABI build, published into package dirs shared across worktrees, wins over build/Release. ++const orcaNativeBuild = `${__dirname}/../build/orca-napi-win32-${process.arch}/windows_process_tree.node`; ++const native = process.platform === 'win32' ? require(require('fs').existsSync(orcaNativeBuild) ? orcaNativeBuild : '../build/Release/windows_process_tree.node') : undefined; +exports.getProcessCreationTime = native === undefined ? undefined : native.getProcessCreationTime; +exports.supportedProcessDataFlags = native === undefined ? undefined : native.supportedProcessDataFlags; var ProcessDataFlag; @@ -50,7 +54,7 @@ index e586cd6..0ee62c3 100644 })(ProcessDataFlag = exports.ProcessDataFlag || (exports.ProcessDataFlag = {})); // requestInProgress is used for any function that uses CreateToolhelp32Snapshot, as multiple calls // to this cannot be done at the same time. -@@ -66,11 +69,12 @@ +@@ -66,11 +71,12 @@ function buildProcessTree(rootPid, processList, maxDepth = MAX_FILTER_DEPTH) { // • the properties are inlined/splatted // • the 'ppid' field is omitted // • the depth of the tree is limited by `maxDepth` @@ -65,13 +69,17 @@ index e586cd6..0ee62c3 100644 }); return buildNode(root, maxDepth); diff --git a/lib/index.ts b/lib/index.ts -index 7b53aad..284dae1 100644 +index 7b53aad05c3682a5c7d814d81a39c3f391ae97e3..364b384e7d55f3682515431839352b5b4a9ffa93 100644 --- a/lib/index.ts +++ b/lib/index.ts -@@ -6,12 +6,16 @@ +@@ -5,13 +5,19 @@ + import { promisify } from 'util'; - const native = process.platform === 'win32' ? require('../build/Release/windows_process_tree.node') : undefined; +-const native = process.platform === 'win32' ? require('../build/Release/windows_process_tree.node') : undefined; ++// Orca's per-ABI build, published into package dirs shared across worktrees, wins over build/Release. ++const orcaNativeBuild = `${__dirname}/../build/orca-napi-win32-${process.arch}/windows_process_tree.node`; ++const native = process.platform === 'win32' ? require(require('fs').existsSync(orcaNativeBuild) ? orcaNativeBuild : '../build/Release/windows_process_tree.node') : undefined; +export const getProcessCreationTime: ((pid: number) => number | undefined) | undefined = native?.getProcessCreationTime; +/** The flag bits this compiled addon reports; undefined off win32. */ +export const supportedProcessDataFlags: number | undefined = native?.supportedProcessDataFlags; @@ -86,7 +94,7 @@ index 7b53aad..284dae1 100644 } type RequestCallback = (processList: IProcessInfo[]) => void; -@@ -81,11 +85,12 @@ +@@ -81,11 +87,12 @@ export function buildProcessTree(rootPid: number, processList: Iterable !existsSync(resolve(runtimeDir, filename)) ) @@ -333,14 +340,6 @@ function assertNodePtyWindowsConptyRuntime(nativeDir) { } } -function getNodePtyNativeModuleName() { - if (process.platform !== 'win32') { - return 'pty' - } - - return getWindowsBuildNumber() >= 18309 ? 'conpty' : 'pty' -} - function getPatchedNodePtyRebuildReason() { if (!requiresPatchedNodePtySourceBuild()) { return null @@ -349,7 +348,7 @@ function getPatchedNodePtyRebuildReason() { // Why: a loadable upstream node-pty prebuild is not enough; Orca's Unix and // Windows patches only land in the source-built build/Release artifacts. const nodePtyDir = resolve(projectDir, 'node_modules', 'node-pty') - const artifactPaths = patchedNodePtyArtifactPaths(nodePtyDir) + const artifactPaths = nodePtyBuildArtifactPaths(nodePtyDir) const missingArtifact = artifactPaths.find((artifactPath) => !existsSync(artifactPath)) if (!missingArtifact) { @@ -359,23 +358,6 @@ function getPatchedNodePtyRebuildReason() { return 'Patched node-pty build artifacts are missing; rebuilding native deps.' } -function patchedNodePtyArtifactPaths(nodePtyDir) { - if (process.platform === 'win32') { - const releaseDir = resolve(nodePtyDir, 'build', 'Release') - return [ - resolve(releaseDir, 'conpty.node'), - ...NODE_PTY_CONPTY_RUNTIME_FILES.map((filename) => resolve(releaseDir, 'conpty', filename)) - ] - } - - const artifactPaths = [resolve(nodePtyDir, 'build', 'Release', 'pty.node')] - // Why: node-pty only builds spawn-helper on macOS; Linux builds only pty.node. - if (process.platform === 'darwin') { - artifactPaths.push(resolve(nodePtyDir, 'build', 'Release', 'spawn-helper')) - } - return artifactPaths -} - function requiresPatchedNodePtySourceBuild() { const nodePtyPatchPath = resolve(projectDir, 'config', 'patches', 'node-pty@1.1.0.patch') if (!existsSync(nodePtyPatchPath)) { @@ -385,18 +367,14 @@ function requiresPatchedNodePtySourceBuild() { return existsSync(resolve(projectDir, 'node_modules', 'node-pty')) } -function isNodePtyReleaseBuildDir(nativeDir) { - return typeof nativeDir === 'string' && nativeDir.replace(/\\/g, '/').includes('build/Release/') -} - -function getWindowsBuildNumber() { - const match = /(\d+)\.(\d+)\.(\d+)/g.exec(release()) - return match && match.length === 4 ? Number.parseInt(match[3], 10) : 0 -} - function rebuildNodeRuntimeModules(moduleNames) { for (const moduleName of moduleNames) { let moduleDir = dirname(require.resolve(`${moduleName}/package.json`)) + if (isSharedNativePackage(moduleName, projectDir)) { + // Why: other worktrees load this very dir, so build beside it instead of over it. + buildSharedNodeRuntimeModule(moduleName, resolveNativePackageDir(moduleName, projectDir)) + continue + } if (moduleName === '@vscode/windows-process-tree') { // Why before node-gyp: this module is rebuilt precisely because the // binary was the unpatched one, and pnpm materializes it unpatched often @@ -425,11 +403,23 @@ function rebuildNodeRuntimeModules(moduleNames) { } } +function buildSharedNodeRuntimeModule(moduleName, packageDir) { + try { + ensureSharedNativeBuild({ + moduleName, + packageDir, + checkBuildDir: (dir) => checkNativeBuildDirLoads(moduleName, dir), + nodeGypEntry: process.env.npm_config_node_gyp || undefined, + log: (message) => console.warn(`[native-runtime] ${message}`) + }) + } catch (error) { + console.error(`[native-runtime] ${moduleName}: ${formatError(error)}`) + process.exit(1) + } +} + function runNodeGyp(moduleName, { args, cwd }) { - const env = - process.platform === 'linux' - ? { ...process.env, CXXFLAGS: `${process.env.CXXFLAGS ?? ''} -std=gnu++2a`.trim() } - : disableMsbuildFileTrackingOnWindows({ ...process.env }) + const env = nativeBuildEnvironment() const result = runProcessSync({ program: process.execPath, args, diff --git a/config/scripts/native-build-entry-probe.cjs b/config/scripts/native-build-entry-probe.cjs new file mode 100644 index 00000000000..f628f8b7a26 --- /dev/null +++ b/config/scripts/native-build-entry-probe.cjs @@ -0,0 +1,53 @@ +'use strict' + +// Loads one native build dir in a fresh Node or Electron process: ` this.cjs `. +// Why a child: a failed addon load can poison the process that attempted it. + +const { accessSync, constants, existsSync } = require('node:fs') +const { join } = require('node:path') +const { assertNodePtyJobOwnership } = require('./node-pty-job-ownership.cjs') +const { assertWindowsProcessTreeCreationTime } = require('./windows-process-tree-creation-time.cjs') +const { nodePtyNativeModuleName } = require('./native-build-entry.cjs') + +const NODE_PTY_CONPTY_RUNTIME_FILES = ['conpty.dll', 'OpenConsole.exe'] + +function probeNativeBuildDir(moduleName, dir) { + if (moduleName === 'node-pty') { + const nativeName = nodePtyNativeModuleName() + const addonPath = join(dir, `${nativeName}.node`) + const native = { dir, module: require(addonPath) } + assertNodePtyJobOwnership({ nativeName, native, addonPath }) + if (process.platform === 'darwin') { + accessSync(join(dir, 'spawn-helper'), constants.X_OK) + } + if (nativeName === 'conpty') { + for (const filename of NODE_PTY_CONPTY_RUNTIME_FILES) { + if (!existsSync(join(dir, 'conpty', filename))) { + throw new Error( + `node-pty ConPTY runtime file is missing: ${join(dir, 'conpty', filename)}` + ) + } + } + } + return + } + if (moduleName === '@vscode/windows-process-tree') { + assertWindowsProcessTreeCreationTime({ + module: require(join(dir, 'windows_process_tree.node')) + }) + return + } + throw new Error(`no probe for ${moduleName}`) +} + +if (require.main === module) { + const [moduleName, dir] = process.argv.slice(2) + try { + probeNativeBuildDir(moduleName, dir) + } catch (error) { + console.error(`${moduleName}: ${error instanceof Error ? error.message : String(error)}`) + process.exit(1) + } +} + +module.exports = { probeNativeBuildDir } diff --git a/config/scripts/native-build-entry.cjs b/config/scripts/native-build-entry.cjs new file mode 100644 index 00000000000..c2e1a11b9bb --- /dev/null +++ b/config/scripts/native-build-entry.cjs @@ -0,0 +1,141 @@ +'use strict' + +const { existsSync, realpathSync } = require('node:fs') +const { release } = require('node:os') +const { join, relative, isAbsolute } = require('node:path') + +/** + * Per-ABI native build entries inside a package dir that other worktrees share. + * + * pnpm's global virtual store gives every worktree the same physical package dir, so an + * in-place `build/Release` rebuild in one worktree replaces the binary another is running. + * A shared package instead gets one immutable `build/` per ABI, published atomically + * and never rebuilt. The patched loaders (config/patches/node-pty@1.1.0.patch and + * @vscode__windows-process-tree@0.8.0.patch) try that dir before build/Release, and must + * compute the same name -- `native-build-entry.test.mjs` holds them to it. + */ + +const ENTRY_PREFIX = 'orca-' + +/** How each shared native package is built and which runtime its binary is tied to. */ +const NATIVE_BUILD_PACKAGES = { + // Both are node-addon-api (N-API): one binary loads in Node and Electron alike. + 'node-pty': { abi: 'napi' }, + '@vscode/windows-process-tree': { abi: 'napi' } +} + +/** + * The entry dir name for `moduleName` on this runtime. N-API binaries are keyed by + * platform-arch only; anything else is tied to NODE_MODULE_VERSION, which already differs + * between Node and every Electron major. + */ +function nativeBuildEntryName( + moduleName, + { platform = process.platform, arch = process.arch, modules = process.versions.modules } = {} +) { + const descriptor = NATIVE_BUILD_PACKAGES[moduleName] + if (!descriptor) { + throw new Error(`${moduleName} has no shared native build descriptor`) + } + const abi = descriptor.abi === 'napi' ? 'napi' : `abi${modules}` + return `${ENTRY_PREFIX}${abi}-${platform}-${arch}` +} + +function nativeBuildEntryDir(packageDir, moduleName, target) { + return join(packageDir, 'build', nativeBuildEntryName(moduleName, target)) +} + +/** The dir the patched loader reads: the published entry when there is one, else build/Release. */ +function loadedNativeBuildDir(packageDir, moduleName, target) { + const entryDir = nativeBuildEntryDir(packageDir, moduleName, target) + return existsSync(entryDir) ? entryDir : join(packageDir, 'build', 'Release') +} + +/** node-pty's own choice: ConPTY from Windows build 18309, winpty below it. */ +function nodePtyNativeModuleName(platform = process.platform, osRelease = release()) { + if (platform !== 'win32') { + return 'pty' + } + const match = /(\d+)\.(\d+)\.(\d+)/.exec(osRelease) + return match && Number.parseInt(match[3], 10) >= 18309 ? 'conpty' : 'pty' +} + +/** Every published, staging and quarantined entry name, for packaging to strip. */ +function isNativeBuildEntryName(name) { + return name.startsWith(ENTRY_PREFIX) +} + +/** electron-builder ignore globs for in-flight staging and quarantine trees (shared-tree-publish.mjs). */ +const TRANSIENT_NATIVE_BUILD_ENTRY_FILTER = [`!build/${ENTRY_PREFIX}*.{staging,unusable}-*{,/**/*}`] + +/** + * Whether `packageDir` physically lives outside this project, i.e. another worktree can load it. + * Why realpath: under the global virtual store node_modules/ is a link into the store. + */ +function isSharedPackageDir(packageDir, projectDir) { + try { + const rel = relative(realpathSync(projectDir), realpathSync(packageDir)) + return rel.startsWith('..') || isAbsolute(rel) + } catch { + return false + } +} + +/** Whether `moduleName` is installed in a package dir other worktrees share and can be built beside. */ +function isSharedNativePackage(moduleName, projectDir) { + if (!NATIVE_BUILD_PACKAGES[moduleName]) { + return false + } + try { + return isSharedPackageDir(resolveNativePackageDir(moduleName, projectDir), projectDir) + } catch { + return false // Not installed: an optional dependency off its platform. + } +} + +/** The project's own top-level link, the same path every Orca native script already reads. */ +function resolveNativePackageDir(moduleName, projectDir) { + const packageDir = join(projectDir, 'node_modules', ...moduleName.split('/')) + if (!existsSync(join(packageDir, 'package.json'))) { + throw new Error(`${moduleName} is not installed in ${projectDir}`) + } + return packageDir +} + +/** The files a node-pty source build must leave in whichever dir the loader reads. */ +function nodePtyBuildArtifactPaths(nodePtyDir, platform = process.platform) { + const buildDir = loadedNativeBuildDir(nodePtyDir, 'node-pty') + if (platform === 'win32') { + return ['conpty.node', 'conpty/conpty.dll', 'conpty/OpenConsole.exe'].map((file) => + join(buildDir, file) + ) + } + // Why: node-pty only builds spawn-helper on macOS; Linux builds only pty.node. + return ['pty.node', ...(platform === 'darwin' ? ['spawn-helper'] : [])].map((file) => + join(buildDir, file) + ) +} + +/** A `loadNativeModule` dir holding Orca's source build rather than an upstream prebuild. */ +function isSourceBuildDir(nativeDir) { + if (typeof nativeDir !== 'string') { + return false + } + const dir = nativeDir.replace(/\\/g, '/') + return dir.includes('build/Release/') || dir.includes(`build/${ENTRY_PREFIX}`) +} + +module.exports = { + NATIVE_BUILD_PACKAGES, + TRANSIENT_NATIVE_BUILD_ENTRY_FILTER, + isNativeBuildEntryName, + isSharedNativePackage, + isSharedPackageDir, + isSourceBuildDir, + loadedNativeBuildDir, + nativeBuildEntryDir, + nativeBuildEntryName, + nodePtyBuildArtifactPaths, + nodePtyNativeModuleName, + resolveNativePackageDir +} diff --git a/config/scripts/native-build-entry.test.mjs b/config/scripts/native-build-entry.test.mjs new file mode 100644 index 00000000000..2aea450cdfd --- /dev/null +++ b/config/scripts/native-build-entry.test.mjs @@ -0,0 +1,188 @@ +import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs' +import { createRequire } from 'node:module' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' + +const require = createRequire(import.meta.url) +const { + isNativeBuildEntryName, + isSharedPackageDir, + isSourceBuildDir, + loadedNativeBuildDir, + nativeBuildEntryName, + nodePtyNativeModuleName +} = require('./native-build-entry.cjs') +const { adoptPackagedNativeBuildEntries } = require('../packaged-native-build-entries.cjs') + +const PATCHES = path.join(import.meta.dirname, '..', 'patches') +const roots = [] + +afterEach(() => { + while (roots.length > 0) { + rmSync(roots.pop(), { recursive: true, force: true }) + } +}) + +function makeRoot() { + const root = mkdtempSync(path.join(tmpdir(), 'orca-native-entry-')) + roots.push(root) + return root +} + +function addedPatchLine(patchFile, pattern) { + const line = readFileSync(path.join(PATCHES, patchFile), 'utf8') + .split('\n') + .find((candidate) => candidate.startsWith('+') && pattern.test(candidate)) + expect(line, `${patchFile} has no added line matching ${pattern}`).toBeDefined() + return line +} + +describe('nativeBuildEntryName', () => { + it('keys an N-API addon by platform and arch only, so Node and Electron share it', () => { + const node = nativeBuildEntryName('node-pty', { + platform: 'darwin', + arch: 'arm64', + modules: '137' + }) + const electron = nativeBuildEntryName('node-pty', { + platform: 'darwin', + arch: 'arm64', + modules: '148' + }) + expect(node).toBe('orca-napi-darwin-arm64') + expect(electron).toBe(node) + expect( + nativeBuildEntryName('@vscode/windows-process-tree', { platform: 'win32', arch: 'x64' }) + ).toBe('orca-napi-win32-x64') + }) + + it('separates arches and refuses a package with no descriptor', () => { + expect(nativeBuildEntryName('node-pty', { platform: 'darwin', arch: 'x64' })).not.toBe( + nativeBuildEntryName('node-pty', { platform: 'darwin', arch: 'arm64' }) + ) + expect(() => nativeBuildEntryName('cpu-features')).toThrow(/no shared native build/) + }) + + // The patched loaders cannot require this module, so they spell the name out themselves. + it('matches the dir the patched node-pty loader tries first', () => { + const line = addedPatchLine('node-pty@1.1.0.patch', /var dirs = \[/) + const firstDir = /\[([^,]+),/.exec(line)[1] + const evaluate = new Function('process', `return ${firstDir}`) + for (const [platform, arch] of [ + ['darwin', 'arm64'], + ['linux', 'x64'], + ['win32', 'arm64'] + ]) { + expect(evaluate({ platform, arch })).toBe( + `build/${nativeBuildEntryName('node-pty', { platform, arch })}` + ) + } + }) + + it('matches the dir the patched windows-process-tree loader prefers', () => { + const line = addedPatchLine('@vscode__windows-process-tree@0.8.0.patch', /orcaNativeBuild = /) + const template = /= `(.*)`;/.exec(line)[1] + for (const arch of ['x64', 'arm64']) { + const resolved = template.replace('${__dirname}', '/pkg/lib').replace('${process.arch}', arch) + expect(path.posix.dirname(path.posix.normalize(resolved))).toBe( + `/pkg/build/${nativeBuildEntryName('@vscode/windows-process-tree', { platform: 'win32', arch })}` + ) + } + }) +}) + +describe('isSharedPackageDir', () => { + it('treats a package linked in from outside the project as shared', () => { + const root = makeRoot() + const store = path.join(root, 'store', 'node-pty') + const project = path.join(root, 'project') + mkdirSync(store, { recursive: true }) + mkdirSync(path.join(project, 'node_modules'), { recursive: true }) + symlinkSync(store, path.join(project, 'node_modules', 'node-pty'), 'junction') + expect(isSharedPackageDir(path.join(project, 'node_modules', 'node-pty'), project)).toBe(true) + }) + + it('treats the per-project virtual store as private', () => { + const root = makeRoot() + const real = path.join( + root, + 'node_modules', + '.pnpm', + 'node-pty@1.1.0', + 'node_modules', + 'node-pty' + ) + mkdirSync(real, { recursive: true }) + symlinkSync(real, path.join(root, 'node_modules', 'node-pty'), 'junction') + expect(isSharedPackageDir(path.join(root, 'node_modules', 'node-pty'), root)).toBe(false) + }) + + it('treats a missing package as private', () => { + const root = makeRoot() + expect(isSharedPackageDir(path.join(root, 'nope'), root)).toBe(false) + }) +}) + +describe('loadedNativeBuildDir', () => { + it('prefers a published entry and otherwise reads build/Release', () => { + const root = makeRoot() + const target = { platform: 'linux', arch: 'x64' } + expect(loadedNativeBuildDir(root, 'node-pty', target)).toBe(path.join(root, 'build', 'Release')) + mkdirSync(path.join(root, 'build', 'orca-napi-linux-x64'), { recursive: true }) + expect(loadedNativeBuildDir(root, 'node-pty', target)).toBe( + path.join(root, 'build', 'orca-napi-linux-x64') + ) + }) +}) + +describe('isSourceBuildDir', () => { + it('accepts build/Release and a published entry, never a prebuild', () => { + expect(isSourceBuildDir('../build/Release/')).toBe(true) + expect(isSourceBuildDir('..\\build\\orca-napi-win32-x64\\')).toBe(true) + expect(isSourceBuildDir('../prebuilds/darwin-arm64/')).toBe(false) + expect(isSourceBuildDir(undefined)).toBe(false) + }) +}) + +describe('nodePtyNativeModuleName', () => { + it('follows node-pty: ConPTY from Windows build 18309', () => { + expect(nodePtyNativeModuleName('darwin', '25.3.0')).toBe('pty') + expect(nodePtyNativeModuleName('win32', '10.0.26100')).toBe('conpty') + expect(nodePtyNativeModuleName('win32', '10.0.17763')).toBe('pty') + }) +}) + +describe('adoptPackagedNativeBuildEntries', () => { + function packagedNodePty(root, dirs) { + const buildDir = path.join(root, 'node_modules', 'node-pty', 'build') + for (const [dir, payload] of Object.entries(dirs)) { + mkdirSync(path.join(buildDir, dir), { recursive: true }) + writeFileSync(path.join(buildDir, dir, 'pty.node'), payload) + } + return buildDir + } + + it('ships the target entry as build/Release and drops every other entry', () => { + const root = makeRoot() + const buildDir = packagedNodePty(root, { + Release: 'pnpm-install-build', + 'orca-napi-darwin-arm64': 'entry-arm64', + 'orca-napi-darwin-x64': 'entry-x64', + 'orca-napi-darwin-arm64.staging-1-x': 'partial' + }) + adoptPackagedNativeBuildEntries(root, 'darwin', 'x64') + expect(readFileSync(path.join(buildDir, 'Release', 'pty.node'), 'utf8')).toBe('entry-x64') + expect( + ['Release', 'orca-napi-darwin-arm64'].map((name) => isNativeBuildEntryName(name)) + ).toEqual([false, true]) + expect(require('node:fs').readdirSync(buildDir)).toEqual(['Release']) + }) + + it('leaves a CI layout with no entries untouched', () => { + const root = makeRoot() + const buildDir = packagedNodePty(root, { Release: 'ci-build' }) + adoptPackagedNativeBuildEntries(root, 'linux', 'x64') + expect(readFileSync(path.join(buildDir, 'Release', 'pty.node'), 'utf8')).toBe('ci-build') + }) +}) diff --git a/config/scripts/pr-code-change-scope.mjs b/config/scripts/pr-code-change-scope.mjs index 2f772be3cae..0227b909bf3 100644 --- a/config/scripts/pr-code-change-scope.mjs +++ b/config/scripts/pr-code-change-scope.mjs @@ -281,6 +281,8 @@ const NATIVE_RUNTIME_PREFIXES = [ 'config/scripts/windows-process-tree-creation-time', 'config/scripts/windows-process-tree-gyp-rebuild', 'config/scripts/electron-builder-native-rebuild', + 'config/scripts/native-build-entry', + 'config/scripts/shared-native-build', 'config/patches/node-pty@', 'config/patches/@vscode__windows-process-tree' ] @@ -306,7 +308,11 @@ const NATIVE_CACHE_FILES = new Set([ 'config/scripts/zip-extractor-command.mjs', 'src/shared/zip-extractor-command.ts', 'config/scripts/shared-electron-dist-cache.mjs', - 'config/scripts/space-sharing-copy.mjs' + 'config/scripts/shared-tree-publish.mjs', + 'config/scripts/space-sharing-copy.mjs', + 'config/scripts/native-build-entry.cjs', + 'config/scripts/native-build-entry-probe.cjs', + 'config/scripts/shared-native-build.mjs' ]) const NATIVE_CACHE_PREFIXES = [ @@ -321,6 +327,7 @@ const SHARED_PACKAGE_PREFIXES = [ 'electron.vite.config.ts', 'config/electron-builder', 'config/packaged-runtime', + 'config/packaged-native-build-entries', 'config/build-plugins/', 'config/scripts/build-', 'config/scripts/smoke-packaged', diff --git a/config/scripts/rebuild-native-deps.mjs b/config/scripts/rebuild-native-deps.mjs index ac9cf10bc96..50669561c08 100644 --- a/config/scripts/rebuild-native-deps.mjs +++ b/config/scripts/rebuild-native-deps.mjs @@ -31,6 +31,11 @@ import { windowsProcessTreeAddonPath } from './windows-process-tree-gyp-rebuild.mjs' import { disableMsbuildFileTrackingOnWindows } from './msbuild-file-tracking.mjs' +import { + isSharedNativePackage, + nodePtyBuildArtifactPaths, + resolveNativePackageDir +} from './native-build-entry.cjs' import { copyFileSync, existsSync, @@ -89,6 +94,11 @@ const NATIVE_MODULES = [ : []) ] const onlyModules = NATIVE_MODULES.filter((m) => !ignoreModules.includes(m)) +// Why: these package dirs are shared with other worktrees (pnpm's global virtual store), so they +// get a published per-ABI build beside build/Release instead of an in-place rebuild. +const sharedModules = new Set(onlyModules.filter((m) => isSharedNativePackage(m, projectDir))) +// Loaded only when needed, so a normal install runs exactly the code it always did. +const sharedNativeBuild = sharedModules.size > 0 ? await import('./shared-native-build.mjs') : null /** Whether this rebuild targets something other than the machine running it. */ const isCrossHostRebuild = rebuildPlatform !== osPlatform() || rebuildArch !== process.arch const forceRebuild = @@ -159,6 +169,10 @@ try { // the catch below is written for. Outside, it aborted `pnpm install` with a // raw stack instead of the "close running Orca/Electron processes" message. assertNodePtyConptySourceDeniesMsysBreakaway() + for (const moduleName of modulesToRebuild.filter((m) => sharedModules.has(m))) { + buildSharedElectronModule(moduleName) + } + modulesToRebuild = modulesToRebuild.filter((m) => !sharedModules.has(m)) if ( rebuildPlatform === 'win32' && modulesToRebuild.includes('@vscode/windows-process-tree') && @@ -170,20 +184,22 @@ try { } } disableMsbuildFileTrackingOnWindows() - await rebuild({ - buildPath: projectDir, - electronVersion, - platform: rebuildPlatform, - arch: rebuildArch, - ignoreModules, - onlyModules: modulesToRebuild, - // Why: without force, @electron/rebuild skips modules it considers - // "already built" — even when they were compiled for the wrong ABI - // (e.g., system Node instead of Electron's embedded Node). This is - // common after pnpm install, which compiles native modules for system - // Node before postinstall runs this script. - force: true - }) + if (modulesToRebuild.length > 0) { + await rebuild({ + buildPath: projectDir, + electronVersion, + platform: rebuildPlatform, + arch: rebuildArch, + ignoreModules, + onlyModules: modulesToRebuild, + // Why: without force, @electron/rebuild skips modules it considers + // "already built" — even when they were compiled for the wrong ABI + // (e.g., system Node instead of Electron's embedded Node). This is + // common after pnpm install, which compiles native modules for system + // Node before postinstall runs this script. + force: true + }) + } restoreNodePtyWindowsConptyRuntime() assertWindowsProcessTreeAddonIsPatched() assertNodePtyConptyDeniesMsysBreakaway() @@ -282,8 +298,29 @@ function assertNodePtyConptySourceDeniesMsysBreakaway() { }) } +function buildSharedElectronModule(moduleName) { + const target = { platform: rebuildPlatform, arch: rebuildArch } + // An N-API build serves Electron and Node alike, so either can vouch for it on this host. + const executable = electronPackageIsUsable() ? getElectronExecutablePath() : process.execPath + sharedNativeBuild.ensureSharedNativeBuild({ + moduleName, + packageDir: resolveNativePackageDir(moduleName, projectDir), + ...target, + checkBuildDir: (dir) => + isCrossHostRebuild + ? sharedNativeBuild.checkNativeBuildDirArtifacts(moduleName, dir, target) + : sharedNativeBuild.checkNativeBuildDirLoads(moduleName, dir, { executable }), + log: (message) => console.log(`[rebuild] ${message}`) + }) +} + function restoreNodePtyWindowsConptyRuntime() { - if (rebuildPlatform !== 'win32' || !onlyModules.includes('node-pty')) { + // A shared node-pty's published build carries its own ConPTY runtime. + if ( + rebuildPlatform !== 'win32' || + !onlyModules.includes('node-pty') || + sharedModules.has('node-pty') + ) { return } @@ -534,21 +571,10 @@ function getPatchedNodePtyRebuildReason() { // Why: Orca patches node-pty's native Unix spawn path and Windows job-object // exports; upstream prebuilds can load while missing those patches. - const nodePtyDir = resolve(projectDir, 'node_modules', 'node-pty') - const artifactPaths = - rebuildPlatform === 'win32' - ? [ - resolve(nodePtyDir, 'build', 'Release', 'conpty.node'), - ...NODE_PTY_CONPTY_RUNTIME_FILES.map((filename) => - resolve(nodePtyDir, 'build', 'Release', 'conpty', filename) - ) - ] - : [ - resolve(nodePtyDir, 'build', 'Release', 'pty.node'), - ...(osPlatform() === 'darwin' - ? [resolve(nodePtyDir, 'build', 'Release', 'spawn-helper')] - : []) - ] + const artifactPaths = nodePtyBuildArtifactPaths( + resolve(projectDir, 'node_modules', 'node-pty'), + rebuildPlatform + ) const missingArtifact = artifactPaths.find((artifactPath) => !existsSync(artifactPath)) if (!missingArtifact) { @@ -583,9 +609,11 @@ function probeElectronNativeModules(moduleNames) { const probeSource = ` const { createRequire } = require('node:module') const { existsSync } = require('node:fs') -const { release } = require('node:os') const { resolve } = require('node:path') const projectRequire = createRequire(resolve(process.cwd(), 'package.json')) +const { isSourceBuildDir, nodePtyNativeModuleName: getNodePtyNativeModuleName } = projectRequire( + './config/scripts/native-build-entry.cjs' +) const moduleNames = ${JSON.stringify(moduleNames)} const requirePatchedNodePtySourceBuild = ${JSON.stringify(requiresPatchedNodePtySourceBuild())} const failures = [] @@ -623,17 +651,14 @@ function loadNativeModule(moduleName) { const { loadNativeModule } = projectRequire('node-pty/lib/utils') const nativeName = getNodePtyNativeModuleName() const native = loadNativeModule(nativeName) - assertNodePtyWindowsConptyRuntime(native.dir) - assertNodePtyJobOwnership({ - nativeName, + const addonPath = nodePtyAddonPath( + projectRequire.resolve('node-pty/lib/utils'), native, - addonPath: nodePtyAddonPath( - projectRequire.resolve('node-pty/lib/utils'), - native, - nativeName - ) - }) - if (requirePatchedNodePtySourceBuild && !isNodePtyReleaseBuildDir(native.dir)) { + nativeName + ) + assertNodePtyWindowsConptyRuntime(native.dir, require('node:path').dirname(addonPath)) + assertNodePtyJobOwnership({ nativeName, native, addonPath }) + if (requirePatchedNodePtySourceBuild && !isSourceBuildDir(native.dir)) { throw new Error( 'node-pty resolved to ' + native.dir + @@ -654,18 +679,11 @@ function loadNativeModule(moduleName) { projectRequire(moduleName) } -function assertNodePtyWindowsConptyRuntime(nativeDir) { - if (process.platform !== 'win32' || !isNodePtyReleaseBuildDir(nativeDir)) { +function assertNodePtyWindowsConptyRuntime(nativeDir, loadedDir) { + if (process.platform !== 'win32' || !isSourceBuildDir(nativeDir)) { return } - const runtimeDir = resolve( - process.cwd(), - 'node_modules', - 'node-pty', - 'build', - 'Release', - 'conpty' - ) + const runtimeDir = resolve(loadedDir, 'conpty') for (const filename of ${JSON.stringify(NODE_PTY_CONPTY_RUNTIME_FILES)}) { const runtimeFile = resolve(runtimeDir, filename) if (!existsSync(runtimeFile)) { @@ -674,18 +692,6 @@ function assertNodePtyWindowsConptyRuntime(nativeDir) { } } -function isNodePtyReleaseBuildDir(nativeDir) { - return typeof nativeDir === 'string' && nativeDir.replace(/\\\\/g, '/').includes('build/Release/') -} - -function getNodePtyNativeModuleName() { - if (process.platform !== 'win32') { - return 'pty' - } - const match = /(\\d+)\\.(\\d+)\\.(\\d+)/g.exec(release()) - const buildNumber = match && match.length === 4 ? Number.parseInt(match[3], 10) : 0 - return buildNumber >= 18309 ? 'conpty' : 'pty' -} function formatError(error) { return error instanceof Error ? error.message : String(error) diff --git a/config/scripts/shared-electron-dist-cache.mjs b/config/scripts/shared-electron-dist-cache.mjs index 0562e926b95..545edb3c5b9 100644 --- a/config/scripts/shared-electron-dist-cache.mjs +++ b/config/scripts/shared-electron-dist-cache.mjs @@ -1,8 +1,9 @@ import { execFileSync } from 'node:child_process' import { randomUUID } from 'node:crypto' -import { existsSync, lstatSync, mkdirSync, readFileSync, renameSync, rmSync } from 'node:fs' +import { existsSync, lstatSync, mkdirSync, readFileSync, rmSync } from 'node:fs' import path from 'node:path' import { makeTreeReadOnly, shareTree } from './space-sharing-copy.mjs' +import { publishSharedTree, sharedTreeStagingPath } from './shared-tree-publish.mjs' const IDENTITY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/ // Sibling of path.txt, never inside dist: an install replaces dist wholesale. @@ -99,7 +100,7 @@ export function publishSharedElectronDist(distPath, entry, options = {}) { return false } - const stagePath = `${entry.entryPath}.staging-${process.pid}-${uuid()}` + const stagePath = sharedTreeStagingPath(entry.entryPath, uuid) try { mkdirSync(entry.cacheRoot, { recursive: true }) ;(options.share ?? shareTree)(distPath, stagePath) @@ -111,65 +112,14 @@ export function publishSharedElectronDist(distPath, entry, options = {}) { return false } - return swapInElectronDistEntry(entry, stagePath, { - canValidate, - version, - platformPath, + // An entry we cannot judge counts as good: another worktree got there first, so keep theirs. + return publishSharedTree(stagePath, entry.entryPath, { + isUsable: () => !canValidate || isUsable(entry, version, platformPath), uuid, - rename: options.rename ?? renameSync + rename: options.rename }) } -/** - * Replace the entry with a staged tree, re-checking first. - * - * Sharing the tree above takes seconds, and a sibling worktree can publish a perfectly good entry - * in that time. Re-validating here, immediately before the destructive rename, keeps us from - * discarding that entry -- and restoring the quarantine on a failed swap keeps a losing publisher - * from leaving the cache empty. - */ -function swapInElectronDistEntry(entry, stagePath, options) { - const { canValidate, version, platformPath, uuid, rename } = options - let quarantinePath = null - if (existsSync(entry.entryPath)) { - // Same rule as before staging: an entry we cannot judge, or one that is good, is never - // displaced. Both mean another worktree got there first, so keep theirs. - if (!canValidate || isUsable(entry, version, platformPath)) { - rmSync(stagePath, { recursive: true, force: true }) - return false - } - quarantinePath = `${entry.entryPath}.unusable-${process.pid}-${uuid()}` - try { - renameSync(entry.entryPath, quarantinePath) - } catch { - rmSync(stagePath, { recursive: true, force: true }) - return false // Another worktree is already replacing it. - } - } - - try { - rename(stagePath, entry.entryPath) - } catch { - rmSync(stagePath, { recursive: true, force: true }) - if (quarantinePath !== null) { - // Put it back rather than leave no entry at all; a bad entry still beats an empty cache, - // because the next publisher re-validates and replaces it. - try { - renameSync(quarantinePath, entry.entryPath) - return false - } catch { - rmSync(quarantinePath, { recursive: true, force: true }) - } - } - return false - } - - if (quarantinePath !== null) { - rmSync(quarantinePath, { recursive: true, force: true }) - } - return true -} - function isUsable(entry, version, platformPath) { return isUsableElectronDist(entry.entryPath, version, platformPath) } diff --git a/config/scripts/shared-native-build.mjs b/config/scripts/shared-native-build.mjs new file mode 100644 index 00000000000..0a9770172a9 --- /dev/null +++ b/config/scripts/shared-native-build.mjs @@ -0,0 +1,212 @@ +/** + * Build a native addon for a package dir other worktrees share, without touching what they load. + * + * Under pnpm's global virtual store the package dir is one store slot for every worktree, so an + * in-place node-gyp rebuild swaps the binary out from under running processes. Instead the + * source is compiled in a private scratch copy, the outputs are validated in a fresh process, + * and they are published as the immutable `build/` from native-build-entry.cjs. A usable + * entry is reused as-is and never rebuilt, which is also why a forced rebuild cannot displace it. + */ +import { cpSync, existsSync, mkdirSync, mkdtempSync, realpathSync, rmSync } from 'node:fs' +import { createRequire } from 'node:module' +import { tmpdir } from 'node:os' +import { dirname, join, relative } from 'node:path' +import { runProcessSync } from '@orca/process-host' +import { disableMsbuildFileTrackingOnWindows } from './msbuild-file-tracking.mjs' +import { stageNodeAddonApi } from './orcad-prebuild-node-gyp.mjs' +import { describeProcessFailure } from './process-failure-message.mjs' +import { publishSharedTree, sharedTreeStagingPath } from './shared-tree-publish.mjs' +import { nativeBuildEntryDir } from './native-build-entry.cjs' +import { + ensureWindowsProcessTreeCommandLinePatch, + inspectWindowsProcessTreeAddon, + nodeGypRebuildInvocation, + nodeGypRebuildTimeoutMs, + stageWindowsProcessTreeNodeAddonApiHeaders +} from './windows-process-tree-gyp-rebuild.mjs' + +const require = createRequire(import.meta.url) +const { assertNodePtySourceDeniesMsysBreakaway } = require('./node-pty-job-ownership.cjs') +const { PE_MACHINE, describePeMachine, readPeMachine } = require('./windows-pe-machine.cjs') +const PROBE_SCRIPT = join(import.meta.dirname, 'native-build-entry-probe.cjs') +// Prior in-place build outputs; the scratch copy compiles from source alone. +const SCRATCH_SKIPPED_TOP_LEVEL = new Set(['build', 'bin', 'node_modules']) +const NODE_PTY_CONPTY_RUNTIME_FILES = ['conpty.dll', 'OpenConsole.exe'] + +/** The compiler environment every Orca node-gyp build of these addons uses. */ +export function nativeBuildEnvironment(env = process.env, platform = process.platform) { + return platform === 'linux' + ? { ...env, CXXFLAGS: `${env.CXXFLAGS ?? ''} -std=gnu++2a`.trim() } + : disableMsbuildFileTrackingOnWindows({ ...env }) +} + +/** + * Why `dir` cannot serve this runtime, or null when it can. Loads it in a fresh `executable` + * (Node, or Electron as Node), the only proof an addon is usable. + */ +export function checkNativeBuildDirLoads(moduleName, dir, { executable = process.execPath } = {}) { + if ( + moduleName === '@vscode/windows-process-tree' && + inspectWindowsProcessTreeAddon(join(dir, 'windows_process_tree.node')) !== 'clean' + ) { + return `${dir} has no addon built from the patched command-line reader` + } + let result + try { + result = runProcessSync({ + program: executable, + args: [PROBE_SCRIPT, moduleName, dir], + env: { ...process.env, ELECTRON_RUN_AS_NODE: '1' }, + timeoutMs: 60_000 + }) + } catch (error) { + return `the load probe could not start: ${error.message}` + } + return result.code === 0 + ? null + : (result.stderr || result.stdout || describeProcessFailure(result)).trim() +} + +/** For a cross-host target nothing here can load: the artifacts must exist and match the arch. */ +export function checkNativeBuildDirArtifacts(moduleName, dir, { platform, arch }) { + const files = + moduleName === '@vscode/windows-process-tree' + ? ['windows_process_tree.node'] + : platform === 'win32' + ? ['conpty.node', ...NODE_PTY_CONPTY_RUNTIME_FILES.map((name) => join('conpty', name))] + : ['pty.node', ...(platform === 'darwin' ? ['spawn-helper'] : [])] + const missing = files.find((file) => !existsSync(join(dir, file))) + if (missing) { + return `${join(dir, missing)} is missing` + } + const machine = platform === 'win32' ? readPeMachine(join(dir, files[0])) : undefined + if (machine !== undefined && machine !== PE_MACHINE[arch]) { + return `${join(dir, files[0])} is ${describePeMachine(machine)}, not ${arch}` + } + return null +} + +/** + * Make sure `packageDir` holds a usable published build of `moduleName` for platform-arch. + * + * `checkBuildDir(dir)` returns why a dir is unusable, or null. Returns the entry dir and whether + * this call compiled. Throws when no usable entry exists afterwards. + */ +export function ensureSharedNativeBuild({ + moduleName, + packageDir, + platform = process.platform, + arch = process.arch, + checkBuildDir, + nodeGypEntry, + log = console.warn +}) { + // Why: pinned once, so a concurrent relink can't publish this source's build into another slot. + const physicalDir = realpathSync(packageDir) + const entryDir = nativeBuildEntryDir(physicalDir, moduleName, { platform, arch }) + if (existsSync(entryDir)) { + const reason = checkBuildDir(entryDir) + if (reason === null) { + log(`Reusing the shared ${moduleName} build at ${entryDir}.`) + return { entryDir, built: false } + } + log(`The shared ${moduleName} build at ${entryDir} is unusable (${reason}); replacing it.`) + } + // node-gyp only compiles for the host OS; publishing that under another platform's name poisons the slot. + if (platform !== process.platform) { + throw new Error( + `cannot build ${moduleName} for ${platform} on ${process.platform}; build it on ${platform}` + ) + } + + const workDir = mkdtempSync(join(tmpdir(), 'orca-native-build-')) + try { + const releaseDir = compileInScratchCopy({ + moduleName, + physicalDir, + sourceDir: join(workDir, 'package'), + arch, + nodeGypEntry, + log + }) + const stagingDir = sharedTreeStagingPath(entryDir) + mkdirSync(dirname(entryDir), { recursive: true }) + cpSync(releaseDir, stagingDir, { + recursive: true, + // windows-process-tree leaves MSBuild intermediates beside its one runtime file. + filter: (path) => + moduleName !== '@vscode/windows-process-tree' || + path === releaseDir || + path.endsWith('.node') + }) + const stagedReason = checkBuildDir(stagingDir) + if (stagedReason !== null) { + rmSync(stagingDir, { recursive: true, force: true }) + throw new Error(`the freshly built ${moduleName} is unusable: ${stagedReason}`) + } + const published = publishSharedTree(stagingDir, entryDir, { + isUsable: (dir) => checkBuildDir(dir) === null + }) + if (!published) { + const reason = checkBuildDir(entryDir) + if (reason !== null) { + throw new Error(`could not publish ${entryDir}, and what is there is unusable: ${reason}`) + } + log(`Another worktree published ${entryDir} first; using it.`) + } else { + log(`Published the shared ${moduleName} build at ${entryDir}.`) + } + return { entryDir, built: true } + } finally { + rmSync(workDir, { recursive: true, force: true }) + } +} + +function compileInScratchCopy({ moduleName, physicalDir, sourceDir, arch, nodeGypEntry, log }) { + cpSync(physicalDir, sourceDir, { + recursive: true, + dereference: true, + filter: (path) => !SCRATCH_SKIPPED_TOP_LEVEL.has(relative(physicalDir, path)) + }) + if (moduleName === 'node-pty') { + stageNodeAddonApi(physicalDir, sourceDir) + if (process.platform === 'win32') { + assertNodePtySourceDeniesMsysBreakaway({ nodePtyDir: sourceDir }) + } + } else if (moduleName === '@vscode/windows-process-tree') { + // Repairs land in the private copy: the shared slot's source stays exactly as pnpm wrote it. + ensureWindowsProcessTreeCommandLinePatch(sourceDir) + stageWindowsProcessTreeNodeAddonApiHeaders(sourceDir, physicalDir) + } + + log(`Building ${moduleName} for ${arch} in ${sourceDir}.`) + const env = { ...nativeBuildEnvironment(), npm_config_arch: arch } + const { args, cwd } = nodeGypRebuildInvocation(arch, sourceDir, nodeGypEntry) + runOrThrow(`node-gyp rebuild of ${moduleName}`, { + program: process.execPath, + args, + cwd, + env, + stdio: 'inherit', + timeoutMs: nodeGypRebuildTimeoutMs(moduleName) + }) + if (moduleName === 'node-pty') { + // node-pty's own post-install trims build/Release to its runtime files and stages ConPTY. + runOrThrow('node-pty post-install', { + program: process.execPath, + args: [join(sourceDir, 'scripts', 'post-install.js')], + cwd: sourceDir, + env, + stdio: 'inherit', + timeoutMs: 60_000 + }) + } + return join(sourceDir, 'build', 'Release') +} + +function runOrThrow(label, options) { + const result = runProcessSync(options) + if (result.code !== 0) { + throw new Error(`${label} failed in ${options.cwd}: ${describeProcessFailure(result)}`) + } +} diff --git a/config/scripts/shared-native-build.test.mjs b/config/scripts/shared-native-build.test.mjs new file mode 100644 index 00000000000..70abe2f7e29 --- /dev/null +++ b/config/scripts/shared-native-build.test.mjs @@ -0,0 +1,97 @@ +import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { checkNativeBuildDirArtifacts, ensureSharedNativeBuild } from './shared-native-build.mjs' + +const roots = [] + +afterEach(() => { + while (roots.length > 0) { + rmSync(roots.pop(), { recursive: true, force: true }) + } +}) + +function makePackage() { + const root = mkdtempSync(path.join(tmpdir(), 'orca-shared-native-')) + roots.push(root) + return realpathSync(root) +} + +describe('ensureSharedNativeBuild', () => { + it('reuses a usable published entry without compiling, even when forced', () => { + const packageDir = makePackage() + const entryDir = path.join(packageDir, 'build', 'orca-napi-linux-x64') + mkdirSync(entryDir, { recursive: true }) + const checked = [] + const result = ensureSharedNativeBuild({ + moduleName: 'node-pty', + packageDir, + platform: 'linux', + arch: 'x64', + checkBuildDir: (dir) => { + checked.push(dir) + return null + }, + // No source in packageDir: any compile attempt would throw. + log: () => {} + }) + expect(result).toEqual({ entryDir, built: false }) + expect(checked).toEqual([entryDir]) + }) + + it('publishes beside the physical package dir, not through the project link', () => { + const packageDir = makePackage() + const entryDir = path.join(packageDir, 'build', 'orca-napi-linux-x64') + mkdirSync(entryDir, { recursive: true }) + const link = path.join(makePackage(), 'node-pty') + symlinkSync(packageDir, link, 'junction') + const result = ensureSharedNativeBuild({ + moduleName: 'node-pty', + packageDir: link, + platform: 'linux', + arch: 'x64', + checkBuildDir: () => null, + log: () => {} + }) + expect(result.entryDir).toBe(entryDir) + }) + + it('refuses to compile for another platform', () => { + const otherPlatform = process.platform === 'linux' ? 'darwin' : 'linux' + expect(() => + ensureSharedNativeBuild({ + moduleName: 'node-pty', + packageDir: makePackage(), + platform: otherPlatform, + arch: 'x64', + checkBuildDir: () => null, + log: () => {} + }) + ).toThrow(`cannot build node-pty for ${otherPlatform}`) + }) +}) + +describe('checkNativeBuildDirArtifacts', () => { + it('requires spawn-helper on macOS only', () => { + const dir = makePackage() + writeFileSync(path.join(dir, 'pty.node'), '') + expect(checkNativeBuildDirArtifacts('node-pty', dir, { platform: 'linux', arch: 'x64' })).toBe( + null + ) + expect( + checkNativeBuildDirArtifacts('node-pty', dir, { platform: 'darwin', arch: 'x64' }) + ).toMatch(/spawn-helper is missing/) + }) + + it('rejects a Windows addon built for another arch', () => { + const dir = makePackage() + mkdirSync(path.join(dir, 'conpty')) + for (const file of ['conpty.node', 'conpty/conpty.dll', 'conpty/OpenConsole.exe']) { + writeFileSync(path.join(dir, file), 'not-a-pe') + } + expect( + checkNativeBuildDirArtifacts('node-pty', dir, { platform: 'win32', arch: 'arm64' }) + ).toMatch(/not arm64/) + }) +}) diff --git a/config/scripts/shared-tree-publish.mjs b/config/scripts/shared-tree-publish.mjs new file mode 100644 index 00000000000..26597d7837c --- /dev/null +++ b/config/scripts/shared-tree-publish.mjs @@ -0,0 +1,85 @@ +import { randomUUID } from 'node:crypto' +import { existsSync, renameSync, rmSync, statSync } from 'node:fs' + +/** A uniquely named sibling of `entryDir`, so concurrent publishers never share a staging tree. */ +export function sharedTreeStagingPath(entryDir, uuid = randomUUID) { + return `${entryDir}.staging-${process.pid}-${uuid()}` +} + +/** + * Move a fully built, validated `stagingDir` into place as `entryDir`, lock-free. + * + * Every worktree that shares the entry may be reading it, so a usable entry is never replaced, + * and the entry is either absent, the old tree, or the new tree -- never half-written. A rename + * onto a populated directory fails, so of two racing publishers one wins and the other just + * removes its own staging tree. An unusable entry is quarantined first, and put back if the swap + * fails, because a bad entry the next publisher can re-judge beats an empty one. The quarantined + * tree must be the one judged unusable; a sibling's fresh publish that landed mid-judgement is + * put back untouched. + * + * Re-checks the entry here, immediately before the rename, because staging takes long enough for + * a sibling to publish a good one in the meantime. Returns true only when this call published. + */ +export function publishSharedTree(stagingDir, entryDir, options) { + const { isUsable } = options + const uuid = options.uuid ?? randomUUID + const rename = options.rename ?? renameSync + let quarantinePath = null + if (existsSync(entryDir)) { + const judged = treeIdentity(entryDir) + if (isUsable(entryDir)) { + rmSync(stagingDir, { recursive: true, force: true }) + return false + } + // Why: the probe takes long enough for a sibling to publish; don't even briefly move theirs. + if (judged === null || treeIdentity(entryDir) !== judged) { + rmSync(stagingDir, { recursive: true, force: true }) + return false + } + quarantinePath = `${entryDir}.unusable-${process.pid}-${uuid()}` + try { + renameSync(entryDir, quarantinePath) + } catch { + rmSync(stagingDir, { recursive: true, force: true }) + return false // Another publisher is already replacing it, or (Windows) it is loaded. + } + if (judged === null || treeIdentity(quarantinePath) !== judged) { + rmSync(stagingDir, { recursive: true, force: true }) + try { + renameSync(quarantinePath, entryDir) + } catch { + rmSync(quarantinePath, { recursive: true, force: true }) // A third publisher refilled it. + } + return false + } + } + + try { + rename(stagingDir, entryDir) + } catch { + rmSync(stagingDir, { recursive: true, force: true }) + if (quarantinePath !== null) { + try { + renameSync(quarantinePath, entryDir) + return false + } catch { + rmSync(quarantinePath, { recursive: true, force: true }) + } + } + return false + } + + if (quarantinePath !== null) { + rmSync(quarantinePath, { recursive: true, force: true }) + } + return true +} + +function treeIdentity(dir) { + try { + const { dev, ino } = statSync(dir, { bigint: true }) + return `${dev}:${ino}` + } catch { + return null + } +} diff --git a/config/scripts/shared-tree-publish.test.mjs b/config/scripts/shared-tree-publish.test.mjs new file mode 100644 index 00000000000..13995fff213 --- /dev/null +++ b/config/scripts/shared-tree-publish.test.mjs @@ -0,0 +1,123 @@ +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + renameSync, + rmSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { publishSharedTree, sharedTreeStagingPath } from './shared-tree-publish.mjs' + +const roots = [] + +afterEach(() => { + while (roots.length > 0) { + rmSync(roots.pop(), { recursive: true, force: true }) + } +}) + +function makeRoot() { + const root = mkdtempSync(path.join(tmpdir(), 'orca-shared-tree-')) + roots.push(root) + return root +} + +function writeTree(dir, content) { + mkdirSync(dir, { recursive: true }) + writeFileSync(path.join(dir, 'payload'), content) + return dir +} + +const readPayload = (dir) => readFileSync(path.join(dir, 'payload'), 'utf8') +const isGood = (dir) => existsSync(path.join(dir, 'payload')) && readPayload(dir) !== 'bad' + +describe('publishSharedTree', () => { + it('names staging trees uniquely beside the entry', () => { + expect(sharedTreeStagingPath('/x/entry', () => 'u1')).toBe(`/x/entry.staging-${process.pid}-u1`) + }) + + it('renames a staged tree into an empty slot', () => { + const root = makeRoot() + const entry = path.join(root, 'entry') + const staging = writeTree(sharedTreeStagingPath(entry), 'new') + expect(publishSharedTree(staging, entry, { isUsable: isGood })).toBe(true) + expect(readPayload(entry)).toBe('new') + expect(readdirSync(root)).toEqual(['entry']) + }) + + it('never replaces a usable entry, and drops its own staging tree', () => { + const root = makeRoot() + const entry = writeTree(path.join(root, 'entry'), 'old') + const staging = writeTree(sharedTreeStagingPath(entry), 'new') + expect(publishSharedTree(staging, entry, { isUsable: isGood })).toBe(false) + expect(readPayload(entry)).toBe('old') + expect(readdirSync(root)).toEqual(['entry']) + }) + + it('lets exactly one of two racing publishers win', () => { + const root = makeRoot() + const entry = path.join(root, 'entry') + const first = writeTree( + sharedTreeStagingPath(entry, () => 'a'), + 'first' + ) + const second = writeTree( + sharedTreeStagingPath(entry, () => 'b'), + 'second' + ) + // The second passes its pre-check, then the first lands just before its own rename does. + const raced = publishSharedTree(second, entry, { + isUsable: isGood, + rename: (from, to) => { + expect(publishSharedTree(first, entry, { isUsable: isGood })).toBe(true) + renameSync(from, to) + } + }) + expect(raced).toBe(false) + expect(readPayload(entry)).toBe('first') + expect(readdirSync(root)).toEqual(['entry']) + }) + + it('quarantines an unusable entry and publishes over it', () => { + const root = makeRoot() + const entry = writeTree(path.join(root, 'entry'), 'bad') + const staging = writeTree(sharedTreeStagingPath(entry), 'new') + expect(publishSharedTree(staging, entry, { isUsable: isGood })).toBe(true) + expect(readPayload(entry)).toBe('new') + expect(readdirSync(root)).toEqual(['entry']) + }) + + it('puts the quarantined entry back when the swap fails', () => { + const root = makeRoot() + const entry = writeTree(path.join(root, 'entry'), 'bad') + const staging = writeTree(sharedTreeStagingPath(entry), 'new') + const failingRename = () => { + throw new Error('EPERM') + } + expect(publishSharedTree(staging, entry, { isUsable: isGood, rename: failingRename })).toBe( + false + ) + expect(readPayload(entry)).toBe('bad') + expect(readdirSync(root)).toEqual(['entry']) + }) + it('never quarantines a good entry a sibling published while the bad one was judged', () => { + const root = makeRoot() + const entry = writeTree(path.join(root, 'entry'), 'bad') + const staging = writeTree(sharedTreeStagingPath(entry), 'mine') + const sibling = writeTree(path.join(root, 'sibling'), 'sibling') + const judgeThenRace = (dir) => { + const usable = isGood(dir) + rmSync(entry, { recursive: true, force: true }) + renameSync(sibling, entry) + return usable + } + expect(publishSharedTree(staging, entry, { isUsable: judgeThenRace })).toBe(false) + expect(readPayload(entry)).toBe('sibling') + expect(readdirSync(root)).toEqual(['entry']) + }) +}) diff --git a/config/scripts/windows-process-tree-gyp-rebuild.mjs b/config/scripts/windows-process-tree-gyp-rebuild.mjs index 77063badeb6..a005a2aaf41 100644 --- a/config/scripts/windows-process-tree-gyp-rebuild.mjs +++ b/config/scripts/windows-process-tree-gyp-rebuild.mjs @@ -13,6 +13,7 @@ import { execFileSync } from 'node:child_process' import { copyFileSync, existsSync, mkdirSync, readFileSync, realpathSync, rmSync } from 'node:fs' import { createRequire } from 'node:module' import { dirname, join, resolve } from 'node:path' +import { loadedNativeBuildDir } from './native-build-entry.cjs' const ROOT = resolve(import.meta.dirname, '..', '..') @@ -75,9 +76,12 @@ export function nodeGypRebuildTimeoutMs( : 300_000 } -/** The binary the addon actually loads. */ +/** The binary the addon actually loads: the patched lib/index.js prefers a published entry. */ export function windowsProcessTreeAddonPath(packageDir = WINDOWS_PROCESS_TREE_PACKAGE_DIR) { - return join(packageDir, 'build', 'Release', 'windows_process_tree.node') + return join( + loadedNativeBuildDir(packageDir, '@vscode/windows-process-tree', { platform: 'win32' }), + 'windows_process_tree.node' + ) } /** The import whose absence tells the patched binary from the published prebuilt. */ @@ -263,11 +267,13 @@ export function ensureWindowsProcessTreeCommandLinePatch( } // Patched binding.gyp includes deps/node-addon-api; the tarball does not ship those headers. +// `resolveFromDir` is the installed package when `packageDir` is a scratch copy outside the tree. export function stageWindowsProcessTreeNodeAddonApiHeaders( - packageDir = WINDOWS_PROCESS_TREE_PACKAGE_DIR + packageDir = WINDOWS_PROCESS_TREE_PACKAGE_DIR, + resolveFromDir = packageDir ) { const nodeAddonApiDir = dirname( - createRequire(join(packageDir, 'package.json')).resolve('node-addon-api/package.json') + createRequire(join(resolveFromDir, 'package.json')).resolve('node-addon-api/package.json') ) const stagedHeaderDir = join(packageDir, 'deps', 'node-addon-api') mkdirSync(stagedHeaderDir, { recursive: true }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 401c71c30d1..6294310e520 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -170,7 +170,7 @@ overrides: packageExtensionsChecksum: sha256-TrarseUCmMKP0lVjkfyLP2TAvkGZMZzvzPxjKY48nrY= patchedDependencies: - '@vscode/windows-process-tree@0.8.0': 9da74aa3d17243aa53dcdc95c9f06e97437e7fbccf098aeb017579e2d24cbac2 + '@vscode/windows-process-tree@0.8.0': a09f2e8b2b4e73ef20c317af91f9e06c09abf62fa2c891d99b16f53047f596fd '@xterm/addon-image@0.10.0-beta.301': f088c9207dac42844a542b9da8078be52351cee60585725ef189b9fd30f33f34 '@xterm/addon-ligatures@0.11.0-beta.301': 47405b9994b5acf1b4e90b49250358c1ca03649854d59560e7732b72fe336920 '@xterm/addon-search@0.17.0-beta.301': eee5338dd2621ece46e79c61ec06766cd7fadaf79ffdb24e2a8ab68e97ef31f0 @@ -180,7 +180,7 @@ patchedDependencies: i18next-cli@1.74.2: 2ee32eef742615cd133e1a6940f027deb3723a8430bb7953bba3a6769b0aac47 lint-staged@16.4.0: 7333b3837f80a7fbd045964db6d76ba4fc118e49134bdbabb00585b6b7b60673 monaco-editor@0.55.1: 53b71f7d8a7c00b76ffdb73ce83d18e31692f193f5fafa6a390e48f5b3fdd636 - node-pty@1.1.0: c2fe19cde7aef223738a7503ae429dd9b03432130b064aa64a601900b33c151f + node-pty@1.1.0: 2aa312c1b29d733bfd3e3ece2fbbb4eed7750384d7d9f91effd27318a75b7a95 oxlint-plugin-anti-slop@0.1.2: c10c871702365a0c2c59e9f1c5fb92df97cae5755d534c0a3d83c6368f6feafe ws@8.22.0: 9393e04dd9a90b01eb7aa35fcd5b1431b5038ce999fbbd82651136193ba51f14 @@ -229,7 +229,7 @@ importers: version: 3.3.1 node-pty: specifier: ^1.1.0 - version: 1.1.0(patch_hash=c2fe19cde7aef223738a7503ae429dd9b03432130b064aa64a601900b33c151f) + version: 1.1.0(patch_hash=2aa312c1b29d733bfd3e3ece2fbbb4eed7750384d7d9f91effd27318a75b7a95) posthog-node: specifier: ^5.52.5 version: 5.52.5 @@ -645,7 +645,7 @@ importers: version: link:native/windows-security '@vscode/windows-process-tree': specifier: 0.8.0 - version: 0.8.0(patch_hash=9da74aa3d17243aa53dcdc95c9f06e97437e7fbccf098aeb017579e2d24cbac2) + version: 0.8.0(patch_hash=a09f2e8b2b4e73ef20c317af91f9e06c09abf62fa2c891d99b16f53047f596fd) sherpa-onnx-darwin-arm64: specifier: 1.12.37 version: 1.12.37 @@ -11601,7 +11601,7 @@ snapshots: '@vscode/ripgrep-universal@1.18.0': {} - '@vscode/windows-process-tree@0.8.0(patch_hash=9da74aa3d17243aa53dcdc95c9f06e97437e7fbccf098aeb017579e2d24cbac2)': + '@vscode/windows-process-tree@0.8.0(patch_hash=a09f2e8b2b4e73ef20c317af91f9e06c09abf62fa2c891d99b16f53047f596fd)': dependencies: node-addon-api: 7.1.0 optional: true @@ -14441,7 +14441,7 @@ snapshots: node-int64@0.4.0: {} - node-pty@1.1.0(patch_hash=c2fe19cde7aef223738a7503ae429dd9b03432130b064aa64a601900b33c151f): + node-pty@1.1.0(patch_hash=2aa312c1b29d733bfd3e3ece2fbbb4eed7750384d7d9f91effd27318a75b7a95): dependencies: node-addon-api: 7.1.1 diff --git a/src/main/orcad/node-pty-precondition.ts b/src/main/orcad/node-pty-precondition.ts index 85c1e4b7886..48de8af5853 100644 --- a/src/main/orcad/node-pty-precondition.ts +++ b/src/main/orcad/node-pty-precondition.ts @@ -197,7 +197,7 @@ export function buildNodePtyLoadProbeScript(nodePtyDir: string): string { `const fs=require('fs'),p=require('path');`, `const n=process.platform==='win32'&&Number(require('os').release().split('.')[2])>=18309?'conpty':'pty';`, `let f=null;`, - `for(const d of ['build/Release','build/Debug','prebuilds/'+process.platform+'-'+process.arch]){`, + `for(const d of ['build/orca-napi-'+process.platform+'-'+process.arch,'build/Release','build/Debug','prebuilds/'+process.platform+'-'+process.arch]){`, `for(const r of [${root},p.join(${root},'lib')]){`, `const c=p.join(r,d,n+'.node');if(fs.existsSync(c)){f=c;break}}if(f)break}`, `if(!f){console.log(${JSON.stringify(NO_BINARY_TOKEN)});process.exit(3)}`, diff --git a/src/main/providers/local-pty-utils.ts b/src/main/providers/local-pty-utils.ts index e0bde30fabe..5bdd4567d1f 100644 --- a/src/main/providers/local-pty-utils.ts +++ b/src/main/providers/local-pty-utils.ts @@ -1,7 +1,10 @@ import { basename, isAbsolute, join } from 'node:path' import { existsSync, accessSync, statSync, chmodSync, constants as fsConstants } from 'node:fs' import type * as pty from 'node-pty' -import { usesNodePtySpawnHelper } from '../../shared/node-pty-spawn-helper' +import { + nodePtyNativeBuildEntryDir, + usesNodePtySpawnHelper +} from '../../shared/node-pty-spawn-helper' import { TERMINAL_SPAWN_ISSUE_REQUEST } from '../../shared/terminal-spawn-error-copy' import { hostReportsChildExitStatus, @@ -34,6 +37,7 @@ export function getNodePtySpawnHelperCandidates(): string[] { : unixTerminalPath return [ + join(packageRoot, nodePtyNativeBuildEntryDir(process.platform, process.arch), 'spawn-helper'), join(packageRoot, 'build', 'Release', 'spawn-helper'), join(packageRoot, 'build', 'Debug', 'spawn-helper'), join(packageRoot, 'prebuilds', `${process.platform}-${process.arch}`, 'spawn-helper') diff --git a/src/shared/node-pty-spawn-helper.ts b/src/shared/node-pty-spawn-helper.ts index c3e40df101a..e854bd004e5 100644 --- a/src/shared/node-pty-spawn-helper.ts +++ b/src/shared/node-pty-spawn-helper.ts @@ -9,3 +9,11 @@ export function usesNodePtySpawnHelper(platform: string): boolean { return platform === 'darwin' } + +/** + * node-pty's per-ABI build dir, which the patched loader tries before build/Release. Matches + * config/scripts/native-build-entry.cjs; only dev installs sharing a package dir have it. + */ +export function nodePtyNativeBuildEntryDir(platform: string, arch: string): string { + return `build/orca-napi-${platform}-${arch}` +}