mirror of
https://github.com/stablyai/orca.git
synced 2026-10-31 00:57:45 +00:00
perf(serve): clone the supervisor's code copy in one step on macOS (#31303)
* perf(serve): copy the supervisor's code aside with one cp clone on macOS orca serve --supervise copies the app's unpacked code and node_modules out of the app before re-running itself from the copy. Node's fs.cp clones those ~6,000 files one at a time, which took 8-15 s in the foreground and around 35 s under the service manager's throttled I/O. One /bin/cp -cR per tree does the same copy in about 1.5 s. The clone-then-plain-copy fallback the terminal host copy already used now lives in src/shared/macos-tree-clone.ts, and both copies call it. Linux keeps fs.cp with reflink cloning, and Windows does not support --supervise. Claude-Session: serve-snapshot-clone * fix(serve): let the macOS tree copy run without a time limit and throw on failure The shared cp helper ran under runProcess's 30 s default time limit, which the old fs.cp copy never had. At background I/O priority a large tree can outlast it, so the supervisor would quietly give up and run from the app itself. The helper now runs cp with no limit; the terminal helper copy stays bounded by its deadline signal. The helper moves to src/main beside the asar-safe rm and uses it directly, so it no longer takes an rm option. It throws instead of returning a boolean, so copyBundle is gone. The relocation reads process.platform instead of a platform dependency, and its copy test now runs on every platform but Windows. Claude-Session: serve-snapshot-clone * fix(build): keep the macOS tree copy module where the packaged CLI loads it The serve supervisor's relocation code in the CLI now imports the macOS tree copy module from the main-process sources. The CLI is compiled file by file, but the later main-process build replaces out/main and keeps only the files it is told to build as entries, so the packaged CLI failed at load with "Cannot find module '../../main/macos-tree-clone'" on every platform, and the CLI runtime verifier rejected the build. Register the module as a CLI main entry, like the other main modules the CLI imports, and unpack it from the app archive. Its asar-safe removal dependency is bundled into a shared main chunk, which is already unpacked. The build output contract test now covers both. Claude-Session: serve-snapshot-clone * fix(build): unpack the asar-safe rm the CLI's tree clone imports The compiled CLI layout keeps out/main/macos-tree-clone.js requiring ./asar-transparent-fs directly, and that file was still packed, so the packaged CLI failed to load (caught by packaged-cli-assets in CI). The bundled build only passed because the main build inlines it into an unpacked chunk. Claude-Session: serve-snapshot-clone
This commit is contained in:
@@ -23,6 +23,7 @@ export const CLI_MAIN_ENTRY_NAMES = [
|
||||
'gitlab/project-ref-parser',
|
||||
'orca-profiles/profile-index-store',
|
||||
'claude-accounts/keychain',
|
||||
'macos-tree-clone',
|
||||
...[
|
||||
'access',
|
||||
'active-location',
|
||||
|
||||
@@ -375,6 +375,8 @@ module.exports = {
|
||||
'out/main/gitlab/project-ref-parser.js',
|
||||
'out/main/grok/**',
|
||||
'out/main/hermes/**',
|
||||
'out/main/asar-transparent-fs.js',
|
||||
'out/main/macos-tree-clone.js',
|
||||
'out/main/orca-profiles/profile-index-store.js',
|
||||
'out/main/persistence/profile-state/**',
|
||||
'out/main/startup/http1-compatibility-marker.js',
|
||||
|
||||
@@ -190,6 +190,7 @@ describe('Electron Vite output contract', () => {
|
||||
|
||||
for (const name of [
|
||||
'gitlab/project-ref-parser',
|
||||
'macos-tree-clone',
|
||||
'orca-profiles/profile-index-store',
|
||||
'persistence/profile-state/profile-state-access',
|
||||
'persistence/profile-state/profile-state-active-location',
|
||||
@@ -206,6 +207,8 @@ describe('Electron Vite output contract', () => {
|
||||
expect(input).toHaveProperty(name)
|
||||
}
|
||||
expect(electronBuilderConfig.asarUnpack).toContain('out/main/gitlab/project-ref-parser.js')
|
||||
expect(electronBuilderConfig.asarUnpack).toContain('out/main/macos-tree-clone.js')
|
||||
expect(electronBuilderConfig.asarUnpack).toContain('out/main/asar-transparent-fs.js')
|
||||
expect(electronBuilderConfig.asarUnpack).toContain('out/main/persistence/profile-state/**')
|
||||
expect(electronBuilderConfig.asarUnpack).toContain(
|
||||
'out/main/orca-profiles/profile-index-store.js'
|
||||
|
||||
@@ -1,4 +1,15 @@
|
||||
import { chmod, mkdir, mkdtemp, readdir, rm, writeFile } from 'node:fs/promises'
|
||||
import {
|
||||
chmod,
|
||||
lstat,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
readdir,
|
||||
readlink,
|
||||
rm,
|
||||
symlink,
|
||||
writeFile
|
||||
} from 'node:fs/promises'
|
||||
import { existsSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
@@ -145,6 +156,40 @@ describe('relocateServeSupervisor', () => {
|
||||
}
|
||||
)
|
||||
|
||||
it.skipIf(process.platform === 'win32')(
|
||||
'copies contents, symlinks and executable bits verbatim',
|
||||
async () => {
|
||||
const modules = join(resources, 'node_modules')
|
||||
await mkdir(join(modules, '.bin'))
|
||||
await writeFile(join(modules, 'ws', 'cli.js'), '#!/usr/bin/env node\n')
|
||||
await chmod(join(modules, 'ws', 'cli.js'), 0o755)
|
||||
await symlink('../ws/cli.js', join(modules, '.bin', 'ws'))
|
||||
await symlink('../gone/index.js', join(modules, '.bin', 'dangling'))
|
||||
await symlink(join(root, 'outside'), join(modules, 'absolute'))
|
||||
await symlink('ws', join(modules, 'ws-alias'))
|
||||
const d = deps()
|
||||
await expect(relocateServeSupervisor(d)).resolves.toBe('relocated')
|
||||
|
||||
const snapshot = join(String(vi.mocked(d.execve).mock.calls[0][1][1]), '..', '..', '..', '..')
|
||||
const copied = join(snapshot, 'node_modules')
|
||||
expect(await readFile(join(copied, 'ws', 'index.js'), 'utf8')).toBe('ws')
|
||||
expect((await lstat(join(copied, 'ws', 'cli.js'))).mode & 0o777).toBe(0o755)
|
||||
expect(await readlink(join(copied, '.bin', 'ws'))).toBe('../ws/cli.js')
|
||||
expect(await readlink(join(copied, '.bin', 'dangling'))).toBe('../gone/index.js')
|
||||
expect(await readlink(join(copied, 'absolute'))).toBe(join(root, 'outside'))
|
||||
expect(await readlink(join(copied, 'ws-alias'))).toBe('ws')
|
||||
expect((await lstat(join(copied, 'ws-alias'))).isSymbolicLink()).toBe(true)
|
||||
// Why: `cp -R` into an existing directory would nest the tree one level down.
|
||||
expect(await readdir(snapshot)).toEqual(
|
||||
expect.arrayContaining(['app.asar.unpacked', 'node_modules'])
|
||||
)
|
||||
expect(existsSync(join(snapshot, 'node_modules', 'node_modules'))).toBe(false)
|
||||
expect(
|
||||
await readFile(join(snapshot, 'app.asar.unpacked', 'out', 'cli', 'index.js'), 'utf8')
|
||||
).toBe('cli v1')
|
||||
}
|
||||
)
|
||||
|
||||
const releaseCopies = async (): Promise<string[]> =>
|
||||
(await readdir(serveSupervisorSnapshotRoot(userData))).filter((name) => !name.startsWith('.'))
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import { createHash } from 'node:crypto'
|
||||
import { constants, readFileSync, statSync } from 'node:fs'
|
||||
import { cp, mkdir, readdir, rename, rm, stat, writeFile } from 'node:fs/promises'
|
||||
import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path'
|
||||
import { cloneMacTree } from '../../main/macos-tree-clone'
|
||||
import { isProcessAlive } from '../../shared/serve-bundle-coordination'
|
||||
|
||||
/** Set on the re-run supervisor: the app root its own `__dirname` no longer points at. */
|
||||
@@ -62,6 +63,19 @@ async function exists(path: string): Promise<boolean> {
|
||||
/** Who runs from each copy, kept beside the copies so it never blocks publishing one. */
|
||||
const usersDir = (root: string, id: string): string => join(root, USERS_DIR, id)
|
||||
|
||||
async function copyTree(source: string, destination: string): Promise<void> {
|
||||
// Why not fs.cp on macOS: it clones file by file, several times slower than one `cp -c`.
|
||||
if (process.platform === 'darwin') {
|
||||
return cloneMacTree(source, destination)
|
||||
}
|
||||
// Why FICLONE: on reflink filesystems the copy shares blocks with the app.
|
||||
await cp(source, destination, {
|
||||
recursive: true,
|
||||
verbatimSymlinks: true,
|
||||
mode: constants.COPYFILE_FICLONE
|
||||
})
|
||||
}
|
||||
|
||||
async function buildSnapshot(
|
||||
resources: string,
|
||||
root: string,
|
||||
@@ -78,15 +92,11 @@ async function buildSnapshot(
|
||||
const temporary = join(root, `${TEMPORARY_PREFIX}${pid}-${id}`)
|
||||
await rm(temporary, { recursive: true, force: true })
|
||||
try {
|
||||
await mkdir(temporary)
|
||||
// Why both: bare requires from out/ walk up into Resources/node_modules.
|
||||
for (const name of [PACKAGED_APP_ROOT_NAME, 'node_modules']) {
|
||||
if (await exists(join(resources, name))) {
|
||||
// Why FICLONE: on APFS and reflink filesystems the copy shares blocks with the app.
|
||||
await cp(join(resources, name), join(temporary, name), {
|
||||
recursive: true,
|
||||
verbatimSymlinks: true,
|
||||
mode: constants.COPYFILE_FICLONE
|
||||
})
|
||||
await copyTree(join(resources, name), join(temporary, name))
|
||||
}
|
||||
}
|
||||
await writeFile(join(temporary, COMPLETE_MARKER), '')
|
||||
|
||||
@@ -178,7 +178,7 @@ it('stops at the shared deadline and removes the staging copy', async () => {
|
||||
return controller.signal.aborted ? { ...ok, code: null } : (signed?.(spec) ?? ok)
|
||||
})
|
||||
await expect(materializeMacDaemonBundle(copies, controller.signal)).rejects.toThrow(
|
||||
'Could not copy the macOS terminal host'
|
||||
'Could not copy Orca Terminal Host.app'
|
||||
)
|
||||
expect(await readdir(copies)).toEqual([])
|
||||
})
|
||||
|
||||
@@ -2,6 +2,7 @@ import { existsSync } from 'node:fs'
|
||||
import { mkdir, mkdtemp, realpath, rename } from 'node:fs/promises'
|
||||
import { dirname, isAbsolute, join, resolve } from 'node:path'
|
||||
import { runProcess } from '@orca/process-host'
|
||||
import { cloneMacTree } from '../macos-tree-clone'
|
||||
import { rm } from '../asar-transparent-fs'
|
||||
import { ensurePrivateDir } from './daemon-private-file-modes'
|
||||
import { inspectMacProcessCodeIdentity } from './daemon-mac-code-identity'
|
||||
@@ -71,23 +72,6 @@ async function readCodesignField(
|
||||
return line.slice(prefix.length)
|
||||
}
|
||||
|
||||
async function copyBundle(source: string, destination: string, signal: AbortSignal): Promise<void> {
|
||||
// APFS clone first; a failed clone can leave a partial tree on other filesystems.
|
||||
for (const args of [['-cR'], ['-R']]) {
|
||||
await rm(destination, { recursive: true, force: true })
|
||||
const result = await runProcess({
|
||||
program: '/bin/cp',
|
||||
args: [...args, source, destination],
|
||||
maxOutputBytes: 8192,
|
||||
signal
|
||||
})
|
||||
if (result.code === 0 && !result.timedOut) {
|
||||
return
|
||||
}
|
||||
}
|
||||
throw new Error('Could not copy the macOS terminal host')
|
||||
}
|
||||
|
||||
/** The copy of the running app's helper for its build, published once and reused after. */
|
||||
export async function materializeMacDaemonBundle(
|
||||
root: string,
|
||||
@@ -116,7 +100,7 @@ export async function materializeMacDaemonBundle(
|
||||
try {
|
||||
const stagedBundle = macDaemonCopyBundlePath(staging)
|
||||
await mkdir(dirname(stagedBundle))
|
||||
await copyBundle(source, stagedBundle, signal)
|
||||
await cloneMacTree(source, stagedBundle, signal)
|
||||
// The copy must satisfy the running app's requirement to inherit Orca's privacy grants.
|
||||
const requirement = await readCodesignField(
|
||||
['--display', '-r-', appBundle],
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
import { existsSync } from 'node:fs'
|
||||
import {
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
readdir,
|
||||
readlink,
|
||||
rm,
|
||||
symlink,
|
||||
writeFile
|
||||
} from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type * as ProcessHost from '@orca/process-host'
|
||||
import type { ProcessResult, ProcessSpec } from '@orca/process-host/process-spec'
|
||||
|
||||
const { runProcessMock } = vi.hoisted(() => ({
|
||||
runProcessMock: vi.fn<(spec: ProcessSpec) => Promise<ProcessResult>>()
|
||||
}))
|
||||
vi.mock('@orca/process-host', async (importActual) => {
|
||||
const actual = await importActual<typeof ProcessHost>()
|
||||
runProcessMock.mockImplementation((spec) => actual.runProcess(spec))
|
||||
return { runProcess: runProcessMock }
|
||||
})
|
||||
|
||||
import { cloneMacTree } from './macos-tree-clone'
|
||||
|
||||
const ok: ProcessResult = { code: 0, signal: null, stdout: '', stderr: '', timedOut: false }
|
||||
|
||||
describe('cloneMacTree', () => {
|
||||
let root: string
|
||||
let source: string
|
||||
let destination: string
|
||||
|
||||
beforeEach(async () => {
|
||||
root = await mkdtemp(join(tmpdir(), 'orca-mac-tree-clone-'))
|
||||
source = join(root, 'source')
|
||||
destination = join(root, 'copy')
|
||||
await mkdir(join(source, 'ws'), { recursive: true })
|
||||
await writeFile(join(source, 'ws', 'index.js'), 'ws')
|
||||
await symlink('../gone/index.js', join(source, 'dangling'))
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
vi.clearAllMocks()
|
||||
await rm(root, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it.runIf(process.platform === 'darwin')(
|
||||
'clones a real tree in one step, replacing an existing destination instead of nesting',
|
||||
async () => {
|
||||
await mkdir(join(destination, 'stale'), { recursive: true })
|
||||
await cloneMacTree(source, destination)
|
||||
expect(runProcessMock).toHaveBeenCalledTimes(1)
|
||||
expect((await readdir(destination)).sort()).toEqual(['dangling', 'ws'])
|
||||
expect(await readFile(join(destination, 'ws', 'index.js'), 'utf8')).toBe('ws')
|
||||
expect(await readlink(join(destination, 'dangling'))).toBe('../gone/index.js')
|
||||
}
|
||||
)
|
||||
|
||||
it('falls back to a plain copy over a cleared destination when the clone fails', async () => {
|
||||
const seenAtCopy: boolean[] = []
|
||||
runProcessMock.mockImplementation(async (spec) => {
|
||||
seenAtCopy.push(existsSync(destination))
|
||||
if (spec.args?.[0] === '-cR') {
|
||||
await mkdir(destination)
|
||||
await writeFile(join(destination, 'partial'), 'partial-clone')
|
||||
return { ...ok, code: 1 }
|
||||
}
|
||||
return ok
|
||||
})
|
||||
const signal = new AbortController().signal
|
||||
await cloneMacTree('/src', destination, signal)
|
||||
expect(runProcessMock.mock.calls.map(([spec]) => spec)).toMatchObject([
|
||||
{ program: '/bin/cp', args: ['-cR', '/src', destination], timeoutMs: null, signal },
|
||||
{ program: '/bin/cp', args: ['-R', '/src', destination], timeoutMs: null, signal }
|
||||
])
|
||||
expect(seenAtCopy).toEqual([false, false])
|
||||
})
|
||||
|
||||
it('throws when neither copy succeeds', async () => {
|
||||
runProcessMock.mockResolvedValue({ ...ok, code: 1 })
|
||||
await expect(cloneMacTree('/src/node_modules', destination)).rejects.toThrow(
|
||||
'Could not copy node_modules'
|
||||
)
|
||||
expect(runProcessMock).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,31 @@
|
||||
import { basename } from 'node:path'
|
||||
import { runProcess } from '@orca/process-host'
|
||||
import { rm } from './asar-transparent-fs'
|
||||
|
||||
/**
|
||||
* Copies `source` to `destination`, whose parent must exist, keeping symlinks as links. One
|
||||
* `/bin/cp` per tree is several times faster than Node's file-by-file `fs.cp` on APFS.
|
||||
*/
|
||||
export async function cloneMacTree(
|
||||
source: string,
|
||||
destination: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<void> {
|
||||
// APFS clone first; a failed clone can leave a partial tree on other filesystems.
|
||||
// Clearing also keeps `cp -R` from nesting `source` inside an existing `destination`.
|
||||
for (const flags of ['-cR', '-R']) {
|
||||
await rm(destination, { recursive: true, force: true })
|
||||
// Why no timeout: a large tree at background I/O priority can outlast the 30 s default.
|
||||
const result = await runProcess({
|
||||
program: '/bin/cp',
|
||||
args: [flags, source, destination],
|
||||
timeoutMs: null,
|
||||
maxOutputBytes: 8192,
|
||||
signal
|
||||
})
|
||||
if (result.code === 0) {
|
||||
return
|
||||
}
|
||||
}
|
||||
throw new Error(`Could not copy ${basename(source)}`)
|
||||
}
|
||||
Reference in New Issue
Block a user