diff --git a/.gitignore b/.gitignore index efaee0a2e38..a08e1058eeb 100644 --- a/.gitignore +++ b/.gitignore @@ -103,6 +103,7 @@ docs/** !docs/reference/git-compatibility.md !docs/reference/headless-linux-server.md !docs/reference/linux-glibc-compatibility.md +!docs/reference/orcad-operations.md !docs/reference/relay-grace-time-reconfiguration.md !docs/reference/windows-process-enumeration.md !docs/reference/wsl-runner-verification.md diff --git a/config/scripts/build-orcad-prebuilds.mjs b/config/scripts/build-orcad-prebuilds.mjs new file mode 100644 index 00000000000..679625b9f6d --- /dev/null +++ b/config/scripts/build-orcad-prebuilds.mjs @@ -0,0 +1,212 @@ +#!/usr/bin/env node +/** + * Build one node-pty prebuilt for the CURRENT platform/arch/libc and file it in orcad's + * prebuilds matrix, so a deployment target needs no C/C++ toolchain. + * + * node-pty is the only ABI-sensitive native module orcad requires. It is also PATCHED in + * this repo (config/patches/node-pty@1.1.0.patch), and that patch is the glibc-floor fix: + * `.symver` pins on openpty/forkpty/pthread_sigmask plus the `--no-as-needed` ldflags that + * keep libutil/libpthread in DT_NEEDED. An upstream prebuilt has none of it and reproduces + * #9902. So the matrix is compiled from patched sources here, and this script refuses to + * run if the patch is not in the tree it is about to compile. + * + * orcad pins its own Node runtime, so the ABI dimension is fixed and the matrix varies + * only platform/arch/libc: + * linux-x64-glibc, linux-arm64-glibc, linux-x64-musl, linux-arm64-musl, + * darwin-x64, darwin-arm64 + * + * CI runs this once per slot, each inside the container that owns that libc/arch, and + * merges the resulting `out/orcad/prebuilds` trees. `--slot=` forces the label so + * the glibc/musl distinction is recorded from the container rather than detected. + * + * Usage: + * node config/scripts/build-orcad-prebuilds.mjs [--slot=linux-x64-musl] + * node config/scripts/build-orcad-prebuilds.mjs --require-slots # release gate + */ +import { copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { createRequire } from 'node:module' +import { dirname, join } from 'node:path' +import process from 'node:process' +import { spawnSync } from 'node:child_process' + +const require = createRequire(import.meta.url) +const ROOT = join(import.meta.dirname, '..', '..') +const PREBUILDS_DIR = join(ROOT, 'out', 'orcad', 'prebuilds') + +/** Every slot a shipped matrix must fill. The single source of truth for the matrix. */ +export const MATRIX_SLOTS = [ + 'linux-x64-glibc', + 'linux-arm64-glibc', + 'linux-x64-musl', + 'linux-arm64-musl', + 'darwin-x64', + 'darwin-arm64' +] + +/** + * Why the report header and not `ldd`: `glibcVersionRuntime` is present only when the + * process is linked against glibc, and musl images have no `ldd` worth parsing. + */ +export function detectLibc(platform = process.platform, header = readReportHeader()) { + if (platform !== 'linux') { + return 'none' + } + return header && typeof header === 'object' && 'glibcVersionRuntime' in header ? 'glibc' : 'musl' +} + +function readReportHeader() { + try { + return process.report?.getReport?.()?.header + } catch { + return undefined + } +} + +export function slotName(argv = process.argv, platform = process.platform, arch = process.arch) { + const forced = argv.find((arg) => arg.startsWith('--slot=')) + if (forced) { + return forced.slice('--slot='.length) + } + const libc = detectLibc(platform) + return libc === 'none' ? `${platform}-${arch}` : `${platform}-${arch}-${libc}` +} + +/** + * The patch is what holds the Ubuntu 20.04 floor. Compiling without it produces a binary + * that loads fine on the build host and dies on the target — the exact failure the matrix + * exists to prevent, now baked into a shipped artifact instead of a first-connect error. + */ +export function assertNodePtyPatchApplied(nodePtyDir) { + const bindingGyp = readFileSync(join(nodePtyDir, 'binding.gyp'), 'utf8') + const ptySource = readFileSync(join(nodePtyDir, 'src', 'unix', 'pty.cc'), 'utf8') + const missing = [] + if (!bindingGyp.includes('--no-as-needed,-l:libutil.so.1')) { + missing.push("binding.gyp is missing the '--no-as-needed,-l:libutil.so.1' ldflag") + } + if (!ptySource.includes('.symver openpty,openpty@')) { + missing.push('src/unix/pty.cc is missing the .symver glibc pins') + } + if (missing.length > 0) { + throw new Error( + [ + '[orcad-prebuilds] refusing to build: config/patches/node-pty@1.1.0.patch is not applied.', + ...missing.map((line) => ` - ${line}`), + 'A prebuilt compiled without it will not load on Ubuntu 20.04 (see', + 'docs/reference/linux-glibc-compatibility.md and #9902). Run `pnpm install` to apply patches.' + ].join('\n') + ) + } +} + +export function readManifest(prebuildsDir) { + try { + return JSON.parse(readFileSync(join(prebuildsDir, 'manifest.json'), 'utf8')) + } catch { + return null + } +} + +/** + * Why merge rather than overwrite: CI builds one slot per container and merges the trees. + * A manifest that records only the last slot would erase every other container's record, + * and `--require-slots` would then reject a complete matrix. + */ +export function mergeManifest(existing, next) { + const slots = new Set([...(existing?.slots ?? []), next.slot]) + return { + module: 'node-pty', + version: next.version, + nodeAbi: next.nodeAbi, + slots: [...slots].sort() + } +} + +function nodePtyDir() { + return dirname(require.resolve('node-pty/package.json')) +} + +function compileNodePty(dir) { + const built = join(dir, 'build', 'Release', 'pty.node') + if (existsSync(built)) { + console.log(`[orcad-prebuilds] reusing existing build at ${built}`) + return built + } + console.log('[orcad-prebuilds] compiling node-pty from patched source ...') + const result = spawnSync(process.platform === 'win32' ? 'npx.cmd' : 'npx', ['node-gyp', 'rebuild'], { + cwd: dir, + stdio: 'inherit', + env: process.env, + windowsHide: true + }) + if (result.status !== 0) { + throw new Error(`[orcad-prebuilds] node-gyp rebuild failed (status ${result.status})`) + } + if (!existsSync(built)) { + throw new Error(`[orcad-prebuilds] node-gyp succeeded but ${built} is missing`) + } + return built +} + +function requireSlots() { + const manifest = readManifest(PREBUILDS_DIR) + const have = new Set(manifest?.slots ?? []) + const missing = MATRIX_SLOTS.filter((slot) => !have.has(slot)) + if (missing.length > 0) { + console.error( + `[orcad-prebuilds] matrix incomplete — missing ${missing.join(', ')}. ` + + 'Hosts on those slots fall back to a source build and need a C/C++ toolchain.' + ) + process.exitCode = 1 + return + } + console.log(`[orcad-prebuilds] matrix complete — ${MATRIX_SLOTS.length} slots`) +} + +function build() { + const dir = nodePtyDir() + assertNodePtyPatchApplied(dir) + const slot = slotName() + const slotDir = join(PREBUILDS_DIR, slot) + mkdirSync(slotDir, { recursive: true }) + + const builtBinary = compileNodePty(dir) + copyFileSync(builtBinary, join(slotDir, 'pty.node')) + console.log(`[orcad-prebuilds] stored ${slot}/pty.node`) + + // Why spawn-helper ships too: on Unix node-pty posix_spawns build/Release/spawn-helper, + // so a slot without it installs cleanly and then fails ENOENT the first time a user + // opens a terminal. Windows has no spawn-helper. + if (process.platform !== 'win32') { + const helperSource = join(dirname(builtBinary), 'spawn-helper') + if (!existsSync(helperSource)) { + throw new Error(`[orcad-prebuilds] spawn-helper missing at ${helperSource}`) + } + copyFileSync(helperSource, join(slotDir, 'spawn-helper')) + console.log(`[orcad-prebuilds] stored ${slot}/spawn-helper`) + } + + // The static floor gate, applied to the artifact we are about to ship rather than only + // to the packaged desktop app. objdump is Linux-only, which is where the floor lives. + if (process.platform === 'linux') { + const { verifyLinuxGlibcFloor } = require('./verify-linux-glibc-floor.cjs') + verifyLinuxGlibcFloor(slotDir) + } + + const manifest = mergeManifest(readManifest(PREBUILDS_DIR), { + slot, + version: require('node-pty/package.json').version, + nodeAbi: process.versions.modules + }) + writeFileSync(join(PREBUILDS_DIR, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`) + console.log( + `[orcad-prebuilds] manifest: node-pty ${manifest.version}, ABI ${manifest.nodeAbi}, slots ${manifest.slots.join(', ')}` + ) +} + +if (process.argv[1] && process.argv[1].endsWith('build-orcad-prebuilds.mjs')) { + if (process.argv.includes('--require-slots')) { + requireSlots() + } else { + build() + } +} diff --git a/config/scripts/build-orcad-prebuilds.test.mjs b/config/scripts/build-orcad-prebuilds.test.mjs new file mode 100644 index 00000000000..84c0e83c18f --- /dev/null +++ b/config/scripts/build-orcad-prebuilds.test.mjs @@ -0,0 +1,121 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + assertNodePtyPatchApplied, + detectLibc, + MATRIX_SLOTS, + mergeManifest, + readManifest, + slotName +} from './build-orcad-prebuilds.mjs' + +const PATCHED_BINDING_GYP = "'ldflags': ['-Wl,--no-as-needed,-l:libutil.so.1,-l:libpthread.so.0,--as-needed']" +const PATCHED_PTY_CC = '__asm__(".symver openpty,openpty@" ORCA_GLIBC_COMPAT_VERSION);' + +const dirs = [] +const stage = (bindingGyp, ptyCc) => { + const dir = mkdtempSync(join(tmpdir(), 'orcad-prebuild-src-')) + dirs.push(dir) + mkdirSync(join(dir, 'src', 'unix'), { recursive: true }) + writeFileSync(join(dir, 'binding.gyp'), bindingGyp) + writeFileSync(join(dir, 'src', 'unix', 'pty.cc'), ptyCc) + return dir +} +afterEach(() => { + for (const dir of dirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }) + } +}) + +describe('assertNodePtyPatchApplied', () => { + it('accepts a tree with both halves of the glibc-floor fix', () => { + expect(() => + assertNodePtyPatchApplied(stage(PATCHED_BINDING_GYP, PATCHED_PTY_CC)) + ).not.toThrow() + }) + + it('refuses to build when the ldflags half is missing', () => { + // The .symver pins alone let gcc's --as-needed drop libutil/libpthread from + // DT_NEEDED, which loads on the build host and fails on Ubuntu 20.04 — #9902 again, + // this time baked into a shipped prebuilt. + expect(() => assertNodePtyPatchApplied(stage("'ldflags': []", PATCHED_PTY_CC))).toThrow( + /--no-as-needed,-l:libutil\.so\.1/ + ) + }) + + it('refuses to build when the .symver pins are missing', () => { + expect(() => assertNodePtyPatchApplied(stage(PATCHED_BINDING_GYP, '// upstream'))).toThrow( + /\.symver glibc pins/ + ) + }) + + it('names the patch and the doc so the fix is findable', () => { + expect(() => assertNodePtyPatchApplied(stage("'ldflags': []", '// upstream'))).toThrow( + /config\/patches\/node-pty@1\.1\.0\.patch/ + ) + }) +}) + +describe('slot naming', () => { + it('covers every platform orcad ships to', () => { + expect([...MATRIX_SLOTS].sort()).toEqual([ + 'darwin-arm64', + 'darwin-x64', + 'linux-arm64-glibc', + 'linux-arm64-musl', + 'linux-x64-glibc', + 'linux-x64-musl' + ]) + }) + + it('lets CI force the label so the container decides glibc vs musl', () => { + // Detection inside a container that happens to run a differently-linked Node would + // file the build under the wrong slot. The forced label must beat detection outright, + // so assert against one detection could never produce for this platform/arch. + expect(slotName(['node', 'x', '--slot=linux-x64-glibc'], 'linux', 'arm64')).toBe( + 'linux-x64-glibc' + ) + }) + + it('omits the libc dimension off Linux', () => { + expect(slotName([], 'darwin', 'arm64')).toBe('darwin-arm64') + }) + + it('reads glibc from the report header and musl from its absence', () => { + expect(detectLibc('linux', { glibcVersionRuntime: '2.31' })).toBe('glibc') + expect(detectLibc('linux', {})).toBe('musl') + expect(detectLibc('darwin', { glibcVersionRuntime: '2.31' })).toBe('none') + }) +}) + +describe('mergeManifest', () => { + it('accumulates slots across the per-container CI runs that build them', () => { + // Overwriting would erase every other container's record, and the release gate would + // then reject a matrix that is actually complete. + const first = mergeManifest(null, { slot: 'linux-x64-glibc', version: '1.1.0', nodeAbi: '127' }) + const second = mergeManifest(first, { + slot: 'linux-arm64-musl', + version: '1.1.0', + nodeAbi: '127' + }) + + expect(second.slots).toEqual(['linux-arm64-musl', 'linux-x64-glibc']) + expect(second).toMatchObject({ module: 'node-pty', version: '1.1.0', nodeAbi: '127' }) + }) + + it('does not duplicate a slot rebuilt twice', () => { + const once = mergeManifest(null, { slot: 'darwin-arm64', version: '1.1.0', nodeAbi: '127' }) + expect(mergeManifest(once, { slot: 'darwin-arm64', version: '1.1.0', nodeAbi: '127' }).slots) + .toEqual(['darwin-arm64']) + }) +}) + +describe('readManifest', () => { + it('returns null instead of throwing when no matrix has been built', () => { + const dir = mkdtempSync(join(tmpdir(), 'orcad-prebuild-manifest-')) + dirs.push(dir) + expect(readManifest(dir)).toBeNull() + }) +}) diff --git a/config/scripts/build-orcad.mjs b/config/scripts/build-orcad.mjs index 05948acd529..69c12f0ecc9 100644 --- a/config/scripts/build-orcad.mjs +++ b/config/scripts/build-orcad.mjs @@ -9,10 +9,25 @@ */ import { fork, spawnSync } from 'node:child_process' import { build } from 'esbuild' -import { chmodSync, copyFileSync, mkdirSync, mkdtempSync, rmSync } from 'node:fs' +import { createHash } from 'node:crypto' +import { + chmodSync, + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync +} from 'node:fs' import { arch, platform, tmpdir } from 'node:os' import { join } from 'node:path' import process from 'node:process' +import { + ORCAD_VERSION, + ORCAD_VERSION_FILENAME, + orcadArtifactFilenames +} from '../../src/shared/orcad-artifacts.ts' const ROOT = join(import.meta.dirname, '..', '..') const OUT_DIR = join(ROOT, 'out', 'orcad') @@ -22,6 +37,11 @@ const ENTRY = join(ROOT, 'src/main/orcad/main.ts') // looks for it in the app root. A deployment has no desktop out/main to fall back to. const WATCHER_ENTRY = join(ROOT, 'src/main/ipc/parcel-watcher-process-entry.ts') const WATCHER_OUT_FILE = join(OUT_DIR, 'parcel-watcher-process-entry.js') +// Why beside orcad.js: orcad forks the terminal daemon so PTYs outlive the runtime process, +// and `getDaemonEntryPath()` probes the app root for this exact filename. Without it every +// orcad restart would SIGKILL every running terminal. +const DAEMON_ENTRY = join(ROOT, 'src/main/daemon/daemon-entry.ts') +const DAEMON_OUT_FILE = join(OUT_DIR, 'daemon-entry.js') const AGENT_BROWSER_NAME = `agent-browser-${platform()}-${arch()}${process.platform === 'win32' ? '.exe' : ''}` const OUT_FILE = join(OUT_DIR, 'orcad.js') const AGENT_BROWSER_SOURCE = join(ROOT, 'node_modules', 'agent-browser', 'bin', AGENT_BROWSER_NAME) @@ -64,20 +84,31 @@ if (process.platform !== 'win32') { chmodSync(AGENT_BROWSER_OUTPUT, 0o755) } -await build({ - entryPoints: [WATCHER_ENTRY], - bundle: true, - platform: 'node', - target: 'node18', - format: 'cjs', - outfile: WATCHER_OUT_FILE, - external: EXTERNAL, - plugins: [externalNativeAddons], - minify: true, - sourcemap: false, - define: { 'process.env.NODE_ENV': '"production"' }, - logLevel: 'error' -}) +/** Why one call per child and not one `outdir` build: esbuild mirrors each entry's source + * directory under `outdir`, and both children must land flat beside orcad.js — that is where + * their runtime resolvers look for them. */ +function buildForkedChild(entryPoint, outfile) { + return build({ + entryPoints: [entryPoint], + bundle: true, + platform: 'node', + target: 'node18', + format: 'cjs', + outfile, + external: EXTERNAL, + plugins: [externalNativeAddons], + metafile: true, + minify: true, + sourcemap: false, + define: { 'process.env.NODE_ENV': '"production"' }, + logLevel: 'error' + }) +} + +const childResults = await Promise.all([ + buildForkedChild(WATCHER_ENTRY, WATCHER_OUT_FILE), + buildForkedChild(DAEMON_ENTRY, DAEMON_OUT_FILE) +]) const result = await build({ entryPoints: [ENTRY], @@ -95,29 +126,36 @@ const result = await build({ logLevel: 'error' }) -const output = Object.values(result.metafile.outputs).find((o) => o.entryPoint) +const output = Object.values(result.metafile.outputs).find( + (o) => o.entryPoint === 'src/main/orcad/main.ts' +) // Why check `original` and not just `path`: when electron is bundleable, esbuild // rewrites `path` to the resolved file under node_modules and the naive check passes // while the package is very much in the bundle. -const electronImporters = new Set() -for (const [file, info] of Object.entries(result.metafile.inputs)) { - for (const imported of info.imports ?? []) { - const specifier = imported.original ?? imported.path - if (specifier === 'electron' || specifier.startsWith('electron/')) { - electronImporters.add(file) - } - } -} -const sqliteImporters = new Set() -for (const [file, info] of Object.entries(result.metafile.inputs)) { - for (const imported of info.imports ?? []) { - const specifier = imported.original ?? imported.path - if (specifier === 'node:sqlite') { - sqliteImporters.add(file) +// Why both metafiles: the forked children ship in the same deployment and run under the +// same plain Node. A daemon-entry that reached electron would fail at fork time, on the +// path whose whole point is that terminals survive. +function collectImporters(metafiles, matches) { + const importers = new Set() + for (const metafile of metafiles) { + for (const [file, info] of Object.entries(metafile.inputs)) { + for (const imported of info.imports ?? []) { + if (matches(imported.original ?? imported.path)) { + importers.add(file) + } + } } } + return importers } +const metafiles = [result.metafile, ...childResults.map((child) => child.metafile)] +const electronImporters = collectImporters( + metafiles, + (specifier) => specifier === 'electron' || specifier.startsWith('electron/') +) +const sqliteImporters = collectImporters(metafiles, (specifier) => specifier === 'node:sqlite') + const graphErrors = [] if (electronImporters.size > 0) { graphErrors.push( @@ -147,23 +185,53 @@ if (graphErrors.length > 0) { // dynamic require, a missing native, a top-level throw. The plain-node-entry-guard // smoke-loads its entries for exactly this reason, and orcad cannot join that guard // because it is an esbuild artifact rather than a rollup input. + // Why an exit code and not a message match: these bundles are minified onto one line, so + // Node's uncaught-exception report echoes that whole line — which contains every string + // literal in the bundle. A crash therefore "matches" any expected message, and a textual + // assertion passes against a bundle that never loaded. const smoke = spawnSync(process.execPath, [OUT_FILE, '--orcad-smoke-load-check'], { encoding: 'utf8', timeout: 60_000 }) const smokeOutput = `${smoke.stdout ?? ''}${smoke.stderr ?? ''}` - if ( - smoke.error || - smoke.signal || - !/Unknown argument: --orcad-smoke-load-check/.test(smokeOutput) - ) { + if (smoke.error || smoke.signal || smoke.status !== 0) { console.error( `[build-orcad] the bundle did not load under plain Node.\n` + - `Expected argv rejection, got signal=${smoke.signal ?? 'none'} ` + + `Expected a clean load-check exit, got status=${smoke.status ?? 'none'} ` + + `signal=${smoke.signal ?? 'none'} ` + `error=${smoke.error?.message ?? 'none'}\n${smokeOutput.slice(0, 2000)}` ) process.exitCode = 1 } + // Why require + parseArgs and not a real daemon: requiring the bundle evaluates every + // top-level import, and calling its exported argv parser proves the entry's own code is + // there rather than a graph that merely resolved. Booting one would need a socket, a + // token and a PTY — `smoke:orcad-terminal` does that end to end, through orcad. + // The verdict is carried by the exit code for the same minification reason as above. + const daemonSmoke = spawnSync( + process.execPath, + [ + '-e', + `const mod = require(${JSON.stringify(DAEMON_OUT_FILE)})\n` + + `if (typeof mod.parseArgs !== 'function') { process.exit(3) }\n` + + `try { mod.parseArgs([]); process.exit(4) } catch { process.exit(0) }` + ], + { + encoding: 'utf8', + timeout: 60_000, + env: { ...process.env, ORCA_DAEMON_ENTRY_LOAD_CHECK: '1' } + } + ) + const daemonSmokeOutput = `${daemonSmoke.stdout ?? ''}${daemonSmoke.stderr ?? ''}` + if (daemonSmoke.error || daemonSmoke.signal || daemonSmoke.status !== 0) { + console.error( + `[build-orcad] the daemon child did not load under plain Node.\n` + + `Expected a clean load check, got status=${daemonSmoke.status ?? 'none'} ` + + `signal=${daemonSmoke.signal ?? 'none'} ` + + `error=${daemonSmoke.error?.message ?? 'none'}\n${daemonSmokeOutput.slice(0, 2000)}` + ) + process.exitCode = 1 + } const watcherFailure = await smokeLoadWatcherChild() if (watcherFailure) { console.error( @@ -173,9 +241,26 @@ if (graphErrors.length > 0) { } } +// Why a content hash and not ORCAD_VERSION alone: the remote install directory is keyed on +// this string, so two different builds carrying one version would share a directory — and an +// already-`.install-complete` dir is never re-uploaded. The deploy would silently run stale +// bytes while reporting the new version. if (process.exitCode !== 1) { + const hash = createHash('sha256') + for (const filename of orcadArtifactFilenames()) { + const artifactPath = join(OUT_DIR, filename) + if (!existsSync(artifactPath)) { + throw new Error( + `orcad declares ${filename} in ORCAD_ARTIFACTS but never emitted it. Add the build ` + + 'step, or drop it from src/shared/orcad-artifacts.ts.' + ) + } + hash.update(readFileSync(artifactPath)) + } + const fullVersion = `${ORCAD_VERSION}+${hash.digest('hex').slice(0, 12)}` + writeFileSync(join(OUT_DIR, ORCAD_VERSION_FILENAME), fullVersion) console.log( - `[build-orcad] ok — ${(output.bytes / 1024 / 1024).toFixed(2)} MB, ${Object.keys(output.inputs).length} modules, zero electron and node:sqlite imports.` + `[build-orcad] ok — ${fullVersion}, ${(output.bytes / 1024 / 1024).toFixed(2)} MB, ${Object.keys(output.inputs).length} modules, zero electron and node:sqlite imports.` ) } diff --git a/docs/reference/linux-glibc-compatibility.md b/docs/reference/linux-glibc-compatibility.md index 60d855c7372..bf98dbdcb80 100644 --- a/docs/reference/linux-glibc-compatibility.md +++ b/docs/reference/linux-glibc-compatibility.md @@ -82,6 +82,32 @@ libstdc++ floor — its glibc needs are still checked. Speech-to-text therefore needs a host with libstdc++ from GCC 11+ (Ubuntu 21.10 / 22.04 LTS or newer); the app itself still launches on stock 20.04. +**3. Check before loading, on hosts that ship without a compiler (`orcad`).** +The two gates above protect the packaged desktop app, where the binary is built and +verified by the same pipeline. `orcad` is deployed to hosts Orca never built on, so it +adds a runtime precondition +([`src/main/orcad/node-pty-precondition.ts`](../../src/main/orcad/node-pty-precondition.ts)), +run from `main.ts` before anything requires `node-pty`. It loads the addon in a **child +process**, so a binary the loader refuses — or one that aborts outright — is data rather +than this process's death, and the operator gets a sentence naming the host's libc, its +Node ABI, its prebuild slot and the command to run. A proven-unloadable binary exits 78 +(`EX_CONFIG`) instead of reaching the `require`; a probe that never answered is reported +as unverifiable and boots anyway, because a silent probe is not evidence. Whatever it +finds is published in `status.get`'s `degradations[]` under `terminal_unavailable`. + +**4. Ship the binary, built from patched sources.** +[`config/scripts/build-orcad-prebuilds.mjs`](../../config/scripts/build-orcad-prebuilds.mjs) +(`pnpm run build:orcad-prebuilds`, after `build:orcad`) compiles node-pty for the current +host and files it under `out/orcad/prebuilds//`, where a slot is +`linux-{x64,arm64}-{glibc,musl}` or `darwin-{x64,arm64}`. libc is part of the slot name +because node-pty's own loader falls back to `prebuilds/-` and cannot tell +glibc from musl — a glibc binary parked there is loaded on Alpine and dies at `dlopen`. +The script refuses to compile a tree where `config/patches/node-pty@1.1.0.patch` is not +applied: without the patch the prebuilt is a #9902 crash shipped as an artifact rather +than a first-connect error. CI runs it once per slot inside the matching container +(`--slot=` forces the label), merges the trees, and `--require-slots` fails a release with +a hole in the matrix. + ## Adding or upgrading a native dependency - Prefer packages that ship prebuilt binaries compiled against an old toolchain diff --git a/docs/reference/orcad-operations.md b/docs/reference/orcad-operations.md new file mode 100644 index 00000000000..3ef3fdea37a --- /dev/null +++ b/docs/reference/orcad-operations.md @@ -0,0 +1,190 @@ +# Running orcad + +`orcad` is the Orca runtime served from plain Node. This is the contract between it and +whatever supervises it: what it binds, what it owns on disk, who restarts what, and what its +readiness payload actually proves. + +Design background: `docs/design/shipping-orcad.html` §00c and §04. + +## Two long-lived processes, not one + +A deployment is **orcad** plus **the terminal daemon**. + +| | orcad | terminal daemon | +| --- | --- | --- | +| Started by | the supervisor | orcad, detached | +| Owns | RPC, git, worktrees, persistence | every local PTY | +| Lifetime | one supervised run | **outlives orcad** | +| Endpoint | `ws://:` | `/daemon/daemon-v.sock` | + +The daemon outliving orcad is the property the whole peer model is recommended for +(`docs/reference/ssh-execution-boundary.md`): daemon-backed PTYs stay `live` across a runtime +restart, so a restart, an update or a rollback does not destroy running work. Everything +below exists to keep that true. + +**Consequence for supervision:** orcad's shutdown path calls `disconnectDaemon()`, never +`shutdownDaemon()`. A supervisor that reaps orcad's whole process group — systemd's +`KillMode=control-group` — kills the daemon too and turns every restart back into data loss. +Use `KillMode=mixed` (the default) or `process`, and never `--send-sigkill` on the group. + +## Bind policy + +`--bind `, **default `127.0.0.1`**. + +Only literal IPs are accepted; hostnames are refused because DNS would decide which +interface got bound. `localhost` maps to `127.0.0.1`. `0.0.0.0` / `::` are the explicit +opt-ins to network reach, and the startup log says so on every launch. + +The bind is **pinned**, not defaulted. Two things widen the desktop's listener on their own — +`orca serve`'s wide default, and a startup where some device has connected before — and an +unattended host's exposure must be exactly what the operator asked for on every launch. A +mobile pairing offer, which normally rebinds to all interfaces, is refused while the bind is +pinned to loopback and reports `network_exposure_failed` rather than advertising an endpoint +nothing can reach. + +Under the shipping design a client reaches a remote orcad over an SSH local port-forward, so +loopback is the correct default and the pairing credential travels over SSH. + +## Data root and the instance lock + +The data root is `$ORCA_USER_DATA`, else `$XDG_DATA_HOME/Orca`, else `~/.orca`. + +Before the profile index or the store is touched, orcad takes `/orcad.lock`. +It refuses to start when: + +| Code | Meaning | +| --- | --- | +| `orcad_data_root_wrong_owner` | the root is owned by another uid (POSIX) | +| `orcad_data_root_shared` | the root is group/world accessible and could not be tightened | +| `orcad_instance_lock_held` | another live orcad owns this root | +| `orcad_instance_lock_foreign_identity` | the lock belongs to a different identity | +| `orcad_data_root_unusable` | the root cannot be created, stat'd or written | + +A root that is merely too permissive and that we own is tightened to `0700` rather than +refused — orcad stores credentials there unsealed (no OS keyring on this host), so the goal +is a private root, and refusing when we could just fix it helps nobody. We refuse when the +permissions are not ours to fix. Windows is exempt from the owner and mode checks: ACLs are +not expressible as a POSIX mode, and `statSync().mode` there reports a synthesized one. + +A dead holder's record is reclaimed (PID plus process start time, so a recycled PID does not +read as alive). A record belonging to a different identity is never reclaimed. + +**The lock scopes one role — who is the runtime.** It deliberately says nothing about the +daemon, which lives under `/daemon` and fences its own endpoint with its own PID +record. A lock that asked "is any process using this root" would refuse exactly the restarts +a live daemon makes worthwhile. + +## Supervision + +### Who supervises orcad + +An external supervisor (systemd, launchd, a process manager). orcad conforms to it: + +- **Readiness.** One JSON line on stdout (`--json`), `type: "orca_server_ready"`, published + after the listener is bound and the daemon verdict is in. There is no separate readiness + socket; the line is the signal. Set the supervisor's start timeout generously — the daemon + launch has its own retries and can take tens of seconds on a cold host. +- **Shutdown.** `SIGTERM` or `SIGINT` starts a graceful stop. A **second** signal exits + immediately with code 1 rather than being swallowed — a supervisor's second signal means + its first deadline elapsed, and waiting silently is what turns a stop into a `SIGKILL`, + the one teardown that skips the daemon handoff. orcad also imposes its own 15s deadline + and exits 1, so the failure stays attributable instead of arriving as an unlogged kill. +- **Exit codes.** + + | Code | Meaning | Supervisor should | + | --- | --- | --- | + | 0 | clean shutdown | restart per policy | + | 1 | startup or shutdown failure | restart with backoff | + | 78 | configuration fault (bind address, data root, instance lock) | **not** restart | + + 78 is `EX_CONFIG`. Put it in systemd's `RestartPreventExitStatus`: restarting on a data + root owned by someone else is a restart-spin, not a recovery. +- **Logs.** orcad writes human-readable diagnostics to **stderr** and its readiness contract + to **stdout**; the supervisor owns capture and rotation. The daemon, being detached, writes + its own NDJSON lifecycle log to `/logs/daemon.log` (suppressed by + `ORCA_DIAGNOSTICS_DISABLED=1`). Rotation of that file is not implemented — see + [What is not covered](#what-is-not-covered). + +### orcad supervising the daemon + +- **Launch.** Forked detached from `daemon-entry.js` beside `orcad.js`, with its own PID + record, token and socket under `/daemon`. +- **Adoption before spawn.** A daemon already answering the endpoint is adopted, not + replaced, unless it is unhealthy, foreign, or built from a superseded bundle *and* owns no + live sessions. Replacing a healthy daemon kills its PTYs, so code freshness always defers + to live work. +- **Restart.** The adapter respawns the daemon on death, transparently to callers. +- **Crash-loop containment.** At most **5 launches per 60s rolling window** per orcad run; + past that, launches are refused with `daemon_crash_loop` and terminals fail with that + message instead of the process forking forever. The window slides, so a repaired host + recovers without restarting orcad. An operator-initiated daemon restart clears it — that + is the deliberate "try again". +- **No macOS login-session watch.** That watch retires the daemon when the spawning GUI login + session dies. An orcad daemon must survive its SSH session ending. +- **Shutdown.** orcad never stops the daemon. A daemon that was never adopted retires itself + after its adoption window; an adopted one stays resident (see Decommissioning). + +### Decommissioning + +The daemon outliving orcad is deliberate, so stopping orcad does **not** leave the host with +zero Orca processes. A daemon that has been adopted stays resident after its runtime +disconnects — that is what makes the next start a reattach rather than a cold restore. To +retire a host completely, stop orcad and then stop the daemon named by +`health.terminalDaemon.pid`, or delete the data root and let the endpoint go stale. + +## Health + +The readiness payload carries a `health` object: + +``` +buildHash sha256 (16 hex) of the running orcad bundle — build identity that a version + string cannot give, so a rollback that did not replace the file is visible +buildVersion ORCA_VERSION +nodeVersion / nodeAbi process.versions.node / .modules — the ABI native addons must match +platform / arch / pid +terminalDaemon: + state live | degraded | absent + ownsFreshSessions whether NEW terminals are daemon-owned, i.e. survive an orcad restart + pid the live daemon's pid, from its own PID record + buildVersion the build the LIVE daemon was forked from (may legitimately predate + this orcad after an update — reporting orcad's version for both would + hide exactly that) + entryPath / protocolVersion + selfTest { ok, coverage, verdict, durationMs } +``` + +### What the self-test proves + +`selfTest` runs `checkDaemonHealth` against the daemon's socket. It is green only when the +daemon **opened its socket, completed the protocol handshake, and ran `ptySpawnHealth` — a +real short-lived PTY spawned inside the daemon's own process**. It therefore spans both +processes: orcad drives it, the daemon performs it, the verdict crosses the socket. + +- `coverage: 'pty-spawn'` — the full round trip above. +- `coverage: 'handshake'` — **win32 only**, where `checkPtySpawnHealth` returns without + spawning anything. A green verdict there covers the handshake and nothing more. It is + reported separately rather than folded into `ok` so nobody reads it as a PTY round trip. + +`state` is `live` only when the self-test passed **and** `ownsFreshSessions` is true. A +daemon that answers but has fallen back to local spawning for new terminals is `degraded`, +because those terminals die with orcad. A daemon that answered and then failed its spawn +probe is also `degraded`, not `absent`: it still holds live sessions, and calling those +exited would be the verdict `ssh-execution-boundary.md` forbids guessing. + +## What is not covered + +Named here so nothing reads as implemented that is not: + +- **A continuous health endpoint.** `health` is published once, in the readiness payload. A + supervisor's periodic liveness/readiness probe needs an HTTP or RPC surface over the same + `collectOrcadHealth()`; that surface does not exist yet. +- **libc slot.** §04 asks for it in the health payload. It belongs to the native strategy + (plan item 5), which owns libc detection; there is no honest value to publish until then. +- **`degradations[]`.** Plan item 2's contract, not this one. +- **Credential administration** (list / revoke / rotate devices, expiring pending offers, + structured security logging) — §04, not delivered here. +- **Pinned-port fail-closed.** A pinned `--port` still falls back to an OS-assigned port on + conflict. +- **Reconciling `webClientUrl` with reachability** under the loopback default. +- **State-schema rollback rules.** +- **Daemon log rotation.** `/logs/daemon.log` grows unbounded. diff --git a/package.json b/package.json index b87bb21a49a..7b46b369406 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "check:max-lines-ratchet": "node config/scripts/check-max-lines-ratchet.mjs", "check:runtime-electron-ratchet": "node config/scripts/check-runtime-electron-ratchet.mjs", "build:orcad": "node config/scripts/build-orcad.mjs", + "build:orcad-prebuilds": "node config/scripts/build-orcad-prebuilds.mjs", "smoke:orcad-terminal": "node config/scripts/ensure-native-runtime.mjs --runtime=node && pnpm run build:cli && pnpm run build:orcad && node config/scripts/runtime-serve-terminal-smoke.mjs --target orcad", "smoke:serve-terminal": "node config/scripts/runtime-serve-terminal-smoke.mjs", "check:feature-wall-assets": "node config/scripts/check-feature-wall-assets.mjs", diff --git a/src/main/daemon/daemon-entry-path-layouts.test.ts b/src/main/daemon/daemon-entry-path-layouts.test.ts new file mode 100644 index 00000000000..b960cbf9d8f --- /dev/null +++ b/src/main/daemon/daemon-entry-path-layouts.test.ts @@ -0,0 +1,115 @@ +/** + * Which daemon-entry.js the launcher forks, per deployment layout. + * + * The layout that matters here is orcad's: a packaged host with NO asar, whose bundle root + * holds `orcad.js` and `daemon-entry.js` side by side (config/scripts/build-orcad.mjs emits + * exactly that). Resolving against `out/main` there would fork a path that does not exist, + * and the failure would surface as "terminals do not persist" rather than as a missing file. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { join } from 'node:path' +import { FAKE_APP_PATH, FAKE_DAEMON_ENTRY_PATH } from './daemon-init-test-harness' + +const { + getAppPathMock, + isPackagedMock, + probeSocketExistsMock, + forkMock, + checkDaemonHealthMock, + spawnerInstances, + importFresh, + installDefaultNetConnectStub, + moduleFactories +} = await vi.hoisted(async () => + (await import('./daemon-init-test-harness')).createDaemonInitMocks() +) + +vi.mock('fs', () => moduleFactories.fs()) +vi.mock('child_process', async (importOriginal) => + moduleFactories.childProcess(await importOriginal>()) +) +vi.mock('net', () => moduleFactories.net()) +vi.mock('./daemon-health', () => moduleFactories.daemonHealth()) +vi.mock('./daemon-pid-identity', () => moduleFactories.daemonPidIdentity()) +vi.mock('./daemon-tcc-attribution', () => moduleFactories.daemonTccAttribution()) +vi.mock('./daemon-bundle-staleness', () => moduleFactories.daemonBundleStaleness()) +vi.mock('./daemon-stale-kill', () => moduleFactories.daemonStaleKill()) +vi.mock('./daemon-process-start-time', () => moduleFactories.daemonProcessStartTime()) +vi.mock('./daemon-pid-file-parse', () => moduleFactories.daemonPidFileParse()) +vi.mock('./client', () => moduleFactories.client()) +vi.mock('./daemon-lifecycle-event', () => moduleFactories.daemonLifecycleEvent()) +vi.mock('./daemon-spawner', () => moduleFactories.daemonSpawner()) +vi.mock('./daemon-pty-adapter', () => moduleFactories.daemonPtyAdapter()) +vi.mock('../ipc/pty', () => moduleFactories.ipcPty()) + +const ASAR_APP_PATH = join('/packaged', 'resources', 'app.asar') +const ASAR_UNPACKED_ENTRY = join( + '/packaged', + 'resources', + 'app.asar.unpacked', + 'out', + 'main', + 'daemon-entry.js' +) +const ORCAD_ROOT = join('/opt', 'orcad') +const ORCAD_ADJACENT_ENTRY = join(ORCAD_ROOT, 'daemon-entry.js') + +/** Drive one launch under the given layout and return the entry path that was forked. */ +async function forkedDaemonEntryPath(layout: { + appPath: string + isPackaged: boolean + existingEntry?: string +}): Promise { + // Why after importFresh: it resets these mocks to their defaults on every fresh import. + const mod = await importFresh() + getAppPathMock.mockReturnValue(layout.appPath) + isPackagedMock.mockReturnValue(layout.isPackaged) + probeSocketExistsMock.mockImplementation((p?: string) => p === layout.existingEntry) + checkDaemonHealthMock.mockResolvedValue('unreachable') + await mod.initDaemonPtyProvider() + const launcher = spawnerInstances[0].launcher as ( + socketPath: string, + tokenPath: string + ) => Promise + forkMock.mockImplementationOnce(() => { + throw new Error('stop after entry resolution') + }) + await expect(launcher('/fake/socket', '/fake/token')).rejects.toThrow( + 'stop after entry resolution' + ) + return forkMock.mock.calls.at(-1)?.[0] as string +} + +describe('daemon entry path per deployment layout', () => { + beforeEach(() => { + installDefaultNetConnectStub() + }) + + afterEach(() => { + vi.clearAllMocks() + }) + + it('resolves the dev checkout entry under out/main', async () => { + expect(await forkedDaemonEntryPath({ appPath: FAKE_APP_PATH, isPackaged: false })).toBe( + FAKE_DAEMON_ENTRY_PATH + ) + }) + + it('redirects a packaged Electron asar root to app.asar.unpacked', async () => { + expect(await forkedDaemonEntryPath({ appPath: ASAR_APP_PATH, isPackaged: true })).toBe( + ASAR_UNPACKED_ENTRY + ) + }) + + it('forks the entry beside orcad.js on a packaged host with no asar', async () => { + // orcad answers isPackaged() true; the question the resolver must ask is whether the app + // root is an asar archive, not whether the build is packaged. + expect( + await forkedDaemonEntryPath({ + appPath: ORCAD_ROOT, + isPackaged: true, + existingEntry: ORCAD_ADJACENT_ENTRY + }) + ).toBe(ORCAD_ADJACENT_ENTRY) + }) +}) diff --git a/src/main/daemon/daemon-entry.ts b/src/main/daemon/daemon-entry.ts index addfa50b16a..72537f64cbf 100644 --- a/src/main/daemon/daemon-entry.ts +++ b/src/main/daemon/daemon-entry.ts @@ -327,8 +327,9 @@ async function main(): Promise { warmWindowsConptyOnce() } -// Only auto-run when executed directly (not imported for testing) -const isDirectExecution = !process.env.VITEST +// Only auto-run when executed directly (not imported for testing, or for the build guard's +// load check — see config/scripts/build-orcad.mjs). +const isDirectExecution = !process.env.VITEST && !process.env.ORCA_DAEMON_ENTRY_LOAD_CHECK if (isDirectExecution) { main().catch((err) => { console.error('[daemon] Fatal:', err) diff --git a/src/main/daemon/daemon-host-relocation.test.ts b/src/main/daemon/daemon-host-relocation.test.ts index 1d3ab98db90..78e4787c0f6 100644 --- a/src/main/daemon/daemon-host-relocation.test.ts +++ b/src/main/daemon/daemon-host-relocation.test.ts @@ -9,20 +9,30 @@ import { } from 'node:fs' import os from 'node:os' import { dirname, join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' -// Mutable Electron app stub, hoisted so the vi.mock factory closes over it. -const { electronApp } = vi.hoisted(() => ({ - electronApp: { - isPackaged: true, - userDataPath: '', - version: '9.9.9', - getPath: (): string => electronApp.userDataPath, - getVersion: (): string => electronApp.version - } -})) +import { setAppEnvironment, type AppEnvironment } from '../../shared/app-environment' -vi.mock('electron', () => ({ app: electronApp })) +// Mutable host stub. Relocation now reads the AppEnvironment port rather than electron's +// `app`, so orcad's daemon launch path can resolve without Electron in the graph. +const hostApp = { + isPackaged: true, + userDataPath: '', + appPath: '', + version: '9.9.9' +} + +function installHostApp(): void { + setAppEnvironment({ + getPath: () => hostApp.userDataPath, + getAppPath: () => hostApp.appPath, + getVersion: () => hostApp.version, + isPackaged: () => hostApp.isPackaged, + onWillQuit: () => {}, + exit: () => {}, + getAppMetrics: () => [] + } as AppEnvironment) +} import { buildDaemonHostManifest, @@ -88,9 +98,11 @@ beforeEach(() => { mkdirSync(localAppDataDir, { recursive: true }) process.env.LOCALAPPDATA = localAppDataDir buildInstallFixture(installDir) - electronApp.isPackaged = true - electronApp.userDataPath = userDataDir - electronApp.version = '9.9.9' + hostApp.isPackaged = true + hostApp.userDataPath = userDataDir + hostApp.appPath = join(installDir, 'resources', 'app.asar') + hostApp.version = '9.9.9' + installHostApp() setProcessProp('platform', 'win32') setProcessProp('execPath', join(installDir, 'Orca.exe')) setProcessProp('resourcesPath', join(installDir, 'resources')) @@ -214,6 +226,18 @@ describe('materializeRelocatedDaemonHost', () => { expect(materializeRelocatedDaemonHost()).toBeNull() expect(existsSync(join(localAppDataDir, 'Orca', 'daemon-host'))).toBe(false) }) + + it('does nothing for a packaged host with no asar root (orcad on win32)', () => { + // orcad answers isPackaged() true — it is a shipped build — but it is plain Node: no + // asar, no resourcesPath, and no NSIS updater to escape. Relocation staging a copy of + // an Electron tree that is not there is the isPackaged-honesty defect, and it would + // silently produce a null host on a path whose failures are meant to be visible. + hostApp.appPath = join(installDir, 'resources', 'app') + installHostApp() + expect(materializeRelocatedDaemonHost()).toBeNull() + expect(getRelocatedDaemonHost()).toBeNull() + expect(existsSync(join(localAppDataDir, 'Orca', 'daemon-host'))).toBe(false) + }) }) describe('getRelocatedDaemonHost', () => { @@ -249,6 +273,17 @@ describe('pruneOldDaemonHosts', () => { expect(existsSync(join(root, '2.0.0'))).toBe(true) expect(existsSync(join(root, '1.0.0'))).toBe(false) }) + + it('reclaims nothing for a packaged host with no asar root (orcad on win32)', () => { + const root = join(localAppDataDir, 'Orca', 'daemon-host') + mkdirSync(join(root, '1.0.0'), { recursive: true }) + hostApp.appPath = join(installDir, 'resources', 'app') + installHostApp() + pruneOldDaemonHosts(new Set()) + // A Node host owns no daemon-host tree, so deleting under it would be reaching into a + // directory layout it never created. + expect(existsSync(join(root, '1.0.0'))).toBe(true) + }) }) describe('collectPinnedDaemonVersions', () => { diff --git a/src/main/daemon/daemon-host-relocation.ts b/src/main/daemon/daemon-host-relocation.ts index 4242b292de1..79e8bcf0ad7 100644 --- a/src/main/daemon/daemon-host-relocation.ts +++ b/src/main/daemon/daemon-host-relocation.ts @@ -10,7 +10,7 @@ import { writeFileSync } from 'node:fs' import { dirname, join, win32 as winPath } from 'node:path' -import { app } from 'electron' +import { getAppEnvironment } from '../../shared/app-environment' import { parseDaemonPidFile } from './daemon-pid-file-parse' import { startTimeMatches } from './daemon-process-start-time' @@ -85,9 +85,27 @@ function resolveEntrySourcePath(resourcesPath: string): string { return join(unpackedRoot, 'out', 'main', 'daemon-entry.js') } +/** + * Whether this process is a packaged ELECTRON app on win32 — the only shape relocation + * addresses, because what it escapes is the NSIS updater's kill zone. + * + * Why asar and not isPackaged alone: orcad answers isPackaged() true (it is a shipped build, + * not a dev checkout) while having no asar, no resourcesPath and no NSIS installer. Asking + * whether the app root is an asar archive is the same honesty fix the watcher path uses, and + * it keeps a Node host from staging a copy of an Electron tree it does not have. + */ +function isPackagedElectronWin32(): boolean { + const environment = getAppEnvironment() + return ( + process.platform === 'win32' && + environment.isPackaged() && + environment.getAppPath().includes('app.asar') + ) +} + // Relocation inputs from the live packaged process, or null when it doesn't apply (non-win32, dev, or missing resourcesPath). function collectDaemonHostSources(): DaemonHostSources | null { - if (process.platform !== 'win32' || !app.isPackaged) { + if (!isPackagedElectronWin32()) { return null } const resourcesPath = process.resourcesPath @@ -206,7 +224,7 @@ function hostRootDir(): string { const base = typeof localAppData === 'string' && localAppData.length > 0 ? join(localAppData, LOCAL_HOST_ROOT_NAME) - : app.getPath('userData') + : getAppEnvironment().getPath('userData') return join(base, HOST_SUBDIR) } @@ -219,7 +237,7 @@ export function getRelocatedDaemonHost(): RelocatedDaemonHost | null { if (!sources) { return null } - const version = app.getVersion() + const version = getAppEnvironment().getVersion() const dest = join(hostRootDir(), version) const marker = readMarker(dest) if (!marker || marker.version !== version) { @@ -246,7 +264,7 @@ export function materializeRelocatedDaemonHost(): RelocatedDaemonHost | null { if (!sources) { return null } - const version = app.getVersion() + const version = getAppEnvironment().getVersion() const root = hostRootDir() const dest = join(root, version) const staging = join(root, `${version}.staging-${randomBytes(6).toString('hex')}`) @@ -319,10 +337,10 @@ export function collectPinnedDaemonVersions(runtimeDir: string): Set { * Best-effort — never throws; a locked/staging dir is retried on a future launch. */ export function pruneOldDaemonHosts(pinnedVersions: ReadonlySet): void { - if (process.platform !== 'win32' || !app.isPackaged) { + if (!isPackagedElectronWin32()) { return } - const version = app.getVersion() + const version = getAppEnvironment().getVersion() const root = hostRootDir() let entries try { diff --git a/src/main/daemon/daemon-init-child-readiness.test.ts b/src/main/daemon/daemon-init-child-readiness.test.ts index 132727fff5f..60d3c9c596d 100644 --- a/src/main/daemon/daemon-init-child-readiness.test.ts +++ b/src/main/daemon/daemon-init-child-readiness.test.ts @@ -16,7 +16,6 @@ const { (await import('./daemon-init-test-harness')).createDaemonInitMocks() ) -vi.mock('electron', () => moduleFactories.electron()) vi.mock('fs', () => moduleFactories.fs()) vi.mock('child_process', async (importOriginal) => moduleFactories.childProcess(await importOriginal>()) diff --git a/src/main/daemon/daemon-init-child-startup-failure.test.ts b/src/main/daemon/daemon-init-child-startup-failure.test.ts index 182307d075b..27d23a400d2 100644 --- a/src/main/daemon/daemon-init-child-startup-failure.test.ts +++ b/src/main/daemon/daemon-init-child-startup-failure.test.ts @@ -14,7 +14,6 @@ const { (await import('./daemon-init-test-harness')).createDaemonInitMocks() ) -vi.mock('electron', () => moduleFactories.electron()) vi.mock('fs', () => moduleFactories.fs()) vi.mock('child_process', async (importOriginal) => moduleFactories.childProcess(await importOriginal>()) diff --git a/src/main/daemon/daemon-init-dependency-mocks.ts b/src/main/daemon/daemon-init-dependency-mocks.ts index 40e7ffa1ae6..d920a13866e 100644 --- a/src/main/daemon/daemon-init-dependency-mocks.ts +++ b/src/main/daemon/daemon-init-dependency-mocks.ts @@ -17,10 +17,7 @@ export type { MockAdapter, MockSpawner } from './daemon-init-mock-types' /** The module objects each test file's own hoisted `vi.mock` factories return. */ export function createDaemonInitModuleFactories(state: DaemonInitMockState) { const { - getPathMock, - getAppPathMock, forkMock, - isPackagedMock, probeSocketExistsMock, writeFileSyncMock, readFileSyncMock, @@ -62,6 +59,7 @@ export function createDaemonInitModuleFactories(state: DaemonInitMockState) { readonly launcher: unknown readonly ensureRunning: Mock readonly resetHandle: Mock + readonly resetRespawnWindow: Mock readonly shutdown: Mock readonly getHandle: Mock private socketCounter: number @@ -99,6 +97,7 @@ export function createDaemonInitModuleFactories(state: DaemonInitMockState) { } }) this.resetHandle = vi.fn() + this.resetRespawnWindow = vi.fn() this.shutdown = vi.fn(async () => {}) this.getHandle = vi.fn(() => this.handle) spawnerInstances.push(this as unknown as MockSpawner) @@ -159,16 +158,6 @@ export function createDaemonInitModuleFactories(state: DaemonInitMockState) { } return { - electron: () => ({ - app: { - get isPackaged() { - return isPackagedMock() - }, - getPath: getPathMock, - getAppPath: getAppPathMock, - getVersion: () => '1.2.3' - } - }), fs: () => ({ mkdirSync: vi.fn<(...args: unknown[]) => void>(), existsSync: (p: string) => probeSocketExistsMock(p) || p.includes('.pid'), diff --git a/src/main/daemon/daemon-init-endpoint-adoption.test.ts b/src/main/daemon/daemon-init-endpoint-adoption.test.ts index 2566ba0996e..5503188d430 100644 --- a/src/main/daemon/daemon-init-endpoint-adoption.test.ts +++ b/src/main/daemon/daemon-init-endpoint-adoption.test.ts @@ -25,7 +25,6 @@ const { (await import('./daemon-init-test-harness')).createDaemonInitMocks() ) -vi.mock('electron', () => moduleFactories.electron()) vi.mock('fs', () => moduleFactories.fs()) vi.mock('child_process', async (importOriginal) => moduleFactories.childProcess(await importOriginal>()) diff --git a/src/main/daemon/daemon-init-fresh-import.ts b/src/main/daemon/daemon-init-fresh-import.ts index 5515e5d213e..1d4f35d655c 100644 --- a/src/main/daemon/daemon-init-fresh-import.ts +++ b/src/main/daemon/daemon-init-fresh-import.ts @@ -1,9 +1,11 @@ import { vi } from 'vitest' +import { setAppEnvironment, type AppEnvironment } from '../../shared/app-environment' import type { DaemonInitMockState } from './daemon-init-test-harness' /** Resets every mock plus the module registry, then re-imports daemon-init so its module-level spawner/adapter/restartInFlight start fresh. */ export async function importFreshDaemonInit(state: DaemonInitMockState) { const { + getPathMock, getAppPathMock, isPackagedMock, probeSocketExistsMock, @@ -108,6 +110,17 @@ export async function importFreshDaemonInit(state: DaemonInitMockState) { launchedStartedAtMs.current = 1_000_000 getProcessStartedAtMsMock.mockReset() getProcessStartedAtMsMock.mockReturnValue(1_000_000) + // Why the real port rather than a module mock: daemon-init reads AppEnvironment, whose + // installed instance is anchored to a realm symbol precisely so it survives resetModules. + setAppEnvironment({ + getPath: getPathMock, + getAppPath: getAppPathMock, + getVersion: () => '1.2.3', + isPackaged: isPackagedMock, + onWillQuit: () => {}, + exit: () => {}, + getAppMetrics: () => [] + } as unknown as AppEnvironment) // Why: import after resetModules so module-level spawner/adapter/restartInFlight start fresh — needed to test first-init and the coalescer. return import('./daemon-init') } diff --git a/src/main/daemon/daemon-init-live-session-preservation.test.ts b/src/main/daemon/daemon-init-live-session-preservation.test.ts index 06c926386b9..2a3eece9937 100644 --- a/src/main/daemon/daemon-init-live-session-preservation.test.ts +++ b/src/main/daemon/daemon-init-live-session-preservation.test.ts @@ -23,7 +23,6 @@ const { (await import('./daemon-init-test-harness')).createDaemonInitMocks() ) -vi.mock('electron', () => moduleFactories.electron()) vi.mock('fs', () => moduleFactories.fs()) vi.mock('child_process', async (importOriginal) => moduleFactories.childProcess(await importOriginal>()) diff --git a/src/main/daemon/daemon-init-mock-types.ts b/src/main/daemon/daemon-init-mock-types.ts index c71c19d7e4f..8c8b805740f 100644 --- a/src/main/daemon/daemon-init-mock-types.ts +++ b/src/main/daemon/daemon-init-mock-types.ts @@ -4,6 +4,7 @@ import type { Mock } from 'vitest' export type MockSpawner = { ensureRunning: Mock resetHandle: Mock + resetRespawnWindow: Mock shutdown: Mock getHandle: Mock launcher: unknown diff --git a/src/main/daemon/daemon-init-packaged-bundle-staleness.test.ts b/src/main/daemon/daemon-init-packaged-bundle-staleness.test.ts index f3ee88e7b70..6683e627e6d 100644 --- a/src/main/daemon/daemon-init-packaged-bundle-staleness.test.ts +++ b/src/main/daemon/daemon-init-packaged-bundle-staleness.test.ts @@ -24,7 +24,6 @@ const { (await import('./daemon-init-test-harness')).createDaemonInitMocks() ) -vi.mock('electron', () => moduleFactories.electron()) vi.mock('fs', () => moduleFactories.fs()) vi.mock('child_process', async (importOriginal) => moduleFactories.childProcess(await importOriginal>()) diff --git a/src/main/daemon/daemon-init-provider-installation.test.ts b/src/main/daemon/daemon-init-provider-installation.test.ts index e8038894d69..ceb7e9dfa69 100644 --- a/src/main/daemon/daemon-init-provider-installation.test.ts +++ b/src/main/daemon/daemon-init-provider-installation.test.ts @@ -28,7 +28,6 @@ const { (await import('./daemon-init-test-harness')).createDaemonInitMocks() ) -vi.mock('electron', () => moduleFactories.electron()) vi.mock('fs', () => moduleFactories.fs()) vi.mock('child_process', async (importOriginal) => moduleFactories.childProcess(await importOriginal>()) @@ -255,6 +254,28 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => { expect(adapterInstances[0].listProcesses).toHaveBeenCalled() }) + it('answers daemonOwnsFreshPersistentPtys honestly for each installed provider', async () => { + // What orcad publishes as `canRecoverPersistentLocalPtys` and as the readiness health + // verdict. Answering true under degraded routing would advertise recovery for terminals + // that die with the runtime process. + const mod = await importFresh() + expect(mod.daemonOwnsFreshPersistentPtys()).toBe(false) + + await mod.initDaemonPtyProvider() + expect(mod.daemonOwnsFreshPersistentPtys()).toBe(true) + + const degraded = await importFresh() + ensureRunningOverrides.push(async () => ({ + socketPath: '/fake/degraded-socket', + tokenPath: '/fake/degraded-token', + mode: 'degraded-new-pty-fallback' + })) + await degraded.initDaemonPtyProvider() + const { DegradedDaemonPtyProvider } = await import('./degraded-daemon-pty-provider') + expect(degraded.getDaemonProvider()).toBeInstanceOf(DegradedDaemonPtyProvider) + expect(degraded.daemonOwnsFreshPersistentPtys()).toBe(false) + }) + it('rechecks the preserved daemon endpoint before recovering fresh-spawn routing', async () => { const mod = await importFresh() ensureRunningOverrides.push(async () => ({ diff --git a/src/main/daemon/daemon-init-replacement-reporting.test.ts b/src/main/daemon/daemon-init-replacement-reporting.test.ts index 5cc3f33a02a..92bed823fb3 100644 --- a/src/main/daemon/daemon-init-replacement-reporting.test.ts +++ b/src/main/daemon/daemon-init-replacement-reporting.test.ts @@ -26,7 +26,6 @@ const { (await import('./daemon-init-test-harness')).createDaemonInitMocks() ) -vi.mock('electron', () => moduleFactories.electron()) vi.mock('fs', () => moduleFactories.fs()) vi.mock('child_process', async (importOriginal) => moduleFactories.childProcess(await importOriginal>()) diff --git a/src/main/daemon/daemon-init-restart-sequence.test.ts b/src/main/daemon/daemon-init-restart-sequence.test.ts index 5f1468342ac..71417db986c 100644 --- a/src/main/daemon/daemon-init-restart-sequence.test.ts +++ b/src/main/daemon/daemon-init-restart-sequence.test.ts @@ -24,7 +24,6 @@ const { (await import('./daemon-init-test-harness')).createDaemonInitMocks() ) -vi.mock('electron', () => moduleFactories.electron()) vi.mock('fs', () => moduleFactories.fs()) vi.mock('child_process', async (importOriginal) => moduleFactories.childProcess(await importOriginal>()) diff --git a/src/main/daemon/daemon-init-wedged-daemon-grace.test.ts b/src/main/daemon/daemon-init-wedged-daemon-grace.test.ts index 7ad1b96abb7..18debd1342a 100644 --- a/src/main/daemon/daemon-init-wedged-daemon-grace.test.ts +++ b/src/main/daemon/daemon-init-wedged-daemon-grace.test.ts @@ -18,7 +18,6 @@ const { (await import('./daemon-init-test-harness')).createDaemonInitMocks() ) -vi.mock('electron', () => moduleFactories.electron()) vi.mock('fs', () => moduleFactories.fs()) vi.mock('child_process', async (importOriginal) => moduleFactories.childProcess(await importOriginal>()) diff --git a/src/main/daemon/daemon-init.ts b/src/main/daemon/daemon-init.ts index d6c4240b19b..77edbed6027 100644 --- a/src/main/daemon/daemon-init.ts +++ b/src/main/daemon/daemon-init.ts @@ -2,7 +2,7 @@ restart, teardown); the "swap the provider atomically" invariant keeps restart + singletons co-located. */ import { join } from 'node:path' import { randomUUID } from 'node:crypto' -import { app } from 'electron' +import { getAppEnvironment } from '../../shared/app-environment' import { mkdirSync, existsSync, readFileSync, unlinkSync } from 'node:fs' import { fork, type ChildProcess } from 'node:child_process' import { connect } from 'node:net' @@ -36,7 +36,7 @@ import { import { getDaemonLaunchIdentity } from './daemon-pid-identity' import { isDaemonStaleForCurrentBundle } from './daemon-bundle-staleness' import { killStaleDaemon } from './daemon-stale-kill' -import { parseDaemonPidFile } from './daemon-pid-file-parse' +import { parseDaemonPidFile, type ParsedDaemonPid } from './daemon-pid-file-parse' import { collectPinnedDaemonVersions, materializeRelocatedDaemonHost, @@ -102,21 +102,25 @@ let adapter: DaemonProvider | null = null let restartInFlight: Promise | null = null function getRuntimeDir(): string { - const dir = join(app.getPath('userData'), 'daemon') + const dir = join(getAppEnvironment().getPath('userData'), 'daemon') mkdirSync(dir, { recursive: true }) return dir } function getHistoryDir(): string { - const dir = join(app.getPath('userData'), 'terminal-history') + const dir = join(getAppEnvironment().getPath('userData'), 'terminal-history') mkdirSync(dir, { recursive: true }) return dir } function getDaemonEntryPath(): string { - const appPath = app.getAppPath() - // Why: packaged app.getAppPath() points at app.asar, so redirect to app.asar.unpacked where daemon-entry.js is fork-executable. - const basePath = app.isPackaged ? appPath.replace('app.asar', 'app.asar.unpacked') : appPath + const appPath = getAppEnvironment().getAppPath() + // Why: packaged getAppPath() points at app.asar, so redirect to app.asar.unpacked where daemon-entry.js is fork-executable. + // Why asar and not isPackaged: orcad is a packaged non-Electron host whose bundle root holds + // orcad.js and daemon-entry.js side by side with no asar to redirect (see parcel-watcher-entry-path.ts). + const basePath = appPath.includes('app.asar') + ? appPath.replace('app.asar', 'app.asar.unpacked') + : appPath const directEntryPath = join(basePath, 'daemon-entry.js') if (existsSync(directEntryPath)) { return directEntryPath @@ -124,6 +128,12 @@ function getDaemonEntryPath(): string { return join(basePath, 'out', 'main', 'daemon-entry.js') } +// macOS TCC attribution pins the daemon to a packaged app bundle; there is none on a Node host. +function resolvePackagedDarwinAppVersion(): string | null { + const environment = getAppEnvironment() + return process.platform === 'darwin' && environment.isPackaged() ? environment.getVersion() : null +} + // Why: pass a log-file arg so field failures are diagnosable, but honor the ORCA_DIAGNOSTICS_DISABLED privacy switch. function daemonLogArgs(): string[] { const disabled = (process.env.ORCA_DIAGNOSTICS_DISABLED ?? '').trim().toLowerCase() @@ -519,12 +529,12 @@ function createOutOfProcessLauncher( entryPath ) const stalePackagedBundle = - app.isPackaged && + getAppEnvironment().isPackaged() && (await isDaemonStaleForCurrentBundle( runtimeDir, socketPath, tokenPath, - app.getVersion() + getAppEnvironment().getVersion() )) if (identity === 'mismatch' || stalePackagedBundle) { // Why: replacing a healthy daemon kills its child PTYs; defer code freshness until no live sessions would be lost. @@ -662,7 +672,7 @@ function createOutOfProcessLauncher( trackDaemonReplaced(pendingReplacement.reason, pendingReplacement.liveSessionCount) } - const userDataPath = app.getPath('userData') + const userDataPath = getAppEnvironment().getPath('userData') // Why: on win32 packaged, stage a daemon-host copy in userData so its image escapes the NSIS updater's kill zone; lazy so it's off first-paint. Fail-open: null → in-dir host. const relocatedHost = materializeRelocatedDaemonHost() // Fork the relocated entry when available; otherwise the install-dir entry. @@ -681,7 +691,7 @@ function createOutOfProcessLauncher( '--entry-path', entryPath, '--app-version', - app.getVersion(), + getAppEnvironment().getVersion(), '--spawner-exec-path', process.execPath, ...(macosLoginSessionWatch ? ['--login-session-watch'] : []), @@ -699,7 +709,7 @@ function createOutOfProcessLauncher( env: { ...process.env, ELECTRON_RUN_AS_NODE: '1', - // Why: the detached plain-Node daemon can't call app.getPath(), but shell rcfiles must live outside swept tmp. + // Why: the detached plain-Node daemon has no AppEnvironment, but shell rcfiles must live outside swept tmp. ORCA_USER_DATA_PATH: userDataPath } } @@ -962,7 +972,7 @@ export async function initDaemonPtyProvider( pidPath: getDaemonPidPath(runtimeDir), profileScope: runtimeDir, runtimeDir, - packagedAppVersion: process.platform === 'darwin' && app.isPackaged ? app.getVersion() : null, + packagedAppVersion: resolvePackagedDarwinAppVersion(), historyPath: getHistoryDir(), // Why: on daemon death, ensureConnected() detects the dead socket and calls this to fork a replacement before retrying. respawn: async (reason: DaemonRespawnReason) => { @@ -1066,6 +1076,60 @@ async function reconcileSeededClaudeLivePtys(provider: DaemonProvider): Promise< } // Why: a narrow getter (not a raw export) keeps the "swap on restart" invariant in one place (replaceDaemonProvider). +/** + * Whether the installed provider is a daemon that will own FRESH terminals too. + * + * Why not `getDaemonProvider() !== null`: DegradedDaemonPtyProvider routes the daemon's + * EXISTING sessions to the daemon but spawns new ones on the in-process local provider, so + * those die with this process. A host that answered "I can recover persistent local PTYs" + * from that state would be advertising recovery for terminals that cannot be recovered. + */ +export function daemonOwnsFreshPersistentPtys(): boolean { + return adapter !== null && !(adapter instanceof DegradedDaemonPtyProvider) +} + +/** Endpoint coordinates of the daemon this process installed, for out-of-band health probes. */ +export type DaemonEndpointFacts = { + runtimeDir: string + socketPath: string + tokenPath: string + pidPath: string + protocolVersion: number +} + +export function getDaemonEndpointFacts(): DaemonEndpointFacts | null { + if (!adapter) { + return null + } + const runtimeDir = getRuntimeDir() + return { + runtimeDir, + socketPath: getDaemonSocketPath(runtimeDir), + tokenPath: getDaemonTokenPath(runtimeDir), + pidPath: getDaemonPidPath(runtimeDir), + protocolVersion: PROTOCOL_VERSION + } +} + +/** + * What the live daemon's own PID record says about the build it was forked from. + * + * Why the record and not this process's version: the daemon deliberately outlives the + * runtime, so after an update the two can legitimately disagree — and a health surface that + * reported orcad's version for both would hide exactly that. + */ +export function readDaemonPidRecord(): ParsedDaemonPid | null { + const facts = getDaemonEndpointFacts() + if (!facts) { + return null + } + try { + return parseDaemonPidFile(readFileSync(facts.pidPath, 'utf8')) + } catch { + return null + } +} + export function getDaemonProvider(): DaemonProvider | null { return adapter } @@ -1154,6 +1218,9 @@ async function runRestartDaemon(): Promise { } const runtimeDir = getRuntimeDir() + // An operator asking for a restart is the deliberate "try again" that clears crash-loop + // containment; without this a wedged host could never be recovered from the UI. + currentSpawner.resetRespawnWindow() const currentOnly = getCurrentDaemonAdapter(currentAdapter) const legacyAdapters = getLegacyDaemonAdapters(currentAdapter) @@ -1198,7 +1265,7 @@ async function runRestartDaemon(): Promise { pidPath: getDaemonPidPath(runtimeDir), profileScope: runtimeDir, runtimeDir, - packagedAppVersion: process.platform === 'darwin' && app.isPackaged ? app.getVersion() : null, + packagedAppVersion: resolvePackagedDarwinAppVersion(), historyPath: getHistoryDir(), respawn: async (reason: DaemonRespawnReason) => { // Why: attribute rather than emit — the launcher below is the one that completes the diff --git a/src/main/daemon/daemon-respawn-throttle.test.ts b/src/main/daemon/daemon-respawn-throttle.test.ts new file mode 100644 index 00000000000..70806f999e8 --- /dev/null +++ b/src/main/daemon/daemon-respawn-throttle.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it, vi } from 'vitest' +import { DaemonSpawner, type DaemonProcessHandle } from './daemon-spawner' +import { DaemonCrashLoopError, DaemonRespawnThrottle } from './daemon-respawn-throttle' + +describe('DaemonRespawnThrottle', () => { + it('admits up to the cap, then refuses with the time left in the window', () => { + let now = 0 + const throttle = new DaemonRespawnThrottle({ + maxAttempts: 3, + windowMs: 1_000, + now: () => now + }) + expect(throttle.admit().allowed).toBe(true) + now = 200 + expect(throttle.admit().allowed).toBe(true) + now = 400 + expect(throttle.admit().allowed).toBe(true) + now = 500 + const refused = throttle.admit() + expect(refused).toEqual({ + allowed: false, + reason: 'crash_loop', + attemptsInWindow: 3, + retryAfterMs: 500 + }) + }) + + it('admits again once the window slides past the oldest attempt', () => { + let now = 0 + const throttle = new DaemonRespawnThrottle({ maxAttempts: 2, windowMs: 1_000, now: () => now }) + throttle.admit() + throttle.admit() + expect(throttle.admit().allowed).toBe(false) + now = 1_500 + expect(throttle.admit().allowed).toBe(true) + }) + + it('clears the window on an explicit reset (the operator-restart escape hatch)', () => { + const throttle = new DaemonRespawnThrottle({ maxAttempts: 1, windowMs: 60_000 }) + expect(throttle.admit().allowed).toBe(true) + expect(throttle.admit().allowed).toBe(false) + throttle.reset() + expect(throttle.admit().allowed).toBe(true) + }) +}) + +describe('DaemonSpawner crash-loop containment', () => { + const handle: DaemonProcessHandle = { shutdown: async () => {} } + + it('stops forking once the daemon has died repeatedly inside the window', async () => { + const launcher = vi.fn(async () => handle) + const spawner = new DaemonSpawner({ + runtimeDir: '/tmp/orcad-throttle-test', + launcher, + respawnThrottle: new DaemonRespawnThrottle({ maxAttempts: 3, windowMs: 60_000 }) + }) + for (let i = 0; i < 3; i += 1) { + await spawner.ensureRunning() + // What a dead daemon looks like to the adapter's respawn path. + spawner.resetHandle() + } + expect(launcher).toHaveBeenCalledTimes(3) + await expect(spawner.ensureRunning()).rejects.toBeInstanceOf(DaemonCrashLoopError) + // The refusal must actually prevent the fork, not just annotate it. + expect(launcher).toHaveBeenCalledTimes(3) + }) + + it('does not count a cached handle as a new attempt', async () => { + const launcher = vi.fn(async () => handle) + const spawner = new DaemonSpawner({ + runtimeDir: '/tmp/orcad-throttle-test', + launcher, + respawnThrottle: new DaemonRespawnThrottle({ maxAttempts: 2, windowMs: 60_000 }) + }) + await spawner.ensureRunning() + await spawner.ensureRunning() + await spawner.ensureRunning() + expect(launcher).toHaveBeenCalledTimes(1) + }) + + it('lets an operator restart clear containment', async () => { + const launcher = vi.fn(async () => handle) + const spawner = new DaemonSpawner({ + runtimeDir: '/tmp/orcad-throttle-test', + launcher, + respawnThrottle: new DaemonRespawnThrottle({ maxAttempts: 1, windowMs: 60_000 }) + }) + await spawner.ensureRunning() + spawner.resetHandle() + await expect(spawner.ensureRunning()).rejects.toBeInstanceOf(DaemonCrashLoopError) + spawner.resetRespawnWindow() + await expect(spawner.ensureRunning()).resolves.toBeDefined() + expect(launcher).toHaveBeenCalledTimes(2) + }) +}) diff --git a/src/main/daemon/daemon-respawn-throttle.ts b/src/main/daemon/daemon-respawn-throttle.ts new file mode 100644 index 00000000000..9988a6b9fac --- /dev/null +++ b/src/main/daemon/daemon-respawn-throttle.ts @@ -0,0 +1,84 @@ +/** + * Crash-loop containment for the terminal daemon. + * + * The respawn path is driven by `ensureConnected()`: every reconnect attempt against a dead + * socket forks a replacement. A daemon that dies during startup — a broken node-pty, an + * unwritable runtime dir, a wrong libc — therefore forks forever, as fast as the caller + * retries. On the desktop that burns CPU; under orcad, where a supervisor is watching a + * process that never reports failure, it is the "restart-spin while the deploy reports + * success" shape the ops contract has to rule out. + * + * Sliding window, not a permanent trip: the failure is usually environmental, and an + * environment can be repaired without restarting the runtime. Once the window drains, the + * next attempt is admitted and a repaired host recovers on its own. + */ +export type DaemonRespawnAdmission = + | { allowed: true } + | { allowed: false; reason: 'crash_loop'; attemptsInWindow: number; retryAfterMs: number } + +export type DaemonRespawnThrottleOptions = { + /** Attempts allowed inside `windowMs` before the next one is refused. */ + maxAttempts?: number + windowMs?: number + now?: () => number +} + +// Generous on purpose: a user manually restarting the daemon a few times, or a laptop +// waking to a stale socket, must never trip this. Five failures inside a minute is a +// daemon that cannot start, not a daemon having a bad moment. +export const DEFAULT_DAEMON_RESPAWN_MAX_ATTEMPTS = 5 +export const DEFAULT_DAEMON_RESPAWN_WINDOW_MS = 60_000 + +export class DaemonRespawnThrottle { + private readonly maxAttempts: number + private readonly windowMs: number + private readonly now: () => number + private attempts: number[] = [] + + constructor(options: DaemonRespawnThrottleOptions = {}) { + this.maxAttempts = options.maxAttempts ?? DEFAULT_DAEMON_RESPAWN_MAX_ATTEMPTS + this.windowMs = options.windowMs ?? DEFAULT_DAEMON_RESPAWN_WINDOW_MS + this.now = options.now ?? Date.now + } + + /** Record and admit one respawn attempt, or refuse it as a crash loop. */ + admit(): DaemonRespawnAdmission { + const now = this.now() + this.attempts = this.attempts.filter((at) => now - at < this.windowMs) + if (this.attempts.length >= this.maxAttempts) { + const oldest = this.attempts[0] as number + return { + allowed: false, + reason: 'crash_loop', + attemptsInWindow: this.attempts.length, + retryAfterMs: Math.max(0, this.windowMs - (now - oldest)) + } + } + this.attempts.push(now) + return { allowed: true } + } + + /** + * Forget the recorded attempts. + * + * Why not automatic on a successful fork: a crash loop IS a sequence of successful forks + * followed by immediate deaths, so "the fork returned" is not evidence of recovery. Only + * a caller that knows the daemon stayed up — or a deliberate operator restart — may clear + * the window. + */ + reset(): void { + this.attempts = [] + } +} + +export class DaemonCrashLoopError extends Error { + readonly code = 'daemon_crash_loop' + constructor(admission: Extract) { + super( + `The terminal daemon has failed ${admission.attemptsInWindow} times in a row; refusing to ` + + `respawn it for another ${Math.ceil(admission.retryAfterMs / 1000)}s. Terminals will not ` + + 'start until the underlying failure is fixed (check the daemon log).' + ) + this.name = 'DaemonCrashLoopError' + } +} diff --git a/src/main/daemon/daemon-spawner.ts b/src/main/daemon/daemon-spawner.ts index 6a6d377cc2f..a0376ef0fc0 100644 --- a/src/main/daemon/daemon-spawner.ts +++ b/src/main/daemon/daemon-spawner.ts @@ -10,6 +10,7 @@ import { } from 'node:fs' import { join } from 'node:path' import { PROTOCOL_VERSION } from './types' +import { DaemonCrashLoopError, DaemonRespawnThrottle } from './daemon-respawn-throttle' export type DaemonConnectionInfo = { socketPath: string @@ -44,6 +45,8 @@ export type DaemonLauncher = ( export type DaemonSpawnerOptions = { runtimeDir: string launcher: DaemonLauncher + /** Test seam; production uses the default window. */ + respawnThrottle?: DaemonRespawnThrottle } export class DaemonSpawner { @@ -53,10 +56,12 @@ export class DaemonSpawner { private socketPath: string private tokenPath: string private pidPath: string + private respawnThrottle: DaemonRespawnThrottle constructor(opts: DaemonSpawnerOptions) { this.runtimeDir = opts.runtimeDir this.launcher = opts.launcher + this.respawnThrottle = opts.respawnThrottle ?? new DaemonRespawnThrottle() this.socketPath = getDaemonSocketPath(this.runtimeDir) this.tokenPath = getDaemonTokenPath(this.runtimeDir) this.pidPath = getDaemonPidPath(this.runtimeDir) @@ -67,6 +72,14 @@ export class DaemonSpawner { return { socketPath: this.socketPath, tokenPath: this.tokenPath } } + // Why here and not in the respawn callback: every launch — first, post-death, and + // post-restart — funnels through this method, so this is the only place a crash loop + // cannot route around. + const admission = this.respawnThrottle.admit() + if (!admission.allowed) { + throw new DaemonCrashLoopError(admission) + } + // Why: a detached daemon may clean up after its parent exits. A unique // launch identity keeps it from deleting a replacement daemon's PID file. this.handle = await this.launcher(this.socketPath, this.tokenPath, this.pidPath, randomUUID()) @@ -85,6 +98,17 @@ export class DaemonSpawner { this.handle = null } + /** + * Forget the crash-loop window. + * + * Why an explicit call and not "a fork succeeded": a crash loop is a run of successful + * forks whose daemons then die, so the fork returning proves nothing. An operator asking + * for a restart does mean "try again", and that is the only thing that clears it. + */ + resetRespawnWindow(): void { + this.respawnThrottle.reset() + } + async shutdown(): Promise { if (!this.handle) { return diff --git a/src/main/observability/logs-directory.ts b/src/main/observability/logs-directory.ts index 94a1b7baae8..20ad104af74 100644 --- a/src/main/observability/logs-directory.ts +++ b/src/main/observability/logs-directory.ts @@ -1,27 +1,29 @@ // Single source of truth for the app's logs directory and the files inside it. // macOS convention is `~/Library/Application Support/Orca/logs/`; Windows and -// Linux resolve the same intent via Electron's `userData` dir. Falls back to a -// homedir-derived path when Electron's `app` is unavailable (unit tests). +// Linux resolve the same intent via the host's `userData` dir. Falls back to a +// homedir-derived path when no AppEnvironment is installed (unit tests). -import { app } from 'electron' +import { getAppEnvironment, hasAppEnvironment } from '../../shared/app-environment' import { homedir, platform } from 'node:os' import { join } from 'node:path' +// Why the port and not electron's `app`: the daemon launch path reads this for --log-file, +// and that path has to resolve under plain Node (orcad) as well as the desktop. +// Why 'userData' + 'logs' rather than getPath('logs'): electron's 'logs' is ~/Library/Logs +// on macOS, which is NOT where this app has ever written. Changing it would strand +// existing log bundles. function getUserDataDir(): string { - try { - return app.getPath('userData') - } catch { - // Tests — Electron's `app` may not be initialized. Use an OS-conventional - // fallback so callers can resolve the path without the Electron runtime. - const home = homedir() - if (platform() === 'darwin') { - return join(home, 'Library', 'Application Support', 'Orca') - } - if (platform() === 'win32') { - return join(process.env.APPDATA ?? home, 'Orca') - } - return join(home, '.config', 'Orca') + if (hasAppEnvironment()) { + return getAppEnvironment().getPath('userData') } + const home = homedir() + if (platform() === 'darwin') { + return join(home, 'Library', 'Application Support', 'Orca') + } + if (platform() === 'win32') { + return join(process.env.APPDATA ?? home, 'Orca') + } + return join(home, '.config', 'Orca') } export function getLogsDirectory(): string { diff --git a/src/main/orcad/main-preflight-order.test.ts b/src/main/orcad/main-preflight-order.test.ts new file mode 100644 index 00000000000..bd130bfb33f --- /dev/null +++ b/src/main/orcad/main-preflight-order.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it, vi } from 'vitest' + +/** + * The precondition is only worth anything if it runs first. A loader failure is not + * catchable, so a preflight that lands after `main()` has already reached + * `await import('../ipc/pty')` prevents nothing. + */ +const order: string[] = [] + +vi.mock('./orcad-native-preflight', () => ({ + runOrcadNativePreflight: () => { + order.push('preflight') + return true + } +})) + +vi.mock('./orcad-entry', () => ({ + main: async () => { + order.push('main') + } +})) + +describe('orcad entry', () => { + it('runs the native preflight before starting the runtime', async () => { + await import('./main') + await vi.waitFor(() => expect(order).toContain('main')) + + expect(order).toEqual(['preflight', 'main']) + }) +}) diff --git a/src/main/orcad/main.ts b/src/main/orcad/main.ts index e76ed873cb0..73a973c7b24 100644 --- a/src/main/orcad/main.ts +++ b/src/main/orcad/main.ts @@ -1,8 +1,27 @@ /** Executable entry for `orcad`. See `./orcad-entry.ts`. */ import process from 'node:process' -import { main } from './orcad-entry' +import { main, resolveOrcadExitCode } from './orcad-entry' +import { runOrcadNativePreflight } from './orcad-native-preflight' + +// Why exit before the preflight: reaching this line means the whole module graph resolved +// under plain Node, which is all the build guard needs to prove. Probing natives or +// starting a server to prove it would bind a port and take a data-root lock on a build +// machine. +if (process.argv.includes('--orcad-smoke-load-check')) { + process.exit(0) +} + +// Why here and not inside startOrcad: this must run before anything requires node-pty, +// and `orcad-entry` reaches it through `await import('../ipc/pty')`. Static imports are +// evaluated before this statement, so the guarantee is that no module in the graph +// requires node-pty at import time — which the bundle's lazy `require("node-pty")` in +// local-pty-provider satisfies. See ./node-pty-precondition.ts for why a child process. +runOrcadNativePreflight() main().catch((error: unknown) => { console.error('orcad: failed to start:', error) - process.exit(1) + // Why a resolved code and not a bare 1: a data-root or bind-address refusal is a + // configuration fault that restarting cannot fix, and a supervisor needs to tell the two + // apart to avoid restart-spinning on it. + process.exit(resolveOrcadExitCode(error)) }) diff --git a/src/main/orcad/native-host-abi.test.ts b/src/main/orcad/native-host-abi.test.ts new file mode 100644 index 00000000000..44f469f0faa --- /dev/null +++ b/src/main/orcad/native-host-abi.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it } from 'vitest' +import { + compareDottedVersions, + detectLibcFromReportHeader, + detectNativeHostAbi, + GLIBC_FLOOR, + isBelowGlibcFloor, + nativeSlotName, + parseNodeAbiMismatch, + parseUnmetGlibcVersion +} from './native-host-abi' + +describe('detectLibcFromReportHeader', () => { + it('reads glibc and its version from a glibc host report', () => { + expect(detectLibcFromReportHeader('linux', { glibcVersionRuntime: '2.31' })).toEqual({ + libc: 'glibc', + glibcVersion: '2.31' + }) + }) + + it('calls a Linux report with no glibcVersionRuntime musl', () => { + // Alpine's Node omits the key entirely; that absence is the only signal available + // without shelling out to ldd, which musl images do not usefully provide. + expect(detectLibcFromReportHeader('linux', { arch: 'x64' })).toEqual({ + libc: 'musl', + glibcVersion: null + }) + }) + + it('does not invent a libc dimension for macOS or Windows', () => { + // A darwin-arm64-glibc slot would never match anything CI builds. + expect(detectLibcFromReportHeader('darwin', { glibcVersionRuntime: '2.31' }).libc).toBe('none') + expect(detectLibcFromReportHeader('win32', undefined).libc).toBe('none') + }) + + it('treats an unreadable report as glibc with an unknown version, not as musl', () => { + // Guessing musl would send a glibc host looking for a slot that does not exist. + expect(detectLibcFromReportHeader('linux', undefined)).toEqual({ + libc: 'glibc', + glibcVersion: null + }) + }) +}) + +describe('nativeSlotName', () => { + it('carries libc on Linux and omits it elsewhere', () => { + expect(nativeSlotName({ platform: 'linux', arch: 'x64', libc: 'glibc' })).toBe( + 'linux-x64-glibc' + ) + expect(nativeSlotName({ platform: 'linux', arch: 'arm64', libc: 'musl' })).toBe( + 'linux-arm64-musl' + ) + expect(nativeSlotName({ platform: 'darwin', arch: 'arm64', libc: 'none' })).toBe('darwin-arm64') + }) + + it('never lets a glibc slot answer for a musl host', () => { + // node-pty's own loader checks prebuilds/- with no libc, which is the + // exact confusion this name exists to prevent. + expect(nativeSlotName({ platform: 'linux', arch: 'x64', libc: 'glibc' })).not.toBe( + nativeSlotName({ platform: 'linux', arch: 'x64', libc: 'musl' }) + ) + }) +}) + +describe('glibc floor', () => { + it('compares dotted versions numerically, not lexically', () => { + // '2.9' > '2.31' under string compare; that ordering would pass a broken host. + expect(compareDottedVersions('2.9', '2.31')).toBe(-1) + expect(compareDottedVersions('2.31', '2.31.0')).toBe(0) + expect(compareDottedVersions('2.34', GLIBC_FLOOR)).toBe(1) + }) + + it('answers null when the version is unknown rather than claiming the floor is met', () => { + expect(isBelowGlibcFloor(null)).toBeNull() + expect(isBelowGlibcFloor('2.28')).toBe(true) + expect(isBelowGlibcFloor('2.31')).toBe(false) + }) +}) + +describe('loader error parsing', () => { + it('extracts the unmet symbol version from the #9902 message', () => { + expect( + parseUnmetGlibcVersion( + "/lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.34' not found (required by /app/node_modules/node-pty/build/Release/pty.node)" + ) + ).toBe('2.34') + }) + + it('ignores unrelated loader noise', () => { + expect(parseUnmetGlibcVersion('Error: Cannot find module ./pty.node')).toBeNull() + }) + + it('extracts both ABI numbers from a NODE_MODULE_VERSION mismatch', () => { + expect( + parseNodeAbiMismatch( + 'was compiled against a different Node.js version using NODE_MODULE_VERSION 115. This version of Node.js requires NODE_MODULE_VERSION 127.' + ) + ).toEqual({ built: '115', host: '127' }) + }) +}) + +describe('detectNativeHostAbi', () => { + it('describes the host this test is running on', () => { + const abi = detectNativeHostAbi() + expect(abi.platform).toBe(process.platform) + expect(abi.arch).toBe(process.arch) + expect(abi.nodeAbi).toBe(process.versions.modules) + expect(abi.libc).toBe(process.platform === 'linux' ? abi.libc : 'none') + }) +}) diff --git a/src/main/orcad/native-host-abi.ts b/src/main/orcad/native-host-abi.ts new file mode 100644 index 00000000000..4afb8ac7bfa --- /dev/null +++ b/src/main/orcad/native-host-abi.ts @@ -0,0 +1,127 @@ +/** + * What this host would load a native addon against: platform, arch, libc flavour and + * Node's ABI number, plus the prebuild slot name those four pick. + * + * Why libc is a first-class dimension: node-pty's own loader + * (`node_modules/node-pty/lib/utils.js`) falls back to `prebuilds/-`, + * which does NOT distinguish glibc from musl. A glibc binary dropped in that directory + * is loaded on Alpine and dies inside the dynamic loader. Slot names carry the libc so + * a mismatch is a miss rather than a crash. + * + * Everything here is pure apart from `detectNativeHostAbi`, so the classification can be + * tested for hosts this machine is not. + */ +import process from 'node:process' + +export type LibcFlavor = 'glibc' | 'musl' | 'none' + +export type NativeHostAbi = { + platform: NodeJS.Platform + arch: string + libc: LibcFlavor + /** Runtime glibc version ('2.31'), or null on musl, non-Linux, and unreadable reports. */ + glibcVersion: string | null + /** `NODE_MODULE_VERSION` — the addon ABI this runtime accepts. */ + nodeAbi: string +} + +/** Stock Ubuntu 20.04. See docs/reference/linux-glibc-compatibility.md. */ +export const GLIBC_FLOOR = '2.31' + +type ReportHeader = { glibcVersionRuntime?: unknown } + +/** + * Why absence means musl: `glibcVersionRuntime` is written by Node's report only when the + * process is linked against glibc. Alpine's Node omits it. This avoids shelling out to + * `ldd`, which is not present on every image and prints to stderr on musl. + * + * Non-Linux hosts get 'none': macOS and Windows have one system libc, so the dimension + * carries no information and must not widen the slot matrix. + */ +export function detectLibcFromReportHeader( + platform: NodeJS.Platform, + header: unknown +): { libc: LibcFlavor; glibcVersion: string | null } { + if (platform !== 'linux') { + return { libc: 'none', glibcVersion: null } + } + if (!header || typeof header !== 'object') { + // Why glibc and not 'unknown': an unreadable report is not evidence of musl, and + // glibc is the overwhelmingly common Linux case. The null version keeps the floor + // check from claiming a number it does not have. + return { libc: 'glibc', glibcVersion: null } + } + const runtime = (header as ReportHeader).glibcVersionRuntime + if (typeof runtime === 'string' && runtime.length > 0) { + return { libc: 'glibc', glibcVersion: runtime } + } + if ('glibcVersionRuntime' in (header as object)) { + return { libc: 'glibc', glibcVersion: null } + } + return { libc: 'musl', glibcVersion: null } +} + +export function detectNativeHostAbi(): NativeHostAbi { + let header: unknown + try { + header = (process.report?.getReport?.() as { header?: unknown } | undefined)?.header + } catch { + header = undefined + } + const { libc, glibcVersion } = detectLibcFromReportHeader(process.platform, header) + return { + platform: process.platform, + arch: process.arch, + libc, + glibcVersion, + nodeAbi: process.versions.modules + } +} + +/** `linux-x64-glibc`, `linux-arm64-musl`, `darwin-arm64`, `win32-x64`. */ +export function nativeSlotName(abi: Pick): string { + return abi.libc === 'none' + ? `${abi.platform}-${abi.arch}` + : `${abi.platform}-${abi.arch}-${abi.libc}` +} + +/** Numeric dotted compare: -1 / 0 / 1. Missing components read as 0, so '2.31' === '2.31.0'. */ +export function compareDottedVersions(left: string, right: string): number { + const a = left.split('.').map((part) => Number.parseInt(part, 10) || 0) + const b = right.split('.').map((part) => Number.parseInt(part, 10) || 0) + for (let i = 0; i < Math.max(a.length, b.length); i += 1) { + const diff = (a[i] ?? 0) - (b[i] ?? 0) + if (diff !== 0) { + return diff > 0 ? 1 : -1 + } + } + return 0 +} + +/** Null when the version is unknown — an unread report must not read as "below the floor". */ +export function isBelowGlibcFloor(glibcVersion: string | null): boolean | null { + if (!glibcVersion) { + return null + } + return compareDottedVersions(glibcVersion, GLIBC_FLOOR) < 0 +} + +/** + * The symbol version node's loader complained about, e.g. + * `libc.so.6: version 'GLIBC_2.34' not found (required by .../pty.node)` -> '2.34'. + * This is the fingerprint of #9902: a binary built on a newer glibc than the target. + */ +export function parseUnmetGlibcVersion(loaderError: string): string | null { + const match = loaderError.match(/version `?GLIBC_([0-9][0-9.]*)'? not found/) + return match ? match[1] : null +} + +/** `NODE_MODULE_VERSION 115 ... requires NODE_MODULE_VERSION 127` -> { built: '115', host: '127' }. */ +export function parseNodeAbiMismatch( + loaderError: string +): { built: string; host: string } | null { + const match = loaderError.match( + /NODE_MODULE_VERSION\s+(\d+)\D+NODE_MODULE_VERSION\s+(\d+)/ + ) + return match ? { built: match[1], host: match[2] } : null +} diff --git a/src/main/orcad/node-pty-prebuilt-slot.test.ts b/src/main/orcad/node-pty-prebuilt-slot.test.ts new file mode 100644 index 00000000000..00880eee1e5 --- /dev/null +++ b/src/main/orcad/node-pty-prebuilt-slot.test.ts @@ -0,0 +1,119 @@ +import { existsSync, mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + installPrebuiltSlot, + readPrebuiltSlotManifest, + resolveOrcadPrebuildsDir +} from './node-pty-prebuilt-slot' +import type { NativeHostAbi } from './native-host-abi' + +const LINUX_GLIBC: NativeHostAbi = { + platform: 'linux', + arch: 'x64', + libc: 'glibc', + glibcVersion: '2.31', + nodeAbi: '127' +} + +const dirs: string[] = [] +const temp = (): string => { + const dir = mkdtempSync(join(tmpdir(), 'orcad-slot-')) + dirs.push(dir) + return dir +} +afterEach(() => { + for (const dir of dirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }) + } + delete process.env.ORCA_ORCAD_PREBUILDS_DIR +}) + +const stageSlot = (prebuilds: string, slot: string): void => { + mkdirSync(join(prebuilds, slot), { recursive: true }) + writeFileSync(join(prebuilds, slot, 'pty.node'), 'binary') + writeFileSync(join(prebuilds, slot, 'spawn-helper'), 'helper') +} + +describe('resolveOrcadPrebuildsDir', () => { + it('looks beside the running bundle', () => { + expect(resolveOrcadPrebuildsDir('/opt/orcad/orcad.js')).toBe(join('/opt/orcad', 'prebuilds')) + }) + + it('honours an explicit override', () => { + process.env.ORCA_ORCAD_PREBUILDS_DIR = '/custom/prebuilds' + expect(resolveOrcadPrebuildsDir('/opt/orcad/orcad.js')).toBe('/custom/prebuilds') + }) +}) + +describe('installPrebuiltSlot', () => { + it('installs the slot binary and spawn-helper into build/Release', () => { + const prebuilds = temp() + const nodePtyDir = temp() + stageSlot(prebuilds, 'linux-x64-glibc') + + const outcome = installPrebuiltSlot({ abi: LINUX_GLIBC, nodePtyDir, prebuildsDir: prebuilds }) + + expect(outcome).toEqual({ installed: true, slot: 'linux-x64-glibc', spawnHelper: true }) + expect(existsSync(join(nodePtyDir, 'build', 'Release', 'pty.node'))).toBe(true) + // Without the executable bit every spawn fails EACCES at the moment a user opens a terminal. + const helper = statSync(join(nodePtyDir, 'build', 'Release', 'spawn-helper')) + expect(helper.mode & 0o111).not.toBe(0) + }) + + it('will not load a glibc slot on a musl host', () => { + // node-pty's own loader cannot tell these apart; the slot name is the only thing that can. + const prebuilds = temp() + const nodePtyDir = temp() + stageSlot(prebuilds, 'linux-x64-glibc') + + const outcome = installPrebuiltSlot({ + abi: { ...LINUX_GLIBC, libc: 'musl' }, + nodePtyDir, + prebuildsDir: prebuilds + }) + + expect(outcome).toEqual({ installed: false, slot: 'linux-x64-musl', why: 'no-slot' }) + expect(existsSync(join(nodePtyDir, 'build', 'Release', 'pty.node'))).toBe(false) + }) + + it('refuses an ABI-mismatched matrix instead of installing a binary that cannot load', () => { + // Installing it would turn "no prebuilt for this host" into a loader failure that + // reads as a corrupt install. + const prebuilds = temp() + const nodePtyDir = temp() + stageSlot(prebuilds, 'linux-x64-glibc') + writeFileSync( + join(prebuilds, 'manifest.json'), + JSON.stringify({ module: 'node-pty', version: '1.1.0', nodeAbi: '115', slots: [] }) + ) + + const outcome = installPrebuiltSlot({ abi: LINUX_GLIBC, nodePtyDir, prebuildsDir: prebuilds }) + + expect(outcome).toMatchObject({ installed: false, why: 'abi-mismatch' }) + expect(existsSync(join(nodePtyDir, 'build', 'Release', 'pty.node'))).toBe(false) + }) + + it('reports a missing prebuilds directory distinctly from a missing slot', () => { + // They mean different things: no matrix shipped at all, versus a matrix with a hole. + expect( + installPrebuiltSlot({ + abi: LINUX_GLIBC, + nodePtyDir: temp(), + prebuildsDir: join(temp(), 'absent') + }) + ).toEqual({ installed: false, slot: 'linux-x64-glibc', why: 'no-prebuilds-dir' }) + }) +}) + +describe('readPrebuiltSlotManifest', () => { + it('returns null for absent or malformed manifests rather than a half-built object', () => { + const prebuilds = temp() + expect(readPrebuiltSlotManifest(prebuilds)).toBeNull() + writeFileSync(join(prebuilds, 'manifest.json'), '{ not json') + expect(readPrebuiltSlotManifest(prebuilds)).toBeNull() + writeFileSync(join(prebuilds, 'manifest.json'), JSON.stringify({ version: '1.1.0' })) + expect(readPrebuiltSlotManifest(prebuilds)).toBeNull() + }) +}) diff --git a/src/main/orcad/node-pty-prebuilt-slot.ts b/src/main/orcad/node-pty-prebuilt-slot.ts new file mode 100644 index 00000000000..992cbe705ee --- /dev/null +++ b/src/main/orcad/node-pty-prebuilt-slot.ts @@ -0,0 +1,113 @@ +/** + * Install the shipped node-pty prebuilt that matches this host, so a deployment needs + * no C/C++ toolchain. + * + * Why copy into `build/Release` rather than leave it in `prebuilds/`: node-pty's own + * loader falls back to `prebuilds/-` with no libc in the name, so a + * glibc binary parked there is loaded on Alpine and dies in the dynamic loader. Putting + * the chosen slot's binary in `build/Release` is what makes the libc dimension real — + * node-pty only ever sees the one we picked. + * + * The prebuilds themselves are built from the PATCHED source (config/patches/node-pty@1.1.0.patch) + * by config/scripts/build-orcad-prebuilds.mjs. An upstream tarball would not do: the patch + * carries the `.symver` pins and the `--no-as-needed` libutil/libpthread flags that hold the + * Ubuntu 20.04 / glibc 2.31 floor (docs/reference/linux-glibc-compatibility.md). + */ +import { chmodSync, copyFileSync, existsSync, mkdirSync, readFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import process from 'node:process' +import { nativeSlotName, type NativeHostAbi } from './native-host-abi' + +export type PrebuiltSlotManifest = { + module: string + version: string + nodeAbi: string + slots: string[] +} + +export type PrebuiltSlotOutcome = + | { installed: true; slot: string; spawnHelper: boolean } + | { installed: false; slot: string; why: 'no-slot' | 'no-prebuilds-dir' | 'abi-mismatch'; detail?: string } + +/** + * Where a deployment's prebuilds live: beside the bundle that is running. `argv[1]` is + * `orcad.js` itself, so this stays correct wherever the install directory ends up. + */ +export function resolveOrcadPrebuildsDir(entryScript = process.argv[1]): string | null { + const override = process.env.ORCA_ORCAD_PREBUILDS_DIR + if (override) { + return override + } + return entryScript ? join(dirname(entryScript), 'prebuilds') : null +} + +export function readPrebuiltSlotManifest(prebuildsDir: string): PrebuiltSlotManifest | null { + try { + const parsed = JSON.parse(readFileSync(join(prebuildsDir, 'manifest.json'), 'utf8')) as unknown + if (!parsed || typeof parsed !== 'object') { + return null + } + const manifest = parsed as Partial + if (typeof manifest.nodeAbi !== 'string' || typeof manifest.version !== 'string') { + return null + } + return { + module: typeof manifest.module === 'string' ? manifest.module : 'node-pty', + version: manifest.version, + nodeAbi: manifest.nodeAbi, + slots: Array.isArray(manifest.slots) ? manifest.slots.filter((s) => typeof s === 'string') : [] + } + } catch { + return null + } +} + +/** + * Copy `//pty.node` (and `spawn-helper`) into node-pty's `build/Release`. + * + * Refuses on an ABI mismatch instead of copying: a binary built for another + * `NODE_MODULE_VERSION` cannot load, and installing it would replace a "no prebuilt" + * diagnosis with a loader failure that reads as a corrupt install. + */ +export function installPrebuiltSlot(options: { + abi: NativeHostAbi + nodePtyDir: string + prebuildsDir?: string | null +}): PrebuiltSlotOutcome { + const slot = nativeSlotName(options.abi) + const prebuildsDir = options.prebuildsDir ?? resolveOrcadPrebuildsDir() + if (!prebuildsDir || !existsSync(prebuildsDir)) { + return { installed: false, slot, why: 'no-prebuilds-dir' } + } + const manifest = readPrebuiltSlotManifest(prebuildsDir) + if (manifest && manifest.nodeAbi !== options.abi.nodeAbi) { + return { + installed: false, + slot, + why: 'abi-mismatch', + detail: `shipped prebuilds target Node ABI ${manifest.nodeAbi}, this host runs ABI ${options.abi.nodeAbi}` + } + } + const source = join(prebuildsDir, slot, 'pty.node') + if (!existsSync(source)) { + return { installed: false, slot, why: 'no-slot' } + } + const releaseDir = join(options.nodePtyDir, 'build', 'Release') + mkdirSync(releaseDir, { recursive: true }) + copyFileSync(source, join(releaseDir, 'pty.node')) + + // Why this matters as much as pty.node: on Unix node-pty posix_spawns + // build/Release/spawn-helper. Without it every spawn fails with ENOENT at the moment + // a user opens a terminal, long after the "install succeeded" line. + let spawnHelper = false + if (options.abi.platform !== 'win32') { + const helperSource = join(prebuildsDir, slot, 'spawn-helper') + if (existsSync(helperSource)) { + const helperDest = join(releaseDir, 'spawn-helper') + copyFileSync(helperSource, helperDest) + chmodSync(helperDest, 0o755) + spawnHelper = true + } + } + return { installed: true, slot, spawnHelper } +} diff --git a/src/main/orcad/node-pty-precondition.test.ts b/src/main/orcad/node-pty-precondition.test.ts new file mode 100644 index 00000000000..fadb144d1ff --- /dev/null +++ b/src/main/orcad/node-pty-precondition.test.ts @@ -0,0 +1,358 @@ +import { cpSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { spawnSync } from 'node:child_process' +import { createRequire } from 'node:module' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + buildNodePtyLoadProbeScript, + checkNodePtyPrecondition, + classifyNodePtyProbeResult, + formatNodePtyPreconditionReport, + probeLocalBuildToolchainHints +} from './node-pty-precondition' +import { detectNativeHostAbi } from './native-host-abi' + +const require = createRequire(import.meta.url) +const REAL_NODE_PTY = dirname(require.resolve('node-pty/package.json')) +const REAL_PTY_NODE = join(REAL_NODE_PTY, 'build', 'Release', 'pty.node') + +/** + * Whether this host's compiled node-pty actually loads under plain Node. + * + * Why not existsSync: CI ships a pty.node built for Electron's ABI, so the file is + * present and `require` still fails. Gating on existence ran the load-dependent tests + * on a host that could never satisfy them. Probed in a child so a bad binding cannot + * take the test runner down with it. + */ +const REAL_SPAWN_HELPER = join(REAL_NODE_PTY, 'build', 'Release', 'spawn-helper') + +const realNodePtyLoads = ((): boolean => { + if (!existsSync(REAL_PTY_NODE)) { + return false + } + // Why spawn-helper too: a slot without it is legitimately 'degraded', so a test that + // expects 'ok' has an unsatisfiable premise on a host that lacks it. CI has the + // binding but not the helper, which is what made the previous gate insufficient. + if (process.platform !== 'win32' && !existsSync(REAL_SPAWN_HELPER)) { + return false + } + const probe = spawnSync(process.execPath, ['-e', `require(${JSON.stringify(REAL_PTY_NODE)})`], { + encoding: 'utf8', + timeout: 30_000 + }) + return !probe.error && probe.status === 0 +})() + +const probe = (overrides: Partial[0]> = {}) => + classifyNodePtyProbeResult({ + code: 1, + signal: null, + stdout: '', + stderr: '', + timedOut: false, + ...overrides + }) + +const reported = (message: string) => + classifyNodePtyProbeResult({ + code: 4, + signal: null, + stdout: `ORCA_NODE_PTY_LOAD_ERROR ${JSON.stringify(message)}\n`, + stderr: '', + timedOut: false + }) + +describe('classifyNodePtyProbeResult', () => { + it('accepts only a clean exit that printed the token', () => { + expect(probe({ code: 0, stdout: 'ORCA_NODE_PTY_LOAD_OK /x/build/Release' })).toBeNull() + // A zero exit with no token means the probe never reached the load. + expect(probe({ code: 0, stdout: '' })?.reason).toBe('load_failed') + }) + + it('names the glibc floor for the #9902 loader message', () => { + expect( + reported( + "/lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.34' not found (required by /app/node_modules/node-pty/build/Release/pty.node)" + ) + ).toEqual({ + status: 'blocked', + reason: 'libc_floor', + detail: 'the binary requires GLIBC_2.34' + }) + }) + + it('names a libstdc++ floor break too', () => { + expect(reported("version `GLIBCXX_3.4.29' not found")?.reason).toBe('libc_floor') + }) + + it('separates a Node ABI mismatch from a libc mismatch', () => { + // These need different fixes — rebuild against this Node vs. build on an older libc — + // so collapsing them sends the operator to the wrong one. + expect( + reported( + 'was compiled against a different Node.js version using NODE_MODULE_VERSION 115. This version of Node.js requires NODE_MODULE_VERSION 127.' + ) + ).toEqual({ + status: 'blocked', + reason: 'abi_mismatch', + detail: 'built for Node ABI 115, this host runs ABI 127' + }) + }) + + it('reports a signalled probe as a crash, even with no output at all', () => { + // The uncatchable case: a binary that aborts inside the loader never reaches the + // child's catch and often prints nothing. + expect(probe({ code: null, signal: 'SIGSEGV' })).toEqual({ + status: 'blocked', + reason: 'load_crashed', + detail: 'the load probe was killed by SIGSEGV' + }) + }) + + it('distinguishes no binary anywhere from a binary the loader refused', () => { + // "install node-pty" and "rebuild node-pty for this libc" are different instructions. + expect(probe({ code: 3, stdout: 'ORCA_NODE_PTY_NO_BINARY\n' })).toEqual({ + status: 'blocked', + reason: 'dependency_missing', + detail: 'node-pty is installed but has no compiled binary for this platform' + }) + expect(reported('dlopen(...): slice is not valid mach-o file')?.reason).toBe('load_failed') + }) + + it('ignores its own token strings echoed back inside the child stderr', () => { + // node prints the whole `-e` source above the stack trace, and that source contains + // every token below. Matching on stderr made a refused binary read as "not installed". + const echoedSource = + '[eval]:1\nif(!f){console.log("ORCA_NODE_PTY_NO_BINARY");process.exit(3)}\n' + + ' ^\n\nError: dlopen(/app/pty.node): slice is not valid mach-o file\n' + + expect( + classifyNodePtyProbeResult({ + code: 4, + signal: null, + stdout: `ORCA_NODE_PTY_LOAD_ERROR ${JSON.stringify('dlopen(/app/pty.node): slice is not valid mach-o file')}`, + stderr: echoedSource, + timedOut: false + }) + ).toMatchObject({ reason: 'load_failed' }) + }) + + it('reads past node\u2019s echoed source line when it can only use stderr', () => { + const failure = probe({ + stderr: + '[eval]:1\nprocess.dlopen({exports:{}},f);\n ^\n\nError: something specific went wrong\n' + }) + expect(failure?.detail).toBe('Error: something specific went wrong') + }) + + it('calls a timeout unverifiable rather than blocked', () => { + // A probe that never answered is not evidence that node-pty is broken, and refusing + // to boot on it would take down hosts that work. + expect(probe({ timedOut: true })).toEqual({ + status: 'unverifiable', + reason: 'unknown', + detail: 'the node-pty load probe did not finish in time, so nothing was established' + }) + }) +}) + +describe('buildNodePtyLoadProbeScript', () => { + it('loads through absolute paths so the child cannot resolve a different copy', () => { + const script = buildNodePtyLoadProbeScript('/opt/app/node_modules/node-pty') + expect(script).toContain('"/opt/app/node_modules/node-pty/lib/index.js"') + expect(script).toContain('"/opt/app/node_modules/node-pty/lib/utils.js"') + expect(script).toContain('loadNativeModule') + // Windows defers conpty.node to first spawn, so requiring the package proves nothing there. + expect(script).toContain('conpty') + // The raw dlopen must precede requiring the package, or node-pty's own loader + // re-wraps the loader error into a misleading "Cannot find module". + expect(script.indexOf('process.dlopen')).toBeLessThan(script.indexOf('lib/index.js')) + }) +}) + +describe('checkNodePtyPrecondition', () => { + const temporaryDirs: string[] = [] + const stageNodePty = (): string => { + const root = mkdtempSync(join(tmpdir(), 'orcad-node-pty-')) + temporaryDirs.push(root) + const dir = join(root, 'node-pty') + mkdirSync(join(dir, 'build', 'Release'), { recursive: true }) + cpSync(join(REAL_NODE_PTY, 'lib'), join(dir, 'lib'), { recursive: true }) + cpSync(join(REAL_NODE_PTY, 'package.json'), join(dir, 'package.json')) + return dir + } + + afterEach(() => { + for (const dir of temporaryDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it('survives a native binary that the dynamic loader refuses', () => { + // The whole reason the probe is a child process: this file is loaded with dlopen, and + // an in-process require of it can take the host down before any handler runs. Reaching + // the assertion below at all is the evidence. + const dir = stageNodePty() + writeFileSync(join(dir, 'build', 'Release', 'pty.node'), Buffer.from('not a native addon')) + + const verdict = checkNodePtyPrecondition({ nodePtyDir: dir, prebuildsDir: null }) + + expect(verdict.status).toBe('blocked') + expect(verdict.reason).toBeDefined() + expect(verdict.reason).not.toBe('spawn_helper_missing') + }) + + it('blocks when node-pty is not resolvable at all', () => { + const verdict = checkNodePtyPrecondition({ nodePtyDir: null, prebuildsDir: null }) + expect(verdict).toMatchObject({ status: 'blocked', reason: 'dependency_missing' }) + }) + + it('returns a self-consistent verdict against the real host', () => { + // Why not a predicted status: this depends on how the host was prepared. CI's test + // shard runs `vitest` directly, so `ensure-native-runtime --runtime=node` never + // builds node-pty for the Node ABI and `degraded` is correct there; a prepared + // checkout gives 'ok'. Predicting either encodes an environment. + // + // Why not require('node-pty') as ground truth: that resolves the JS wrapper while + // the native binding loads lazily, so it proves strictly less than this checks — + // that was the first version of this test and it failed on CI for that reason. + // + // What is invariant: on a host where node-pty is installed at all, the verdict is + // never 'blocked' and never carries an unestablished reason. + const verdict = checkNodePtyPrecondition({ prebuildsDir: null }) + + // Why not a fixed status: a prepared host gives 'ok', CI's unprepared shard gives + // 'degraded', and a corrupt binding gives 'blocked' — all three are honest. What is + // invariant is that anything other than 'ok' names an established cause, so the + // host can never decline a terminal for a reason it did not work out. + expect(['ok', 'degraded', 'blocked', 'unverifiable']).toContain(verdict.status) + if (verdict.status !== 'ok') { + expect(verdict.reason).toBeDefined() + expect(verdict.reason).not.toBe('unknown') + } + expect(verdict.slot).toBe( + process.platform === 'linux' + ? `linux-${process.arch}-${verdict.abi.libc}` + : `${process.platform}-${process.arch}` + ) + }) + + // Why gated on the real binding: this asserts a LOAD outcome, so it needs a pty.node + // built for the Node ABI. CI's shard never runs ensure-native-runtime, so the copy + // ENOENT'd there. + it.runIf(process.platform !== 'win32' && realNodePtyLoads)( + 'degrades rather than blocks when only spawn-helper is missing', + () => { + // node-pty posix_spawns spawn-helper, so this host loads fine and then fails ENOENT + // the first time someone opens a terminal. Everything else it serves still works. + const dir = stageNodePty() + cpSync( + join(REAL_NODE_PTY, 'build', 'Release', 'pty.node'), + join(dir, 'build', 'Release', 'pty.node') + ) + + const verdict = checkNodePtyPrecondition({ nodePtyDir: dir, prebuildsDir: null }) + + expect(verdict).toMatchObject({ status: 'degraded', reason: 'spawn_helper_missing' }) + } + ) + + // Why split: the "ok" half needs a REAL loadable pty.node, which only exists after + // `ensure-native-runtime --runtime=node`. CI's shard runs vitest directly, so copying + // from node_modules ENOENT'd there. Slot *placement* is the logic worth checking on + // every host; the load verdict needs a prepared one. + it('places the matching slot even when the payload is not loadable', () => { + const dir = stageNodePty() + const abi = detectNativeHostAbi() + const slot = + abi.libc === 'none' + ? `${abi.platform}-${abi.arch}` + : `${abi.platform}-${abi.arch}-${abi.libc}` + const prebuildsDir = mkdtempSync(join(tmpdir(), 'orcad-prebuilds-')) + temporaryDirs.push(prebuildsDir) + mkdirSync(join(prebuildsDir, slot), { recursive: true }) + writeFileSync(join(prebuildsDir, slot, 'pty.node'), 'not a real binding') + if (process.platform !== 'win32') { + writeFileSync(join(prebuildsDir, slot, 'spawn-helper'), '#!/bin/sh\nexit 0\n') + } + + const verdict = checkNodePtyPrecondition({ nodePtyDir: dir, prebuildsDir }) + + // Installed from the right slot, and honest that the payload does not load. + expect(verdict.prebuilt).toMatchObject({ installed: true, slot }) + expect(verdict.status).not.toBe('ok') + expect(verdict.reason).toBeDefined() + }) + + it.runIf(realNodePtyLoads)( + 'reports ok once a loadable slot is installed (needs a Node-ABI build)', + () => { + const dir = stageNodePty() + const abi = detectNativeHostAbi() + const slot = + abi.libc === 'none' + ? `${abi.platform}-${abi.arch}` + : `${abi.platform}-${abi.arch}-${abi.libc}` + const prebuildsDir = mkdtempSync(join(tmpdir(), 'orcad-prebuilds-')) + temporaryDirs.push(prebuildsDir) + mkdirSync(join(prebuildsDir, slot), { recursive: true }) + cpSync(REAL_PTY_NODE, join(prebuildsDir, slot, 'pty.node')) + const helper = REAL_SPAWN_HELPER + if (process.platform !== 'win32' && existsSync(helper)) { + cpSync(helper, join(prebuildsDir, slot, 'spawn-helper')) + } + + const verdict = checkNodePtyPrecondition({ nodePtyDir: dir, prebuildsDir }) + + expect(verdict.prebuilt).toMatchObject({ installed: true, slot }) + expect(verdict.status).toBe('ok') + } + ) +}) + +describe('formatNodePtyPreconditionReport', () => { + it('names the host, the slot and the action', () => { + const report = formatNodePtyPreconditionReport( + { + status: 'blocked', + slot: 'linux-x64-musl', + abi: { + platform: 'linux', + arch: 'x64', + libc: 'musl', + glibcVersion: null, + nodeAbi: '127' + }, + reason: 'dependency_missing', + prebuilt: { installed: false, slot: 'linux-x64-musl', why: 'no-slot' } + }, + 'Terminals are unavailable on this host.', + [' sudo apk add build-base python3'] + ) + + expect(report).toContain('platform linux/x64') + expect(report).toContain('libc musl') + expect(report).toContain('prebuild slot linux-x64-musl') + expect(report).toContain('No shipped prebuilt matches slot linux-x64-musl.') + expect(report).toContain('sudo apk add build-base python3') + }) +}) + +describe('probeLocalBuildToolchainHints', () => { + it('gives macOS the Xcode command line tools, not a Linux package manager', () => { + // The relay's hint list answers with a cross-distro apt/dnf/pacman/apk menu when it + // finds no package manager. On macOS every line of that menu is wrong. + const hints = probeLocalBuildToolchainHints('darwin') + expect(hints).toEqual([' xcode-select --install']) + expect(hints.join('\n')).not.toMatch(/apt-get|dnf|pacman|apk/) + }) + + it('says nothing on Windows, where node-pty ships prebuilds', () => { + expect(probeLocalBuildToolchainHints('win32')).toEqual([]) + }) + + it.runIf(process.platform === 'linux')('reuses the relay diagnosis on Linux', () => { + expect(probeLocalBuildToolchainHints('linux').length).toBeGreaterThan(0) + }) +}) diff --git a/src/main/orcad/node-pty-precondition.ts b/src/main/orcad/node-pty-precondition.ts new file mode 100644 index 00000000000..2857449c736 --- /dev/null +++ b/src/main/orcad/node-pty-precondition.ts @@ -0,0 +1,365 @@ +/** + * Prove `node-pty` can be loaded on this host BEFORE anything in the process requires it. + * + * Why this exists: of the two ways node-pty fails, only one is catchable. A missing + * module throws `MODULE_NOT_FOUND` and a caller can degrade. A module that is present but + * built against the wrong libc or Node ABI is refused by the dynamic loader, and in the + * worst case takes the process down before any handler exists — that is #9902, which + * crashed the desktop app on Ubuntu 20.04 before a window appeared + * (docs/reference/linux-glibc-compatibility.md). + * + * So the load happens in a CHILD process. Whatever the child does — throw, abort, die on + * a signal — is data to us rather than our own death, and the operator gets a sentence + * naming what to change instead of a loader stack trace. + * + * The cost is one short-lived `node -e` at startup. That is the price of turning an + * uncatchable failure into a catchable one, and it is paid once per boot. + */ +import { existsSync, accessSync, constants } from 'node:fs' +import { dirname, join } from 'node:path' +import process from 'node:process' +import { runProcessSync, type ProcessResult } from '../../shared/child-process/run-process' +import type { RuntimeTerminalUnavailableReason } from '../../shared/runtime-types' +import { + buildToolchainProbeCommand, + parseBuildToolchainProbe, + toolchainInstallHintLines +} from '../ssh/build-toolchain-diagnosis' +import { + detectNativeHostAbi, + nativeSlotName, + parseNodeAbiMismatch, + parseUnmetGlibcVersion, + type NativeHostAbi +} from './native-host-abi' +import { installPrebuiltSlot, type PrebuiltSlotOutcome } from './node-pty-prebuilt-slot' + +// Why every verdict travels on STDOUT: node echoes the whole `-e` source into stderr +// before the stack trace, so any substring test against stderr also matches this file's +// own token strings. stdout carries only what the child chose to print. +const PROBE_OK_TOKEN = 'ORCA_NODE_PTY_LOAD_OK' +const NO_BINARY_TOKEN = 'ORCA_NODE_PTY_NO_BINARY' +const LOAD_ERROR_TOKEN = 'ORCA_NODE_PTY_LOAD_ERROR' +const PROBE_TIMEOUT_MS = 20_000 + +/** + * `ok` — proved loadable. `degraded` — loads, but something only spawn-time needs is + * broken, so the host should still serve everything else. `blocked` — proved unloadable, + * so nothing in this process may require it. `unverifiable` — the probe itself did not + * answer, which is not evidence either way. + * + * Why `unverifiable` is separate from `blocked`: a probe that times out or cannot spawn + * says nothing about node-pty, and refusing to boot on it would brick working hosts for + * a reason that was never established. Same verdict discipline as + * docs/reference/ssh-execution-boundary.md — loss of contact is not proof of death. + */ +export type NodePtyPreconditionStatus = 'ok' | 'degraded' | 'blocked' | 'unverifiable' + +export type NodePtyPreconditionVerdict = { + status: NodePtyPreconditionStatus + slot: string + abi: NativeHostAbi + reason?: RuntimeTerminalUnavailableReason + detail?: string + /** What the slot install did, when one was attempted. */ + prebuilt?: PrebuiltSlotOutcome +} + +export type NodePtyProbeFailure = { + status: 'blocked' | 'unverifiable' + reason: RuntimeTerminalUnavailableReason + detail: string +} + +/** + * Read the child's exit into a cause. Pure, so every failure shape is testable from a + * host that cannot reproduce it — the whole point, since the shapes that matter belong + * to Alpine and Ubuntu 20.04. + */ +export function classifyNodePtyProbeResult( + result: Pick +): NodePtyProbeFailure | null { + const stdout = result.stdout + if (result.code === 0 && stdout.includes(PROBE_OK_TOKEN)) { + return null + } + if (result.timedOut) { + return { + status: 'unverifiable', + reason: 'unknown', + detail: 'the node-pty load probe did not finish in time, so nothing was established' + } + } + // Why signal before anything the child said: a binary that aborts or segfaults inside + // the loader never reaches the catch, and often prints nothing at all. That silence is + // exactly the uncatchable case this probe is a separate process for. + if (result.signal) { + return { + status: 'blocked', + reason: 'load_crashed', + detail: `the load probe was killed by ${result.signal}` + } + } + if (stdout.includes(NO_BINARY_TOKEN)) { + return { + status: 'blocked', + reason: 'dependency_missing', + detail: 'node-pty is installed but has no compiled binary for this platform' + } + } + const reported = readReportedLoadError(stdout) + if (reported !== null) { + return classifyLoaderMessage(reported) + } + return { + status: 'blocked', + reason: 'load_failed', + detail: firstLine(result.stderr) || `the load probe exited with code ${result.code}` + } +} + +/** The message the child caught, or null when it never got that far. */ +function readReportedLoadError(stdout: string): string | null { + const line = stdout.split('\n').find((candidate) => candidate.startsWith(LOAD_ERROR_TOKEN)) + if (!line) { + return null + } + try { + return JSON.parse(line.slice(LOAD_ERROR_TOKEN.length).trim()) as string + } catch { + return line.slice(LOAD_ERROR_TOKEN.length).trim() + } +} + +/** Read a dynamic-loader message. Pure, so shapes this host cannot reproduce are testable. */ +export function classifyLoaderMessage(message: string): NodePtyProbeFailure { + const abiMismatch = parseNodeAbiMismatch(message) + if (abiMismatch) { + return { + status: 'blocked', + reason: 'abi_mismatch', + detail: `built for Node ABI ${abiMismatch.built}, this host runs ABI ${abiMismatch.host}` + } + } + const unmetGlibc = parseUnmetGlibcVersion(message) + if (unmetGlibc) { + return { + status: 'blocked', + reason: 'libc_floor', + detail: `the binary requires GLIBC_${unmetGlibc}` + } + } + if (/(GLIBCXX_|CXXABI_)[0-9.]+'? not found/.test(message)) { + return { status: 'blocked', reason: 'libc_floor', detail: firstLine(message) } + } + if (/MODULE_NOT_FOUND|Cannot find module/.test(message)) { + return { status: 'blocked', reason: 'dependency_missing', detail: firstLine(message) } + } + return { status: 'blocked', reason: 'load_failed', detail: firstLine(message) } +} + +/** + * Why not simply the first non-empty line: when the child dies without catching, node + * prints the offending source line and a caret before the error, so line one is the + * script rather than the diagnosis. Prefer the first line that reads as an error. + */ +function firstLine(text: string): string { + const lines = text.split('\n').filter((candidate) => candidate.trim().length > 0) + const errorLine = lines.find((candidate) => /^[A-Za-z]*(Error|Exception):/.test(candidate.trim())) + return (errorLine ?? lines[0] ?? text).trim().slice(0, 400) +} + +/** + * The script the child runs. + * + * Why it dlopens the file itself rather than trusting node-pty's loader: that loader + * tries several directories and rethrows only the LAST error, so a `pty.node` the + * dynamic loader refused is reported as `Cannot find module './prebuilds/...'`. Acting on + * that sends the operator to install a module that is already there. The dlopen has to + * come BEFORE `require(index.js)` for the same reason: node-pty's unixTerminal calls the + * loader at module scope, so requiring the package first re-wraps the error we came for. + * + * Why it catches and prints instead of throwing: a thrown error reaches us as a stack + * trace with the script source echoed above it, and the message we need is then one line + * inside a blob that also contains these very tokens. What the child cannot catch — a + * loader that aborts the process — still reaches us as a signal, which is the case this + * whole indirection exists for. + */ +export function buildNodePtyLoadProbeScript(nodePtyDir: string): string { + const entry = JSON.stringify(join(nodePtyDir, 'lib', 'index.js')) + const utils = JSON.stringify(join(nodePtyDir, 'lib', 'utils.js')) + const root = JSON.stringify(nodePtyDir) + // Same directory order node-pty's own loader walks, so the file opened here is the file + // it would load. Windows defers conpty.node to the first spawn, which is why the name is + // chosen the way node-pty chooses it rather than always being 'pty'. + return [ + `const fs=require('fs'),p=require('path');`, + `const n=process.platform==='win32'&&Number(require('os').release().split('.')[2])>=18309?'conpty':'pty';`, + `let f=null;`, + `for(const d of ['build/Release','build/Debug','prebuilds/'+process.platform+'-'+process.arch]){`, + `for(const r of [${root},p.join(${root},'lib')]){`, + `const c=p.join(r,d,n+'.node');if(fs.existsSync(c)){f=c;break}}if(f)break}`, + `if(!f){console.log(${JSON.stringify(NO_BINARY_TOKEN)});process.exit(3)}`, + `try{`, + `process.dlopen({exports:{}},f);`, + `require(${entry});`, + `require(${utils}).loadNativeModule(n);`, + `console.log(${JSON.stringify(PROBE_OK_TOKEN)}+' '+p.dirname(f));`, + `}catch(e){`, + `console.log(${JSON.stringify(LOAD_ERROR_TOKEN)}+' '+JSON.stringify(String((e&&e.message)||e)));`, + `process.exit(4)}` + ].join('') +} + +function resolveNodePtyDir(): string | null { + try { + // Why require.resolve and not import: resolution only — the load itself happens in + // the child process, which is the whole point of the precondition. + return dirname(require.resolve('node-pty/package.json')) + } catch { + return null + } +} + +/** Local equivalent of the relay's remote toolchain probe, reusing its pure half. */ +export function probeLocalBuildToolchainHints(platform: NodeJS.Platform): string[] { + if (platform === 'win32') { + return [] + } + // Why macOS is not routed through the relay's hints: that function answers with a + // cross-distro apt/dnf/pacman/apk menu when it finds no package manager, and none of + // those lines is the macOS answer. Printing them here would be confidently wrong. + if (platform === 'darwin') { + return [' xcode-select --install'] + } + try { + const result = runProcessSync({ + program: '/bin/sh', + args: ['-c', buildToolchainProbeCommand()], + timeoutMs: 10_000 + }) + return toolchainInstallHintLines(parseBuildToolchainProbe(result.stdout)) + } catch { + return [] + } +} + +export function checkNodePtyPrecondition( + options: { nodePtyDir?: string | null; abi?: NativeHostAbi; prebuildsDir?: string | null } = {} +): NodePtyPreconditionVerdict { + const abi = options.abi ?? detectNativeHostAbi() + const slot = nativeSlotName(abi) + // Why `in` and not `??`: an explicit `null` means "this host cannot resolve node-pty", + // which is a case tests must be able to state. `??` would silently re-detect instead. + const nodePtyDir = 'nodePtyDir' in options ? options.nodePtyDir : resolveNodePtyDir() + if (!nodePtyDir) { + return { + status: 'blocked', + slot, + abi, + reason: 'dependency_missing', + detail: 'node-pty is not resolvable from this install' + } + } + + // Why install before probing: on a toolchain-free deployment the compiled binary does + // not exist yet, and the shipped slot is the only thing that can make the probe pass. + let prebuilt: PrebuiltSlotOutcome | undefined + if (!existsSync(join(nodePtyDir, 'build', 'Release', 'pty.node'))) { + prebuilt = installPrebuiltSlot({ + abi, + nodePtyDir, + ...(options.prebuildsDir === undefined ? {} : { prebuildsDir: options.prebuildsDir }) + }) + } + + let result: ProcessResult + try { + result = runProcessSync({ + program: process.execPath, + args: ['-e', buildNodePtyLoadProbeScript(nodePtyDir)], + timeoutMs: PROBE_TIMEOUT_MS + }) + } catch (error) { + return { + status: 'unverifiable', + slot, + abi, + reason: 'unknown', + detail: `the node-pty load probe could not be started: ${(error as Error).message}`, + ...(prebuilt ? { prebuilt } : {}) + } + } + const failure = classifyNodePtyProbeResult(result) + if (failure) { + return { + status: failure.status, + slot, + abi, + reason: failure.reason, + detail: failure.detail, + ...(prebuilt ? { prebuilt } : {}) + } + } + + // Loaded. The remaining way terminals fail is spawn-time: node-pty posix_spawns + // build/Release/spawn-helper, and a missing one turns every terminal.create into ENOENT + // on a host that otherwise looks healthy. That is a degradation, not a boot blocker. + const loadedDir = result.stdout.split(PROBE_OK_TOKEN)[1]?.trim().split('\n')[0]?.trim() + if (abi.platform !== 'win32') { + const helper = join(loadedDir || join(nodePtyDir, 'build', 'Release'), 'spawn-helper') + if (!isExecutableFile(helper)) { + return { + status: 'degraded', + slot, + abi, + reason: 'spawn_helper_missing', + detail: `expected an executable at ${helper}`, + ...(prebuilt ? { prebuilt } : {}) + } + } + } + return { status: 'ok', slot, abi, ...(prebuilt ? { prebuilt } : {}) } +} + +function isExecutableFile(path: string): boolean { + try { + accessSync(path, constants.X_OK) + return true + } catch { + return false + } +} + +/** The operator-facing report. Names the host, the cause, and the next action. */ +export function formatNodePtyPreconditionReport( + verdict: NodePtyPreconditionVerdict, + message: string, + toolchainHints: string[] = [] +): string { + const { abi } = verdict + const host = [ + `platform ${abi.platform}/${abi.arch}`, + abi.libc === 'none' + ? null + : `libc ${abi.libc}${abi.glibcVersion ? ` ${abi.glibcVersion}` : ''}`, + `Node ABI ${abi.nodeAbi}`, + `prebuild slot ${verdict.slot}` + ] + .filter((part): part is string => part !== null) + .join(', ') + const lines = [message, '', `Host: ${host}`] + if (verdict.prebuilt && !verdict.prebuilt.installed) { + lines.push( + verdict.prebuilt.why === 'no-slot' + ? `No shipped prebuilt matches slot ${verdict.slot}.` + : verdict.prebuilt.why === 'no-prebuilds-dir' + ? 'This install ships no prebuilds directory.' + : `Shipped prebuilds are unusable here: ${verdict.prebuilt.detail ?? 'ABI mismatch'}.` + ) + } + if (toolchainHints.length > 0) { + lines.push('', 'To build node-pty on this host, install a C/C++ toolchain:', ...toolchainHints) + } + return lines.join('\n') +} diff --git a/src/main/orcad/orcad-bind-address.test.ts b/src/main/orcad/orcad-bind-address.test.ts new file mode 100644 index 00000000000..2ddbcc56343 --- /dev/null +++ b/src/main/orcad/orcad-bind-address.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'vitest' +import { + bindHostIsNetworkExposed, + describeOrcadBindExposure, + ORCAD_LOOPBACK_BIND_HOST, + OrcadBindAddressError, + resolveOrcadBindHost +} from './orcad-bind-address' + +describe('resolveOrcadBindHost', () => { + it('defaults to loopback when the operator asked for nothing', () => { + expect(resolveOrcadBindHost()).toBe(ORCAD_LOOPBACK_BIND_HOST) + expect(ORCAD_LOOPBACK_BIND_HOST).toBe('127.0.0.1') + }) + + it('accepts literal IPv4 and IPv6 addresses, including explicit wide binds', () => { + expect(resolveOrcadBindHost('0.0.0.0')).toBe('0.0.0.0') + expect(resolveOrcadBindHost('10.1.2.3')).toBe('10.1.2.3') + expect(resolveOrcadBindHost('::1')).toBe('::1') + expect(resolveOrcadBindHost('localhost')).toBe('127.0.0.1') + expect(resolveOrcadBindHost(' 127.0.0.1 ')).toBe('127.0.0.1') + }) + + it('refuses hostnames, because DNS would decide which interface got bound', () => { + expect(() => resolveOrcadBindHost('internal.example')).toThrow(OrcadBindAddressError) + expect(() => resolveOrcadBindHost('')).toThrow(OrcadBindAddressError) + expect(() => resolveOrcadBindHost('0.0.0.0:80')).toThrow(OrcadBindAddressError) + }) +}) + +describe('bindHostIsNetworkExposed', () => { + it('separates local-only addresses from network-reachable ones', () => { + expect(bindHostIsNetworkExposed('127.0.0.1')).toBe(false) + expect(bindHostIsNetworkExposed('127.5.5.5')).toBe(false) + expect(bindHostIsNetworkExposed('::1')).toBe(false) + expect(bindHostIsNetworkExposed('0.0.0.0')).toBe(true) + expect(bindHostIsNetworkExposed('::')).toBe(true) + expect(bindHostIsNetworkExposed('10.1.2.3')).toBe(true) + }) + + it('says out loud when a deployment is reachable from the network', () => { + expect(describeOrcadBindExposure('0.0.0.0')).toContain('reachable from the network') + expect(describeOrcadBindExposure('127.0.0.1')).toContain('local only') + }) +}) diff --git a/src/main/orcad/orcad-bind-address.ts b/src/main/orcad/orcad-bind-address.ts new file mode 100644 index 00000000000..d5fc4548dd1 --- /dev/null +++ b/src/main/orcad/orcad-bind-address.ts @@ -0,0 +1,66 @@ +/** + * The address `orcad` binds its RPC listener to. + * + * Why loopback by default: the desktop stays on loopback until the user pairs, and the + * shipping design reaches a remote orcad over an SSH local port-forward — so the wide + * bind orcad had was both a departure from the desktop's posture and unnecessary for the + * deploy model. Exposure is now something an operator asks for by name. + */ +import { isIP } from 'node:net' + +export const ORCAD_LOOPBACK_BIND_HOST = '127.0.0.1' +const ALL_INTERFACES_V4 = '0.0.0.0' +const ALL_INTERFACES_V6 = '::' + +export class OrcadBindAddressError extends Error { + readonly code = 'orcad_invalid_bind_address' +} + +/** + * Resolve `--bind`. Literal IPs only. + * + * Why not hostnames: `listen()` resolves a name through DNS, so the interface actually + * bound is decided by resolver configuration this process cannot see. An operator who + * writes `--bind internal.example` would have no way to know whether the service came up + * on a private interface or a public one. + */ +export function resolveOrcadBindHost(raw?: string): string { + if (raw === undefined) { + return ORCAD_LOOPBACK_BIND_HOST + } + const value = raw.trim() + if (value === '') { + throw new OrcadBindAddressError('--bind expects an address') + } + if (value === 'localhost') { + return ORCAD_LOOPBACK_BIND_HOST + } + if (isIP(value) === 0) { + throw new OrcadBindAddressError( + `--bind expects a literal IP address (got '${value}'). Hostnames are refused because ` + + 'DNS decides which interface would be bound. Use 127.0.0.1 for loopback, or ' + + '0.0.0.0 to expose every interface.' + ) + } + return value +} + +/** True when this address reaches beyond the local machine. */ +export function bindHostIsNetworkExposed(host: string): boolean { + if (host === ALL_INTERFACES_V4 || host === ALL_INTERFACES_V6) { + return true + } + if (isIP(host) === 4) { + return !host.startsWith('127.') + } + return host !== '::1' +} + +/** One line for the startup log, so an exposed deployment is never a silent default. */ +export function describeOrcadBindExposure(host: string): string { + return bindHostIsNetworkExposed(host) + ? `orcad is bound to ${host} and is reachable from the network. Anything that can reach ` + + 'this port can attempt pairing.' + : `orcad is bound to ${host} (local only). Reach it from another machine with an SSH ` + + 'local port-forward, or re-launch with --bind to expose it.' +} diff --git a/src/main/orcad/orcad-bundle-native-load-order.test.ts b/src/main/orcad/orcad-bundle-native-load-order.test.ts new file mode 100644 index 00000000000..1c26501b6a6 --- /dev/null +++ b/src/main/orcad/orcad-bundle-native-load-order.test.ts @@ -0,0 +1,91 @@ +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { runProcessSync } from '../../shared/child-process/run-process' + +/** + * The preflight only prevents a loader crash if nothing in the bundle's import graph has + * already required node-pty by the time it runs. esbuild wraps `local-pty-provider` in a + * lazy initializer because `orcad-entry` reaches it through `await import('../ipc/pty')`, + * and that laziness is load-bearing rather than incidental — a single top-level static + * import anywhere in the graph would hoist `require("node-pty")` above every statement in + * `main.ts`, including the preflight. + * + * Why this builds the bundle instead of skipping without one: no CI job builds orcad and + * runs vitest. `smoke:orcad-terminal` builds it in the static-analysis job, which never + * runs vitest; the `orcad_browser` job runs vitest but deliberately does not build orcad. + * A `runIf(existsSync(...))` guard therefore skips in every shard, forever — the same way + * an unset ORCA_BROWSER_EXECUTABLE kept the browser provider uncovered. + * + * Why not fail-when-CI instead: the wiring that would satisfy it lives in `.github/`, so + * that turns a silent gap into a red build someone else has to fix. Building costs well + * under a second (esbuild), works in every shard and on every machine, and needs no job + * to cooperate. What it must never do is skip. + */ +const REPO_ROOT = join(__dirname, '..', '..', '..') +const BUNDLE = join(REPO_ROOT, 'out', 'orcad', 'orcad.js') +const BUILD_SCRIPT = join(REPO_ROOT, 'config', 'scripts', 'build-orcad.mjs') + +/** + * Why it throws rather than skipping when the build fails: a bundle that cannot be built + * is a louder problem than the one this test checks, and swallowing it here is exactly + * how the assertion would go missing. + */ +function ensureOrcadBundle(): void { + if (existsSync(BUNDLE)) { + return + } + const build = runProcessSync({ + program: process.execPath, + args: [BUILD_SCRIPT], + cwd: REPO_ROOT, + timeoutMs: 300_000 + }) + if (!existsSync(BUNDLE)) { + const output = `${build.stdout}${build.stderr}`.slice(0, 4000) + throw new Error( + `could not build ${BUNDLE} (exit ${build.code}); the load-order assertion cannot run:\n${output}` + ) + } +} + +describe('orcad bundle native load order', () => { + const dirs: string[] = [] + afterEach(() => { + for (const dir of dirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it('does not require node-pty in-process before the entry rejects its argv', () => { + ensureOrcadBundle() + const dir = mkdtempSync(join(tmpdir(), 'orcad-load-order-')) + dirs.push(dir) + const harness = join(dir, 'harness.cjs') + writeFileSync( + harness, + [ + "const Module = require('module')", + 'const original = Module._load', + 'Module._load = function (request, ...rest) {', + " if (request === 'node-pty') { console.log('IN_PROCESS_NODE_PTY_REQUIRE') }", + ' return original.call(this, request, ...rest)', + '}', + "process.argv.push('--orcad-load-order-check')", + `require(${JSON.stringify(BUNDLE)})` + ].join('\n') + ) + + const result = runProcessSync({ + program: process.execPath, + args: [harness], + timeoutMs: 120_000 + }) + const output = `${result.stdout}${result.stderr}` + + // Proof the graph fully loaded and reached argv parsing rather than dying early. + expect(output).toContain('Unknown argument: --orcad-load-order-check') + expect(output).not.toContain('IN_PROCESS_NODE_PTY_REQUIRE') + }, 360_000) +}) diff --git a/src/main/orcad/orcad-daemon-supervision.test.ts b/src/main/orcad/orcad-daemon-supervision.test.ts new file mode 100644 index 00000000000..19616c9d5ef --- /dev/null +++ b/src/main/orcad/orcad-daemon-supervision.test.ts @@ -0,0 +1,85 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { + initDaemonPtyProviderMock, + disconnectDaemonMock, + shutdownDaemonMock, + daemonOwnsFreshPersistentPtysMock, + readDaemonPidRecordMock +} = vi.hoisted(() => ({ + initDaemonPtyProviderMock: vi.fn<(signal?: unknown, options?: unknown) => Promise>(), + disconnectDaemonMock: vi.fn<() => Promise>(), + shutdownDaemonMock: vi.fn<() => Promise>(), + daemonOwnsFreshPersistentPtysMock: vi.fn<() => boolean>(), + readDaemonPidRecordMock: vi.fn<() => { pid: number } | null>() +})) + +vi.mock('../daemon/daemon-init', () => ({ + initDaemonPtyProvider: initDaemonPtyProviderMock, + disconnectDaemon: disconnectDaemonMock, + // Exported here purely so the test can prove it is never reached — killing the daemon on + // orcad shutdown is what would make an orcad restart destructive again. + shutdownDaemon: shutdownDaemonMock, + daemonOwnsFreshPersistentPtys: daemonOwnsFreshPersistentPtysMock, + readDaemonPidRecord: readDaemonPidRecordMock +})) + +const { startOrcadDaemon, stopOrcadDaemon } = await import('./orcad-daemon-supervision') + +beforeEach(() => { + initDaemonPtyProviderMock.mockResolvedValue() + disconnectDaemonMock.mockResolvedValue() + shutdownDaemonMock.mockResolvedValue() + daemonOwnsFreshPersistentPtysMock.mockReturnValue(true) + readDaemonPidRecordMock.mockReturnValue({ pid: 4242 }) + vi.spyOn(console, 'warn').mockImplementation(() => {}) + vi.spyOn(console, 'error').mockImplementation(() => {}) +}) + +afterEach(() => { + vi.restoreAllMocks() + vi.clearAllMocks() +}) + +describe('startOrcadDaemon', () => { + it('reports live with the daemon pid once the provider is installed', async () => { + await expect(startOrcadDaemon()).resolves.toEqual({ state: 'live', pid: 4242 }) + }) + + it('does not arm the macOS login-session death watch', async () => { + await startOrcadDaemon() + // That watch retires the daemon when the spawning GUI login session dies. An orcad + // daemon must survive its SSH session ending — arming it would kill every terminal the + // moment the operator logged out, which is the opposite of the property being bought. + expect(initDaemonPtyProviderMock).toHaveBeenCalledWith(undefined, { + macosLoginSessionWatch: false + }) + }) + + it('reports degraded when fresh terminals would fall back to the local provider', async () => { + daemonOwnsFreshPersistentPtysMock.mockReturnValue(false) + const result = await startOrcadDaemon() + expect(result.state).toBe('degraded') + }) + + it('fails open when the daemon cannot start at all', async () => { + initDaemonPtyProviderMock.mockRejectedValue(new Error('node-pty is missing')) + daemonOwnsFreshPersistentPtysMock.mockReturnValue(false) + // Fail-open, like the desktop: git, worktrees and non-persistent terminals must still + // serve. What must not happen is a thrown startup or a claim of persistence. + await expect(startOrcadDaemon()).resolves.toEqual({ + state: 'unavailable', + reason: 'node-pty is missing' + }) + }) +}) + +describe('stopOrcadDaemon', () => { + it('disconnects and never shuts the daemon down', async () => { + await stopOrcadDaemon() + expect(disconnectDaemonMock).toHaveBeenCalledTimes(1) + // The whole point of item 4: an orcad restart is non-destructive only if the daemon + // outlives it. shutdownDaemon() kills the daemon and every PTY under it. + expect(shutdownDaemonMock).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/orcad/orcad-daemon-supervision.ts b/src/main/orcad/orcad-daemon-supervision.ts new file mode 100644 index 00000000000..d35b03048d3 --- /dev/null +++ b/src/main/orcad/orcad-daemon-supervision.ts @@ -0,0 +1,67 @@ +/** + * orcad's half of the supervision contract: it spawns and supervises the terminal daemon. + * + * The whole reason the peer model is recommended over an SSH target is that daemon-backed + * PTYs stay `live` across a runtime restart (docs/reference/ssh-execution-boundary.md). + * Without a daemon here, every orcad restart, update and rollback is a SIGKILL for every + * running terminal — on the host whose selling point is that work survives the client going + * away. + * + * The constraint that makes a restart non-destructive lives in `stopOrcadDaemon` below: the + * daemon is DETACHED and must outlive this process. Anything that tears it down on the way + * out silently converts a restart back into data loss. + */ +import { + disconnectDaemon, + daemonOwnsFreshPersistentPtys, + initDaemonPtyProvider, + readDaemonPidRecord +} from '../daemon/daemon-init' + +export type OrcadDaemonStartup = + | { state: 'live'; pid: number | null } + | { state: 'degraded'; reason: string } + | { state: 'unavailable'; reason: string } + +/** + * Bring the daemon up and install it as the local PTY provider. + * + * Fail-open, like the desktop: a host that cannot start a daemon must still serve git, + * worktrees and non-persistent terminals. What it must NOT do is keep claiming persistence + * — `daemonOwnsFreshPersistentPtys()` is what the runtime reads for that, and it answers + * false here without any extra bookkeeping. + */ +export async function startOrcadDaemon(): Promise { + try { + // Why no login-session watch: that retires the daemon when the spawning macOS GUI login + // session dies. An orcad daemon must survive its SSH session ending — that is the point. + await initDaemonPtyProvider(undefined, { macosLoginSessionWatch: false }) + } catch (error) { + const reason = error instanceof Error ? error.message : String(error) + console.error( + `[orcad] The terminal daemon did not start: ${reason}\n` + + '[orcad] Terminals will run in-process and WILL NOT survive an orcad restart.' + ) + return { state: 'unavailable', reason } + } + if (!daemonOwnsFreshPersistentPtys()) { + const reason = 'daemon adopted in degraded mode; fresh terminals run on the local provider' + console.warn( + `[orcad] ${reason}. Existing daemon sessions keep working, but new terminals will not ` + + 'survive an orcad restart until the daemon is restarted.' + ) + return { state: 'degraded', reason } + } + return { state: 'live', pid: readDaemonPidRecord()?.pid ?? null } +} + +/** + * Release the daemon without killing it. + * + * Why `disconnectDaemon` and never `shutdownDaemon`: shutdown kills the daemon process and + * every PTY under it. Calling it here would make orcad's own restart destructive, which is + * the exact property this whole item exists to buy. + */ +export async function stopOrcadDaemon(): Promise { + await disconnectDaemon() +} diff --git a/src/main/orcad/orcad-entry.ts b/src/main/orcad/orcad-entry.ts index 6afdcdb50e3..4af2a87c45d 100644 --- a/src/main/orcad/orcad-entry.ts +++ b/src/main/orcad/orcad-entry.ts @@ -18,6 +18,16 @@ import type { ServeReadiness } from '../server/serve-readiness' import { setRuntimeBrowserCommandsFactory } from '../runtime/runtime-browser-commands-factory' import { resolveOrcadBrowserProvider, type OrcadBrowserProvider } from './orcad-browser-provider' import { resolveOrcadInstallRoot, resolveOrcadPath, resolveUserDataPath } from './orcad-app-paths' +import { + describeOrcadBindExposure, + OrcadBindAddressError, + resolveOrcadBindHost +} from './orcad-bind-address' +import { + acquireOrcadInstanceLock, + OrcadInstanceLockError, + type OrcadInstanceLock +} from './orcad-instance-lock' let runOrcadQuitHandlers = (): void => {} @@ -80,6 +90,8 @@ export type OrcadOptions = { json?: boolean noPairing?: boolean pairingAddress?: string + /** Literal IP to bind. Defaults to loopback; see orcad-bind-address.ts. */ + bind?: string } export type OrcadHandle = { @@ -95,24 +107,30 @@ export type OrcadHandle = { export async function startOrcad(options: OrcadOptions = {}): Promise { installOrcadHostAdapters() const userDataPath = resolveUserDataPath() + // Why before anything else touches the root: the profile index, the store and the daemon + // runtime dir all live under it, and two orcads sharing them corrupt state silently. This + // is also the last point at which refusing costs nothing. + const instanceLock = acquireOrcadInstanceLock(userDataPath) const browserProvider = await resolveOrcadBrowserProvider({ userDataPath }) setRuntimeBrowserCommandsFactory(browserProvider?.factory ?? null, { headless: browserProvider !== null, ...(browserProvider ? { isAvailable: () => browserProvider.isAvailable() } : {}) }) try { - return await startOrcadRuntime(options, browserProvider) + return await startOrcadRuntime(options, browserProvider, instanceLock) } catch (error) { await browserProvider?.stop() setRuntimeBrowserCommandsFactory(null) runOrcadQuitHandlers() + instanceLock.release() throw error } } async function startOrcadRuntime( options: OrcadOptions, - browserProvider: OrcadBrowserProvider | null + browserProvider: OrcadBrowserProvider | null, + instanceLock: OrcadInstanceLock ): Promise { const { OrcaRuntimeService } = await import('../runtime/orca-runtime') const { OrcaRuntimeRpcServer } = await import('../runtime/runtime-rpc') @@ -125,6 +143,9 @@ async function startOrcadRuntime( const { ensureActiveOrcaProfile, initOrcaProfilePaths } = await import('../orca-profiles/profile-index-store') const { initSshHostKeyStoreFile } = await import('../ssh/ssh-host-key-store') + const { startOrcadDaemon, stopOrcadDaemon } = await import('./orcad-daemon-supervision') + const { daemonOwnsFreshPersistentPtys } = await import('../daemon/daemon-init') + const { collectOrcadHealth } = await import('./orcad-health') const runtimeUserDataPath = getAppEnvironment().getPath('userData') initOrcaProfilePaths() @@ -139,6 +160,11 @@ async function startOrcadRuntime( // which is safe but silently discards accept records on every launch. initSshHostKeyStoreFile(profile.dataFile) + // Why before the runtime and the PTY handlers: `setLocalPtyProvider` installs the daemon + // adapter as THE local provider, and the registry's contract is that it lands before + // registerPtyHandlers so the IPC layer routes through the daemon from the first call. + await startOrcadDaemon() + const runtime = new OrcaRuntimeService(store, undefined, { // Why lazy: a daemon swap replaces the provider after construction, so an eager // reference would freeze the pre-daemon one. @@ -146,10 +172,11 @@ async function startOrcadRuntime( // Why: destructive worktree removal refuses to run without a provider to stop // processes through — correctly, since it cannot otherwise verify the tree is idle. getSshProvider: (connectionId) => getSshPtyProvider(connectionId), - // Why false: this host does not run the terminal daemon, so persistent local PTYs - // cannot be recovered. The constructor defaults this to true, which would claim a - // capability orcad does not have. - canRecoverPersistentLocalPtys: () => false, + // Why the daemon predicate and not a constant: orcad now spawns the terminal daemon, so + // its PTYs DO survive an orcad restart — but only while a daemon that owns fresh + // sessions is installed. A failed or degraded launch has to answer false, and this reads + // that live rather than snapshotting it at construction. + canRecoverPersistentLocalPtys: () => daemonOwnsFreshPersistentPtys(), // Why 'blocked': `'openable'` means a desktop window can be opened here, which is // what powers serve→desktop promotion. A Node host can never do that, and the // constructor's default would advertise it. @@ -173,14 +200,20 @@ async function startOrcadRuntime( await runtime.refreshRestoredOrchestrationAuthority() await runtime.reconcileLegacyWorkerTerminals() + const bindHost = resolveOrcadBindHost(options.bind) const rpc = new OrcaRuntimeRpcServer({ runtime, userDataPath: runtimeUserDataPath, enableWebSocket: true, - exposeNetworkByDefault: true, + // Why pinned and not `exposeNetworkByDefault`: an unattended host's exposure must be + // exactly what the operator asked for, on every launch. The default path widens itself + // once a device has connected, so a loopback deployment would silently go wide one + // restart after its first client paired. + pinnedBindHost: bindHost, ...(options.port !== undefined ? { wsPort: options.port, preferPinnedWsPort: true } : {}) }) await rpc.start() + console.error(`[orcad] ${describeOrcadBindExposure(bindHost)}`) const boundEndpoint = rpc.getWebSocketEndpoint() const advertised = boundEndpoint @@ -215,7 +248,11 @@ async function startOrcadRuntime( scope: 'runtime', qr: null } - : offer + : offer, + // Why in the readiness payload: this is the one message a supervisor and a deploy + // transaction both read, and a green orcad with a dead daemon is exactly the + // looks-healthy-but-useless state they must not activate. + health: await collectOrcadHealth(getAppEnvironment().getVersion()) } await new ServeReadinessPublisher().publish(readiness, { @@ -228,15 +265,20 @@ async function startOrcadRuntime( try { await rpc.stop() } finally { + // Why disconnect and not shut down: the daemon must outlive this process, or an + // orcad restart goes back to killing every running terminal. See + // orcad-daemon-supervision.ts. + await stopOrcadDaemon() await browserProvider?.stop() setRuntimeBrowserCommandsFactory(null) runOrcadQuitHandlers() + instanceLock.release() } } } } -function parseArgs(argv: string[]): OrcadOptions { +export function parseArgs(argv: string[]): OrcadOptions { const options: OrcadOptions = {} for (let i = 0; i < argv.length; i += 1) { const arg = argv[i] @@ -252,6 +294,13 @@ function parseArgs(argv: string[]): OrcadOptions { options.json = true } else if (arg === '--no-pairing') { options.noPairing = true + } else if (arg === '--bind') { + const value = argv[i + 1] + if (value === undefined) { + throw new Error('--bind expects a value') + } + options.bind = value + i += 1 } else if (arg === '--pairing-address') { const value = argv[i + 1] if (!value) { @@ -266,22 +315,56 @@ function parseArgs(argv: string[]): OrcadOptions { return options } +/** + * Exit codes a supervisor can act on. Closed set — see docs/reference/orcad-operations.md. + * + * `ORCAD_EXIT_CONFIGURATION` is the load-bearing one: a data root owned by someone else, or + * held by another orcad, is not fixed by restarting. Restarting on it is the crash-loop the + * supervision contract has to prevent, so systemd's `RestartPreventExitStatus` needs a code + * that means "do not retry" and nothing else does. + */ +export const ORCAD_EXIT_OK = 0 +export const ORCAD_EXIT_FAILED = 1 +export const ORCAD_EXIT_CONFIGURATION = 78 + +/** Bounded so a wedged transport cannot hold a supervisor's stop past its own deadline. */ +export const ORCAD_SHUTDOWN_DEADLINE_MS = 15_000 + +export function resolveOrcadExitCode(error: unknown): number { + return error instanceof OrcadInstanceLockError || error instanceof OrcadBindAddressError + ? ORCAD_EXIT_CONFIGURATION + : ORCAD_EXIT_FAILED +} + export async function main(argv: string[] = process.argv.slice(2)): Promise { const handle = await startOrcad(parseArgs(argv)) let stopping = false const shutdown = (signal: NodeJS.Signals): void => { if (stopping) { - return + // Why escalate rather than ignore: a supervisor's second signal means the first + // deadline elapsed. Continuing to wait silently is what makes a stop hang until + // SIGKILL, which is the one teardown that skips the daemon handoff entirely. + console.error(`orcad: second ${signal} during shutdown — exiting immediately`) + process.exit(ORCAD_EXIT_FAILED) } stopping = true + // Why a self-imposed deadline as well: the supervisor's SIGKILL leaves no exit code and + // no log line. Exiting ourselves keeps the failure attributable. + const deadline = setTimeout(() => { + console.error( + `orcad: shutdown after ${signal} exceeded ${ORCAD_SHUTDOWN_DEADLINE_MS}ms — exiting` + ) + process.exit(ORCAD_EXIT_FAILED) + }, ORCAD_SHUTDOWN_DEADLINE_MS) + deadline.unref() handle .stop() - .then(() => process.exit(0)) + .then(() => process.exit(ORCAD_EXIT_OK)) // Why not rethrow: we are already tearing down on a signal, and an exit code is // the only thing a supervisor can act on. .catch((error) => { console.error(`orcad: shutdown after ${signal} failed:`, error) - process.exit(1) + process.exit(ORCAD_EXIT_FAILED) }) } process.on('SIGINT', () => shutdown('SIGINT')) diff --git a/src/main/orcad/orcad-health.test.ts b/src/main/orcad/orcad-health.test.ts new file mode 100644 index 00000000000..7a1a2a269ff --- /dev/null +++ b/src/main/orcad/orcad-health.test.ts @@ -0,0 +1,152 @@ +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { ParsedDaemonPid } from '../daemon/daemon-pid-file-parse' +import type { DaemonHealth } from '../daemon/daemon-health' + +const { + checkDaemonHealthMock, + getDaemonEndpointFactsMock, + readDaemonPidRecordMock, + daemonOwnsFreshPersistentPtysMock +} = vi.hoisted(() => ({ + checkDaemonHealthMock: vi.fn<() => Promise>(), + getDaemonEndpointFactsMock: vi.fn<() => unknown>(), + readDaemonPidRecordMock: vi.fn<() => ParsedDaemonPid | null>(), + daemonOwnsFreshPersistentPtysMock: vi.fn<() => boolean>() +})) + +vi.mock('../daemon/daemon-health', () => ({ checkDaemonHealth: checkDaemonHealthMock })) +vi.mock('../daemon/daemon-init', () => ({ + getDaemonEndpointFacts: getDaemonEndpointFactsMock, + readDaemonPidRecord: readDaemonPidRecordMock, + daemonOwnsFreshPersistentPtys: daemonOwnsFreshPersistentPtysMock +})) + +const { collectOrcadHealth, collectTerminalDaemonHealth, computeOrcadBuildHash } = + await import('./orcad-health') + +const LIVE_FACTS = { + runtimeDir: '/data/daemon', + socketPath: '/data/daemon/daemon-v36.sock', + tokenPath: '/data/daemon/daemon-v36.token', + pidPath: '/data/daemon/daemon-v36.pid', + protocolVersion: 36 +} + +const PID_RECORD: ParsedDaemonPid = { + pid: 4242, + startedAtMs: 1_000, + entryPath: '/opt/orcad/daemon-entry.js', + appVersion: '1.2.2', + launchNonce: 'n', + linuxStartTicks: null, + bootId: null, + spawnerExecPath: null +} + +const originalPlatform = Object.getOwnPropertyDescriptor(process, 'platform')! + +beforeEach(() => { + getDaemonEndpointFactsMock.mockReturnValue(LIVE_FACTS) + readDaemonPidRecordMock.mockReturnValue(PID_RECORD) + daemonOwnsFreshPersistentPtysMock.mockReturnValue(true) + checkDaemonHealthMock.mockResolvedValue('healthy') +}) + +afterEach(() => { + Object.defineProperty(process, 'platform', originalPlatform) + vi.clearAllMocks() +}) + +describe('collectTerminalDaemonHealth', () => { + it('reports live only when the daemon answered its own PTY spawn probe', async () => { + const health = await collectTerminalDaemonHealth() + expect(health.state).toBe('live') + expect(health.selfTest).toMatchObject({ ok: true, verdict: 'healthy', coverage: 'pty-spawn' }) + expect(health.pid).toBe(4242) + // The build the LIVE daemon came from, which can legitimately predate this orcad. + expect(health.buildVersion).toBe('1.2.2') + expect(health.entryPath).toBe('/opt/orcad/daemon-entry.js') + expect(health.protocolVersion).toBe(36) + // Why assert the coordinates: a self-test that probed some other endpoint would prove + // nothing about the daemon this process installed. + expect(checkDaemonHealthMock).toHaveBeenCalledWith(LIVE_FACTS.socketPath, LIVE_FACTS.tokenPath) + }) + + it('is not green when the daemon is up but cannot spawn a PTY', async () => { + checkDaemonHealthMock.mockResolvedValue('pty-spawn-unhealthy') + const health = await collectTerminalDaemonHealth() + expect(health.selfTest.ok).toBe(false) + expect(health.selfTest.verdict).toBe('pty-spawn-unhealthy') + // Degraded, not absent: it still owns live sessions, and calling those exited would be + // the verdict the execution-boundary vocabulary forbids guessing. + expect(health.state).toBe('degraded') + }) + + it('is not green when the daemon stopped answering entirely', async () => { + checkDaemonHealthMock.mockResolvedValue('unreachable') + const health = await collectTerminalDaemonHealth() + expect(health.selfTest.ok).toBe(false) + expect(health.state).toBe('degraded') + }) + + it('is not green when fresh terminals fall back to the local provider', async () => { + daemonOwnsFreshPersistentPtysMock.mockReturnValue(false) + const health = await collectTerminalDaemonHealth() + // The socket answers and the probe passes, but new terminals would die with this + // process — reporting live here is precisely the looks-healthy-but-useless shape. + expect(health.selfTest.ok).toBe(true) + expect(health.ownsFreshSessions).toBe(false) + expect(health.state).toBe('degraded') + }) + + it('reports absent, and probes nothing, when no daemon was ever installed', async () => { + getDaemonEndpointFactsMock.mockReturnValue(null) + daemonOwnsFreshPersistentPtysMock.mockReturnValue(false) + const health = await collectTerminalDaemonHealth() + expect(health.state).toBe('absent') + expect(health.selfTest).toMatchObject({ ok: false, verdict: 'no-daemon' }) + expect(health.pid).toBeNull() + expect(checkDaemonHealthMock).not.toHaveBeenCalled() + }) + + it('declares handshake-only coverage on win32, where the spawn probe is a no-op', async () => { + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + const health = await collectTerminalDaemonHealth() + // `checkPtySpawnHealth` returns immediately on win32 without spawning anything, so a + // green verdict there must not be reported as a PTY round trip. + expect(health.selfTest.coverage).toBe('handshake') + }) +}) + +describe('collectOrcadHealth', () => { + it('carries build identity and the Node ABI native addons must match', async () => { + const health = await collectOrcadHealth('1.2.3') + expect(health.buildVersion).toBe('1.2.3') + expect(health.nodeVersion).toBe(process.versions.node) + expect(health.nodeAbi).toBe(process.versions.modules) + expect(health.platform).toBe(process.platform) + expect(health.terminalDaemon.state).toBe('live') + }) +}) + +describe('computeOrcadBuildHash', () => { + it('changes when the bundle bytes change, even at the same version string', () => { + const dir = mkdtempSync(join(tmpdir(), 'orcad-build-hash-')) + const entry = join(dir, 'orcad.js') + writeFileSync(entry, 'build-a') + const first = computeOrcadBuildHash(entry) + writeFileSync(entry, 'build-b') + // A rollback that did not actually replace the file is what this has to catch, and a + // version string cannot. + expect(computeOrcadBuildHash(entry)).not.toBe(first) + }) + + it('answers unknown rather than throwing when the entry cannot be read', () => { + expect(computeOrcadBuildHash(join(tmpdir(), 'definitely-absent-orcad.js'))).toBe('unknown') + // A process with no argv[1] (an embedded host) still has to publish a readiness payload. + expect(computeOrcadBuildHash('')).toBe('unknown') + }) +}) diff --git a/src/main/orcad/orcad-health.ts b/src/main/orcad/orcad-health.ts new file mode 100644 index 00000000000..9f271c55d84 --- /dev/null +++ b/src/main/orcad/orcad-health.ts @@ -0,0 +1,155 @@ +/** + * orcad's health surface: the facts a supervisor needs to decide whether this deployment + * is actually serving, as opposed to merely listening. + * + * The load-bearing one is the terminal-daemon verdict. orcad answers RPC from its own + * process, so "the port is open" stays true while the daemon that owns every terminal is + * dead — a green host that cannot run a single command. The self-test below therefore has + * to cross the process boundary: orcad drives it, the daemon performs it, and the verdict + * travels back over the daemon's socket. + */ +import { createHash } from 'node:crypto' +import { readFileSync } from 'node:fs' +import process from 'node:process' +import { checkDaemonHealth, type DaemonHealth } from '../daemon/daemon-health' +import { + daemonOwnsFreshPersistentPtys, + getDaemonEndpointFacts, + readDaemonPidRecord +} from '../daemon/daemon-init' + +/** + * How much a green self-test actually proves. + * + * `pty-spawn` — the daemon spawned a real PTY inside its own process and it worked. + * `handshake` — the daemon answered its protocol handshake, but its spawn probe is a no-op + * on this platform (win32: `checkPtySpawnHealth` returns without spawning). Reported + * separately rather than folded into `ok`, because claiming a PTY round trip we did not + * perform is the failure mode this surface exists to prevent. + */ +export type PtySelfTestCoverage = 'pty-spawn' | 'handshake' + +export type PtySelfTest = { + ok: boolean + coverage: PtySelfTestCoverage + /** The daemon's own verdict word, so a failure is diagnosable without re-probing. */ + verdict: DaemonHealth | 'no-daemon' + durationMs: number +} + +export type TerminalDaemonHealth = { + /** `live` requires the daemon to have answered; absence is never inferred from silence. */ + state: 'live' | 'degraded' | 'absent' + /** True only when FRESH terminals are daemon-owned, i.e. survive an orcad restart. */ + ownsFreshSessions: boolean + pid: number | null + /** The build the LIVE daemon was forked from, which may predate this orcad after an update. */ + buildVersion: string | null + entryPath: string | null + protocolVersion: number | null + selfTest: PtySelfTest +} + +export type OrcadHealth = { + /** Content hash of the running orcad bundle — the deployed build's identity. */ + buildHash: string + buildVersion: string + nodeVersion: string + /** `process.versions.modules`: the ABI every native addon on this host must match. */ + nodeAbi: string + platform: NodeJS.Platform + arch: string + pid: number + terminalDaemon: TerminalDaemonHealth +} + +/** + * Identity of the exact bytes running. + * + * Why hash the entry and not read a version string: `ORCA_VERSION` is whatever the deploy + * exported, so two different builds can carry one version. A rollback that did not actually + * replace the file is precisely what this has to catch. + */ +export function computeOrcadBuildHash(entryPath = process.argv[1]): string { + if (!entryPath) { + return 'unknown' + } + try { + return createHash('sha256').update(readFileSync(entryPath)).digest('hex').slice(0, 16) + } catch { + return 'unknown' + } +} + +/** + * Probe the daemon across the process boundary. + * + * `checkDaemonHealth` is the cross-process test: it opens the daemon's socket, completes the + * protocol handshake, and asks the daemon to run `ptySpawnHealth` — a real short-lived PTY + * spawned inside the daemon. Only a daemon that is alive AND can create terminals answers + * `healthy`; a wedged one times out to `unreachable`, and one whose node-pty or login session + * is broken answers `pty-spawn-unhealthy`. + */ +export async function runTerminalDaemonSelfTest( + now: () => number = () => Date.now() +): Promise { + const startedAt = now() + // Why: `checkPtySpawnHealth` returns immediately on win32 without spawning anything, so a + // green verdict there covers the handshake only. Say so instead of overclaiming. + const coverage: PtySelfTestCoverage = process.platform === 'win32' ? 'handshake' : 'pty-spawn' + const facts = getDaemonEndpointFacts() + if (!facts) { + return { ok: false, coverage, verdict: 'no-daemon', durationMs: now() - startedAt } + } + const verdict = await checkDaemonHealth(facts.socketPath, facts.tokenPath) + return { ok: verdict === 'healthy', coverage, verdict, durationMs: now() - startedAt } +} + +export async function collectTerminalDaemonHealth(): Promise { + const facts = getDaemonEndpointFacts() + const selfTest = await runTerminalDaemonSelfTest() + if (!facts) { + return { + state: 'absent', + ownsFreshSessions: false, + pid: null, + buildVersion: null, + entryPath: null, + protocolVersion: null, + selfTest + } + } + const record = readDaemonPidRecord() + const ownsFreshSessions = daemonOwnsFreshPersistentPtys() + return { + // Why `degraded` and not `absent` on a failed self-test: a daemon that answered its + // handshake but failed the spawn probe is still holding live sessions. Reporting it gone + // would invite a caller to treat those terminals as exited, which is the one verdict the + // execution-boundary vocabulary forbids guessing. + state: + selfTest.ok && ownsFreshSessions + ? 'live' + : selfTest.verdict === 'no-daemon' + ? 'absent' + : 'degraded', + ownsFreshSessions, + pid: record?.pid ?? null, + buildVersion: record?.appVersion ?? null, + entryPath: record?.entryPath ?? null, + protocolVersion: facts.protocolVersion, + selfTest + } +} + +export async function collectOrcadHealth(buildVersion: string): Promise { + return { + buildHash: computeOrcadBuildHash(), + buildVersion, + nodeVersion: process.versions.node, + nodeAbi: process.versions.modules ?? 'unknown', + platform: process.platform, + arch: process.arch, + pid: process.pid, + terminalDaemon: await collectTerminalDaemonHealth() + } +} diff --git a/src/main/orcad/orcad-instance-lock.test.ts b/src/main/orcad/orcad-instance-lock.test.ts new file mode 100644 index 00000000000..45f7543c515 --- /dev/null +++ b/src/main/orcad/orcad-instance-lock.test.ts @@ -0,0 +1,151 @@ +import { + chmodSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + acquireOrcadInstanceLock, + ORCAD_LOCK_FILE_NAME, + OrcadInstanceLockError, + type OrcadInstanceLockHooks +} from './orcad-instance-lock' + +const roots: string[] = [] + +function makeRoot(): string { + const root = mkdtempSync(join(tmpdir(), 'orcad-lock-')) + roots.push(root) + return root +} + +/** Deterministic identity/liveness so the assertions do not depend on this machine's pids. */ +function hooks(overrides: OrcadInstanceLockHooks = {}): OrcadInstanceLockHooks { + return { + identity: () => 'uid-1000', + version: () => '1.0.0-test', + startedAtMs: () => 1_000, + startTimeMatches: () => true, + processIsAlive: () => false, + ...overrides + } +} + +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +describe('acquireOrcadInstanceLock', () => { + it('publishes a record naming the holder and removes it on release', () => { + const root = makeRoot() + const lock = acquireOrcadInstanceLock(root, hooks()) + const record = JSON.parse(readFileSync(join(root, ORCAD_LOCK_FILE_NAME), 'utf8')) + expect(record.pid).toBe(process.pid) + expect(record.identity).toBe('uid-1000') + expect(record.version).toBe('1.0.0-test') + lock.release() + expect(() => readFileSync(join(root, ORCAD_LOCK_FILE_NAME), 'utf8')).toThrow() + }) + + it('refuses a second instance while the holder is alive', () => { + const root = makeRoot() + acquireOrcadInstanceLock(root, hooks()) + expect(() => acquireOrcadInstanceLock(root, hooks({ processIsAlive: () => true }))).toThrow( + OrcadInstanceLockError + ) + expect(() => acquireOrcadInstanceLock(root, hooks({ processIsAlive: () => true }))).toThrow( + expect.objectContaining({ code: 'orcad_instance_lock_held' }) + ) + }) + + it('reclaims the record of a holder that is gone', () => { + const root = makeRoot() + writeFileSync( + join(root, ORCAD_LOCK_FILE_NAME), + JSON.stringify({ pid: 424242, identity: 'uid-1000', startedAtMs: 1, nonce: 'stale' }) + ) + const lock = acquireOrcadInstanceLock(root, hooks({ processIsAlive: () => false })) + expect(JSON.parse(readFileSync(lock.path, 'utf8')).pid).toBe(process.pid) + }) + + it('treats a live pid whose start time does not match as a recycled pid, not a holder', () => { + const root = makeRoot() + writeFileSync( + join(root, ORCAD_LOCK_FILE_NAME), + JSON.stringify({ pid: 424242, identity: 'uid-1000', startedAtMs: 1, nonce: 'stale' }) + ) + const lock = acquireOrcadInstanceLock( + root, + hooks({ processIsAlive: () => true, startTimeMatches: () => false }) + ) + expect(JSON.parse(readFileSync(lock.path, 'utf8')).pid).toBe(process.pid) + }) + + it('never reclaims a lock held by a different identity, even a dead one', () => { + const root = makeRoot() + writeFileSync( + join(root, ORCAD_LOCK_FILE_NAME), + JSON.stringify({ pid: 424242, identity: 'uid-2000', startedAtMs: 1, nonce: 'other' }) + ) + expect(() => acquireOrcadInstanceLock(root, hooks({ processIsAlive: () => false }))).toThrow( + expect.objectContaining({ code: 'orcad_instance_lock_foreign_identity' }) + ) + }) + + it('does not delete a record that a later instance already replaced', () => { + const root = makeRoot() + const lock = acquireOrcadInstanceLock(root, hooks()) + // A successor reclaimed the root while this process was wedged. + writeFileSync( + lock.path, + JSON.stringify({ pid: 777, identity: 'uid-1000', startedAtMs: 2, nonce: 'successor' }) + ) + lock.release() + expect(JSON.parse(readFileSync(lock.path, 'utf8')).nonce).toBe('successor') + }) + + it.runIf(process.platform !== 'win32')( + 'tightens a group/world-accessible data root rather than refusing when it can', + () => { + const root = makeRoot() + chmodSync(root, 0o755) + acquireOrcadInstanceLock(root, hooks()) + expect(statSync(root).mode & 0o777).toBe(0o700) + } + ) + + it.runIf(process.platform !== 'win32')('refuses a data root owned by another uid', () => { + const root = makeRoot() + // /tmp itself is root-owned and sticky on every supported platform, so it stands in for + // "a data root this process does not own" without needing privileges to create one. + expect(() => acquireOrcadInstanceLock('/tmp', hooks())).toThrow( + expect.objectContaining({ code: 'orcad_data_root_wrong_owner' }) + ) + // And the private root this test made is still acceptable, so the refusal is about + // ownership rather than a blanket rejection. + expect(acquireOrcadInstanceLock(root, hooks()).record.identity).toBe('uid-1000') + }) + + it('leaves the terminal daemon alone: the lock covers only the runtime role', () => { + const root = makeRoot() + // The daemon lives here and deliberately outlives the runtime. Releasing the runtime's + // lock must not touch it, or a restart would stop being non-destructive. + const daemonDir = join(root, 'daemon') + mkdirSync(daemonDir, { recursive: true }) + writeFileSync(join(daemonDir, 'daemon-v36.pid'), JSON.stringify({ pid: 99, startedAtMs: 1 })) + const lock = acquireOrcadInstanceLock(root, hooks()) + lock.release() + expect(JSON.parse(readFileSync(join(daemonDir, 'daemon-v36.pid'), 'utf8')).pid).toBe(99) + // And a fresh instance takes the root back while that daemon record still stands. + const next = acquireOrcadInstanceLock(root, hooks()) + expect(next.record.pid).toBe(process.pid) + }) +}) diff --git a/src/main/orcad/orcad-instance-lock.ts b/src/main/orcad/orcad-instance-lock.ts new file mode 100644 index 00000000000..052140bcd2e --- /dev/null +++ b/src/main/orcad/orcad-instance-lock.ts @@ -0,0 +1,324 @@ +/** + * Single-instance ownership of orcad's data root, taken BEFORE the profile is loaded. + * + * Two orcads on one data root corrupt it quietly: both load the same profile, both write + * the same store file, and the loser's writes disappear on the next flush. The refusal is + * therefore a startup gate, not a warning. + * + * What this lock does NOT cover, and must not: the terminal daemon. The daemon is a second + * long-lived process living under `/daemon`, it deliberately outlives the orcad that + * spawned it, and it fences its own endpoint with a PID record of its own. A lock that + * asked "is any process using this root" would refuse every restart that a live daemon + * makes worthwhile. This lock scopes exactly one role — who is the runtime — so releasing + * it says nothing about the daemon, which is what makes a non-destructive restart possible. + */ +import { randomUUID } from 'node:crypto' +import { + chmodSync, + mkdirSync, + readFileSync, + renameSync, + statSync, + unlinkSync, + writeFileSync +} from 'node:fs' +import { userInfo } from 'node:os' +import { join } from 'node:path' +import process from 'node:process' +import { getProcessStartedAtMs, startTimeMatches } from '../daemon/daemon-process-start-time' + +export const ORCAD_LOCK_FILE_NAME = 'orcad.lock' + +export type OrcadInstanceLockCode = + | 'orcad_data_root_unusable' + | 'orcad_data_root_wrong_owner' + | 'orcad_data_root_shared' + | 'orcad_instance_lock_held' + | 'orcad_instance_lock_foreign_identity' + +export class OrcadInstanceLockError extends Error { + constructor( + readonly code: OrcadInstanceLockCode, + message: string + ) { + super(message) + this.name = 'OrcadInstanceLockError' + } +} + +export type OrcadLockRecord = { + pid: number + /** Null where the platform cannot read it; PID alone is then the (weaker) fence. */ + startedAtMs: number | null + /** POSIX uid, or the Windows username. Compared as an opaque string. */ + identity: string + version: string + acquiredAt: string + /** Distinguishes our record from a replacement written after we lost the race. */ + nonce: string +} + +export type OrcadInstanceLock = { + readonly path: string + readonly record: OrcadLockRecord + release(): void +} + +export type OrcadInstanceLockHooks = { + identity?: () => string + version?: () => string + now?: () => Date + /** Whether a PID is running. EPERM counts as alive: it proves the process exists. */ + processIsAlive?: (pid: number) => boolean + startedAtMs?: (pid: number) => number | null + startTimeMatches?: (pid: number, expected: number | null) => boolean +} + +function defaultIdentity(): string { + // Why uid and not the name on POSIX: two accounts can share a login name across a + // container boundary while the uid is what the filesystem actually enforces. + return process.platform === 'win32' + ? (userInfo().username ?? 'unknown') + : String(process.getuid?.() ?? 'unknown') +} + +function defaultProcessIsAlive(pid: number): boolean { + try { + process.kill(pid, 0) + return true + } catch (error) { + return isErrorCode(error, 'EPERM') + } +} + +function isErrorCode(error: unknown, code: string): boolean { + return typeof error === 'object' && error !== null && 'code' in error && error.code === code +} + +function parseLockRecord(content: string): OrcadLockRecord | null { + try { + const parsed: unknown = JSON.parse(content) + if (!parsed || typeof parsed !== 'object') { + return null + } + const record = parsed as Partial + if (typeof record.pid !== 'number' || typeof record.identity !== 'string') { + return null + } + return { + pid: record.pid, + startedAtMs: typeof record.startedAtMs === 'number' ? record.startedAtMs : null, + identity: record.identity, + version: typeof record.version === 'string' ? record.version : 'unknown', + acquiredAt: typeof record.acquiredAt === 'string' ? record.acquiredAt : '', + nonce: typeof record.nonce === 'string' ? record.nonce : '' + } + } catch { + return null + } +} + +/** + * Fail closed on a data root other identities can read or write. + * + * Why self-heal first and refuse second: orcad stores credentials unsealed (there is no OS + * keyring on this host), so a group- or world-accessible root is a real exposure — but if + * we own the directory, tightening it is strictly better than refusing to start. We refuse + * only when the permissions are not ours to fix. + */ +function assertDataRootIsPrivate(dataRoot: string): void { + // Windows ACLs are not expressible as a POSIX mode, and `statSync().mode` there reports a + // synthesized one. Checking it would refuse correct deployments and pass wrong ones. + if (process.platform === 'win32') { + return + } + let stats + try { + stats = statSync(dataRoot) + } catch (error) { + throw new OrcadInstanceLockError( + 'orcad_data_root_unusable', + `Cannot stat the orcad data root ${dataRoot}: ${(error as Error).message}` + ) + } + const uid = process.getuid?.() + if (uid !== undefined && stats.uid !== uid) { + throw new OrcadInstanceLockError( + 'orcad_data_root_wrong_owner', + `The orcad data root ${dataRoot} is owned by uid ${stats.uid}, not by uid ${uid} running ` + + 'this process. Give orcad its own data root (ORCA_USER_DATA) or chown this one.' + ) + } + if ((stats.mode & 0o077) === 0) { + return + } + try { + chmodSync(dataRoot, 0o700) + } catch { + // Fall through to the re-stat, which produces the actionable message. + } + let mode: number + try { + mode = statSync(dataRoot).mode + } catch (error) { + throw new OrcadInstanceLockError( + 'orcad_data_root_unusable', + `Cannot stat the orcad data root ${dataRoot}: ${(error as Error).message}` + ) + } + if ((mode & 0o077) !== 0) { + throw new OrcadInstanceLockError( + 'orcad_data_root_shared', + `The orcad data root ${dataRoot} is accessible to other users (mode ` + + `${(mode & 0o777).toString(8)}) and could not be tightened. orcad stores credentials ` + + 'there unsealed, so it refuses to start. Run `chmod 700` on it, or point ORCA_USER_DATA ' + + 'at a private directory.' + ) + } +} + +/** + * Take the lock, or throw an `OrcadInstanceLockError` naming why. + * + * A dead holder's record is reclaimed; a live one, or one belonging to a different identity, + * is never touched. + */ +export function acquireOrcadInstanceLock( + dataRoot: string, + hooks: OrcadInstanceLockHooks = {} +): OrcadInstanceLock { + const identity = (hooks.identity ?? defaultIdentity)() + const isAlive = hooks.processIsAlive ?? defaultProcessIsAlive + const readStartedAt = hooks.startedAtMs ?? getProcessStartedAtMs + const matchesStartTime = hooks.startTimeMatches ?? startTimeMatches + + try { + mkdirSync(dataRoot, { recursive: true, mode: 0o700 }) + } catch (error) { + throw new OrcadInstanceLockError( + 'orcad_data_root_unusable', + `Cannot create the orcad data root ${dataRoot}: ${(error as Error).message}` + ) + } + assertDataRootIsPrivate(dataRoot) + + const lockPath = join(dataRoot, ORCAD_LOCK_FILE_NAME) + const record: OrcadLockRecord = { + pid: process.pid, + startedAtMs: readStartedAt(process.pid), + identity, + version: (hooks.version ?? (() => process.env.ORCA_VERSION ?? 'unknown'))(), + acquiredAt: (hooks.now ?? (() => new Date()))().toISOString(), + nonce: randomUUID() + } + const serialized = JSON.stringify(record) + + const publish = (): boolean => { + try { + writeFileSync(lockPath, serialized, { flag: 'wx', mode: 0o600 }) + return true + } catch (error) { + if (isErrorCode(error, 'EEXIST')) { + return false + } + throw new OrcadInstanceLockError( + 'orcad_data_root_unusable', + `Cannot write the orcad instance lock ${lockPath}: ${(error as Error).message}` + ) + } + } + + if (publish()) { + return makeLock(lockPath, record) + } + + const existing = parseLockRecord(safeRead(lockPath) ?? '') + if (existing && existing.identity !== identity) { + throw new OrcadInstanceLockError( + 'orcad_instance_lock_foreign_identity', + `The orcad data root ${dataRoot} is locked by identity ${existing.identity} (pid ` + + `${existing.pid}); this process runs as ${identity}. Two identities sharing one data ` + + 'root corrupts it. Give each its own ORCA_USER_DATA.' + ) + } + if (existing && isAlive(existing.pid) && matchesStartTime(existing.pid, existing.startedAtMs)) { + throw new OrcadInstanceLockError( + 'orcad_instance_lock_held', + `Another orcad (pid ${existing.pid}, started ${existing.acquiredAt || 'unknown'}) already ` + + `owns the data root ${dataRoot}. Stop it before starting another, or use a different ` + + 'ORCA_USER_DATA.' + ) + } + if (!existing) { + console.warn( + `[orcad] The instance lock at ${lockPath} is unreadable; reclaiming it. If another orcad ` + + 'is running on this data root, stop it now.' + ) + } + + // Why rename-and-then-publish rather than unlink-and-write: rename claims one exact + // directory entry, so a replacement written between our read and our write stays at the + // canonical path and wins — we never delete a record we did not inspect. + const claimPath = `${lockPath}.stale-${process.pid}-${randomUUID()}` + try { + renameSync(lockPath, claimPath) + } catch { + throw new OrcadInstanceLockError( + 'orcad_instance_lock_held', + `Could not reclaim the stale orcad instance lock at ${lockPath}; another process is ` + + 'holding it. Retry, or stop the other orcad.' + ) + } + if (!publish()) { + // Someone else claimed it first. Their record is authoritative; ours is not. + try { + unlinkSync(claimPath) + } catch { + // A uniquely named claim is inert. + } + throw new OrcadInstanceLockError( + 'orcad_instance_lock_held', + `Another orcad took the data root ${dataRoot} while this one was reclaiming a stale lock.` + ) + } + try { + unlinkSync(claimPath) + } catch { + // The canonical record is authoritative; the claim is inert. + } + return makeLock(lockPath, record) +} + +function safeRead(path: string): string | null { + try { + return readFileSync(path, 'utf8') + } catch { + return null + } +} + +function makeLock(lockPath: string, record: OrcadLockRecord): OrcadInstanceLock { + let released = false + return { + path: lockPath, + record, + release: () => { + if (released) { + return + } + released = true + // Why re-read before unlinking: a reclaim by a later orcad (after, say, a SIGKILL that + // this process somehow survived enough to run handlers) leaves a record that is not + // ours. Deleting it would unlock a live runtime. + const current = parseLockRecord(safeRead(lockPath) ?? '') + if (!current || current.nonce !== record.nonce) { + return + } + try { + unlinkSync(lockPath) + } catch { + // Best-effort: a leftover record with a dead pid is reclaimed on the next start. + } + } + } +} diff --git a/src/main/orcad/orcad-launch-contract.test.ts b/src/main/orcad/orcad-launch-contract.test.ts new file mode 100644 index 00000000000..b22dc74f0e4 --- /dev/null +++ b/src/main/orcad/orcad-launch-contract.test.ts @@ -0,0 +1,43 @@ +/** + * The two things a supervisor reads off a launch: what the arguments mean, and what an exit + * code means. Both are part of the ops contract in docs/reference/orcad-operations.md. + */ +import { describe, expect, it } from 'vitest' +import { + ORCAD_EXIT_CONFIGURATION, + ORCAD_EXIT_FAILED, + parseArgs, + resolveOrcadExitCode +} from './orcad-entry' +import { OrcadBindAddressError } from './orcad-bind-address' +import { OrcadInstanceLockError } from './orcad-instance-lock' + +describe('parseArgs', () => { + it('accepts --bind and leaves it unset when absent', () => { + expect(parseArgs(['--bind', '0.0.0.0'])).toEqual({ bind: '0.0.0.0' }) + expect(parseArgs([])).toEqual({}) + expect(parseArgs(['--port', '6768', '--bind', '10.0.0.5', '--json'])).toEqual({ + port: 6768, + bind: '10.0.0.5', + json: true + }) + }) + + it('rejects --bind with no value rather than silently binding the default', () => { + expect(() => parseArgs(['--bind'])).toThrow('--bind expects a value') + expect(() => parseArgs(['--bind', '--json'])).not.toThrow() + }) +}) + +describe('resolveOrcadExitCode', () => { + it('separates a configuration fault from a generic failure', () => { + // A supervisor must be able to stop restarting on faults that restarting cannot fix: + // a data root owned by someone else, held by another instance, or a bad bind address. + expect( + resolveOrcadExitCode(new OrcadInstanceLockError('orcad_instance_lock_held', 'held')) + ).toBe(ORCAD_EXIT_CONFIGURATION) + expect(resolveOrcadExitCode(new OrcadBindAddressError('bad'))).toBe(ORCAD_EXIT_CONFIGURATION) + expect(resolveOrcadExitCode(new Error('port in use'))).toBe(ORCAD_EXIT_FAILED) + expect(ORCAD_EXIT_CONFIGURATION).not.toBe(ORCAD_EXIT_FAILED) + }) +}) diff --git a/src/main/orcad/orcad-native-preflight.test.ts b/src/main/orcad/orcad-native-preflight.test.ts new file mode 100644 index 00000000000..6c697a3ecce --- /dev/null +++ b/src/main/orcad/orcad-native-preflight.test.ts @@ -0,0 +1,125 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + ORCAD_NATIVE_PRECONDITION_EXIT_CODE, + runOrcadNativePreflight +} from './orcad-native-preflight' +import { + runtimeTerminalDegradation, + setRuntimeTerminalUnavailableCause +} from '../runtime/native-terminal-availability' +import type { NodePtyPreconditionVerdict } from './node-pty-precondition' + +const ABI = { + platform: 'linux' as NodeJS.Platform, + arch: 'x64', + libc: 'glibc' as const, + glibcVersion: '2.31', + nodeAbi: '127' +} + +const verdict = (over: Partial): NodePtyPreconditionVerdict => ({ + status: 'ok', + slot: 'linux-x64-glibc', + abi: ABI, + ...over +}) + +const harness = (given: NodePtyPreconditionVerdict) => { + const warn = vi.fn() + const fail = vi.fn() + const exit = vi.fn(() => undefined as never) + const continued = runOrcadNativePreflight({ + check: () => given, + toolchainHints: () => [' sudo apt-get install -y build-essential python3'], + warn, + fail, + exit + }) + return { warn, fail, exit, continued } +} + +afterEach(() => { + setRuntimeTerminalUnavailableCause(null) +}) + +describe('runOrcadNativePreflight', () => { + it('stops the boot on a proven-unloadable binary instead of reaching the require', () => { + // Continuing here would hit the very dlopen the probe just proved fatal, and the + // operator would get the loader's stack trace instead of the sentence below. + const { fail, exit, warn } = harness( + verdict({ status: 'blocked', reason: 'libc_floor', detail: 'the binary requires GLIBC_2.34' }) + ) + + expect(exit).toHaveBeenCalledWith(ORCAD_NATIVE_PRECONDITION_EXIT_CODE) + expect(warn).not.toHaveBeenCalled() + const message = fail.mock.calls[0][0] as string + expect(message).toContain('newer C library') + expect(message).toContain('the binary requires GLIBC_2.34') + expect(message).toContain('libc glibc 2.31') + // The toolchain hint is what makes it actionable rather than merely accurate. + expect(message).toContain('sudo apt-get install -y build-essential python3') + }) + + it('exits with EX_CONFIG so a supervisor does not restart an unequippable host forever', () => { + expect(ORCAD_NATIVE_PRECONDITION_EXIT_CODE).toBe(78) + expect(ORCAD_NATIVE_PRECONDITION_EXIT_CODE).not.toBe(1) + }) + + it('publishes the blocked cause as a status degradation', () => { + harness(verdict({ status: 'blocked', reason: 'abi_mismatch', detail: 'built for ABI 115' })) + + expect(runtimeTerminalDegradation()).toEqual({ + code: 'terminal_unavailable', + capability: 'terminal.pty.v1', + reason: 'abi_mismatch', + detail: 'built for ABI 115', + message: + "This host's node-pty binary was built for a different Node ABI than the running Node, so it cannot be loaded. Rebuild node-pty against this Node version. (built for ABI 115)" + }) + }) + + it('boots on a spawn-time-only fault and reports it rather than refusing to serve', () => { + const { continued, exit, warn } = harness( + verdict({ status: 'degraded', reason: 'spawn_helper_missing', detail: 'no spawn-helper' }) + ) + + expect(continued).toBe(true) + expect(exit).not.toHaveBeenCalled() + expect(warn).toHaveBeenCalledOnce() + expect(runtimeTerminalDegradation()?.reason).toBe('spawn_helper_missing') + }) + + it('boots when the probe established nothing, because that is not evidence of a fault', () => { + const { continued, exit } = harness( + verdict({ status: 'unverifiable', reason: 'unknown', detail: 'probe timed out' }) + ) + + expect(continued).toBe(true) + expect(exit).not.toHaveBeenCalled() + // Still reported: an unverifiable host must not look identical to a proven-healthy one. + expect(runtimeTerminalDegradation()?.reason).toBe('unknown') + }) + + it('reports nothing when the load was proved good', () => { + setRuntimeTerminalUnavailableCause({ reason: 'load_failed' }) + + const { continued, warn, fail } = harness(verdict({ status: 'ok' })) + + expect(continued).toBe(true) + expect(warn).not.toHaveBeenCalled() + expect(fail).not.toHaveBeenCalled() + expect(runtimeTerminalDegradation()).toBeNull() + }) + + it('does not run the toolchain probe for a verdict that is not about a missing build', () => { + const toolchainHints = vi.fn(() => [] as string[]) + runOrcadNativePreflight({ + check: () => verdict({ status: 'degraded', reason: 'spawn_helper_missing' }), + toolchainHints, + warn: vi.fn(), + fail: vi.fn(), + exit: vi.fn(() => undefined as never) + }) + expect(toolchainHints).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/orcad/orcad-native-preflight.ts b/src/main/orcad/orcad-native-preflight.ts new file mode 100644 index 00000000000..0f3c2cf8970 --- /dev/null +++ b/src/main/orcad/orcad-native-preflight.ts @@ -0,0 +1,72 @@ +/** + * The native precondition orcad runs before its runtime loads anything native. + * + * Split from `node-pty-precondition.ts` so the decision (what to do about a verdict) is + * testable apart from the detection (what the verdict is). + */ +import process from 'node:process' +import { setRuntimeTerminalUnavailableCause } from '../runtime/native-terminal-availability' +import { terminalUnavailableMessage } from '../../shared/runtime-types' +import { + checkNodePtyPrecondition, + formatNodePtyPreconditionReport, + probeLocalBuildToolchainHints, + type NodePtyPreconditionVerdict +} from './node-pty-precondition' + +/** + * EX_CONFIG. Why not 1: a supervisor that restarts on 1 would restart forever against a + * host that can never load this binary. This code says "the host is not equipped", which + * is a different instruction from "it crashed". + */ +export const ORCAD_NATIVE_PRECONDITION_EXIT_CODE = 78 + +export type NativePreflightHooks = { + check?: () => NodePtyPreconditionVerdict + toolchainHints?: (platform: NodeJS.Platform) => string[] + warn?: (message: string) => void + fail?: (message: string) => void + exit?: (code: number) => never +} + +/** + * Returns true when boot may continue. + * + * A `blocked` verdict never returns: continuing would reach the very `require` the probe + * just proved fatal, and the operator would get the loader's stack trace instead of the + * sentence printed here. + */ +export function runOrcadNativePreflight(hooks: NativePreflightHooks = {}): boolean { + const check = hooks.check ?? checkNodePtyPrecondition + const warn = hooks.warn ?? ((message: string) => console.warn(message)) + const fail = hooks.fail ?? ((message: string) => console.error(message)) + const exit = hooks.exit ?? ((code: number) => process.exit(code) as never) + const verdict = check() + + if (verdict.status === 'ok') { + // Why clear rather than leave alone: a previous run in this process may have recorded + // a cause, and status.get must not keep reporting a degradation that no longer holds. + setRuntimeTerminalUnavailableCause(null) + return true + } + + const reason = verdict.reason ?? 'unknown' + setRuntimeTerminalUnavailableCause({ + reason, + ...(verdict.detail ? { detail: verdict.detail } : {}) + }) + const message = terminalUnavailableMessage(reason, verdict.detail) + + if (verdict.status === 'blocked') { + const hints = (hooks.toolchainHints ?? probeLocalBuildToolchainHints)(verdict.abi.platform) + fail(`orcad: ${formatNodePtyPreconditionReport(verdict, message, hints)}`) + exit(ORCAD_NATIVE_PRECONDITION_EXIT_CODE) + return false + } + + // `degraded` and `unverifiable` both boot. The first is a proven spawn-time fault the + // host can still serve around; the second established nothing, and refusing to boot on + // an inconclusive probe would take down hosts that work. + warn(`orcad: ${formatNodePtyPreconditionReport(verdict, message)}`) + return true +} diff --git a/src/main/ports/port-scan-command-client.test.ts b/src/main/ports/port-scan-command-client.test.ts index d1a15d29774..c2bece624f8 100644 --- a/src/main/ports/port-scan-command-client.test.ts +++ b/src/main/ports/port-scan-command-client.test.ts @@ -298,3 +298,29 @@ describe('PortScanCommandClient on a real worker thread', () => { } }, 30_000) }) + +describe('resolveWorkerEntryPath on a non-Electron host', () => { + // Why: orcad reports isPackaged true (it is a production build), but + // process.resourcesPath is Electron-only and undefined there. Joining undefined threw + // a TypeError instead of failing as a missing worker — a crash where a clean + // "worker unavailable" was the honest outcome. + it('does not join an undefined resourcesPath', () => { + expect(() => + resolveWorkerEntryPath({ + isPackaged: true, + resourcesPath: undefined, + moduleDir: '/opt/orcad' + }) + ).not.toThrow() + }) + + it('falls back to the module directory when there is no resources tree', () => { + expect( + resolveWorkerEntryPath({ + isPackaged: true, + resourcesPath: undefined, + moduleDir: '/opt/orcad' + }) + ).toBe(join('/opt/orcad', 'port-scan-command-worker-entry.js')) + }) +}) diff --git a/src/main/ports/port-scan-command-client.ts b/src/main/ports/port-scan-command-client.ts index c2bc80bc883..9c01b4dccda 100644 --- a/src/main/ports/port-scan-command-client.ts +++ b/src/main/ports/port-scan-command-client.ts @@ -303,7 +303,8 @@ const WORKER_ENTRY_FILENAME = 'port-scan-command-worker-entry.js' /** Where the built worker entry can live: packaged resources or the build dir. */ export type WorkerEntryLayout = { isPackaged: boolean - resourcesPath: string + /** Undefined on a non-Electron host: `process.resourcesPath` is Electron-only. */ + resourcesPath: string | undefined moduleDir: string } @@ -318,7 +319,12 @@ export function resolveWorkerEntryPath(layout: WorkerEntryLayout): string { // the bundler's __dirname, matching the shipped stt/warp/opencode workers. // Split out from the electron read so the packaged branch is testable without // a packaged build. - if (layout.isPackaged) { + // Why the resourcesPath guard: `isPackaged` is true on orcad too, but + // `process.resourcesPath` is Electron-only and undefined under plain Node — joining + // it threw a TypeError rather than failing as a missing worker. A host without an + // Electron resources tree has no asar to look in, so fall back to the module dir and + // let the caller report a missing worker honestly. + if (layout.isPackaged && layout.resourcesPath) { return join(layout.resourcesPath, 'app.asar', 'out', 'main', WORKER_ENTRY_FILENAME) } return join(layout.moduleDir, WORKER_ENTRY_FILENAME) diff --git a/src/main/runtime/native-terminal-availability.ts b/src/main/runtime/native-terminal-availability.ts new file mode 100644 index 00000000000..85eb3df57f0 --- /dev/null +++ b/src/main/runtime/native-terminal-availability.ts @@ -0,0 +1,51 @@ +import { + type RuntimeTerminalUnavailableReason, + terminalUnavailableMessage, + TERMINAL_PTY_DEGRADATION_CAPABILITY, + TERMINAL_UNAVAILABLE_ERROR_CODE, + type RuntimeDegradation +} from '../../shared/runtime-types' + +/** + * A note left by whoever proved this host cannot load or spawn PTYs, read back when + * `status.get` assembles `degradations[]`. + * + * Why a module-level note rather than a constructor argument: only the host entry point + * can run the out-of-process load probe, and it must run before anything requires + * `node-pty` — long before `OrcaRuntimeService` exists. This is the same shape + * `runtime-browser-commands-factory` uses for `browser_unavailable`, for the same reason. + * + * Silence means "nothing proved it broken", never "proved working". Hosts that never + * run a precondition therefore report no degradation, which is the honest answer. + */ +export type RuntimeTerminalUnavailableCause = { + reason: RuntimeTerminalUnavailableReason + detail?: string +} + +let unavailableCause: RuntimeTerminalUnavailableCause | null = null + +export function setRuntimeTerminalUnavailableCause( + cause: RuntimeTerminalUnavailableCause | null +): void { + unavailableCause = cause +} + +export function runtimeTerminalUnavailableCause(): RuntimeTerminalUnavailableCause | null { + return unavailableCause +} + +/** The degradation entry for the recorded cause, or null when nothing is degraded. */ +export function runtimeTerminalDegradation(): RuntimeDegradation | null { + const cause = unavailableCause + if (!cause) { + return null + } + return { + code: TERMINAL_UNAVAILABLE_ERROR_CODE, + capability: TERMINAL_PTY_DEGRADATION_CAPABILITY, + message: terminalUnavailableMessage(cause.reason, cause.detail), + reason: cause.reason, + ...(cause.detail ? { detail: cause.detail } : {}) + } +} diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index 60926fc873c..40c9eafd621 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -5,6 +5,7 @@ import { setRuntimeBrowserCommandsFactory, setRuntimeBrowserUnavailableCause } from './runtime-browser-commands-factory' +import { setRuntimeTerminalUnavailableCause } from './native-terminal-availability' import { setRuntimeDesktopSurface } from './runtime-desktop-surface' import { installFakeAppEnvironment } from '../../../config/scripts/vitest-host-ports-setup' import type * as GitUsernameModule from '../git/git-username' @@ -697,6 +698,7 @@ function resetRuntimeTestMocks(): void { // browser RPCs reject rather than silently succeeding. setRuntimeBrowserCommandsFactory((host) => new RuntimeBrowserCommands(host)) setRuntimeBrowserUnavailableCause(null) + setRuntimeTerminalUnavailableCause(null) // Why: the runtime's notification, window lookup and tab-create-reply channel are // injected now, so the electron mock alone is inert. Back the surface with the same // mocks so every existing expectation still holds. @@ -2769,6 +2771,43 @@ describe('OrcaRuntimeService', () => { ) }) + it('reports a host that cannot load node-pty, instead of that host never answering', () => { + // The alternative to reporting it is the process dying inside the dynamic loader, + // which reaches a client as a dropped connection with no cause attached. + setRuntimeTerminalUnavailableCause({ + reason: 'libc_floor', + detail: 'the binary requires GLIBC_2.34' + }) + + const degradations = createRuntime().getStatus().degradations ?? [] + + expect(degradations).toContainEqual({ + code: 'terminal_unavailable', + capability: 'terminal.pty.v1', + reason: 'libc_floor', + detail: 'the binary requires GLIBC_2.34', + message: + "This host's node-pty binary was built against a newer C library than the host provides, so the dynamic loader refuses it. Rebuild node-pty on this host, or deploy a build whose prebuilt binary matches this platform's libc. (the binary requires GLIBC_2.34)" + }) + }) + + it('reports browser and terminal loss together, because they fail independently', () => { + setRuntimeBrowserCommandsFactory(null) + setRuntimeBrowserUnavailableCause({ reason: 'unconfigured' }) + setRuntimeTerminalUnavailableCause({ reason: 'dependency_missing' }) + + const codes = (createRuntime().getStatus().degradations ?? []).map((entry) => entry.code) + + expect(codes).toEqual(['browser_unavailable', 'terminal_unavailable']) + }) + + it('says nothing about terminals when no precondition proved them broken', () => { + // Silence must mean "nothing proved it broken", never "proved working" — a host that + // never ran the precondition has no verdict to publish. + const degradations = createRuntime().getStatus().degradations ?? [] + expect(degradations.map((entry) => entry.code)).not.toContain('terminal_unavailable') + }) + it('closes a worktree’s offscreen browser pages when its metadata is removed (leak fix)', () => { const runtime = createRuntime() const closeTab = vi.fn().mockResolvedValue(undefined) diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index ee1f977ca88..0b25b3cef46 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -448,6 +448,7 @@ import type { LinearTeamStatesResult, LinearStatusSetResult } from '../../shared/linear/agent-access' +import { runtimeTerminalDegradation } from './native-terminal-availability' import { BROWSER_UNAVAILABLE_ERROR_CODE, browserUnavailableMessage, @@ -6482,6 +6483,12 @@ export class OrcaRuntimeService { } ] : [] + // Why appended rather than merged into the ternary: PTY loss and browser loss are + // independent, and a host can be degraded on both at once. + const terminalDegradation = runtimeTerminalDegradation() + if (terminalDegradation) { + degradations.push(terminalDegradation) + } return { runtimeId: this.runtimeId, rendererGraphEpoch: this.rendererGraphEpoch, diff --git a/src/main/runtime/runtime-rpc-websocket-bind-host.test.ts b/src/main/runtime/runtime-rpc-websocket-bind-host.test.ts index a74c48b896b..8210b0f06d4 100644 --- a/src/main/runtime/runtime-rpc-websocket-bind-host.test.ts +++ b/src/main/runtime/runtime-rpc-websocket-bind-host.test.ts @@ -518,6 +518,109 @@ describe('OrcaRuntimeRpcServer WebSocket bind host (STA-2370)', () => { } }) + it('honours a pinned bind host over exposeNetworkByDefault (orcad --bind)', async () => { + const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-rpc-')) + const server = new OrcaRuntimeRpcServer({ + runtime: new OrcaRuntimeService(), + userDataPath, + enableWebSocket: true, + wsPort: 0, + // Why both: an unattended host passes an explicit answer, and it must outrank every + // implicit widen — otherwise "default loopback" is only true until something else wins. + exposeNetworkByDefault: true, + pinnedBindHost: '127.0.0.1' + }) + + await server.start() + try { + expect(wsTransportOf(server)?.resolvedHost).toBe('127.0.0.1') + expect(new URL(server.getWebSocketEndpoint()!).hostname).toBe('127.0.0.1') + } finally { + await server.stop() + } + }) + + it('stays pinned to loopback even after a device has connected once', async () => { + const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-rpc-')) + // Why this case specifically: the unpinned default widens at the NEXT startup once any + // network-reach device has connected. A loopback orcad would therefore go wide one + // restart after its first client paired — silently, and without the operator asking. + const registry = new DeviceRegistry(userDataPath) + const device = registry.getOrCreatePendingDevice('CLI', 'runtime', 'network') + registry.updateLastSeen(device.deviceId) + + const server = new OrcaRuntimeRpcServer({ + runtime: new OrcaRuntimeService(), + userDataPath, + enableWebSocket: true, + wsPort: 0, + pinnedBindHost: '127.0.0.1' + }) + + await server.start() + try { + expect( + server + .getDeviceRegistry() + ?.listDevices() + .some((d) => d.lastSeenAt > 0) + ).toBe(true) + expect(wsTransportOf(server)?.resolvedHost).toBe('127.0.0.1') + } finally { + await server.stop() + } + }) + + it('refuses a runtime-widening pairing offer while the bind is pinned to loopback', async () => { + const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-rpc-')) + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}) + const server = new OrcaRuntimeRpcServer({ + runtime: new OrcaRuntimeService(), + userDataPath, + enableWebSocket: true, + wsPort: 0, + pinnedBindHost: '127.0.0.1' + }) + + await server.start() + try { + // A paired client can reach this RPC. Without the pin's refusal it would rebind the + // listener to every interface, undoing the operator's bind policy from the outside. + const offer = await server.createMobilePairingOffer({ + address: '100.64.1.20', + connectionMode: 'local-only' + }) + expect(offer.available).toBe(false) + if (!offer.available) { + expect(offer.reason).toBe('network_exposure_failed') + } + expect(wsTransportOf(server)?.resolvedHost).toBe('127.0.0.1') + } finally { + await server.stop() + errorSpy.mockRestore() + } + }) + + it('still widens on request when the operator pinned the wide address', async () => { + const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-rpc-')) + const server = new OrcaRuntimeRpcServer({ + runtime: new OrcaRuntimeService(), + userDataPath, + enableWebSocket: true, + wsPort: 0, + pinnedBindHost: '0.0.0.0' + }) + + await server.start() + try { + expect(wsTransportOf(server)?.resolvedHost).toBe('0.0.0.0') + await server.ensureNetworkExposure() + expect(wsTransportOf(server)?.resolvedHost).toBe('0.0.0.0') + } finally { + await server.stop() + } + }) + it('refuses to widen a pairing offer that arrives after the server has stopped', async () => { const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-rpc-')) const server = new OrcaRuntimeRpcServer({ diff --git a/src/main/runtime/runtime-rpc.ts b/src/main/runtime/runtime-rpc.ts index 1b5899f346c..b28619b2eed 100644 --- a/src/main/runtime/runtime-rpc.ts +++ b/src/main/runtime/runtime-rpc.ts @@ -59,6 +59,12 @@ const DEFAULT_WS_PORT = 6768 const WS_BIND_HOST_LOOPBACK = '127.0.0.1' const WS_BIND_HOST_ALL_INTERFACES = '0.0.0.0' +// Why brackets: `ws://::1:6768` is not a URL, and every consumer of this endpoint parses it +// with `new URL`. An IPv6 bind address would otherwise publish an unparseable endpoint. +function formatWsEndpoint(host: string, port: number): string { + return `ws://${host.includes(':') ? `[${host}]` : host}:${port}` +} + type OrcaRuntimeRpcServerOptions = { runtime: OrcaRuntimeService userDataPath: string @@ -71,6 +77,16 @@ type OrcaRuntimeRpcServerOptions = { // Why: STA-2370 — bind the WS listener to all interfaces at startup instead of loopback-until-paired. // Only `orca serve` (explicit remote opt-in) and E2E set this; the desktop app widens lazily on pairing. exposeNetworkByDefault?: boolean + /** + * Pin the WS listener to exactly this address for the process's whole life. + * + * Why a pin and not another default: the two paths below both widen on their own — + * `exposeNetworkByDefault` at startup, and a device that has connected once at every + * later startup. An unattended host (orcad) whose operator asked for loopback must + * still be on loopback after a client pairs and the service restarts, so the answer + * has to outrank both, and `ensureNetworkExposure()` has to refuse rather than widen. + */ + pinnedBindHost?: string webClientRoot?: string // Why: test-only overrides for the two constants below; production must not pass these (defaults set by §3.1). keepaliveIntervalMs?: number @@ -495,6 +511,7 @@ export class OrcaRuntimeRpcServer { private readonly wsPort: number private readonly preferPinnedWsPort: boolean private readonly exposeNetworkByDefault: boolean + private readonly pinnedBindHost: string | null private readonly webClientRoot: string | undefined // Why: STA-2370 — the host the WS listener is currently bound to, so pairing can widen loopback→all-interfaces once. private wsBoundHost: string | null = null @@ -555,6 +572,7 @@ export class OrcaRuntimeRpcServer { wsPort = DEFAULT_WS_PORT, preferPinnedWsPort = false, exposeNetworkByDefault = false, + pinnedBindHost, webClientRoot, keepaliveIntervalMs = KEEPALIVE_INTERVAL_MS, longPollCap = LONG_POLL_CAP, @@ -570,6 +588,7 @@ export class OrcaRuntimeRpcServer { this.wsPort = wsPort this.preferPinnedWsPort = preferPinnedWsPort this.exposeNetworkByDefault = exposeNetworkByDefault + this.pinnedBindHost = pinnedBindHost ?? null this.webClientRoot = webClientRoot this.keepaliveIntervalMs = keepaliveIntervalMs this.longPollCap = longPollCap @@ -1241,6 +1260,9 @@ export class OrcaRuntimeRpcServer { // A grant minted for "This computer only" is excluded: its client is a browser on this machine, so // counting it would republish the runtime on every interface one restart after the user declined that. private resolveInitialWebSocketBindHost(): string { + if (this.pinnedBindHost) { + return this.pinnedBindHost + } if (this.exposeNetworkByDefault) { return WS_BIND_HOST_ALL_INTERFACES } @@ -1286,7 +1308,7 @@ export class OrcaRuntimeRpcServer { this.wsBoundHost = options.host return { transport: wsTransport, - endpoint: `ws://${options.host}:${wsTransport.resolvedPort}` + endpoint: formatWsEndpoint(options.host, wsTransport.resolvedPort) } } @@ -1359,6 +1381,14 @@ export class OrcaRuntimeRpcServer { // connected when a later LAN/QR offer opts in. Rebinding terminates them (ws cannot move a listener), // so the resolved port is reused — already-issued endpoints stay valid and clients reconnect in place. async ensureNetworkExposure(): Promise { + if (this.pinnedBindHost && this.pinnedBindHost !== WS_BIND_HOST_ALL_INTERFACES) { + // Why throw and not return: callers widen so they can ADVERTISE a LAN endpoint. Returning + // quietly would let them publish one that nothing can reach; the throw lands in their + // existing network_exposure_failed branch, which reports the offer unavailable instead. + throw new Error( + `Runtime bind address is pinned to ${this.pinnedBindHost}; refusing to widen to all interfaces` + ) + } if ( !this.enableWebSocket || this.stopping || diff --git a/src/main/server/serve-readiness.test.ts b/src/main/server/serve-readiness.test.ts index 7e037eba025..3e4b77e695f 100644 --- a/src/main/server/serve-readiness.test.ts +++ b/src/main/server/serve-readiness.test.ts @@ -4,6 +4,7 @@ import { ServeReadinessPublisher, type ServeReadiness } from './serve-readiness' +import type { OrcadHealth } from '../orcad/orcad-health' const ready: ServeReadiness = { runtimeId: 'runtime-1', @@ -21,6 +22,25 @@ const ready: ServeReadiness = { } } +const health: OrcadHealth = { + buildHash: 'abc123def4567890', + buildVersion: '1.4.0', + nodeVersion: '20.11.0', + nodeAbi: '115', + platform: 'linux', + arch: 'x64', + pid: 1234, + terminalDaemon: { + state: 'live', + ownsFreshSessions: true, + pid: 4242, + buildVersion: '1.4.0', + entryPath: '/opt/orcad/daemon-entry.js', + protocolVersion: 36, + selfTest: { ok: true, coverage: 'pty-spawn', verdict: 'healthy', durationMs: 12 } + } +} + describe('ServeReadinessPublisher', () => { it('writes one complete human-readable ready block', async () => { const write = vi.fn(async () => {}) @@ -90,6 +110,50 @@ describe('ServeReadinessPublisher', () => { ).toThrow('websocket_unavailable. Choose an unused --port.') }) + it('omits the health block entirely when a host does not report one', () => { + const payload = JSON.parse(renderServeReadiness(ready, { mode: 'json' })) + // Why absent rather than a null/empty object: a reader must be able to tell "this host + // does not publish health" from "this host published a green verdict". + expect('health' in payload).toBe(false) + expect(renderServeReadiness(ready, { mode: 'human' })).not.toContain('Terminal daemon') + }) + + it('carries build identity, Node ABI and the daemon self-test in the JSON contract', () => { + const payload = JSON.parse(renderServeReadiness({ ...ready, health }, { mode: 'json' })) + expect(payload.health.buildHash).toBe('abc123def4567890') + expect(payload.health.nodeAbi).toBe('115') + expect(payload.health.terminalDaemon.selfTest).toEqual({ + ok: true, + coverage: 'pty-spawn', + verdict: 'healthy', + durationMs: 12 + }) + }) + + it('says out loud when the daemon self-test failed', () => { + const failed: ServeReadiness = { + ...ready, + health: { + ...health, + terminalDaemon: { + ...health.terminalDaemon, + state: 'degraded', + ownsFreshSessions: false, + selfTest: { + ok: false, + coverage: 'pty-spawn', + verdict: 'pty-spawn-unhealthy', + durationMs: 3_000 + } + } + } + } + const human = renderServeReadiness(failed, { mode: 'human' }) + // An operator reading the ready block must not have to infer this from a missing line. + expect(human).toContain('PTY self-test FAILED') + expect(human).toContain('terminals survive an orcad restart: NO') + }) + it('rejects concurrent and later duplicate publications', async () => { let finishWrite: (() => void) | undefined const publisher = new ServeReadinessPublisher( diff --git a/src/main/server/serve-readiness.ts b/src/main/server/serve-readiness.ts index e4ab3db8925..b0bdf14daba 100644 --- a/src/main/server/serve-readiness.ts +++ b/src/main/server/serve-readiness.ts @@ -1,4 +1,5 @@ import type { PairingOfferUnavailableReason } from '../runtime/runtime-rpc' +import type { OrcadHealth } from '../orcad/orcad-health' export type ServePairingUnavailableReason = PairingOfferUnavailableReason | 'disabled_by_operator' @@ -24,6 +25,14 @@ export type ServeReadiness = { advertisedEndpoint: string | null managedWslCliReconciliation: 'pending' | 'settled' | 'failed' pairing: ServePairingReadiness + /** + * Build identity, Node ABI and the cross-process terminal-daemon self-test. + * + * Optional because the Electron `--serve` host does not publish one yet; readers must + * treat its absence as "not reported", never as healthy. Additive, so an older client + * parsing this payload is unaffected. + */ + health?: OrcadHealth } export type ServeReadinessOutput = @@ -77,7 +86,8 @@ export function renderServeReadiness( boundEndpoint: readiness.boundEndpoint, advertisedEndpoint: readiness.advertisedEndpoint, managedWslCliReconciliation: readiness.managedWslCliReconciliation, - pairing: readiness.pairing + pairing: readiness.pairing, + ...(readiness.health ? { health: readiness.health } : {}) }) } return renderHumanReadiness(readiness) @@ -89,6 +99,18 @@ function renderHumanReadiness(readiness: ServeReadiness): string { `Bound endpoint: ${readiness.boundEndpoint ?? 'websocket unavailable'}`, `Advertised endpoint: ${readiness.advertisedEndpoint ?? 'unavailable'}` ] + if (readiness.health) { + const daemon = readiness.health.terminalDaemon + lines.push( + `Build: ${readiness.health.buildVersion} (${readiness.health.buildHash}), Node ` + + `${readiness.health.nodeVersion} ABI ${readiness.health.nodeAbi}` + ) + lines.push( + `Terminal daemon: ${daemon.state} — PTY self-test ${daemon.selfTest.ok ? 'passed' : 'FAILED'}` + + ` (${daemon.selfTest.coverage}: ${daemon.selfTest.verdict})` + + `; terminals survive an orcad restart: ${daemon.ownsFreshSessions ? 'yes' : 'NO'}` + ) + } if (readiness.pairing.available) { if (readiness.pairing.webClientUrl) { lines.push(`Web client URL: ${readiness.pairing.webClientUrl}`) diff --git a/src/main/ssh/build-toolchain-diagnosis.ts b/src/main/ssh/build-toolchain-diagnosis.ts new file mode 100644 index 00000000000..446601518c9 --- /dev/null +++ b/src/main/ssh/build-toolchain-diagnosis.ts @@ -0,0 +1,166 @@ +/** + * Diagnose a host's C/C++ build toolchain from a POSIX-sh probe. + * + * Why this is separate from `ssh-relay-build-toolchain.ts`: everything here is pure — + * it builds a shell command, parses its output, and formats messages. The relay reaches + * it over SSH; `orcad` reaches it over a local shell. Keeping it transport-free is what + * lets the Node-only bundle reuse it without pulling `ssh2` in behind it. + */ +// Why: node-pty@1.1.0 ships no Linux prebuild, so the remote `npm install` falls +// back to `node-gyp rebuild` and needs a C/C++ toolchain. A missing toolchain is +// the dominant first-connect failure on Linux relays (#1693); node-gyp surfaces +// it as an opaque `not found: make`. We probe for the tools so we can replace +// that with an actionable "install build-essential" message. +const PROBED_TOOLS = [ + 'make', + 'gcc', + 'g++', + 'cc', + 'c++', + 'clang', + 'clang++', + 'python3', + 'python' +] as const + +// Package managers mapped to the one-liner that installs a C/C++ toolchain on +// the matching distro family. Ordered by detection priority. +const PACKAGE_MANAGER_HINTS: readonly { bin: string; install: string }[] = [ + { bin: 'apt-get', install: 'sudo apt-get install -y build-essential python3' }, + { bin: 'dnf', install: 'sudo dnf install -y make gcc gcc-c++ python3' }, + { bin: 'yum', install: 'sudo yum install -y make gcc gcc-c++ python3' }, + { bin: 'pacman', install: 'sudo pacman -S --needed base-devel python' }, + { bin: 'apk', install: 'sudo apk add build-base python3' }, + { bin: 'zypper', install: 'sudo zypper install -y gcc gcc-c++ make python3' } +] + +export type BuildToolchainStatus = { + present: string[] + packageManager: string | null + // node-gyp needs make, Python, and a C++ compiler. The caller only uses this + // verdict after npm/node-gyp output already points at a native-build failure, + // so custom Python paths do not make unrelated npm failures look toolchainy. + toolchainMissing: boolean +} + +function hasCxxCompiler(present: ReadonlySet): boolean { + return present.has('g++') || present.has('c++') || present.has('clang++') +} + +function hasPython(present: ReadonlySet): boolean { + return present.has('python3') || present.has('python') +} + +// POSIX-sh probe: echo a `HAVE ` line per resolvable build tool and a +// single `PKG ` line for the host's package manager. Runs under +// `/bin/sh -c` (see wrapRemoteCommandForPosixShell), so it stays portable. +export function buildToolchainProbeCommand(): string { + const toolLoop = `for t in ${PROBED_TOOLS.join( + ' ' + )}; do if command -v "$t" >/dev/null 2>&1; then echo "HAVE $t"; fi; done` + const pkgList = PACKAGE_MANAGER_HINTS.map((hint) => hint.bin).join(' ') + const pkgLoop = `for p in ${pkgList}; do if command -v "$p" >/dev/null 2>&1; then echo "PKG $p"; break; fi; done` + return `${toolLoop}; ${pkgLoop}` +} + +export function parseBuildToolchainProbe(output: string): BuildToolchainStatus { + const present = new Set() + let packageManager: string | null = null + for (const line of output.split('\n')) { + const haveMatch = line.trim().match(/^HAVE (\S+)$/) + if (haveMatch) { + present.add(haveMatch[1]) + continue + } + const pkgMatch = line.trim().match(/^PKG (\S+)$/) + if (pkgMatch && !packageManager) { + packageManager = pkgMatch[1] + } + } + return { + present: PROBED_TOOLS.filter((tool) => present.has(tool)), + packageManager, + toolchainMissing: !present.has('make') || !hasCxxCompiler(present) || !hasPython(present) + } +} + +export function shouldProbeBuildToolchainAfterNativeDepsFailure(message: string): boolean { + const lower = message.toLowerCase() + if (!lower.includes('gyp') && !lower.includes('node-gyp')) { + return false + } + return ( + /\bnot found:\s*(make|gmake|gcc|g\+\+|cc|c\+\+|clang|clang\+\+|python|python3)\b/i.test( + message + ) || + /\b(make|gmake|gcc|g\+\+|cc|c\+\+|clang|clang\+\+|python|python3)\b.*\bnot found\b/i.test( + message + ) || + lower.includes('could not find any python installation') || + lower.includes('no xcode or clt version detected') + ) +} + +function missingToolNames(status: BuildToolchainStatus): string[] { + const present = new Set(status.present) + const missing: string[] = [] + if (!present.has('make')) { + missing.push('make') + } + if (!hasCxxCompiler(present)) { + missing.push('a C++ compiler (g++ or clang++)') + } + if (!hasPython(present)) { + missing.push('python3') + } + return missing +} + +/** Install hint for the host's package manager, or the cross-distro list when it is unknown. */ +export function toolchainInstallHintLines(status: BuildToolchainStatus): string[] { + const tailored = status.packageManager + ? PACKAGE_MANAGER_HINTS.find((hint) => hint.bin === status.packageManager)?.install + : null + if (tailored) { + return [` ${tailored}`] + } + return [ + ' Debian/Ubuntu: sudo apt-get install -y build-essential python3', + ' Fedora/RHEL: sudo dnf install -y make gcc gcc-c++ python3', + ' Arch: sudo pacman -S --needed base-devel python', + ' Alpine: sudo apk add build-base python3' + ] +} + +/** One-line summary for the deploy log when node-pty is skipped rather than compiled. */ +export function formatSkippedNodePtyWarning(status: BuildToolchainStatus): string { + // Why: with no package manager detected the hint list is the cross-distro menu, whose first line + // is Debian's — quoting it alone would name the wrong distro, so stay neutral instead. + const hintLines = toolchainInstallHintLines(status) + const hint = + hintLines.length === 1 + ? hintLines[0].trim() + : 'install a C/C++ toolchain (make, a C++ compiler, python3)' + return ( + `missing build tools (${missingToolNames(status).join(', ')}); skipping node-pty so the ` + + `connection still serves files and git. Remote terminals need: ${hint}` + ) +} + +export function formatMissingToolchainError( + status: BuildToolchainStatus, + underlyingError: string +): string { + const lines = [ + `The remote host is missing the C/C++ build tools (${missingToolNames(status).join(', ')}) ` + + `needed to compile Orca's relay native modules (node-pty, @parcel/watcher). node-pty has no ` + + `prebuilt binary for Linux, so they must be compiled on the remote host.`, + '', + 'Install the build tools on the remote host, then reconnect:', + ...toolchainInstallHintLines(status), + '', + `Underlying install error: ${underlyingError}` + ] + return lines.join('\n') +} + diff --git a/src/main/ssh/orcad-activation-gate.test.ts b/src/main/ssh/orcad-activation-gate.test.ts new file mode 100644 index 00000000000..534d41c08f0 --- /dev/null +++ b/src/main/ssh/orcad-activation-gate.test.ts @@ -0,0 +1,164 @@ +import { describe, expect, it } from 'vitest' + +import { evaluateOrcadActivation } from './orcad-activation-gate' +import type { ServeReadiness } from '../server/serve-readiness' +import type { OrcadHealth, TerminalDaemonHealth } from '../orcad/orcad-health' + +const EXPECTED = { buildHash: 'abc123def4567890', fullVersion: '0.2.0+bb01' } + +function daemon(overrides: Partial = {}): TerminalDaemonHealth { + return { + state: 'live', + ownsFreshSessions: true, + pid: 4242, + buildVersion: '0.2.0+bb01', + entryPath: '/home/u/.orca-remote/orcad-0.2.0+bb01/daemon-entry.js', + protocolVersion: 3, + selfTest: { ok: true, coverage: 'pty-spawn', verdict: 'healthy', durationMs: 12 }, + ...overrides + } +} + +function health(overrides: Partial = {}): OrcadHealth { + return { + buildHash: EXPECTED.buildHash, + buildVersion: EXPECTED.fullVersion, + nodeVersion: '20.11.0', + nodeAbi: '115', + platform: 'linux', + arch: 'x64', + pid: 4200, + terminalDaemon: daemon(), + ...overrides + } +} + +function readiness(overrides: Partial = {}): ServeReadiness { + return { + runtimeId: 'runtime-1', + boundEndpoint: 'ws://127.0.0.1:7777', + advertisedEndpoint: null, + managedWslCliReconciliation: 'settled', + pairing: { available: false, reason: 'disabled_by_operator', guidance: 'n/a' }, + health: health(), + ...overrides + } +} + +describe('evaluateOrcadActivation', () => { + it('activates a candidate that proved a real PTY round trip', () => { + const verdict = evaluateOrcadActivation(readiness(), EXPECTED) + expect(verdict).toEqual({ decision: 'activate', coverage: 'pty-spawn', warnings: [] }) + }) + + it('refuses when the candidate never published readiness', () => { + const verdict = evaluateOrcadActivation(null, EXPECTED) + expect(verdict).toMatchObject({ decision: 'reject', code: 'orcad_activation_no_readiness' }) + }) + + it('refuses a readiness payload with no health, rather than reading silence as healthy', () => { + const { health: _dropped, ...withoutHealth } = readiness() + const verdict = evaluateOrcadActivation(withoutHealth as ServeReadiness, EXPECTED) + expect(verdict).toMatchObject({ decision: 'reject', code: 'orcad_activation_no_health' }) + }) + + it('refuses when a different build answered — a stale process holding the port', () => { + const verdict = evaluateOrcadActivation( + readiness({ health: health({ buildHash: '0000000000000000' }) }), + EXPECTED + ) + expect(verdict).toMatchObject({ decision: 'reject', code: 'orcad_activation_build_mismatch' }) + }) + + it('refuses a listening orcad whose terminal daemon is absent', () => { + const verdict = evaluateOrcadActivation( + readiness({ + health: health({ + terminalDaemon: daemon({ + state: 'absent', + ownsFreshSessions: false, + selfTest: { ok: false, coverage: 'pty-spawn', verdict: 'no-daemon', durationMs: 1 } + }) + }) + }), + EXPECTED + ) + expect(verdict).toMatchObject({ decision: 'reject', code: 'orcad_activation_daemon_absent' }) + }) + + it('refuses a degraded daemon, whose fresh terminals would not survive a restart', () => { + const verdict = evaluateOrcadActivation( + readiness({ + health: health({ terminalDaemon: daemon({ state: 'degraded', ownsFreshSessions: false }) }) + }), + EXPECTED + ) + expect(verdict).toMatchObject({ decision: 'reject', code: 'orcad_activation_daemon_degraded' }) + }) + + it('refuses a live daemon that failed its PTY spawn probe', () => { + const verdict = evaluateOrcadActivation( + readiness({ + health: health({ + terminalDaemon: daemon({ + selfTest: { + ok: false, + coverage: 'pty-spawn', + verdict: 'pty-spawn-unhealthy', + durationMs: 30 + } + }) + }) + }), + EXPECTED + ) + expect(verdict).toMatchObject({ + decision: 'reject', + code: 'orcad_activation_pty_self_test_failed' + }) + }) + + it('refuses a green daemon that does not own fresh sessions', () => { + const verdict = evaluateOrcadActivation( + readiness({ health: health({ terminalDaemon: daemon({ ownsFreshSessions: false }) }) }), + EXPECTED + ) + expect(verdict).toMatchObject({ + decision: 'reject', + code: 'orcad_activation_no_persistent_terminals' + }) + }) + + it('refuses a candidate that is not listening', () => { + const verdict = evaluateOrcadActivation(readiness({ boundEndpoint: null }), EXPECTED) + expect(verdict).toMatchObject({ decision: 'reject', code: 'orcad_activation_not_listening' }) + }) + + it('activates handshake-only coverage but never records it as a proven PTY', () => { + const verdict = evaluateOrcadActivation( + readiness({ + health: health({ + platform: 'win32', + terminalDaemon: daemon({ + selfTest: { ok: true, coverage: 'handshake', verdict: 'healthy', durationMs: 5 } + }) + }) + }), + EXPECTED + ) + expect(verdict).toMatchObject({ decision: 'activate', coverage: 'handshake' }) + expect(verdict.decision === 'activate' && verdict.warnings[0]).toContain( + 'covered the daemon handshake only' + ) + }) + + it('checks identity before health, so a wrong-build green payload cannot pass', () => { + const verdict = evaluateOrcadActivation( + readiness({ + health: health({ buildHash: 'ffffffffffffffff', terminalDaemon: daemon() }) + }), + EXPECTED + ) + expect(verdict).toMatchObject({ code: 'orcad_activation_build_mismatch' }) + }) +}) diff --git a/src/main/ssh/orcad-activation-gate.ts b/src/main/ssh/orcad-activation-gate.ts new file mode 100644 index 00000000000..c9581919117 --- /dev/null +++ b/src/main/ssh/orcad-activation-gate.ts @@ -0,0 +1,155 @@ +/** + * Whether a freshly launched orcad has earned the right to become the active one. + * + * The failure this exists to prevent is the one `docs/design/shipping-orcad.html` names + * throughout: a deployment that reports success because a port opened. orcad answers RPC + * from its own process, so "listening" stays true while the terminal daemon that owns every + * terminal is dead — a green host that cannot run a single command. Activation therefore + * reads the cross-process health payload the candidate published, not the exit code of the + * command that started it. + * + * A refusal here is not a failure to deploy. The bytes are installed and the previous + * version is still active; nothing was lost. Activating on a bad verdict is what loses + * things. + */ +import type { ServeReadiness } from '../server/serve-readiness' + +export type OrcadActivationRejectCode = + | 'orcad_activation_no_readiness' + | 'orcad_activation_no_health' + | 'orcad_activation_build_mismatch' + | 'orcad_activation_not_listening' + | 'orcad_activation_daemon_absent' + | 'orcad_activation_daemon_degraded' + | 'orcad_activation_pty_self_test_failed' + | 'orcad_activation_no_persistent_terminals' + +export type OrcadActivationVerdict = + | { + decision: 'activate' + /** + * `pty-spawn` means a real PTY was created and torn down inside the daemon. + * `handshake` means the daemon answered but its spawn probe is a no-op on this + * platform (win32). Carried through so an activation is never recorded as proving + * more than it did. + */ + coverage: 'pty-spawn' | 'handshake' + warnings: string[] + } + | { decision: 'reject'; code: OrcadActivationRejectCode; reason: string } + +export type OrcadActivationExpectation = { + /** sha256(orcad.js).slice(0,16) computed from the bytes this client just uploaded. */ + buildHash: string + /** The full content-hashed version this deploy installed. */ + fullVersion: string +} + +/** + * Gate an activation on what the candidate actually reported. + * + * Order matters: identity before health. A health payload from the wrong process is worse + * than no payload, because it is green and about something else. + */ +export function evaluateOrcadActivation( + readiness: ServeReadiness | null, + expected: OrcadActivationExpectation +): OrcadActivationVerdict { + if (!readiness) { + return { + decision: 'reject', + code: 'orcad_activation_no_readiness', + reason: + 'The candidate orcad never published an `orca_server_ready` line. It may have exited, ' + + 'failed to bind, or be wedged before readiness. Nothing was activated.' + } + } + const health = readiness.health + if (!health) { + return { + decision: 'reject', + code: 'orcad_activation_no_health', + reason: + 'The candidate published readiness without a health payload, so its terminal daemon ' + + 'is unverified. Absence of a verdict is not a healthy verdict — treat this build as ' + + 'too old to gate on and do not activate it.' + } + } + // Why identity first: a stale orcad already holding the port would answer readiness and + // report its own (healthy) daemon. Activating on that record points the pointer at bytes + // nobody is running. + if (health.buildHash !== expected.buildHash) { + return { + decision: 'reject', + code: 'orcad_activation_build_mismatch', + reason: + `The process that answered is running build ${health.buildHash}, not the ` + + `${expected.buildHash} this deploy installed. Something else owns that port, or the ` + + 'upload did not land. Nothing was activated.' + } + } + if (!readiness.boundEndpoint) { + return { + decision: 'reject', + code: 'orcad_activation_not_listening', + reason: + 'The candidate reported no bound endpoint, so no client could reach it. Nothing was ' + + 'activated.' + } + } + const daemon = health.terminalDaemon + if (daemon.state === 'absent') { + return { + decision: 'reject', + code: 'orcad_activation_daemon_absent', + reason: + 'The candidate has no terminal daemon. Every terminal on this host would run in the ' + + 'orcad process and die with it, which is the exact regression the daemon exists to ' + + 'prevent. Nothing was activated.' + } + } + if (daemon.state === 'degraded') { + return { + decision: 'reject', + code: 'orcad_activation_daemon_degraded', + reason: + `The candidate's terminal daemon is degraded (self-test: ${daemon.selfTest.verdict}). ` + + 'Existing sessions keep working, but fresh terminals would not survive a restart. ' + + 'Nothing was activated; the previous version is still serving.' + } + } + if (!daemon.selfTest.ok) { + return { + decision: 'reject', + code: 'orcad_activation_pty_self_test_failed', + reason: + `The candidate's PTY self-test failed (${daemon.selfTest.verdict}). The host is ` + + 'listening but cannot create a terminal. Nothing was activated.' + } + } + if (!daemon.ownsFreshSessions) { + return { + decision: 'reject', + code: 'orcad_activation_no_persistent_terminals', + reason: + 'The candidate answered healthy but does not own fresh sessions, so new terminals ' + + 'would not survive its own restart. Nothing was activated.' + } + } + const warnings: string[] = [] + if (daemon.selfTest.coverage === 'handshake') { + warnings.push( + 'The PTY self-test covered the daemon handshake only — this platform does not spawn a ' + + 'probe PTY. Terminal creation is unproven on this host.' + ) + } + if (health.buildVersion !== expected.fullVersion) { + // Not a rejection: the hash already proved identity, and ORCA_VERSION is whatever the + // launch command exported. Worth saying, because a mismatch means the launch env is wrong. + warnings.push( + `The candidate reports version ${health.buildVersion} but was installed as ` + + `${expected.fullVersion}; check ORCA_VERSION in the launch command.` + ) + } + return { decision: 'activate', coverage: daemon.selfTest.coverage, warnings } +} diff --git a/src/main/ssh/orcad-activation-record-store.ts b/src/main/ssh/orcad-activation-record-store.ts new file mode 100644 index 00000000000..d60634ec525 --- /dev/null +++ b/src/main/ssh/orcad-activation-record-store.ts @@ -0,0 +1,48 @@ +/** + * Reading the activation record off a host. + * + * Split out from the deploy driver because the rollback path needs it too, and because an + * unreadable record must fail loudly in both: treating "I cannot parse this" as "nothing is + * activated" would deploy over a live install and lose its rollback target. + */ +import type { SshConnection } from './ssh-connection' +import { execCommand } from './ssh-relay-deploy-helpers' +import { RELAY_REMOTE_DIR } from './relay-protocol' +import { + ORCAD_ACTIVATION_FILENAME, + emptyOrcadActivationRecord, + parseOrcadActivationRecord, + type OrcadActivationRecord +} from './orcad-activation-record' +import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' + +export function orcadActivationPath(host: RemoteHostPlatform, remoteHome: string): string { + return joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR, ORCAD_ACTIVATION_FILENAME) +} + +export async function readOrcadActivationRecord(options: { + conn: SshConnection + host: RemoteHostPlatform + remoteHome: string + signal?: AbortSignal +}): Promise { + const path = orcadActivationPath(options.host, options.remoteHome) + const raw = await execCommand(options.conn, `cat ${shellQuote(path)} 2>/dev/null || true`, { + wrapCommand: options.host.commandDialect !== 'powershell', + signal: options.signal + }).catch(() => '') + const parsed = parseOrcadActivationRecord(raw) + if (parsed.state === 'ok') { + return parsed.record + } + if (parsed.state === 'unreadable') { + // Why throw: an unreadable record is not an empty one. Treating it as empty would + // activate over a live install and orphan its rollback target. + throw new Error(`Cannot read this host's orcad activation record: ${parsed.reason}`) + } + return emptyOrcadActivationRecord() +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'\\''`)}'` +} diff --git a/src/main/ssh/orcad-activation-record.test.ts b/src/main/ssh/orcad-activation-record.test.ts new file mode 100644 index 00000000000..dd625238bb1 --- /dev/null +++ b/src/main/ssh/orcad-activation-record.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from 'vitest' + +import { + emptyOrcadActivationRecord, + orcadGcPinnedDirNames, + parseOrcadActivationRecord, + serializeOrcadActivationRecord, + withActivatedVersion, + withRolledBackVersion, + type OrcadStateSnapshot +} from './orcad-activation-record' + +const SNAPSHOT: OrcadStateSnapshot = { + dirName: 'pre-0.2.0+bb01-1000', + takenBeforeVersion: '0.2.0+bb01', + readableByVersion: '0.1.0+aa01', + takenAt: '2026-01-01T00:00:00.000Z' +} +const NOW = new Date('2026-01-02T00:00:00.000Z') + +describe('orcad activation record', () => { + it('round-trips through the host', () => { + const record = withActivatedVersion( + { ...emptyOrcadActivationRecord(), active: '0.1.0+aa01' }, + '0.2.0+bb01', + SNAPSHOT, + NOW + ) + const parsed = parseOrcadActivationRecord(serializeOrcadActivationRecord(record)) + expect(parsed).toEqual({ state: 'ok', record }) + }) + + it('reports an absent record as absent', () => { + expect(parseOrcadActivationRecord(null)).toEqual({ state: 'absent' }) + expect(parseOrcadActivationRecord(' ')).toEqual({ state: 'absent' }) + }) + + it('reports a newer schema as unreadable, never as absent', () => { + const parsed = parseOrcadActivationRecord(JSON.stringify({ schemaVersion: 2, active: 'x' })) + expect(parsed.state).toBe('unreadable') + }) + + it('reports corrupt JSON as unreadable, never as absent', () => { + expect(parseOrcadActivationRecord('{not json').state).toBe('unreadable') + }) + + it('names the outgoing version as the rollback target', () => { + const record = withActivatedVersion( + { ...emptyOrcadActivationRecord(), active: '0.1.0+aa01' }, + '0.2.0+bb01', + SNAPSHOT, + NOW + ) + expect(record).toMatchObject({ active: '0.2.0+bb01', previous: '0.1.0+aa01' }) + }) + + it('does not let a re-deploy of the active version erase the rollback target', () => { + const before = { + ...emptyOrcadActivationRecord(), + active: '0.2.0+bb01', + previous: '0.1.0+aa01', + snapshot: SNAPSHOT + } + const after = withActivatedVersion(before, '0.2.0+bb01', null, NOW) + expect(after).toMatchObject({ active: '0.2.0+bb01', previous: '0.1.0+aa01' }) + expect(after.snapshot).toEqual(SNAPSHOT) + }) + + it('clears the rollback target after rolling back, so it cannot walk into the bad build', () => { + const before = { + ...emptyOrcadActivationRecord(), + active: '0.2.0+bb01', + previous: '0.1.0+aa01', + snapshot: SNAPSHOT + } + expect(withRolledBackVersion(before, NOW)).toMatchObject({ + active: '0.1.0+aa01', + previous: null, + snapshot: null + }) + }) + + it('pins the active version, the rollback target and the live daemon"s bundle against GC', () => { + const pinned = orcadGcPinnedDirNames( + { + ...emptyOrcadActivationRecord(), + active: '0.3.0+cc01', + previous: '0.2.0+bb01' + }, + '0.1.0+aa01' + ) + expect(pinned).toEqual(['orcad-0.3.0+cc01', 'orcad-0.2.0+bb01', 'orcad-0.1.0+aa01']) + }) + + it('deduplicates pins when the live daemon came from the active bundle', () => { + const pinned = orcadGcPinnedDirNames( + { ...emptyOrcadActivationRecord(), active: '0.3.0+cc01', previous: null }, + '0.3.0+cc01' + ) + expect(pinned).toEqual(['orcad-0.3.0+cc01']) + }) +}) diff --git a/src/main/ssh/orcad-activation-record.ts b/src/main/ssh/orcad-activation-record.ts new file mode 100644 index 00000000000..8c32e6c355d --- /dev/null +++ b/src/main/ssh/orcad-activation-record.ts @@ -0,0 +1,175 @@ +/** + * Which installed orcad is the live one, and which one a rollback goes back to. + * + * A versioned install directory decides where bytes land; it does not decide which version + * runs. Without this record, "roll back" means "deploy the old version again" — which needs + * the client that has those bytes, on a host that may be the only thing still working. The + * record is the host-side half: it names an active version, a rollback target, and the + * pre-activation state snapshot that makes going back to that target sound. + * + * It lives beside the version dirs (`~/.orca-remote/orcad-active.json`), not inside one, + * because it has to outlive whichever version GC removes. + */ +import { remoteInstallDirName, ORCAD_INSTALL_MODEL } from './remote-install-model' + +export const ORCAD_ACTIVATION_FILENAME = 'orcad-active.json' +export const ORCAD_ACTIVATION_SCHEMA_VERSION = 1 + +/** Where a pre-activation copy of the shared data root lives, relative to `.orca-remote/`. */ +export const ORCAD_STATE_SNAPSHOT_DIR = 'orcad-state-snapshots' + +export type OrcadStateSnapshot = { + /** Directory name under `ORCAD_STATE_SNAPSHOT_DIR`. */ + dirName: string + /** The version whose activation this snapshot was taken FOR — i.e. taken before it ran. */ + takenBeforeVersion: string + /** The version that produced the state, i.e. the rollback target it is readable by. */ + readableByVersion: string | null + takenAt: string +} + +export type OrcadActivationRecord = { + schemaVersion: typeof ORCAD_ACTIVATION_SCHEMA_VERSION + /** Full content-hashed version, e.g. `0.1.0+9f2a1c`. Null before the first activation. */ + active: string | null + /** The version `active` replaced. The rollback target, and pinned against GC. */ + previous: string | null + activatedAt: string | null + snapshot: OrcadStateSnapshot | null +} + +export function emptyOrcadActivationRecord(): OrcadActivationRecord { + return { + schemaVersion: ORCAD_ACTIVATION_SCHEMA_VERSION, + active: null, + previous: null, + activatedAt: null, + snapshot: null + } +} + +/** + * Parse the record read off the host. + * + * Why a null return and not a throw on a newer schema: a client older than the host must not + * treat "I cannot read this" as "nothing is activated" — that would deploy over a live + * install. Callers distinguish the two through `OrcadActivationReadResult`. + */ +export type OrcadActivationReadResult = + | { state: 'absent' } + | { state: 'ok'; record: OrcadActivationRecord } + | { state: 'unreadable'; reason: string } + +export function parseOrcadActivationRecord(raw: string | null): OrcadActivationReadResult { + if (raw === null || raw.trim() === '') { + return { state: 'absent' } + } + let parsed: unknown + try { + parsed = JSON.parse(raw) + } catch (error) { + return { + state: 'unreadable', + reason: `activation record is not JSON: ${error instanceof Error ? error.message : String(error)}` + } + } + if (typeof parsed !== 'object' || parsed === null) { + return { state: 'unreadable', reason: 'activation record is not an object' } + } + const record = parsed as Partial + if (record.schemaVersion !== ORCAD_ACTIVATION_SCHEMA_VERSION) { + return { + state: 'unreadable', + reason: + `activation record schemaVersion ${String(record.schemaVersion)} is not ` + + `${ORCAD_ACTIVATION_SCHEMA_VERSION}; this client cannot safely interpret it` + } + } + return { + state: 'ok', + record: { + schemaVersion: ORCAD_ACTIVATION_SCHEMA_VERSION, + active: typeof record.active === 'string' ? record.active : null, + previous: typeof record.previous === 'string' ? record.previous : null, + activatedAt: typeof record.activatedAt === 'string' ? record.activatedAt : null, + snapshot: parseSnapshot(record.snapshot) + } + } +} + +function parseSnapshot(value: unknown): OrcadStateSnapshot | null { + if (typeof value !== 'object' || value === null) { + return null + } + const snapshot = value as Partial + if (typeof snapshot.dirName !== 'string' || typeof snapshot.takenBeforeVersion !== 'string') { + return null + } + return { + dirName: snapshot.dirName, + takenBeforeVersion: snapshot.takenBeforeVersion, + readableByVersion: + typeof snapshot.readableByVersion === 'string' ? snapshot.readableByVersion : null, + takenAt: typeof snapshot.takenAt === 'string' ? snapshot.takenAt : '' + } +} + +export function serializeOrcadActivationRecord(record: OrcadActivationRecord): string { + return `${JSON.stringify(record, null, 2)}\n` +} + +/** The record that results from activating `version`, keeping the outgoing one as the target. */ +export function withActivatedVersion( + record: OrcadActivationRecord, + version: string, + snapshot: OrcadStateSnapshot | null, + now: Date +): OrcadActivationRecord { + return { + schemaVersion: ORCAD_ACTIVATION_SCHEMA_VERSION, + active: version, + // Why keep the OLD previous when re-activating the same version: a repeated deploy of + // an already-active build is not a version change, so it must not erase the rollback + // target by naming the active version as its own predecessor. + previous: record.active === version ? record.previous : record.active, + activatedAt: now.toISOString(), + snapshot: record.active === version ? record.snapshot : snapshot + } +} + +/** The record that results from rolling `active` back to `previous`. */ +export function withRolledBackVersion( + record: OrcadActivationRecord, + now: Date +): OrcadActivationRecord { + return { + schemaVersion: ORCAD_ACTIVATION_SCHEMA_VERSION, + active: record.previous, + // Why null and not the version we just left: it is the build we are rolling back FROM, + // so offering it as the next rollback target would walk straight back into the failure. + previous: null, + activatedAt: now.toISOString(), + // The snapshot was taken before `active` ran; once restored it has been consumed. + snapshot: null + } +} + +/** + * Version dirs GC must not remove, as directory names. + * + * `previous` is here because a rollback target that GC deleted is not a rollback target. + * `daemonEntryVersion` is here because an update preserves a live daemon forked from the + * OUTGOING bundle (see orcad-update-plan.ts) — deleting the tree under a running process is + * how a later respawn finds no entry point. + */ +export function orcadGcPinnedDirNames( + record: OrcadActivationRecord, + daemonEntryVersion?: string | null +): string[] { + const versions = [record.active, record.previous, daemonEntryVersion ?? null].filter( + (v): v is string => typeof v === 'string' && v.length > 0 + ) + return [...new Set(versions)].map((version) => + remoteInstallDirName(ORCAD_INSTALL_MODEL, version) + ) +} diff --git a/src/main/ssh/orcad-local-build-hash.ts b/src/main/ssh/orcad-local-build-hash.ts new file mode 100644 index 00000000000..3740ea6d55c --- /dev/null +++ b/src/main/ssh/orcad-local-build-hash.ts @@ -0,0 +1,22 @@ +/** + * The build identity the deploy expects the host to answer with. + * + * It must be computed the same way `computeOrcadBuildHash` computes it on the host — + * sha256 of `orcad.js`, first 16 hex characters — or the activation gate would reject every + * healthy candidate. Keeping the two in one comment is deliberate: they are one contract + * split across a network, and the version string cannot stand in for it, because + * `ORCA_VERSION` is whatever the launch command exported and two builds can carry one value. + */ +import { createHash } from 'node:crypto' +import { readFileSync } from 'node:fs' +import { join } from 'node:path' + +export const ORCAD_BUILD_HASH_LENGTH = 16 + +export function computeLocalOrcadBuildHash(localOrcadDir: string): string { + const entry = join(localOrcadDir, 'orcad.js') + return createHash('sha256') + .update(readFileSync(entry)) + .digest('hex') + .slice(0, ORCAD_BUILD_HASH_LENGTH) +} diff --git a/src/main/ssh/orcad-remote-deploy.test.ts b/src/main/ssh/orcad-remote-deploy.test.ts new file mode 100644 index 00000000000..799193f3384 --- /dev/null +++ b/src/main/ssh/orcad-remote-deploy.test.ts @@ -0,0 +1,264 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('./ssh-relay-deploy-helpers', () => ({ + execCommand: vi.fn(), + isUnconfirmedSshCommandTermination: () => false +})) +vi.mock('./ssh-connection-utils', () => ({ shellEscape: (s: string) => `'${s}'` })) +vi.mock('./ssh-relay-install-lock', () => ({ + acquireInstallLock: vi.fn().mockResolvedValue(undefined), + RELAY_INSTALL_LOCK_NAME: '.install-lock' +})) +vi.mock('./ssh-relay-install-transfers', () => ({ + uploadRelayDirectory: vi.fn().mockResolvedValue(undefined), + writeRelayFile: vi.fn().mockResolvedValue(undefined) +})) +vi.mock('./orcad-local-build-hash', () => ({ + computeLocalOrcadBuildHash: () => 'abc123def4567890' +})) + +import { execCommand } from './ssh-relay-deploy-helpers' +import { acquireInstallLock } from './ssh-relay-install-lock' +import { uploadRelayDirectory, writeRelayFile } from './ssh-relay-install-transfers' +import { deployOrcad, type OrcadDeployOptions } from './orcad-remote-deploy' +import { emptyOrcadActivationRecord, withActivatedVersion } from './orcad-activation-record' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import type { SshConnection } from './ssh-connection' + +const mockExec = vi.mocked(execCommand) +const NEW_VERSION = '0.2.0+bb01' +const OLD_VERSION = '0.1.0+aa01' + +vi.mock('./ssh-relay-versioned-install', async (importOriginal) => ({ + ...(await importOriginal>()), + readLocalFullVersion: () => '0.2.0+bb01', + isRemoteInstallComplete: vi.fn().mockResolvedValue(false), + finalizeInstall: vi.fn().mockResolvedValue(undefined), + abandonInstall: vi.fn().mockResolvedValue(undefined) +})) + +function readyLine(overrides: { + buildHash?: string + daemonState?: 'live' | 'degraded' | 'absent' + selfTestOk?: boolean +}): string { + return JSON.stringify({ + type: 'orca_server_ready', + schemaVersion: 1, + runtimeId: 'r1', + boundEndpoint: 'ws://127.0.0.1:7777', + advertisedEndpoint: null, + managedWslCliReconciliation: 'settled', + pairing: { available: false, reason: 'disabled_by_operator', guidance: 'n/a' }, + health: { + buildHash: overrides.buildHash ?? 'abc123def4567890', + buildVersion: NEW_VERSION, + nodeVersion: '20.11.0', + nodeAbi: '115', + platform: 'linux', + arch: 'x64', + pid: 1, + terminalDaemon: { + state: overrides.daemonState ?? 'live', + ownsFreshSessions: (overrides.daemonState ?? 'live') === 'live', + pid: 2, + buildVersion: NEW_VERSION, + entryPath: '/x/daemon-entry.js', + protocolVersion: 3, + selfTest: { + ok: overrides.selfTestOk ?? true, + coverage: 'pty-spawn', + verdict: (overrides.selfTestOk ?? true) ? 'healthy' : 'pty-spawn-unhealthy', + durationMs: 5 + } + } + } + }) +} + +type HostScript = { + activationRecord: string + /** Readiness content per version dir, keyed by the version in the path. */ + readiness: Record + log: string[] +} + +function scriptHost(script: HostScript): void { + mockExec.mockImplementation(async (_conn, command: string) => { + const text = String(command) + if (text.startsWith('cat ') && text.includes('orcad-active.json')) { + return script.activationRecord + } + if (text.includes('.orcad-readiness') && text.startsWith('cat ')) { + const version = Object.keys(script.readiness).find((v) => text.includes(v)) + return version ? script.readiness[version] : '' + } + if (text.includes('nohup')) { + script.log.push(`launch:${text.includes(NEW_VERSION) ? NEW_VERSION : OLD_VERSION}`) + return '9999' + } + if (text.includes('kill -TERM')) { + script.log.push(`stop:${text.includes(NEW_VERSION) ? NEW_VERSION : OLD_VERSION}`) + return 'STOPPED' + } + if (text.includes('tar -C') && text.includes('-cf')) { + script.log.push('snapshot') + return 'CAPTURED' + } + return '' + }) +} + +function options(overrides: Partial = {}): OrcadDeployOptions { + return { + conn: {} as SshConnection, + host: getRemoteHostPlatform('linux-x64'), + remoteHome: '/home/u', + localOrcadDir: '/local/out/orcad', + nodePath: '/usr/bin/node', + userDataDir: '/home/u/.orca', + bindHost: '127.0.0.1', + port: 7777, + census: { liveSessions: 0, startedSinceActivation: 0 }, + readinessTimeoutMs: 50, + sleep: async () => {}, + now: () => new Date('2026-02-02T00:00:00.000Z'), + ...overrides + } +} + +const ACTIVE_OLD = JSON.stringify( + withActivatedVersion(emptyOrcadActivationRecord(), OLD_VERSION, null, new Date(0)) +) + +describe('deployOrcad', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('installs under the orcad namespace, not the relay one', async () => { + const script: HostScript = { + activationRecord: '', + readiness: { [NEW_VERSION]: readyLine({}) }, + log: [] + } + scriptHost(script) + await deployOrcad(options()) + expect(vi.mocked(acquireInstallLock).mock.calls[0][1]).toBe( + `/home/u/.orca-remote/orcad-${NEW_VERSION}` + ) + expect(vi.mocked(uploadRelayDirectory).mock.calls[0][2]).toContain(`orcad-${NEW_VERSION}`) + }) + + it('activates a healthy candidate and records the outgoing version as the rollback target', async () => { + const script: HostScript = { + activationRecord: ACTIVE_OLD, + readiness: { [NEW_VERSION]: readyLine({}) }, + log: [] + } + scriptHost(script) + const result = await deployOrcad(options()) + expect(result).toMatchObject({ outcome: 'installed-and-activated', fullVersion: NEW_VERSION }) + const written = vi + .mocked(writeRelayFile) + .mock.calls.find((call) => String(call[2]).endsWith('orcad-active.json')) + expect(JSON.parse(String(written?.[3]))).toMatchObject({ + active: NEW_VERSION, + previous: OLD_VERSION + }) + }) + + it('snapshots the shared data root before the candidate ever runs', async () => { + const script: HostScript = { + activationRecord: ACTIVE_OLD, + readiness: { [NEW_VERSION]: readyLine({}) }, + log: [] + } + scriptHost(script) + await deployOrcad(options()) + expect(script.log.indexOf('snapshot')).toBeGreaterThan(-1) + expect(script.log.indexOf('snapshot')).toBeLessThan(script.log.indexOf(`launch:${NEW_VERSION}`)) + }) + + it('installs but does not activate when terminals are running', async () => { + const script: HostScript = { + activationRecord: ACTIVE_OLD, + readiness: { [NEW_VERSION]: readyLine({}) }, + log: [] + } + scriptHost(script) + const result = await deployOrcad( + options({ census: { liveSessions: 2, startedSinceActivation: 0 } }) + ) + expect(result).toMatchObject({ + outcome: 'installed-not-activated', + code: 'orcad_update_terminals_running' + }) + // The bytes landed; nothing was stopped, launched or snapshotted. + expect(vi.mocked(uploadRelayDirectory)).toHaveBeenCalled() + expect(script.log).toEqual([]) + }) + + it('does not write the activation record when the candidate fails its health gate', async () => { + const script: HostScript = { + activationRecord: ACTIVE_OLD, + readiness: { [NEW_VERSION]: readyLine({ daemonState: 'degraded' }) }, + log: [] + } + scriptHost(script) + const result = await deployOrcad(options()) + expect(result).toMatchObject({ code: 'orcad_activation_daemon_degraded' }) + expect( + vi.mocked(writeRelayFile).mock.calls.some((call) => String(call[2]).endsWith('orcad-active.json')) + ).toBe(false) + }) + + it('puts the previous version back after a rejected candidate, rather than leaving the host down', async () => { + const script: HostScript = { + activationRecord: ACTIVE_OLD, + readiness: { + [NEW_VERSION]: readyLine({ selfTestOk: false }), + [OLD_VERSION]: readyLine({}) + }, + log: [] + } + scriptHost(script) + const result = await deployOrcad(options()) + expect(result).toMatchObject({ outcome: 'installed-not-activated' }) + expect(script.log).toEqual([ + 'snapshot', + `stop:${OLD_VERSION}`, + `launch:${NEW_VERSION}`, + `stop:${NEW_VERSION}`, + `launch:${OLD_VERSION}` + ]) + expect(result.outcome === 'installed-not-activated' && result.reason).toContain( + `orcad ${OLD_VERSION} was restarted and is serving again` + ) + }) + + it('refuses to activate when a different build answered the port', async () => { + const script: HostScript = { + activationRecord: ACTIVE_OLD, + readiness: { + [NEW_VERSION]: readyLine({ buildHash: 'deadbeefdeadbeef' }), + [OLD_VERSION]: readyLine({}) + }, + log: [] + } + scriptHost(script) + const result = await deployOrcad(options()) + expect(result).toMatchObject({ code: 'orcad_activation_build_mismatch' }) + }) + + it('refuses to treat an unreadable activation record as an empty one', async () => { + const script: HostScript = { + activationRecord: JSON.stringify({ schemaVersion: 99, active: 'x' }), + readiness: { [NEW_VERSION]: readyLine({}) }, + log: [] + } + scriptHost(script) + await expect(deployOrcad(options())).rejects.toThrow('activation record') + expect(vi.mocked(uploadRelayDirectory)).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/ssh/orcad-remote-deploy.ts b/src/main/ssh/orcad-remote-deploy.ts new file mode 100644 index 00000000000..1f8a4099a82 --- /dev/null +++ b/src/main/ssh/orcad-remote-deploy.ts @@ -0,0 +1,340 @@ +/** + * Installing orcad on a host and, only if it proves itself, making it the active one. + * + * The install half is the relay's transaction, parameterized: the same per-version lock, + * staged SFTP write, `.install-complete` sentinel and stale-lock recovery, under + * `orcad-/` instead of `relay-/`. That is what §02 marks reusable. + * + * The activation half has no relay equivalent, because the relay has no notion of a version + * being *selected*. Bytes landing in a versioned directory neither picks a version nor rolls + * one back; the activation record does, and it is written only after the candidate publishes + * a health payload that survives `evaluateOrcadActivation`. A rejected candidate leaves the + * previous version running and its own bytes on disk — nothing is lost, and a retry costs no + * upload. + */ +import type { SshConnection } from './ssh-connection' +import { execCommand } from './ssh-relay-deploy-helpers' +import { ORCAD_INSTALL_MODEL } from './remote-install-model' +import { acquireInstallLock } from './ssh-relay-install-lock' +import { uploadRelayDirectory, writeRelayFile } from './ssh-relay-install-transfers' +import { + abandonInstall, + computeRemoteInstallDir, + finalizeInstall, + isRemoteInstallComplete, + readLocalFullVersion +} from './ssh-relay-versioned-install' +import { RELAY_REMOTE_DIR } from './relay-protocol' +import { + ORCAD_STATE_SNAPSHOT_DIR, + serializeOrcadActivationRecord, + withActivatedVersion, + type OrcadActivationRecord, + type OrcadStateSnapshot +} from './orcad-activation-record' +import { + orcadActivationPath, + readOrcadActivationRecord +} from './orcad-activation-record-store' +import { evaluateOrcadActivation, type OrcadActivationVerdict } from './orcad-activation-gate' +import { planOrcadUpdate, type OrcadTerminalCensus } from './orcad-update-plan' +import { + ORCAD_LOG_FILENAME, + orcadLaunchCommand, + parseOrcadReadinessOutput, + readOrcadReadinessCommand, + type OrcadLaunchSpec +} from './orcad-remote-launch' +import { + captureOrcadStateSnapshotCommand, + orcadSnapshotDirName, + parseOrcadSnapshotCapture +} from './orcad-state-snapshot' +import { + orcadStopFreedTheHost, + parseOrcadStopOutcome, + stopOrcadCommand +} from './orcad-remote-process-control' +import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { computeLocalOrcadBuildHash } from './orcad-local-build-hash' + +export type OrcadDeployOptions = { + conn: SshConnection + host: RemoteHostPlatform + remoteHome: string + /** Local `out/orcad`, containing the artifacts and the `.version` marker. */ + localOrcadDir: string + nodePath: string + userDataDir: string + bindHost: string + port: number + /** + * Live-terminal counts, supplied by the caller from the runtime it is already connected + * to. Not probed here: counting the daemon's sessions needs its protocol, and a deploy + * that guessed zero from silence would be the "loss of contact means death" mistake. + */ + census: OrcadTerminalCensus + force?: boolean + readinessTimeoutMs?: number + now?: () => Date + sleep?: (ms: number) => Promise + signal?: AbortSignal +} + +export type OrcadDeployResult = + | { outcome: 'installed-and-activated'; fullVersion: string; verdict: OrcadActivationVerdict } + | { outcome: 'already-active'; fullVersion: string } + | { outcome: 'installed-not-activated'; fullVersion: string; code: string; reason: string } + +const DEFAULT_READINESS_TIMEOUT_MS = 90_000 +const READINESS_POLL_MS = 500 +const STOP_WAIT_SECONDS = 20 + +function exec( + options: OrcadDeployOptions, + command: string, + signal = options.signal +): Promise { + return execCommand(options.conn, command, { + wrapCommand: options.host.commandDialect !== 'powershell', + signal + }) +} + +function baseDir(options: OrcadDeployOptions): string { + return joinRemotePath(options.host, options.remoteHome, RELAY_REMOTE_DIR) +} + +/** Install the bytes under `orcad-/`, using the relay's install transaction. */ +async function installOrcadBundle( + options: OrcadDeployOptions, + fullVersion: string, + remoteDir: string +): Promise { + if (await isRemoteInstallComplete(options.conn, ORCAD_INSTALL_MODEL, remoteDir, options.host, { + signal: options.signal + })) { + return + } + await acquireInstallLock(options.conn, remoteDir, options.host, { signal: options.signal }) + try { + // Re-probe under the lock: a sibling deploy may have finished while we waited. + if ( + await isRemoteInstallComplete(options.conn, ORCAD_INSTALL_MODEL, remoteDir, options.host, { + signal: options.signal + }) + ) { + return + } + await uploadRelayDirectory(options.conn, options.localOrcadDir, remoteDir, options.host, { + signal: options.signal + }) + await writeRelayFile( + options.conn, + options.host, + joinRemotePath(options.host, remoteDir, ORCAD_INSTALL_MODEL.versionFilename), + fullVersion, + { signal: options.signal } + ) + await finalizeInstall(options.conn, remoteDir, options.host, { signal: options.signal }) + } catch (error) { + // Leave a recoverable partial rather than a dir that probes complete. + await abandonInstall(options.conn, remoteDir, options.host) + throw error + } +} + +async function captureSnapshot( + options: OrcadDeployOptions, + fullVersion: string, + outgoingVersion: string | null, + takenAt: Date +): Promise { + const dirName = orcadSnapshotDirName(fullVersion, takenAt.getTime()) + const snapshotDir = joinRemotePath(options.host, baseDir(options), ORCAD_STATE_SNAPSHOT_DIR, dirName) + const capture = parseOrcadSnapshotCapture( + await exec( + options, + captureOrcadStateSnapshotCommand(options.host, options.userDataDir, snapshotDir) + ) + ) + if (capture === 'failed') { + throw new Error( + `Could not snapshot ${options.userDataDir} before activating ${fullVersion}. Orca's ` + + 'persisted state carries no schema version, so without a snapshot a rollback has no ' + + 'way back. Refusing to activate.' + ) + } + if (capture === 'empty') { + // Nothing on the host to lose: a first deployment. Rollback will correctly report that + // it has no snapshot, rather than restoring an archive of nothing over a populated root. + return null + } + return { + dirName, + takenBeforeVersion: fullVersion, + readableByVersion: outgoingVersion, + takenAt: takenAt.toISOString() + } +} + +async function launchAndAwaitReadiness( + options: OrcadDeployOptions, + spec: OrcadLaunchSpec +): Promise> { + await exec(options, orcadLaunchCommand(options.host, spec)) + const deadline = Date.now() + (options.readinessTimeoutMs ?? DEFAULT_READINESS_TIMEOUT_MS) + const sleep = options.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))) + let last = parseOrcadReadinessOutput('') + while (Date.now() < deadline) { + options.signal?.throwIfAborted() + last = parseOrcadReadinessOutput( + await exec(options, readOrcadReadinessCommand(options.host, spec.remoteInstallDir)) + ) + if (last.state !== 'pending') { + return last + } + await sleep(READINESS_POLL_MS) + } + return last +} + +/** + * Put the previous version back after a rejected candidate. + * + * Why this exists at all: activating means swapping which process owns the data root and the + * port, so the incumbent has to stop before the candidate can start. A gate that rejected + * and returned would leave the host with nothing running — a careful deploy causing the + * outage it was being careful about. The returned sentence goes into the caller's reason so + * the operator learns the host's actual state, not just why the candidate failed. + */ +async function restoreIncumbent( + options: OrcadDeployOptions, + record: OrcadActivationRecord, + candidateDir: string +): Promise { + const stopped = parseOrcadStopOutcome( + await exec( + options, + stopOrcadCommand(options.host, candidateDir, { waitSeconds: STOP_WAIT_SECONDS }) + ) + ) + if (!orcadStopFreedTheHost(stopped)) { + return `The candidate itself did not stop (${stopped}); the host may still be serving the rejected build.` + } + if (!record.active) { + return 'No previous version was active, so this host is now serving nothing.' + } + const incumbentDir = computeRemoteInstallDir( + ORCAD_INSTALL_MODEL, + options.remoteHome, + record.active + ) + const parsed = await launchAndAwaitReadiness(options, { + remoteInstallDir: incumbentDir, + nodePath: options.nodePath, + fullVersion: record.active, + userDataDir: options.userDataDir, + bindHost: options.bindHost, + port: options.port + }) + return parsed.state === 'ready' + ? `orcad ${record.active} was restarted and is serving again.` + : `orcad ${record.active} was relaunched but has not published readiness; this host may be down.` +} + +/** + * Install, then activate only on a green cross-process health verdict. + * + * Every early return past the install leaves the bytes on disk and the previous version + * serving, which is why they all report `installed-not-activated` rather than throwing: a + * refusal to switch is a successful outcome of a deploy that was asked to be careful. + */ +export async function deployOrcad(options: OrcadDeployOptions): Promise { + const now = options.now ?? ((): Date => new Date()) + const fullVersion = readLocalFullVersion(options.localOrcadDir) + const remoteDir = computeRemoteInstallDir(ORCAD_INSTALL_MODEL, options.remoteHome, fullVersion) + const record = await readOrcadActivationRecord(options) + + await installOrcadBundle(options, fullVersion, remoteDir) + + const plan = planOrcadUpdate({ + record, + candidateVersion: fullVersion, + census: options.census, + ...(options.force !== undefined ? { force: options.force } : {}) + }) + if (plan.action === 'noop') { + return { outcome: 'already-active', fullVersion } + } + if (plan.action === 'defer') { + return { + outcome: 'installed-not-activated', + fullVersion, + code: plan.code, + reason: plan.reason + } + } + + const snapshot = record.active + ? await captureSnapshot(options, fullVersion, record.active, now()) + : null + + if (record.active) { + const outgoingDir = computeRemoteInstallDir( + ORCAD_INSTALL_MODEL, + options.remoteHome, + record.active + ) + const stopped = parseOrcadStopOutcome( + await exec(options, stopOrcadCommand(options.host, outgoingDir, { + waitSeconds: STOP_WAIT_SECONDS + })) + ) + if (!orcadStopFreedTheHost(stopped)) { + return { + outcome: 'installed-not-activated', + fullVersion, + code: 'orcad_outgoing_stop_incomplete', + reason: + `orcad ${record.active} did not exit within ${STOP_WAIT_SECONDS}s of SIGTERM ` + + `(${stopped}). It is still holding the data root and the port, so the candidate ` + + 'cannot start. Not escalating to SIGKILL: that skips the shutdown that releases ' + + 'the instance lock, and the successor would then refuse to start.' + } + } + } + + const parsed = await launchAndAwaitReadiness(options, { + remoteInstallDir: remoteDir, + nodePath: options.nodePath, + fullVersion, + userDataDir: options.userDataDir, + bindHost: options.bindHost, + port: options.port + }) + const verdict = evaluateOrcadActivation(parsed.state === 'ready' ? parsed.readiness : null, { + buildHash: computeLocalOrcadBuildHash(options.localOrcadDir), + fullVersion + }) + if (verdict.decision === 'reject') { + const restored = await restoreIncumbent(options, record, remoteDir) + return { + outcome: 'installed-not-activated', + fullVersion, + code: verdict.code, + reason: + `${verdict.reason} Candidate stderr is at ` + + `${joinRemotePath(options.host, remoteDir, ORCAD_LOG_FILENAME)}. ${restored}` + } + } + + await writeRelayFile( + options.conn, + options.host, + orcadActivationPath(options.host, options.remoteHome), + serializeOrcadActivationRecord(withActivatedVersion(record, fullVersion, snapshot, now())), + { signal: options.signal } + ) + return { outcome: 'installed-and-activated', fullVersion, verdict } +} diff --git a/src/main/ssh/orcad-remote-gc.test.ts b/src/main/ssh/orcad-remote-gc.test.ts new file mode 100644 index 00000000000..3716aaddaf9 --- /dev/null +++ b/src/main/ssh/orcad-remote-gc.test.ts @@ -0,0 +1,153 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('./ssh-relay-deploy-helpers', () => ({ + execCommand: vi.fn(), + isUnconfirmedSshCommandTermination: () => false +})) +vi.mock('./ssh-connection-utils', () => ({ shellEscape: (s: string) => `'${s}'` })) +vi.mock('./ssh-relay-gc-claim', () => ({ + isRelayGcClaimOwned: vi.fn().mockResolvedValue(true), + releaseRelayGcClaimWithRetry: vi.fn().mockResolvedValue('released'), + tryAcquireRelayGcClaim: vi.fn().mockResolvedValue('token') +})) +vi.mock('./ssh-relay-gc-tombstone', () => ({ + cleanupRelayGcTombstones: vi.fn().mockResolvedValue(undefined) +})) +vi.mock('./ssh-relay-install-lock', () => ({ + RELAY_INSTALL_LOCK_NAME: '.install-lock', + isRelayInstallLockStale: vi.fn().mockResolvedValue(false) +})) + +import { execCommand } from './ssh-relay-deploy-helpers' +import { gcOldOrcadVersions } from './orcad-remote-gc' +import { emptyOrcadActivationRecord } from './orcad-activation-record' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import type { SshConnection } from './ssh-connection' + +const conn = {} as SshConnection +const host = getRemoteHostPlatform('linux-x64') +const mockExec = vi.mocked(execCommand) + +/** + * Drive one GC pass against a scripted host. `listing` is what the remote listing command + * returns; every other command answers from `responses`, defaulting to the happy path + * (unlocked, complete, dead) so a candidate is removed unless a test says otherwise. + */ +function scriptHost(options: { + listing: string[] + liveness?: Record + removed: string[] +}): void { + mockExec.mockImplementation(async (_conn, command: string) => { + if (command.includes('-mindepth 1 -maxdepth 1')) { + return options.listing.join('\n') + } + if (command.includes('.install-lock')) { + return 'OPEN' + } + if (command.includes('.install-complete')) { + return 'COMPLETE' + } + if (command.includes('.orcad-pid')) { + const dir = Object.keys(options.liveness ?? {}).find((name) => command.includes(name)) + return dir ? (options.liveness?.[dir] ?? 'DEAD') : 'DEAD' + } + if (command.startsWith('mv ')) { + const match = command.match(/'([^']*)'/) + if (match) { + options.removed.push(match[1].split('/').pop() ?? '') + } + return 'MOVED' + } + return '' + }) +} + +describe('orcad GC', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('scopes its remote listing to the orcad namespace', async () => { + const removed: string[] = [] + scriptHost({ listing: [], removed }) + await gcOldOrcadVersions({ + conn, + host, + remoteHome: '/home/u', + currentDirAbsPath: '/home/u/.orca-remote/orcad-0.2.0+bb', + record: emptyOrcadActivationRecord() + }) + const listCommand = mockExec.mock.calls.map((call) => String(call[1])).find((c) => c.includes('find')) + expect(listCommand).toContain("-name 'orcad-*'") + expect(listCommand).not.toContain("-name 'relay-*'") + }) + + it('leaves relay directories alone even when the host hands them to it', async () => { + // The host is lying — or a future listing bug widened the glob. GC must still refuse. + const removed: string[] = [] + scriptHost({ + listing: ['relay-0.1.0+aa', 'relay-0.0.9+ff', 'orcad-0.1.0+aa'], + removed + }) + await gcOldOrcadVersions({ + conn, + host, + remoteHome: '/home/u', + currentDirAbsPath: '/home/u/.orca-remote/orcad-0.2.0+bb', + record: emptyOrcadActivationRecord() + }) + expect(removed).toEqual(['orcad-0.1.0+aa']) + }) + + it('never removes the active or the previous version', async () => { + const removed: string[] = [] + scriptHost({ + listing: ['orcad-0.1.0+01d', 'orcad-0.2.0+9ee0', 'orcad-0.3.0+cc0'], + removed + }) + await gcOldOrcadVersions({ + conn, + host, + remoteHome: '/home/u', + currentDirAbsPath: '/home/u/.orca-remote/orcad-0.3.0+cc0', + record: { + ...emptyOrcadActivationRecord(), + active: '0.3.0+cc0', + previous: '0.2.0+9ee0' + } + }) + expect(removed).toEqual(['orcad-0.1.0+01d']) + }) + + it('never removes the version a live daemon was forked from', async () => { + const removed: string[] = [] + scriptHost({ listing: ['orcad-0.1.0+01d', 'orcad-0.0.9+01de'], removed }) + await gcOldOrcadVersions({ + conn, + host, + remoteHome: '/home/u', + currentDirAbsPath: '/home/u/.orca-remote/orcad-0.3.0+cc0', + record: { ...emptyOrcadActivationRecord(), active: '0.3.0+cc0' }, + liveDaemonVersion: '0.1.0+01d' + }) + expect(removed).toEqual(['orcad-0.0.9+01de']) + }) + + it('treats an unanswerable liveness probe as in use', async () => { + const removed: string[] = [] + scriptHost({ + listing: ['orcad-0.1.0+bb0', 'orcad-0.0.9+dead'], + liveness: { 'orcad-0.1.0+bb0': 'UNKNOWN', 'orcad-0.0.9+dead': 'DEAD' }, + removed + }) + await gcOldOrcadVersions({ + conn, + host, + remoteHome: '/home/u', + currentDirAbsPath: '/home/u/.orca-remote/orcad-0.3.0+cc0', + record: emptyOrcadActivationRecord() + }) + expect(removed).toEqual(['orcad-0.0.9+dead']) + }) +}) diff --git a/src/main/ssh/orcad-remote-gc.ts b/src/main/ssh/orcad-remote-gc.ts new file mode 100644 index 00000000000..a96b386fc1b --- /dev/null +++ b/src/main/ssh/orcad-remote-gc.ts @@ -0,0 +1,74 @@ +/** + * orcad's garbage collection, and the half of §06 falsifier 1 that says who owns it. + * + * **Each model GCs only its own namespace, permanently.** orcad removes `orcad-/` + * directories; the relay removes `relay-/` directories; neither ever removes the other's, + * and no plan item makes one the winner. That is not a migration compromise — the two models + * serve different users on the same machine (SSH target vs paired peer), so there is no + * moment at which one of them is entitled to clean up after the other. A pass that deleted + * the sibling's tree would be reaching across the execution boundary the whole design exists + * to keep intact. + * + * On top of the ownership rule, orcad pins three directories that are idle-looking but + * load-bearing: the active version, the rollback target, and whichever version the LIVE + * terminal daemon was forked from. + */ +import type { SshConnection } from './ssh-connection' +import { execCommand } from './ssh-relay-deploy-helpers' +import { ORCAD_INSTALL_MODEL } from './remote-install-model' +import { gcOldRemoteInstallVersions } from './ssh-relay-versioned-install' +import { orcadGcPinnedDirNames, type OrcadActivationRecord } from './orcad-activation-record' +import { + orcadLivenessBlocksGc, + orcadLivenessProbeCommand, + parseOrcadLiveness +} from './orcad-remote-launch' +import type { RemoteHostPlatform } from './ssh-remote-platform' + +export type OrcadGcOptions = { + conn: SshConnection + host: RemoteHostPlatform + remoteHome: string + /** Absolute path of the version dir this client just used; never a candidate. */ + currentDirAbsPath: string + record: OrcadActivationRecord + /** + * The full version the live daemon's PID record names, when it can be read. + * + * An update preserves a daemon forked from the OUTGOING bundle whenever terminals are + * live, so this is routinely a version that is neither active nor previous. Deleting it + * would remove the tree under a running process. + */ + liveDaemonVersion?: string | null + signal?: AbortSignal +} + +export async function gcOldOrcadVersions(options: OrcadGcOptions): Promise { + await gcOldRemoteInstallVersions( + options.conn, + ORCAD_INSTALL_MODEL, + options.remoteHome, + options.currentDirAbsPath, + options.host, + { + pinnedDirNames: orcadGcPinnedDirNames(options.record, options.liveDaemonVersion), + isDirLive: async (dir) => { + try { + const probe = await execCommand( + options.conn, + orcadLivenessProbeCommand(options.host, dir), + { + wrapCommand: options.host.commandDialect !== 'powershell', + signal: options.signal + } + ) + return orcadLivenessBlocksGc(parseOrcadLiveness(probe)) + } catch { + // Why true: an unanswered probe is not evidence a tree is idle. Same rule the + // relay's socket probe applies, for the same reason. + return true + } + } + } + ) +} diff --git a/src/main/ssh/orcad-remote-host-support.ts b/src/main/ssh/orcad-remote-host-support.ts new file mode 100644 index 00000000000..dfcb0f4b6e4 --- /dev/null +++ b/src/main/ssh/orcad-remote-host-support.ts @@ -0,0 +1,50 @@ +/** + * Which hosts the orcad launch/lifecycle path actually supports, declared rather than + * discovered at runtime. + * + * The install transaction is host-agnostic — it is the relay's, and the relay runs on + * Windows. The launch, liveness and stop path is not: it uses `nohup`, a redirected stdout, + * `kill -0` and `ps`. Emitting a PowerShell-shaped approximation of that would produce a + * deploy that reports success on a host where nothing is running. + */ +import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform' + +export class OrcadRemoteLaunchUnsupportedError extends Error { + readonly code = 'orcad_remote_launch_unsupported_host' + constructor(hostLabel: string) { + super( + `Deploying orcad to a ${hostLabel} host is not implemented. The install transaction is ` + + 'host-agnostic, but the launch and readiness path is POSIX-only: it uses nohup, a ' + + 'redirected stdout and `kill -0` liveness. Use the relay for this host.' + ) + this.name = 'OrcadRemoteLaunchUnsupportedError' + } +} + +export function assertPosixOrcadHost(host: RemoteHostPlatform): void { + if (isWindowsRemoteHost(host)) { + throw new OrcadRemoteLaunchUnsupportedError('Windows') + } +} + +/** PID of the launched orcad, written into its own version dir at launch. */ +export const ORCAD_PID_FILENAME = '.orcad-pid' + +/** + * A shell function answering whether a PID is a *running* process. + * + * `kill -0` alone is not that question. It succeeds for a zombie — a process that has + * exited but whose parent has not reaped it — so a stop loop built on it reports + * `STILL_RUNNING` for a process that is already gone, and GC reports a dead version dir as + * in use. Verified against a real zombie on macOS; the `ps` state check is what separates + * the two. + * + * A host without `ps` yields an empty state, which falls through to "alive" — the safe + * direction for both callers. + */ +export function posixProcessAliveShellFunction(): string { + return ( + 'orcad_alive() { kill -0 "$1" 2>/dev/null || return 1; ' + + 'case "$(ps -o stat= -p "$1" 2>/dev/null)" in Z*) return 1;; esac; return 0; };' + ) +} diff --git a/src/main/ssh/orcad-remote-launch.test.ts b/src/main/ssh/orcad-remote-launch.test.ts new file mode 100644 index 00000000000..068ae588f65 --- /dev/null +++ b/src/main/ssh/orcad-remote-launch.test.ts @@ -0,0 +1,121 @@ +import { describe, expect, it } from 'vitest' + +import { + ORCAD_READINESS_FILENAME, + orcadLaunchCommand, + orcadLivenessBlocksGc, + orcadLivenessProbeCommand, + OrcadRemoteLaunchUnsupportedError, + parseOrcadLiveness, + parseOrcadReadinessOutput +} from './orcad-remote-launch' +import { + orcadStopFreedTheHost, + parseOrcadStopOutcome, + stopOrcadCommand +} from './orcad-remote-process-control' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +const posix = getRemoteHostPlatform('linux-x64') +const windows = getRemoteHostPlatform('win32-x64') + +const SPEC = { + remoteInstallDir: '/home/u/.orca-remote/orcad-0.2.0+bb01', + nodePath: '/usr/bin/node', + fullVersion: '0.2.0+bb01', + userDataDir: '/home/u/.orca', + bindHost: '127.0.0.1', + port: 7777 +} + +const READY_LINE = JSON.stringify({ + type: 'orca_server_ready', + schemaVersion: 1, + runtimeId: 'r1', + boundEndpoint: 'ws://127.0.0.1:7777', + advertisedEndpoint: null, + managedWslCliReconciliation: 'settled', + pairing: { available: false, reason: 'disabled_by_operator', guidance: 'n/a' }, + health: { buildHash: 'abc', terminalDaemon: { state: 'live' } } +}) + +describe('orcadLaunchCommand', () => { + it('states the bind posture rather than inheriting the build default', () => { + expect(orcadLaunchCommand(posix, SPEC)).toContain("--bind '127.0.0.1'") + }) + + it('truncates the readiness file, so a stale line cannot be activated on', () => { + const command = orcadLaunchCommand(posix, SPEC) + const truncate = command.indexOf(`: > '${SPEC.remoteInstallDir}/${ORCAD_READINESS_FILENAME}'`) + const launch = command.indexOf('nohup') + expect(truncate).toBeGreaterThan(-1) + expect(truncate).toBeLessThan(launch) + }) + + it('exports the version and the shared data root the deploy decided on', () => { + const command = orcadLaunchCommand(posix, SPEC) + expect(command).toContain(`ORCA_VERSION '${SPEC.fullVersion}'`.replace(' ', '=')) + expect(command).toContain(`ORCA_USER_DATA='${SPEC.userDataDir}'`) + }) + + it('declares the Windows refusal instead of emitting a command that cannot work', () => { + expect(() => orcadLaunchCommand(windows, SPEC)).toThrow(OrcadRemoteLaunchUnsupportedError) + }) +}) + +describe('readiness parsing', () => { + it('extracts the orca_server_ready payload', () => { + const parsed = parseOrcadReadinessOutput(`${READY_LINE}\n`) + expect(parsed).toMatchObject({ state: 'ready' }) + expect(parsed.state === 'ready' && parsed.readiness.boundEndpoint).toBe('ws://127.0.0.1:7777') + }) + + it('treats an empty or half-written file as pending, not as a failure', () => { + expect(parseOrcadReadinessOutput('')).toEqual({ state: 'pending' }) + expect(parseOrcadReadinessOutput('{"type":"orca_serv')).toEqual({ state: 'pending' }) + }) + + it('reports a complete JSON line that is not a readiness payload as malformed', () => { + expect(parseOrcadReadinessOutput('{"type":"something_else"}')).toMatchObject({ + state: 'malformed' + }) + }) +}) + +describe('liveness', () => { + it('reads the pid recorded in the version dir', () => { + expect(orcadLivenessProbeCommand(posix, SPEC.remoteInstallDir)).toContain('.orcad-pid') + }) + + it.each([ + ['LIVE', 'LIVE', true], + ['DEAD', 'DEAD', false], + ['', 'UNKNOWN', true], + ['garbage', 'UNKNOWN', true] + ])('parses %s and blocks GC = %s', (output, expected, blocks) => { + expect(parseOrcadLiveness(output)).toBe(expected) + expect(orcadLivenessBlocksGc(parseOrcadLiveness(output))).toBe(blocks) + }) +}) + +describe('stopping a running orcad', () => { + it('sends SIGTERM and never SIGKILL', () => { + const command = stopOrcadCommand(posix, SPEC.remoteInstallDir, { waitSeconds: 20 }) + expect(command).toContain('kill -TERM') + for (const kill of ['kill -9', 'kill -KILL', 'kill -SIGKILL', 'pkill']) { + expect(command).not.toContain(kill) + } + }) + + it.each([ + ['STOPPED', 'stopped', true], + ['ALREADY_EXITED', 'already-exited', true], + ['NO_PID', 'no-pid', true], + ['STILL_RUNNING', 'still-running', false], + ['SIGNAL_FAILED', 'signal-failed', false], + ['', 'unknown', false] + ])('parses %s and frees the host = %s', (output, expected, frees) => { + expect(parseOrcadStopOutcome(output)).toBe(expected) + expect(orcadStopFreedTheHost(parseOrcadStopOutcome(output))).toBe(frees) + }) +}) diff --git a/src/main/ssh/orcad-remote-launch.ts b/src/main/ssh/orcad-remote-launch.ts new file mode 100644 index 00000000000..e82be95faec --- /dev/null +++ b/src/main/ssh/orcad-remote-launch.ts @@ -0,0 +1,178 @@ +/** + * Starting a candidate orcad on the host and reading back what it says about itself. + * + * This is the piece `docs/design/shipping-orcad.html` §02 marks **fork**, not reuse: the + * relay launches detached and proves itself by printing an `ORCA-RELAY` sentinel, and orcad + * has no such interface. It publishes a single `orca_server_ready` JSON line on stdout, + * carrying the health payload activation is gated on — so the handshake here is "capture + * that line", not "match a marker". + * + * The candidate is launched detached with stdout redirected to a file inside its own version + * directory. Reading readiness off the exec channel would mean holding the channel open for + * the process's whole life; redirecting means the deploy can disconnect and the supervisor + * still owns a running service. + */ +import { shellEscape } from './ssh-connection-utils' +import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { + assertPosixOrcadHost as assertPosixHost, + ORCAD_PID_FILENAME, + posixProcessAliveShellFunction +} from './orcad-remote-host-support' +import type { ServeReadiness } from '../server/serve-readiness' + +/** Stdout of the launched candidate: exactly one `orca_server_ready` line, then nothing. */ +export const ORCAD_READINESS_FILENAME = '.orcad-readiness' +/** Stderr, including the bind-exposure line and every supervision message. */ +export const ORCAD_LOG_FILENAME = 'orcad.log' +export { + ORCAD_PID_FILENAME, + OrcadRemoteLaunchUnsupportedError +} from './orcad-remote-host-support' + +export type OrcadLaunchSpec = { + remoteInstallDir: string + nodePath: string + fullVersion: string + /** Shared across versions, and the reason rollback needs a snapshot. */ + userDataDir: string + /** Loopback by default; the client reaches it through an SSH local port-forward. */ + bindHost: string + port: number +} + +/** + * Launch the candidate detached and echo its PID. + * + * Why `--bind` is always passed explicitly: orcad defaults to loopback, but a default is a + * thing a future version can change. The deploy states the posture it intends rather than + * inheriting whatever the installed build happens to default to. + */ +export function orcadLaunchCommand(host: RemoteHostPlatform, spec: OrcadLaunchSpec): string { + assertPosixHost(host) + const dir = shellEscape(spec.remoteInstallDir) + const readiness = shellEscape(joinRemotePath(host, spec.remoteInstallDir, ORCAD_READINESS_FILENAME)) + const log = shellEscape(joinRemotePath(host, spec.remoteInstallDir, ORCAD_LOG_FILENAME)) + const pidFile = shellEscape(joinRemotePath(host, spec.remoteInstallDir, ORCAD_PID_FILENAME)) + const entry = shellEscape(joinRemotePath(host, spec.remoteInstallDir, 'orcad.js')) + return [ + `cd ${dir} &&`, + // Why truncate: a re-launch into a dir that already holds a previous readiness line would + // otherwise let the deploy activate on the OLD process's health payload. + `: > ${readiness} &&`, + 'umask 077 &&', + `ORCA_VERSION=${shellEscape(spec.fullVersion)}`, + `ORCA_USER_DATA=${shellEscape(spec.userDataDir)}`, + `nohup ${shellEscape(spec.nodePath)} ${entry}`, + `--json --bind ${shellEscape(spec.bindHost)} --port ${String(spec.port)}`, + `> ${readiness} 2>> ${log} < /dev/null &`, + `echo $! > ${pidFile} && cat ${pidFile}` + ].join(' ') +} + +export function readOrcadReadinessCommand( + host: RemoteHostPlatform, + remoteInstallDir: string +): string { + assertPosixHost(host) + const readiness = shellEscape(joinRemotePath(host, remoteInstallDir, ORCAD_READINESS_FILENAME)) + return `cat ${readiness} 2>/dev/null || true` +} + +/** + * Is the process recorded in this version dir still running? + * + * Answers `LIVE`, `DEAD`, or `UNKNOWN`. `UNKNOWN` covers a missing or unparseable PID file + * and a `kill -0` that failed for a reason other than "no such process" — a permission + * error means someone else's process holds that PID, which is not evidence of death. + */ +export function orcadLivenessProbeCommand( + host: RemoteHostPlatform, + remoteInstallDir: string +): string { + assertPosixHost(host) + const pidFile = shellEscape(joinRemotePath(host, remoteInstallDir, ORCAD_PID_FILENAME)) + return [ + posixProcessAliveShellFunction(), + `pid=$(cat ${pidFile} 2>/dev/null);`, + 'case "$pid" in', + '"" ) echo UNKNOWN;;', + '*[!0-9]* ) echo UNKNOWN;;', + // Why EPERM is LIVE and not DEAD: a permission error means some process holds that PID, + // and deleting a tree because we could not signal its owner is the wrong direction. + '* ) if orcad_alive "$pid"; then echo LIVE;', + 'elif kill -0 "$pid" 2>&1 | grep -qi "not permitted"; then echo LIVE;', + 'else echo DEAD; fi;;', + 'esac' + ].join(' ') +} + +export type OrcadLiveness = 'LIVE' | 'DEAD' | 'UNKNOWN' + +export function parseOrcadLiveness(output: string): OrcadLiveness { + const value = output.trim().split('\n').pop()?.trim() + return value === 'LIVE' || value === 'DEAD' ? value : 'UNKNOWN' +} + +/** True when GC must leave this directory alone. Inconclusive counts as in use. */ +export function orcadLivenessBlocksGc(liveness: OrcadLiveness): boolean { + return liveness !== 'DEAD' +} + +export type OrcadReadinessParse = + | { state: 'ready'; readiness: ServeReadiness } + | { state: 'pending' } + | { state: 'malformed'; reason: string } + +/** + * Pull the `orca_server_ready` payload out of whatever the candidate has written so far. + * + * Why scan for the type tag rather than parsing the last line: stdout is a file being + * appended to, so a poll can catch a half-written line. A partial JSON line is `pending`, + * not `malformed` — reporting a parse failure for a race would fail deploys that were fine. + */ +export function parseOrcadReadinessOutput(raw: string): OrcadReadinessParse { + const lines = raw.split('\n') + let sawCandidate = false + for (const line of lines) { + const trimmed = line.trim() + if (!trimmed.startsWith('{')) { + continue + } + sawCandidate = true + let parsed: unknown + try { + parsed = JSON.parse(trimmed) + } catch { + continue + } + if (typeof parsed !== 'object' || parsed === null) { + continue + } + const payload = parsed as { type?: unknown } + if (payload.type !== 'orca_server_ready') { + return { + state: 'malformed', + reason: `expected an orca_server_ready line, got type=${JSON.stringify(payload.type)}` + } + } + return { state: 'ready', readiness: toServeReadiness(payload as Record) } + } + return sawCandidate ? { state: 'pending' } : { state: 'pending' } +} + +function toServeReadiness(payload: Record): ServeReadiness { + return { + runtimeId: typeof payload.runtimeId === 'string' ? payload.runtimeId : '', + boundEndpoint: typeof payload.boundEndpoint === 'string' ? payload.boundEndpoint : null, + advertisedEndpoint: + typeof payload.advertisedEndpoint === 'string' ? payload.advertisedEndpoint : null, + managedWslCliReconciliation: + payload.managedWslCliReconciliation === 'pending' || + payload.managedWslCliReconciliation === 'failed' + ? payload.managedWslCliReconciliation + : 'settled', + pairing: payload.pairing as ServeReadiness['pairing'], + ...(payload.health ? { health: payload.health as ServeReadiness['health'] } : {}) + } +} diff --git a/src/main/ssh/orcad-remote-process-control.ts b/src/main/ssh/orcad-remote-process-control.ts new file mode 100644 index 00000000000..6a9038ea3d6 --- /dev/null +++ b/src/main/ssh/orcad-remote-process-control.ts @@ -0,0 +1,73 @@ +/** + * Stopping a running orcad on the host without taking its terminals with it. + * + * `SIGKILL` is absent on purpose. orcad's own escalation contract (`orcad-entry.ts`) is + * SIGTERM, then a second SIGTERM meaning "your deadline elapsed, exit now"; a kill skips the + * teardown that releases the instance lock and disconnects — rather than shuts down — the + * terminal daemon. The daemon is detached and would survive a kill, but a stop that leaves + * the lock file behind makes the successor refuse to start with + * `orcad_data_root_shared`, so the update turns into an outage for no gain. + */ +import { shellEscape } from './ssh-connection-utils' +import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { + assertPosixOrcadHost as assertPosixHost, + ORCAD_PID_FILENAME, + posixProcessAliveShellFunction +} from './orcad-remote-host-support' + +/** + * Signal the orcad recorded in a version dir and wait for it to go. + * + * `escalate` sends the second SIGTERM orcad reads as "exit immediately". Callers use it only + * after the first deadline elapses, so the two signals are never in the same command. + */ +export function stopOrcadCommand( + host: RemoteHostPlatform, + remoteInstallDir: string, + options: { waitSeconds: number } +): string { + assertPosixHost(host) + const pidFile = shellEscape(joinRemotePath(host, remoteInstallDir, ORCAD_PID_FILENAME)) + return [ + posixProcessAliveShellFunction(), + `pid=$(cat ${pidFile} 2>/dev/null);`, + 'case "$pid" in "" | *[!0-9]* ) echo NO_PID; exit 0;; esac;', + 'orcad_alive "$pid" || { echo ALREADY_EXITED; exit 0; };', + 'kill -TERM "$pid" 2>/dev/null || { echo SIGNAL_FAILED; exit 0; };', + `i=0; while [ "$i" -lt ${options.waitSeconds} ]; do`, + 'orcad_alive "$pid" || { echo STOPPED; exit 0; };', + 'sleep 1; i=$((i + 1)); done;', + 'echo STILL_RUNNING' + ].join(' ') +} + +export type OrcadStopOutcome = + | 'stopped' + | 'already-exited' + | 'no-pid' + | 'still-running' + | 'signal-failed' + | 'unknown' + +export function parseOrcadStopOutcome(output: string): OrcadStopOutcome { + switch (output.trim().split('\n').pop()?.trim() ?? '') { + case 'STOPPED': + return 'stopped' + case 'ALREADY_EXITED': + return 'already-exited' + case 'NO_PID': + return 'no-pid' + case 'STILL_RUNNING': + return 'still-running' + case 'SIGNAL_FAILED': + return 'signal-failed' + default: + return 'unknown' + } +} + +/** True when the port is free and a successor may bind. */ +export function orcadStopFreedTheHost(outcome: OrcadStopOutcome): boolean { + return outcome === 'stopped' || outcome === 'already-exited' || outcome === 'no-pid' +} diff --git a/src/main/ssh/orcad-remote-rollback.test.ts b/src/main/ssh/orcad-remote-rollback.test.ts new file mode 100644 index 00000000000..59f3d46e330 --- /dev/null +++ b/src/main/ssh/orcad-remote-rollback.test.ts @@ -0,0 +1,206 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('./ssh-relay-deploy-helpers', () => ({ + execCommand: vi.fn(), + isUnconfirmedSshCommandTermination: () => false +})) +vi.mock('./ssh-connection-utils', () => ({ shellEscape: (s: string) => `'${s}'` })) +vi.mock('./ssh-relay-install-transfers', () => ({ + writeRelayFile: vi.fn().mockResolvedValue(undefined), + uploadRelayDirectory: vi.fn().mockResolvedValue(undefined) +})) + +import { execCommand } from './ssh-relay-deploy-helpers' +import { writeRelayFile } from './ssh-relay-install-transfers' +import { rollbackOrcad, type OrcadRollbackOptions } from './orcad-remote-rollback' +import { + emptyOrcadActivationRecord, + type OrcadActivationRecord +} from './orcad-activation-record' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import type { SshConnection } from './ssh-connection' + +const mockExec = vi.mocked(execCommand) +const ACTIVE = '0.2.0+bb01' +const TARGET = '0.1.0+aa01' +const BUILD_HASH = 'abc123def4567890' + +function record(overrides: Partial = {}): OrcadActivationRecord { + return { + ...emptyOrcadActivationRecord(), + active: ACTIVE, + previous: TARGET, + activatedAt: '2026-01-01T00:00:00.000Z', + snapshot: { + dirName: 'pre-0.2.0+bb01-1000', + takenBeforeVersion: ACTIVE, + readableByVersion: TARGET, + takenAt: '2026-01-01T00:00:00.000Z' + }, + ...overrides + } +} + +function readyLine(version: string): string { + return JSON.stringify({ + type: 'orca_server_ready', + schemaVersion: 1, + runtimeId: 'r1', + boundEndpoint: 'ws://127.0.0.1:7777', + advertisedEndpoint: null, + managedWslCliReconciliation: 'settled', + pairing: { available: false, reason: 'disabled_by_operator', guidance: 'n/a' }, + health: { + buildHash: BUILD_HASH, + buildVersion: version, + nodeVersion: '20.11.0', + nodeAbi: '115', + platform: 'linux', + arch: 'x64', + pid: 1, + terminalDaemon: { + state: 'live', + ownsFreshSessions: true, + pid: 2, + buildVersion: version, + entryPath: '/x/daemon-entry.js', + protocolVersion: 3, + selfTest: { ok: true, coverage: 'pty-spawn', verdict: 'healthy', durationMs: 5 } + } + } + }) +} + +function scriptHost(log: string[], overrides: { restore?: string } = {}): void { + mockExec.mockImplementation(async (_conn, command: string) => { + const text = String(command) + if (text.includes('state.tar') && text.includes('test -f') && !text.includes('tar -C')) { + return 'PRESENT' + } + if (text.includes('find ') && text.includes('stat')) { + return 'UNKNOWN' + } + if (text.includes('kill -TERM')) { + log.push(`stop:${text.includes(ACTIVE) ? ACTIVE : TARGET}`) + return 'STOPPED' + } + if (text.includes('tar -C') && text.includes('-xf')) { + log.push('restore') + return overrides.restore ?? 'RESTORED' + } + if (text.includes('nohup')) { + log.push(`launch:${text.includes(ACTIVE) ? ACTIVE : TARGET}`) + return '9999' + } + if (text.startsWith('cat ') && text.includes('.orcad-readiness')) { + return readyLine(TARGET) + } + return '' + }) +} + +function options(overrides: Partial = {}): OrcadRollbackOptions { + return { + conn: {} as SshConnection, + host: getRemoteHostPlatform('linux-x64'), + remoteHome: '/home/u', + record: record(), + nodePath: '/usr/bin/node', + userDataDir: '/home/u/.orca', + bindHost: '127.0.0.1', + port: 7777, + census: { liveSessions: 0, startedSinceActivation: 0 }, + targetBuildHash: BUILD_HASH, + readinessTimeoutMs: 50, + sleep: async () => {}, + now: () => new Date('2026-02-02T00:00:00.000Z'), + ...overrides + } +} + +describe('rollbackOrcad', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('stops, restores state, then starts the target — in that order', async () => { + const log: string[] = [] + scriptHost(log) + const result = await rollbackOrcad(options()) + expect(result).toMatchObject({ outcome: 'rolled-back', target: TARGET }) + // Restoring under a running orcad would replace the store beneath a process holding it; + // starting first would let the older build migrate the newer build's state. + expect(log).toEqual([`stop:${ACTIVE}`, 'restore', `launch:${TARGET}`]) + }) + + it('refuses before touching anything when terminals started after activation', async () => { + const log: string[] = [] + scriptHost(log) + const result = await rollbackOrcad( + options({ census: { liveSessions: 3, startedSinceActivation: 2 } }) + ) + expect(result).toMatchObject({ + outcome: 'refused', + code: 'orcad_rollback_orphans_live_terminals' + }) + expect(log).toEqual([]) + expect(vi.mocked(writeRelayFile)).not.toHaveBeenCalled() + }) + + it('refuses when the snapshot is gone from the host', async () => { + const log: string[] = [] + mockExec.mockImplementation(async (_conn, command: string) => + String(command).includes('state.tar') ? 'ABSENT' : '' + ) + const result = await rollbackOrcad(options()) + expect(result).toMatchObject({ outcome: 'refused', code: 'orcad_rollback_snapshot_missing' }) + expect(log).toEqual([]) + }) + + it('does not start the old build when the restore failed', async () => { + const log: string[] = [] + scriptHost(log, { restore: 'FAILED' }) + const result = await rollbackOrcad(options()) + expect(result).toMatchObject({ outcome: 'failed', code: 'orcad_rollback_restore_failed' }) + expect(log).toEqual([`stop:${ACTIVE}`, 'restore']) + expect(result.outcome === 'failed' && result.reason).toContain('Do NOT start the older build') + }) + + it('leaves the record naming the newer version when the target fails to come up', async () => { + const log: string[] = [] + scriptHost(log) + mockExec.mockImplementation(async (_conn, command: string) => { + const text = String(command) + if (text.includes('state.tar') && text.includes('test -f') && !text.includes('tar -C')) { + return 'PRESENT' + } + if (text.includes('kill -TERM')) { + return 'STOPPED' + } + if (text.includes('tar -C') && text.includes('-xf')) { + return 'RESTORED' + } + // The target never publishes readiness. + return '' + }) + const result = await rollbackOrcad(options()) + expect(result).toMatchObject({ outcome: 'failed', code: 'orcad_activation_no_readiness' }) + // Until the target is proven serving, `active` must still name the version an operator + // would have to bring back. + expect(vi.mocked(writeRelayFile)).not.toHaveBeenCalled() + }) + + it('records the rollback only after the target answers healthy', async () => { + const log: string[] = [] + scriptHost(log) + await rollbackOrcad(options()) + const written = vi + .mocked(writeRelayFile) + .mock.calls.find((call) => String(call[2]).endsWith('orcad-active.json')) + expect(JSON.parse(String(written?.[3]))).toMatchObject({ + active: TARGET, + previous: null, + snapshot: null + }) + }) +}) diff --git a/src/main/ssh/orcad-remote-rollback.ts b/src/main/ssh/orcad-remote-rollback.ts new file mode 100644 index 00000000000..165ccfc037f --- /dev/null +++ b/src/main/ssh/orcad-remote-rollback.ts @@ -0,0 +1,244 @@ +/** + * Going back to the previously active orcad. + * + * Rollback is a state operation, not a binary swap. The version dirs are immutable and both + * are still on disk, so pointing at the old one is trivial; what is not trivial is that both + * versions share ONE data root, outside either dir. A newer orcad migrates that root on load + * — and Orca's persisted state carries no schema version to migrate against, so the older + * build cannot be shown to read the result. Rollback therefore restores the pre-activation + * snapshot, and refuses when restoring it would orphan work (`assessOrcadRollback`). + * + * The order below is the whole safety argument: stop, then restore, then start. Restoring + * under a running orcad would replace the store beneath a process holding it open, and + * starting before restoring would let the old build migrate the new build's state — the + * failure this is meant to avoid, arrived at from the other side. + */ +import type { SshConnection } from './ssh-connection' +import { execCommand } from './ssh-relay-deploy-helpers' +import { ORCAD_INSTALL_MODEL } from './remote-install-model' +import { computeRemoteInstallDir } from './ssh-relay-versioned-install' +import { writeRelayFile } from './ssh-relay-install-transfers' +import { RELAY_REMOTE_DIR } from './relay-protocol' +import { + ORCAD_STATE_SNAPSHOT_DIR, + serializeOrcadActivationRecord, + withRolledBackVersion, + type OrcadActivationRecord +} from './orcad-activation-record' +import { assessOrcadRollback, type OrcadTerminalCensus } from './orcad-update-plan' +import { evaluateOrcadActivation, type OrcadActivationVerdict } from './orcad-activation-gate' +import { + ORCAD_LOG_FILENAME, + orcadLaunchCommand, + parseOrcadReadinessOutput, + readOrcadReadinessCommand +} from './orcad-remote-launch' +import { + newestStateMtimeCommand, + parseNewestStateMtimeSeconds, + parseOrcadSnapshotRestore, + probeOrcadStateSnapshotCommand, + restoreOrcadStateSnapshotCommand +} from './orcad-state-snapshot' +import { + orcadStopFreedTheHost, + parseOrcadStopOutcome, + stopOrcadCommand +} from './orcad-remote-process-control' +import { orcadActivationPath } from './orcad-activation-record-store' +import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' + +export type OrcadRollbackOptions = { + conn: SshConnection + host: RemoteHostPlatform + remoteHome: string + record: OrcadActivationRecord + nodePath: string + userDataDir: string + bindHost: string + port: number + census: OrcadTerminalCensus + /** Expected build hash of the rollback target, from the client's copy of those bytes. */ + targetBuildHash: string + readinessTimeoutMs?: number + now?: () => Date + sleep?: (ms: number) => Promise + signal?: AbortSignal +} + +export type OrcadRollbackResult = + | { outcome: 'rolled-back'; target: string; discarded: string[]; verdict: OrcadActivationVerdict } + | { outcome: 'refused'; code: string; reason: string } + | { outcome: 'failed'; code: string; reason: string } + +const DEFAULT_READINESS_TIMEOUT_MS = 90_000 +const READINESS_POLL_MS = 500 +const STOP_WAIT_SECONDS = 20 + +function exec(options: OrcadRollbackOptions, command: string): Promise { + return execCommand(options.conn, command, { + wrapCommand: options.host.commandDialect !== 'powershell', + signal: options.signal + }) +} + +function snapshotDirPath(options: OrcadRollbackOptions, dirName: string): string { + return joinRemotePath( + options.host, + options.remoteHome, + RELAY_REMOTE_DIR, + ORCAD_STATE_SNAPSHOT_DIR, + dirName + ) +} + +/** Has the store been written since activation? `null` when it cannot be established. */ +async function readStateWritesSinceActivation( + options: OrcadRollbackOptions +): Promise { + if (!options.record.activatedAt) { + return null + } + const activatedAtSeconds = Math.floor(Date.parse(options.record.activatedAt) / 1000) + if (!Number.isFinite(activatedAtSeconds)) { + return null + } + const newest = parseNewestStateMtimeSeconds( + await exec(options, newestStateMtimeCommand(options.host, options.userDataDir)).catch(() => '') + ) + return newest === null ? null : newest >= activatedAtSeconds +} + +export async function rollbackOrcad( + options: OrcadRollbackOptions +): Promise { + const now = options.now ?? ((): Date => new Date()) + const snapshotPresent = options.record.snapshot + ? ( + await exec( + options, + probeOrcadStateSnapshotCommand( + options.host, + snapshotDirPath(options, options.record.snapshot.dirName) + ) + ).catch(() => 'ABSENT') + ).trim() === 'PRESENT' + : false + + const safety = assessOrcadRollback({ + record: options.record, + snapshotPresent, + census: options.census, + stateWritesSinceActivation: await readStateWritesSinceActivation(options) + }) + if (safety.safety === 'unsafe') { + return { outcome: 'refused', code: safety.code, reason: safety.reason } + } + + if (options.record.active) { + const outgoingDir = computeRemoteInstallDir( + ORCAD_INSTALL_MODEL, + options.remoteHome, + options.record.active + ) + const stopped = parseOrcadStopOutcome( + await exec( + options, + stopOrcadCommand(options.host, outgoingDir, { waitSeconds: STOP_WAIT_SECONDS }) + ) + ) + if (!orcadStopFreedTheHost(stopped)) { + return { + outcome: 'failed', + code: 'orcad_rollback_stop_incomplete', + reason: + `orcad ${options.record.active} did not exit within ${STOP_WAIT_SECONDS}s of SIGTERM ` + + `(${stopped}). Nothing was restored — the store is untouched and the host is still ` + + 'serving the version you tried to leave.' + } + } + } + + // Why between stop and start: the store must be replaced while no orcad holds it, and + // before the older build gets a chance to migrate the newer build's state. + const restored = parseOrcadSnapshotRestore( + await exec( + options, + restoreOrcadStateSnapshotCommand( + options.host, + options.userDataDir, + // Guarded by `assessOrcadRollback`: `unsafe` covers a missing snapshot. + snapshotDirPath(options, options.record.snapshot?.dirName ?? '') + ) + ).catch(() => 'FAILED') + ) + if (restored !== 'restored') { + return { + outcome: 'failed', + code: 'orcad_rollback_restore_failed', + reason: + `The pre-activation snapshot could not be restored (${restored}). orcad is stopped and ` + + 'the data root may be partially replaced. Do NOT start the older build against it; ' + + `re-deploy ${options.record.active ?? 'the newer version'}, which can read what is there.` + } + } + + const targetDir = computeRemoteInstallDir( + ORCAD_INSTALL_MODEL, + options.remoteHome, + safety.target + ) + await exec( + options, + orcadLaunchCommand(options.host, { + remoteInstallDir: targetDir, + nodePath: options.nodePath, + fullVersion: safety.target, + userDataDir: options.userDataDir, + bindHost: options.bindHost, + port: options.port + }) + ) + const deadline = Date.now() + (options.readinessTimeoutMs ?? DEFAULT_READINESS_TIMEOUT_MS) + const sleep = options.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))) + let parsed = parseOrcadReadinessOutput('') + while (Date.now() < deadline && parsed.state === 'pending') { + options.signal?.throwIfAborted() + parsed = parseOrcadReadinessOutput( + await exec(options, readOrcadReadinessCommand(options.host, targetDir)) + ) + if (parsed.state === 'pending') { + await sleep(READINESS_POLL_MS) + } + } + const verdict = evaluateOrcadActivation(parsed.state === 'ready' ? parsed.readiness : null, { + buildHash: options.targetBuildHash, + fullVersion: safety.target + }) + if (verdict.decision === 'reject') { + return { + outcome: 'failed', + code: verdict.code, + reason: + `The rollback target ${safety.target} did not come up healthy: ${verdict.reason} The ` + + `store has been restored to its pre-activation state. Its stderr is at ` + + `${joinRemotePath(options.host, targetDir, ORCAD_LOG_FILENAME)}.` + } + } + + // Why the record is written last: until the target is proven serving, `active` still names + // the version an operator would need to bring back, and `previous` still names this target. + await writeRelayFile( + options.conn, + options.host, + orcadActivationPath(options.host, options.remoteHome), + serializeOrcadActivationRecord(withRolledBackVersion(options.record, now())), + { signal: options.signal } + ) + return { + outcome: 'rolled-back', + target: safety.target, + discarded: safety.safety === 'lossy' ? safety.discards : [], + verdict + } +} diff --git a/src/main/ssh/orcad-remote-shell-commands.integration.test.ts b/src/main/ssh/orcad-remote-shell-commands.integration.test.ts new file mode 100644 index 00000000000..d1f8ac0048f --- /dev/null +++ b/src/main/ssh/orcad-remote-shell-commands.integration.test.ts @@ -0,0 +1,189 @@ +/** + * Runs the generated POSIX commands through a real `/bin/sh`. + * + * The unit tests assert on command *text*, which is exactly the kind of test that stays + * green while the shell it produces does not work — a quoting slip, a `case` pattern that + * never matches, a `tar` invocation that silently captures nothing. These run the strings. + */ +import { execFileSync, spawn } from 'node:child_process' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +import { + orcadLivenessProbeCommand, + ORCAD_PID_FILENAME, + parseOrcadLiveness +} from './orcad-remote-launch' +import { parseOrcadStopOutcome, stopOrcadCommand } from './orcad-remote-process-control' +import { + captureOrcadStateSnapshotCommand, + newestStateMtimeCommand, + parseNewestStateMtimeSeconds, + parseOrcadSnapshotCapture, + parseOrcadSnapshotRestore, + probeOrcadStateSnapshotCommand, + restoreOrcadStateSnapshotCommand +} from './orcad-state-snapshot' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +const host = getRemoteHostPlatform('linux-x64') +let root = '' +let dataDir = '' +let snapshotDir = '' +let versionDir = '' + +function sh(command: string): string { + return execFileSync('/bin/sh', ['-c', command], { encoding: 'utf8' }) +} + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'orcad-shell-')) + dataDir = join(root, '.orca') + snapshotDir = join(root, 'snapshots', 'pre-0.2.0+bb01-1000') + versionDir = join(root, '.orca-remote', 'orcad-0.2.0+bb01') + mkdirSync(join(dataDir, 'profiles', 'p1'), { recursive: true }) + mkdirSync(join(dataDir, 'daemon'), { recursive: true }) + mkdirSync(versionDir, { recursive: true }) + writeFileSync(join(dataDir, 'orca-profile-index.json'), '{"v":"before"}') + writeFileSync(join(dataDir, 'profiles', 'p1', 'orca-data.json'), '{"repos":"before"}') + writeFileSync(join(dataDir, 'daemon', 'daemon.sock.token'), 'live-daemon-token') +}) + +afterEach(() => { + rmSync(root, { recursive: true, force: true }) +}) + +describe('state snapshot commands, run for real', () => { + it('captures, then restores state the newer build overwrote', () => { + expect( + parseOrcadSnapshotCapture(sh(captureOrcadStateSnapshotCommand(host, dataDir, snapshotDir))) + ).toBe('captured') + expect(sh(probeOrcadStateSnapshotCommand(host, snapshotDir)).trim()).toBe('PRESENT') + + // The new version migrates the store and adds a file of its own. + writeFileSync(join(dataDir, 'orca-profile-index.json'), '{"v":"migrated"}') + writeFileSync(join(dataDir, 'profiles', 'p1', 'new-build-only.json'), '{}') + + expect( + parseOrcadSnapshotRestore(sh(restoreOrcadStateSnapshotCommand(host, dataDir, snapshotDir))) + ).toBe('restored') + expect(readFileSync(join(dataDir, 'orca-profile-index.json'), 'utf8')).toBe('{"v":"before"}') + // Removed before extraction, so the older build never sees a file it cannot interpret. + expect(() => readFileSync(join(dataDir, 'profiles', 'p1', 'new-build-only.json'))).toThrow() + }) + + it('leaves the live daemon runtime dir untouched through capture and restore', () => { + sh(captureOrcadStateSnapshotCommand(host, dataDir, snapshotDir)) + // The daemon is running across the rollback and rewrites its token; a restore that + // reached /daemon would break the fence that keeps its terminals adoptable. + writeFileSync(join(dataDir, 'daemon', 'daemon.sock.token'), 'token-after-restart') + sh(restoreOrcadStateSnapshotCommand(host, dataDir, snapshotDir)) + expect(readFileSync(join(dataDir, 'daemon', 'daemon.sock.token'), 'utf8')).toBe( + 'token-after-restart' + ) + }) + + it('reports EMPTY on a data root with nothing to lose, instead of an archive of nothing', () => { + const emptyRoot = join(root, 'fresh') + mkdirSync(emptyRoot) + expect( + parseOrcadSnapshotCapture(sh(captureOrcadStateSnapshotCommand(host, emptyRoot, snapshotDir))) + ).toBe('empty') + expect(sh(probeOrcadStateSnapshotCommand(host, snapshotDir)).trim()).toBe('ABSENT') + }) + + it('reports MISSING rather than claiming a restore it did not perform', () => { + expect( + parseOrcadSnapshotRestore( + sh(restoreOrcadStateSnapshotCommand(host, dataDir, join(root, 'nope'))) + ) + ).toBe('missing') + }) + + it('reads a real mtime for the store', () => { + const seconds = parseNewestStateMtimeSeconds(sh(newestStateMtimeCommand(host, dataDir))) + expect(seconds).toBeGreaterThan(1_600_000_000) + expect(seconds).toBeLessThanOrEqual(Math.floor(Date.now() / 1000) + 5) + }) + + it('survives a data root whose path contains a quote and a space', () => { + const nasty = join(root, `it's a dir`) + mkdirSync(join(nasty, 'profiles'), { recursive: true }) + writeFileSync(join(nasty, 'orca-profile-index.json'), '{"v":"quoted"}') + expect( + parseOrcadSnapshotCapture(sh(captureOrcadStateSnapshotCommand(host, nasty, snapshotDir))) + ).toBe('captured') + writeFileSync(join(nasty, 'orca-profile-index.json'), '{"v":"changed"}') + expect( + parseOrcadSnapshotRestore(sh(restoreOrcadStateSnapshotCommand(host, nasty, snapshotDir))) + ).toBe('restored') + expect(readFileSync(join(nasty, 'orca-profile-index.json'), 'utf8')).toBe('{"v":"quoted"}') + }) +}) + +describe('liveness and stop commands, run for real', () => { + it('reports UNKNOWN with no pid file, and DEAD for a pid that has exited', () => { + expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('UNKNOWN') + writeFileSync(join(versionDir, ORCAD_PID_FILENAME), 'not-a-pid') + expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('UNKNOWN') + // A pid that has certainly exited: our own `sh` child from the line above. + const exited = Number(sh('sh -c "echo $$"').trim()) + writeFileSync(join(versionDir, ORCAD_PID_FILENAME), String(exited)) + expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('DEAD') + }) + + it('reports LIVE for a running process and stops it with SIGTERM', async () => { + const child = spawn('/bin/sh', ['-c', 'sleep 30'], { stdio: 'ignore' }) + try { + writeFileSync(join(versionDir, ORCAD_PID_FILENAME), String(child.pid)) + expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('LIVE') + + const exited = new Promise((resolve) => + child.once('exit', (_code, signal) => resolve(signal)) + ) + expect(parseOrcadStopOutcome(sh(stopOrcadCommand(host, versionDir, { waitSeconds: 10 })))).toBe( + 'stopped' + ) + expect(await exited).toBe('SIGTERM') + expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('DEAD') + } finally { + child.kill('SIGKILL') + } + }) + + // `kill -0` succeeds on a zombie, so a probe built on it alone calls an exited process + // live: the stop loop would time out on a process that is already gone, and GC would keep + // a dead version dir forever. Verified as a real macOS behaviour, not a hypothetical. + it('reports a zombie as DEAD, not as a running process', () => { + const child = spawn('/bin/sh', ['-c', 'exit 0'], { stdio: 'ignore' }) + try { + writeFileSync(join(versionDir, ORCAD_PID_FILENAME), String(child.pid)) + // Block the event loop so Node never reaps it; the process is now a zombie. + sh('sleep 1') + expect(sh(`ps -o stat= -p ${child.pid} || echo GONE`).trim()).toMatch(/^Z/) + expect(sh(`kill -0 ${child.pid} 2>/dev/null && echo LIVE || echo DEAD`).trim()).toBe('LIVE') + expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('DEAD') + expect( + parseOrcadStopOutcome(sh(stopOrcadCommand(host, versionDir, { waitSeconds: 1 }))) + ).toBe('already-exited') + } finally { + child.unref() + } + }) + + it('reports ALREADY_EXITED for a stale pid file rather than signalling a stranger', () => { + const exited = Number(sh('sh -c "echo $$"').trim()) + writeFileSync(join(versionDir, ORCAD_PID_FILENAME), String(exited)) + expect(parseOrcadStopOutcome(sh(stopOrcadCommand(host, versionDir, { waitSeconds: 1 })))).toBe( + 'already-exited' + ) + }) + + it('reports NO_PID when the version dir was never launched', () => { + expect(parseOrcadStopOutcome(sh(stopOrcadCommand(host, versionDir, { waitSeconds: 1 })))).toBe( + 'no-pid' + ) + }) +}) diff --git a/src/main/ssh/orcad-state-snapshot.test.ts b/src/main/ssh/orcad-state-snapshot.test.ts new file mode 100644 index 00000000000..0c50dc55fdb --- /dev/null +++ b/src/main/ssh/orcad-state-snapshot.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from 'vitest' + +import { + ORCAD_SNAPSHOT_EXCLUDED, + ORCAD_SNAPSHOT_MEMBERS, + captureOrcadStateSnapshotCommand, + newestStateMtimeCommand, + orcadSnapshotDirName, + parseNewestStateMtimeSeconds, + parseOrcadSnapshotCapture, + parseOrcadSnapshotRestore, + restoreOrcadStateSnapshotCommand +} from './orcad-state-snapshot' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +const posix = getRemoteHostPlatform('linux-x64') +const windows = getRemoteHostPlatform('win32-x64') +const ROOT = '/home/u/.orca' +const SNAP = '/home/u/.orca-remote/orcad-state-snapshots/pre-0.2.0+bb01-1000' + +describe('capturing the pre-activation snapshot', () => { + it('captures the profile state a rollback needs', () => { + const command = captureOrcadStateSnapshotCommand(posix, ROOT, SNAP) + for (const member of ORCAD_SNAPSHOT_MEMBERS) { + expect(command).toContain(`'${member}'`) + } + }) + + // The live daemon owns /daemon and outlives every restart. Restoring a stale copy of + // its socket, PID record and token would break the fence that keeps its terminals adoptable. + it.each(ORCAD_SNAPSHOT_EXCLUDED)('never captures %s', (excluded) => { + expect(captureOrcadStateSnapshotCommand(posix, ROOT, SNAP)).not.toContain(`'${excluded}'`) + }) + + it.each(ORCAD_SNAPSHOT_EXCLUDED)('never removes or restores over %s', (excluded) => { + expect(restoreOrcadStateSnapshotCommand(posix, ROOT, SNAP)).not.toContain(`'${excluded}'`) + }) + + it('writes the archive under a temp name and renames, so a killed deploy leaves no torn tar', () => { + const command = captureOrcadStateSnapshotCommand(posix, ROOT, SNAP) + expect(command).toContain('.partial') + expect(command.indexOf('tar -C')).toBeLessThan(command.indexOf('mv ')) + }) + + it.each([ + ['CAPTURED', 'captured'], + ['EMPTY', 'empty'], + ['tar: broken', 'failed'], + ['', 'failed'] + ])('parses %s as %s', (output, expected) => { + expect(parseOrcadSnapshotCapture(output)).toBe(expected) + }) + + it('keys the snapshot dir on both version and time, so a retry cannot overwrite one', () => { + expect(orcadSnapshotDirName('0.2.0+bb01', 1000)).not.toBe( + orcadSnapshotDirName('0.2.0+bb01', 2000) + ) + }) +}) + +describe('restoring the snapshot', () => { + it('clears the members before extracting, so files the new build added do not survive', () => { + const command = restoreOrcadStateSnapshotCommand(posix, ROOT, SNAP) + expect(command.indexOf('rm -rf')).toBeLessThan(command.indexOf('tar -C')) + }) + + it('reports a missing archive instead of extracting nothing and claiming success', () => { + expect(restoreOrcadStateSnapshotCommand(posix, ROOT, SNAP)).toContain('echo MISSING') + expect(parseOrcadSnapshotRestore('MISSING')).toBe('missing') + expect(parseOrcadSnapshotRestore('RESTORED')).toBe('restored') + expect(parseOrcadSnapshotRestore('FAILED')).toBe('failed') + }) +}) + +describe('detecting writes since activation', () => { + it.each([ + ['1700000000', 1_700_000_000], + ['UNKNOWN', null], + ['', null] + ])('parses %s', (output, expected) => { + expect(parseNewestStateMtimeSeconds(output)).toBe(expected) + }) + + it('looks at the same members the snapshot covers', () => { + const command = newestStateMtimeCommand(posix, ROOT) + for (const member of ORCAD_SNAPSHOT_MEMBERS) { + expect(command).toContain(`'${member}'`) + } + }) +}) + +describe('Windows hosts', () => { + it.each([ + ['capture', () => captureOrcadStateSnapshotCommand(windows, ROOT, SNAP)], + ['restore', () => restoreOrcadStateSnapshotCommand(windows, ROOT, SNAP)], + ['mtime', () => newestStateMtimeCommand(windows, ROOT)] + ])('refuses %s rather than emitting a POSIX command', (_label, build) => { + expect(build).toThrow('orcad to a Windows host is not implemented') + }) +}) diff --git a/src/main/ssh/orcad-state-snapshot.ts b/src/main/ssh/orcad-state-snapshot.ts new file mode 100644 index 00000000000..78ad8b47d2a --- /dev/null +++ b/src/main/ssh/orcad-state-snapshot.ts @@ -0,0 +1,173 @@ +/** + * The pre-activation copy of shared profile state that makes rollback sound. + * + * `docs/design/shipping-orcad.html` §04's state-schema row asks for "backward-readable + * migrations or a pre-activation snapshot". Only the second is available here, and not as a + * preference: Orca's persisted state carries **no schema version**. Migrations are cohort + * and shape heuristics that run on load and rewrite in place, and the load path rebuilds + * `settings` and `ui` from known fields — so a newer build's nested additions are silently + * dropped by an older one rather than rejected. There is nothing to compare and nothing that + * fails loudly, which rules out proving backward-readability and leaves the snapshot. + * + * What is snapshotted is deliberately narrow. `/daemon` is EXCLUDED: it holds the live + * daemon's socket, PID record and auth token, and that daemon outlives every orcad restart + * by design. Restoring a stale copy of it over a running daemon would break the endpoint + * fence that keeps its terminals adoptable — turning a rollback into the exact terminal + * massacre the daemon exists to prevent. + */ +import { shellEscape } from './ssh-connection-utils' +import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { assertPosixOrcadHost as assertPosixHost } from './orcad-remote-host-support' + +/** + * Root-relative paths a rollback needs restored. Everything else under the data root is + * either regenerable, or owned by a process that survives the rollback. + */ +export const ORCAD_SNAPSHOT_MEMBERS = [ + 'orca-profile-index.json', + // Pre-profiles layout; still read as a migration source. + 'orca-data.json', + 'profiles' +] as const + +/** Never captured and never restored — see the module comment. */ +export const ORCAD_SNAPSHOT_EXCLUDED = ['daemon', 'logs'] as const + +/** + * The member names go into the command unquoted (see `captureOrcadStateSnapshotCommand`), so + * they must be inert. They are compile-time constants; this catches the edit that adds one + * with a space or a metacharacter in it. + */ +function assertPlainMemberName(member: string): string { + if (!/^[A-Za-z0-9._-]+$/.test(member)) { + throw new Error(`Unsafe orcad snapshot member name: ${JSON.stringify(member)}`) + } + return member +} + +export function orcadSnapshotDirName(fullVersion: string, takenAtMs: number): string { + // Why the version and the timestamp: two activations of one version (a re-deploy after a + // rejected activation) must not overwrite each other's snapshot. + return `pre-${fullVersion}-${takenAtMs}` +} + +/** + * Capture the snapshot, or report why there is nothing to capture. + * + * Prints `CAPTURED`, or `EMPTY` when the data root holds none of the members — a first-ever + * deployment, where there is no state to lose and therefore no snapshot to take. `EMPTY` is + * reported rather than fabricating an empty archive, because a rollback that "restored" an + * empty archive would wipe a root that had filled up in between. + */ +export function captureOrcadStateSnapshotCommand( + host: RemoteHostPlatform, + userDataDir: string, + snapshotDir: string +): string { + assertPosixHost(host) + const root = shellEscape(userDataDir) + const dir = shellEscape(snapshotDir) + const archive = shellEscape(joinRemotePath(host, snapshotDir, 'state.tar')) + const memberTests = ORCAD_SNAPSHOT_MEMBERS.map( + // Why the accumulated name is NOT quoted: `$members` is re-split by the shell before it + // reaches tar, so a quoted name arrives as a literal `'profiles'` that tar cannot stat. + // `assertPlainMemberName` is what makes leaving them bare safe. + (member) => + `[ -e ${root}/${shellEscape(member)} ] && members="$members ${assertPlainMemberName(member)}";` + ).join(' ') + return [ + `members=;`, + memberTests, + 'if [ -z "$members" ]; then echo EMPTY; else', + `mkdir -p ${dir} && umask 077 &&`, + // Why a temp name then mv: a deploy killed mid-tar must not leave a truncated archive + // that a later rollback would happily restore. + `tar -C ${root} -cf ${archive}.partial $members && mv ${archive}.partial ${archive} &&`, + 'echo CAPTURED; fi' + ].join(' ') +} + +export type OrcadSnapshotCapture = 'captured' | 'empty' | 'failed' + +export function parseOrcadSnapshotCapture(output: string): OrcadSnapshotCapture { + const value = output.trim().split('\n').pop()?.trim() + if (value === 'CAPTURED') { + return 'captured' + } + return value === 'EMPTY' ? 'empty' : 'failed' +} + +export function probeOrcadStateSnapshotCommand( + host: RemoteHostPlatform, + snapshotDir: string +): string { + assertPosixHost(host) + const archive = shellEscape(joinRemotePath(host, snapshotDir, 'state.tar')) + return `test -f ${archive} && echo PRESENT || echo ABSENT` +} + +/** + * Restore the snapshot over the data root. + * + * Two things make this safe to run: the members are removed before extraction (so a file the + * new version added is gone rather than half-shadowed), and neither the removal nor the + * extraction can reach `/daemon`, because the member list never names it. + * + * The caller must have stopped orcad first. This does not check — it cannot, from a shell — + * so `orcad-remote-deploy.ts` owns that ordering. + */ +export function restoreOrcadStateSnapshotCommand( + host: RemoteHostPlatform, + userDataDir: string, + snapshotDir: string +): string { + assertPosixHost(host) + const root = shellEscape(userDataDir) + const archive = shellEscape(joinRemotePath(host, snapshotDir, 'state.tar')) + const removals = ORCAD_SNAPSHOT_MEMBERS.map( + (member) => `rm -rf ${root}/${shellEscape(member)};` + ).join(' ') + return [ + `test -f ${archive} || { echo MISSING; exit 0; };`, + `test -d ${root} || mkdir -p ${root};`, + removals, + `tar -C ${root} -xf ${archive} && echo RESTORED || echo FAILED` + ].join(' ') +} + +export type OrcadSnapshotRestore = 'restored' | 'missing' | 'failed' + +export function parseOrcadSnapshotRestore(output: string): OrcadSnapshotRestore { + const value = output.trim().split('\n').pop()?.trim() + if (value === 'RESTORED') { + return 'restored' + } + return value === 'MISSING' ? 'missing' : 'failed' +} + +/** + * Has the shared store been written since `activatedAt`? + * + * Prints the newest mtime (epoch seconds) across the snapshot members, or `UNKNOWN`. The + * caller compares; an `UNKNOWN` becomes `null`, which `assessOrcadRollback` treats as "yes, + * assume writes". + */ +export function newestStateMtimeCommand(host: RemoteHostPlatform, userDataDir: string): string { + assertPosixHost(host) + const root = shellEscape(userDataDir) + const paths = ORCAD_SNAPSHOT_MEMBERS.map((member) => `${root}/${shellEscape(member)}`).join(' ') + return [ + `newest=$(find ${paths} -type f -exec stat -c %Y {} + 2>/dev/null ||`, + `find ${paths} -type f -exec stat -f %m {} + 2>/dev/null);`, + 'if [ -z "$newest" ]; then echo UNKNOWN; else', + `echo "$newest" | sort -n | tail -1; fi` + ].join(' ') +} + +export function parseNewestStateMtimeSeconds(output: string): number | null { + const value = output.trim().split('\n').pop()?.trim() + if (!value || !/^\d+$/.test(value)) { + return null + } + return Number.parseInt(value, 10) +} diff --git a/src/main/ssh/orcad-update-plan.test.ts b/src/main/ssh/orcad-update-plan.test.ts new file mode 100644 index 00000000000..5d90c597e3c --- /dev/null +++ b/src/main/ssh/orcad-update-plan.test.ts @@ -0,0 +1,177 @@ +import { describe, expect, it } from 'vitest' + +import { assessOrcadRollback, planOrcadUpdate } from './orcad-update-plan' +import { + emptyOrcadActivationRecord, + type OrcadActivationRecord, + type OrcadStateSnapshot +} from './orcad-activation-record' + +const SNAPSHOT: OrcadStateSnapshot = { + dirName: 'pre-0.2.0+bb01-1000', + takenBeforeVersion: '0.2.0+bb01', + readableByVersion: '0.1.0+aa01', + takenAt: '2026-01-01T00:00:00.000Z' +} + +function record(overrides: Partial = {}): OrcadActivationRecord { + return { + ...emptyOrcadActivationRecord(), + active: '0.2.0+bb01', + previous: '0.1.0+aa01', + activatedAt: '2026-01-01T00:00:01.000Z', + snapshot: SNAPSHOT, + ...overrides + } +} + +describe('planOrcadUpdate', () => { + it('does nothing when the candidate is already active', () => { + const plan = planOrcadUpdate({ + record: record(), + candidateVersion: '0.2.0+bb01', + census: { liveSessions: 0, startedSinceActivation: 0 } + }) + expect(plan).toMatchObject({ action: 'noop' }) + }) + + it('defers rather than restarting a host with live terminals', () => { + const plan = planOrcadUpdate({ + record: record(), + candidateVersion: '0.3.0+cc01', + census: { liveSessions: 3, startedSinceActivation: 1 } + }) + expect(plan).toMatchObject({ action: 'defer', code: 'orcad_update_terminals_running' }) + expect(plan.action === 'defer' && plan.reason).toContain('would not kill them') + }) + + it('defers when the session count cannot be established', () => { + const plan = planOrcadUpdate({ + record: record(), + candidateVersion: '0.3.0+cc01', + census: { liveSessions: null, startedSinceActivation: null } + }) + expect(plan).toMatchObject({ + action: 'defer', + code: 'orcad_update_terminal_census_unavailable' + }) + }) + + it('plans a forced update with an unknown census as if terminals were live', () => { + const plan = planOrcadUpdate({ + record: record(), + candidateVersion: '0.3.0+cc01', + census: { liveSessions: null, startedSinceActivation: null }, + force: true + }) + expect(plan).toMatchObject({ action: 'proceed', preservesLiveDaemon: true }) + }) + + it('carries the daemon across a forced update with live terminals', () => { + const plan = planOrcadUpdate({ + record: record(), + candidateVersion: '0.3.0+cc01', + census: { liveSessions: 2, startedSinceActivation: 0 }, + force: true + }) + expect(plan).toMatchObject({ action: 'proceed', preservesLiveDaemon: true }) + }) + + it('replaces the daemon only when nothing is running under it', () => { + const plan = planOrcadUpdate({ + record: record(), + candidateVersion: '0.3.0+cc01', + census: { liveSessions: 0, startedSinceActivation: 0 } + }) + expect(plan).toMatchObject({ action: 'proceed', preservesLiveDaemon: false }) + }) +}) + +describe('assessOrcadRollback', () => { + it('is clean when the snapshot is intact and nothing happened since activation', () => { + const safety = assessOrcadRollback({ + record: record(), + snapshotPresent: true, + census: { liveSessions: 0, startedSinceActivation: 0 }, + stateWritesSinceActivation: false + }) + expect(safety).toMatchObject({ safety: 'clean', target: '0.1.0+aa01' }) + }) + + it('is lossy, and names what goes, once the store has been written since activation', () => { + const safety = assessOrcadRollback({ + record: record(), + snapshotPresent: true, + census: { liveSessions: 1, startedSinceActivation: 0 }, + stateWritesSinceActivation: true + }) + expect(safety).toMatchObject({ safety: 'lossy', target: '0.1.0+aa01' }) + expect(safety.safety === 'lossy' && safety.discards[0]).toContain('2026-01-01T00:00:01.000Z') + }) + + it('treats an unreadable store mtime as writes, not as a clean rollback', () => { + const safety = assessOrcadRollback({ + record: record(), + snapshotPresent: true, + census: { liveSessions: 0, startedSinceActivation: 0 }, + stateWritesSinceActivation: null + }) + expect(safety).toMatchObject({ safety: 'lossy' }) + }) + + // The point past which rollback is unsafe: the first terminal created after activation. + it('refuses once a terminal started after activation, because restoring would orphan it', () => { + const safety = assessOrcadRollback({ + record: record(), + snapshotPresent: true, + census: { liveSessions: 4, startedSinceActivation: 1 }, + stateWritesSinceActivation: true + }) + expect(safety).toMatchObject({ + safety: 'unsafe', + code: 'orcad_rollback_orphans_live_terminals' + }) + expect(safety.safety === 'unsafe' && safety.reason).toContain('nothing would be able to') + }) + + it('refuses when the snapshot the record names is gone from the host', () => { + const safety = assessOrcadRollback({ + record: record(), + snapshotPresent: false, + census: { liveSessions: 0, startedSinceActivation: 0 }, + stateWritesSinceActivation: false + }) + expect(safety).toMatchObject({ safety: 'unsafe', code: 'orcad_rollback_snapshot_missing' }) + expect(safety.safety === 'unsafe' && safety.reason).toContain('no schema version') + }) + + it('refuses when no snapshot was ever recorded', () => { + const safety = assessOrcadRollback({ + record: record({ snapshot: null }), + snapshotPresent: true, + census: { liveSessions: 0, startedSinceActivation: 0 }, + stateWritesSinceActivation: false + }) + expect(safety).toMatchObject({ safety: 'unsafe', code: 'orcad_rollback_snapshot_missing' }) + }) + + it('refuses when the post-activation session count is unverifiable', () => { + const safety = assessOrcadRollback({ + record: record(), + snapshotPresent: true, + census: { liveSessions: 2, startedSinceActivation: null }, + stateWritesSinceActivation: false + }) + expect(safety).toMatchObject({ safety: 'unsafe', code: 'orcad_rollback_census_unavailable' }) + }) + + it('refuses when there is no previous version to go back to', () => { + const safety = assessOrcadRollback({ + record: record({ previous: null }), + snapshotPresent: true, + census: { liveSessions: 0, startedSinceActivation: 0 }, + stateWritesSinceActivation: false + }) + expect(safety).toMatchObject({ safety: 'unsafe', code: 'orcad_rollback_no_target' }) + }) +}) diff --git a/src/main/ssh/orcad-update-plan.ts b/src/main/ssh/orcad-update-plan.ts new file mode 100644 index 00000000000..2653956ebe5 --- /dev/null +++ b/src/main/ssh/orcad-update-plan.ts @@ -0,0 +1,234 @@ +/** + * When an update may restart orcad, and when going back is still sound. + * + * Two constraints shape everything here. + * + * **The daemon must outlive the restart.** orcad forks the terminal daemon and deliberately + * does not kill it on stop (`orcad-daemon-supervision.ts` uses `disconnectDaemon`, never + * `shutdownDaemon`). An update that killed it would destroy every terminal on the host — + * the thing the daemon exists to prevent. After an update the surviving daemon was forked + * from the OUTGOING bundle, so `daemon-init` sees an entry-path/version mismatch and takes + * its `shouldPreserveDaemonWithLiveSessions` branch: with live sessions it preserves the old + * daemon; at exactly zero it replaces it. Both are correct, and both mean the outgoing + * version's directory is still load-bearing. + * + * **The state root is shared across versions.** `~/.orca/` (or `$ORCA_USER_DATA`) is outside + * every version dir, and Orca's persisted state carries no schema version — migrations run + * on load and rewrite in place. So "is the old version able to read what the new one wrote" + * has no answer that can be computed. That is why rollback is defined against a + * pre-activation snapshot rather than against a version comparison. + */ +import type { OrcadActivationRecord } from './orcad-activation-record' + +export type OrcadTerminalCensus = { + /** + * Sessions the live daemon owns right now. `null` means the probe could not answer — + * never treated as zero, because loss of contact is not evidence of process death + * (docs/reference/ssh-execution-boundary.md). + */ + liveSessions: number | null + /** + * Of those, how many started at or after `record.activatedAt`. These are the sessions the + * pre-activation snapshot does not describe. + */ + startedSinceActivation: number | null +} + +export type OrcadUpdateDecision = + | { action: 'noop'; reason: string } + | { + action: 'proceed' + /** True when a live daemon will be carried across the restart rather than replaced. */ + preservesLiveDaemon: boolean + notes: string[] + } + | { action: 'defer'; code: OrcadUpdateDeferCode; reason: string } + +export type OrcadUpdateDeferCode = + | 'orcad_update_terminals_running' + | 'orcad_update_terminal_census_unavailable' + +/** + * Decide whether to restart orcad onto `candidateVersion`. + * + * Deferring on live terminals is a deliberate choice, not caution. The restart itself is + * non-destructive, but it leaves the host running a NEW orcad against an OLD daemon until + * every one of those terminals exits — a mixed pair whose duration the operator, not the + * deploy, should decide. `force` is how they decide it. + */ +export function planOrcadUpdate(input: { + record: OrcadActivationRecord + candidateVersion: string + census: OrcadTerminalCensus + force?: boolean +}): OrcadUpdateDecision { + if (input.record.active === input.candidateVersion) { + return { + action: 'noop', + reason: `${input.candidateVersion} is already the active version; nothing to restart.` + } + } + const { liveSessions } = input.census + if (liveSessions === null) { + if (!input.force) { + return { + action: 'defer', + code: 'orcad_update_terminal_census_unavailable', + reason: + 'The terminal daemon did not answer a session count, so this update cannot tell ' + + 'whether work is running on the host. Retry, or force the update knowing terminals ' + + 'may be mid-flight.' + } + } + return { + action: 'proceed', + // Why true: an unverifiable census must be planned for as if sessions exist. Assuming + // the daemon is replaceable is the assumption that destroys terminals. + preservesLiveDaemon: true, + notes: [ + 'Forced with an unverifiable session count. Planning as if terminals are live: the ' + + 'daemon will be preserved across the restart, and the outgoing version directory ' + + 'stays pinned against GC.' + ] + } + } + if (liveSessions > 0 && !input.force) { + return { + action: 'defer', + code: 'orcad_update_terminals_running', + reason: + `${liveSessions} terminal${liveSessions === 1 ? ' is' : 's are'} running on this host. ` + + 'The restart would not kill them — the daemon is preserved — but the host would run ' + + `orcad ${input.candidateVersion} against a daemon forked from ` + + `${input.record.active ?? 'the previous build'} until they all exit. Update when the ` + + 'host is idle, or force it.' + } + } + if (liveSessions > 0) { + return { + action: 'proceed', + preservesLiveDaemon: true, + notes: [ + `Forced with ${liveSessions} live terminal${liveSessions === 1 ? '' : 's'}. They survive ` + + 'the restart on the existing daemon; the outgoing version directory stays pinned ' + + 'against GC because that daemon was forked from it.' + ] + } + } + return { + action: 'proceed', + // Zero live sessions is the one case where daemon-init's freshness branch replaces the + // daemon, so nothing is carried across and nothing is lost. + preservesLiveDaemon: false, + notes: [ + 'No terminals are running, so the daemon is replaced by one forked from the new bundle.' + ] + } +} + +export type OrcadRollbackSafety = + | { safety: 'clean'; target: string; notes: string[] } + | { safety: 'lossy'; target: string; discards: string[] } + | { safety: 'unsafe'; code: OrcadRollbackUnsafeCode; reason: string } + +export type OrcadRollbackUnsafeCode = + | 'orcad_rollback_no_target' + | 'orcad_rollback_snapshot_missing' + | 'orcad_rollback_orphans_live_terminals' + | 'orcad_rollback_census_unavailable' + +/** + * How safe it is to switch back to `record.previous`. + * + * **The point past which rollback is unsafe is the first terminal created after + * activation.** Not the first state write, and not any schema comparison: + * + * - Rolling back means restoring the pre-activation snapshot, because there is no schema + * version to prove the old build can read what the new one wrote. + * - The snapshot predates activation, so it does not describe sessions created since. + * - The daemon survives the binary swap and still owns those sessions. After the restore, + * a live daemon holds PTYs that the restored store has no rows for: work that is running, + * that no client can reattach to, and that the host will report as neither `live` nor + * `exited` for any session anyone can name. + * + * Settings and UI churn written after activation are merely discarded, which is `lossy`. + * Orphaning running work is not something a deploy gets to do quietly, so it is `unsafe`. + */ +export function assessOrcadRollback(input: { + record: OrcadActivationRecord + /** Whether the snapshot named by the record is actually still on the host. */ + snapshotPresent: boolean + census: OrcadTerminalCensus + /** + * Whether the shared store has been written since activation, from its mtime against + * `record.activatedAt`. `null` means unknown, which is treated as "yes" — claiming a + * lossless rollback we cannot demonstrate is the failure mode, not the caution. + */ + stateWritesSinceActivation: boolean | null +}): OrcadRollbackSafety { + const target = input.record.previous + if (!target) { + return { + safety: 'unsafe', + code: 'orcad_rollback_no_target', + reason: + 'This host has no previous orcad version recorded, so there is nothing to roll back ' + + 'to. Deploy a known-good build instead.' + } + } + if (!input.record.snapshot || !input.snapshotPresent) { + return { + safety: 'unsafe', + code: 'orcad_rollback_snapshot_missing', + reason: + `The pre-activation state snapshot for ${input.record.active ?? 'the active version'} ` + + 'is gone, and Orca state carries no schema version that could prove the older build ' + + 'can read what the newer one migrated. Switching the binary back would hand ' + + `${target} a store it may not understand. Deploy forward instead.` + } + } + const { startedSinceActivation } = input.census + if (startedSinceActivation === null) { + return { + safety: 'unsafe', + code: 'orcad_rollback_census_unavailable', + reason: + 'The daemon did not answer how many of its terminals started after this version was ' + + 'activated, so a snapshot restore might orphan running work. Retry when the host is ' + + 'reachable.' + } + } + if (startedSinceActivation > 0) { + return { + safety: 'unsafe', + code: 'orcad_rollback_orphans_live_terminals', + reason: + `${startedSinceActivation} terminal${startedSinceActivation === 1 ? '' : 's'} started ` + + 'after this version was activated. The daemon survives the rollback and would keep ' + + 'owning them, but the restored snapshot predates them, so nothing would be able to ' + + 'reattach. Close them (or let them exit) and roll back then.' + } + } + if (input.stateWritesSinceActivation === false) { + return { + safety: 'clean', + target, + notes: [ + 'The pre-activation snapshot is intact, no terminals started since activation, and ' + + 'the store has not been written since. Restoring it changes nothing.' + ] + } + } + return { + safety: 'lossy', + target, + discards: [ + input.stateWritesSinceActivation === null + ? 'Any profile, settings and UI change written since activation — the store mtime ' + + 'could not be read, so assume there are some.' + : `Every profile, settings and UI change written since ${ + input.record.activatedAt ?? 'activation' + }, when ${input.record.active ?? 'the active version'} was activated.` + ] + } +} diff --git a/src/main/ssh/remote-install-coexistence.test.ts b/src/main/ssh/remote-install-coexistence.test.ts new file mode 100644 index 00000000000..ce844587d62 --- /dev/null +++ b/src/main/ssh/remote-install-coexistence.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from 'vitest' + +import { selectRemoteInstallModel } from './remote-install-coexistence' + +const BOTH_INSTALLED = ['relay-0.1.0+aa01', 'orcad-0.2.0+bb01', 'orcad-0.1.0+aa01'] + +describe('what a client does when it finds both models installed', () => { + it('uses the registered model and leaves the other install alone', () => { + const selection = selectRemoteInstallModel({ + registration: 'orcad-peer', + installedDirNames: BOTH_INSTALLED + }) + expect(selection).toMatchObject({ outcome: 'use', model: 'orcad' }) + expect(selection.outcome === 'use' && selection.coexisting).toEqual(['relay-0.1.0+aa01']) + expect(selection.outcome === 'use' && selection.note).toContain( + 'garbage-collects only its own namespace' + ) + }) + + it('does not switch model just because the other one is also on disk', () => { + const selection = selectRemoteInstallModel({ + registration: 'ssh-target', + installedDirNames: BOTH_INSTALLED + }) + expect(selection).toMatchObject({ outcome: 'use', model: 'relay' }) + expect(selection.outcome === 'use' && selection.coexisting).toEqual([ + 'orcad-0.2.0+bb01', + 'orcad-0.1.0+aa01' + ]) + }) + + it('picks the registered model even when only the other one is installed', () => { + // On-disk presence is diagnostic, never a vote: an orcad-registered host with only relay + // dirs is a first orcad deploy, not a reason to fall back to the relay. + const selection = selectRemoteInstallModel({ + registration: 'orcad-peer', + installedDirNames: ['relay-0.1.0+aa01'] + }) + expect(selection).toMatchObject({ outcome: 'use', model: 'orcad' }) + }) + + it('refuses a machine registered under both models', () => { + const selection = selectRemoteInstallModel({ + registration: 'both', + installedDirNames: BOTH_INSTALLED + }) + expect(selection).toMatchObject({ + outcome: 'refuse', + code: 'remote_host_registered_under_both_models' + }) + // The forbidden thing is two registrations, not two directories — say so, or a user will + // "fix" it by deleting an install that is serving someone. + expect(selection.outcome === 'refuse' && selection.reason).toContain( + 'Leaving both install directories on disk is fine' + ) + }) + + it('refuses to infer a model for an unregistered machine', () => { + const selection = selectRemoteInstallModel({ + registration: 'none', + installedDirNames: BOTH_INSTALLED + }) + expect(selection).toMatchObject({ + outcome: 'refuse', + code: 'remote_host_not_registered' + }) + }) + + it('says nothing when there is nothing coexisting', () => { + const selection = selectRemoteInstallModel({ + registration: 'orcad-peer', + installedDirNames: ['orcad-0.2.0+bb01'] + }) + expect(selection).toMatchObject({ outcome: 'use', model: 'orcad', note: null }) + }) +}) diff --git a/src/main/ssh/remote-install-coexistence.ts b/src/main/ssh/remote-install-coexistence.ts new file mode 100644 index 00000000000..a24ac36cfb1 --- /dev/null +++ b/src/main/ssh/remote-install-coexistence.ts @@ -0,0 +1,83 @@ +/** + * What a client does when it finds both a relay and an orcad installed on one host. + * + * `docs/design/shipping-orcad.html` §06 draws the line the boundary doc actually draws: + * two *directories* on disk are fine and permanent; two *registered targets* for one + * machine are forbidden, because that is what splits a machine's worktrees across two + * identities (`docs/reference/ssh-execution-boundary.md`). + * + * So the model is never inferred from the filesystem. It is decided by how the user + * registered the host, and the on-disk inventory is only ever diagnostic. Inferring it — + * "an orcad dir exists, so prefer orcad" — would let a GC pass, a half-finished install or + * a stale tree silently re-point a live connection at a different execution identity. + */ +import { inventoryRemoteInstallDirs, type RemoteInstallModelId } from './remote-install-model' + +/** + * How this host is registered in the client's own records. + * + * `both` is representable on purpose: it is a state a user can reach by adding an SSH + * target for a machine they have already paired, and the point of this module is to refuse + * it loudly instead of picking one. + */ +export type RemoteHostRegistration = 'ssh-target' | 'orcad-peer' | 'both' | 'none' + +export type RemoteInstallSelection = + | { + outcome: 'use' + model: RemoteInstallModelId + /** Other-model dirs present on this host. Left alone; never GC'd by the chosen model. */ + coexisting: string[] + /** Non-null when there is something an operator should know but nothing to refuse over. */ + note: string | null + } + | { + outcome: 'refuse' + code: 'remote_host_registered_under_both_models' | 'remote_host_not_registered' + reason: string + } + +/** + * Choose the execution model for a connection to a host, given its registration and + * whatever happens to be installed there. + */ +export function selectRemoteInstallModel(input: { + registration: RemoteHostRegistration + /** Raw directory names under `~/.orca-remote/`, as listed on the host. */ + installedDirNames: readonly string[] +}): RemoteInstallSelection { + if (input.registration === 'both') { + return { + outcome: 'refuse', + code: 'remote_host_registered_under_both_models', + reason: + 'This machine is registered both as an SSH target and as a paired orcad peer. One ' + + 'machine must have one execution identity, or its worktrees and terminals split ' + + 'across two owners that cannot see each other. Remove one registration. Leaving ' + + 'both install directories on disk is fine and expected.' + } + } + if (input.registration === 'none') { + return { + outcome: 'refuse', + code: 'remote_host_not_registered', + reason: + 'This machine has no execution model registered. Add it as an SSH target or pair it ' + + 'as an orcad peer; what is already installed on it does not decide which it is.' + } + } + const model: RemoteInstallModelId = input.registration === 'ssh-target' ? 'relay' : 'orcad' + const inventory = inventoryRemoteInstallDirs(input.installedDirNames) + const coexisting = model === 'relay' ? inventory.orcad : inventory.relay + return { + outcome: 'use', + model, + coexisting, + note: + coexisting.length > 0 + ? `This host also has ${coexisting.length} ${model === 'relay' ? 'orcad' : 'relay'} ` + + `install directory(ies) (${coexisting.join(', ')}). They are left untouched: each ` + + 'model garbage-collects only its own namespace.' + : null + } +} diff --git a/src/main/ssh/remote-install-gc.ts b/src/main/ssh/remote-install-gc.ts new file mode 100644 index 00000000000..0d649cd0b03 --- /dev/null +++ b/src/main/ssh/remote-install-gc.ts @@ -0,0 +1,286 @@ +/** + * The version-directory garbage collector, shared by the relay and orcad. + * + * It lives apart from `ssh-relay-versioned-install.ts` because it is the one piece both + * models run, and because the ownership rule below is the whole point of separating them: + * a pass only ever sees, and only ever deletes, directories belonging to `model`. + */ +import type { SshConnection } from './ssh-connection' +import { RELAY_REMOTE_DIR } from './relay-protocol' +import { execCommand } from './ssh-relay-deploy-helpers' +import { probeInstallLockExistsCommand } from './ssh-relay-install-lock-commands' +import { isRelayInstallLockStale, RELAY_INSTALL_LOCK_NAME } from './ssh-relay-install-lock' +import { + RELAY_INSTALL_MODEL, + remoteInstallGcPermits, + remoteInstallVersionDirRegex, + type RemoteInstallModel +} from './remote-install-model' +import { + isRelayGcClaimOwned, + releaseRelayGcClaimWithRetry, + tryAcquireRelayGcClaim +} from './ssh-relay-gc-claim' +import { cleanupRelayGcTombstones } from './ssh-relay-gc-tombstone' +import { + listRemoteInstallBaseDirsCommand, + MAX_RELAY_GC_LISTING_ENTRIES, + moveRemoteTreeCommand, + probeFileExistsCommand, + relayLivenessProbeCommand, + removeRemoteTreeCommand +} from './ssh-remote-commands' +import { + getRemoteHostPlatform, + isWindowsRemoteHost, + joinRemotePath, + remoteBasename, + type RemoteHostPlatform +} from './ssh-remote-platform' +import { windowsRelayPipePathsForSocketName } from './ssh-relay-endpoints' +import { isUnconfirmedSshCommandTermination } from './ssh-relay-exec-command' + +// Legacy relay dirs predate `.install-complete`; they need a liveness-only GC check so they +// eventually drain. There is no orcad equivalent — orcad has never shipped without one. +const LEGACY_RELAY_DIR_REGEX = /^relay-v\d+\.\d+\.\d+$/ +const DEFAULT_REMOTE_HOST = getRemoteHostPlatform('linux-x64') + +function execHostCommand( + conn: SshConnection, + host: RemoteHostPlatform, + command: string +): Promise { + return execCommand(conn, command, { wrapCommand: host.commandDialect !== 'powershell' }) +} + +export type RemoteInstallGcOptions = { + windowsNodePath?: string + windowsSockNames?: string[] + /** + * Model-specific "someone is using this directory" probe. It must answer TRUE when + * inconclusive — an unanswered probe is never evidence a tree is idle. + */ + isDirLive: (dir: string) => Promise + /** + * Directories this pass must never remove even when idle and complete, named by directory + * (not absolute path). orcad passes its active and previous versions: the previous one is + * the rollback target, and GC'ing it turns a recoverable bad update into a re-deploy. + */ + pinnedDirNames?: readonly string[] +} + +/** + * Garbage-collect one model's old version directories. + * + * **GC ownership (design §06 falsifier 1):** a pass only ever sees, and only ever deletes, + * directories belonging to `model`. The remote listing is scoped by prefix, and + * `remoteInstallGcPermits` re-checks every candidate locally, so neither a widened glob nor + * a hand-rolled listing can make one model delete the other's live install. + */ +export async function gcOldRemoteInstallVersions( + conn: SshConnection, + model: RemoteInstallModel, + remoteHome: string, + currentDirAbsPath: string, + host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, + options: RemoteInstallGcOptions +): Promise { + const baseDir = joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR) + const currentDirName = remoteBasename(currentDirAbsPath, host) + let listing: string + try { + listing = await execHostCommand( + conn, + host, + listRemoteInstallBaseDirsCommand(host, baseDir, model) + ) + } catch { + return + } + const entries = listing + .split('\n') + .map((s) => s.trim()) + .filter(Boolean) + .slice(0, MAX_RELAY_GC_LISTING_ENTRIES) + + await cleanupRelayGcTombstones(conn, baseDir, entries, host) + + const versionDirRegex = remoteInstallVersionDirRegex(model) + const pinned = new Set([currentDirName, ...(options.pinnedDirNames ?? [])]) + const candidates = entries + // Why re-check ownership after a prefix-scoped listing: this is the one line that stands + // between a parameterized GC and deleting the sibling model's live install. + .filter((name) => remoteInstallGcPermits(model, name)) + .filter((name) => versionDirRegex.test(name)) + .filter((name) => !pinned.has(name)) + + if (candidates.length === 0) { + return + } + + const removed: string[] = [] + const kept: string[] = [] + for (const name of candidates) { + const dir = joinRemotePath(host, baseDir, name) + try { + const safe = await isCandidateSafeToRemove(conn, model, dir, name, host, options) + if (!safe) { + kept.push(name) + continue + } + // Why: the claim is a sibling, so it survives moving/deleting the candidate and lets installers back out first. + const gcClaimToken = await tryAcquireRelayGcClaim(conn, dir, host) + if (!gcClaimToken) { + kept.push(name) + continue + } + let preserveGcClaim = false + let gcClaimReleaseNeeded = true + try { + // Recheck under the stable claim; installers probe it before and after creating their lock, closing both orders. + if (!(await isCandidateSafeToRemove(conn, model, dir, name, host, options))) { + kept.push(name) + continue + } + if (!(await isRelayGcClaimOwned(conn, dir, gcClaimToken, host))) { + kept.push(name) + continue + } + const tombstone = `${dir}.gc-tombstone.${process.pid}.${Date.now()}` + const moved = await execHostCommand(conn, host, moveRemoteTreeCommand(host, dir, tombstone)) + if (moved.trim() !== 'MOVED') { + kept.push(name) + continue + } + // Once renamed, a fresh install at the original path is isolated from the tombstone's deletion, so release the claim. + const release = await releaseRelayGcClaimWithRetry(conn, dir, gcClaimToken, host) + gcClaimReleaseNeeded = release === 'unknown' + await execHostCommand(conn, host, removeRemoteTreeCommand(host, tombstone)) + } catch (err) { + if (isUnconfirmedSshCommandTermination(err)) { + preserveGcClaim = true + } + throw err + } finally { + if (!preserveGcClaim && gcClaimReleaseNeeded) { + await releaseRelayGcClaimWithRetry(conn, dir, gcClaimToken, host) + } + } + removed.push(name) + } catch (err) { + console.warn( + `[${model.id}] GC failed for ${dir}: ${err instanceof Error ? err.message : String(err)}` + ) + kept.push(name) + } + } + + if (removed.length > 0) { + const keptSuffix = kept.length > 0 ? ` (kept: ${kept.join(', ')})` : '' + console.log( + `[${model.id}] GC: removed ${removed.length} stale version dir(s): ${removed.join(', ')}${keptSuffix}` + ) + } +} + +async function isCandidateSafeToRemove( + conn: SshConnection, + model: RemoteInstallModel, + dir: string, + name: string, + host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, + options: RemoteInstallGcOptions +): Promise { + const isLegacy = model.id === 'relay' && LEGACY_RELAY_DIR_REGEX.test(name) + + const lockDir = joinRemotePath(host, dir, RELAY_INSTALL_LOCK_NAME) + let lockProbe: string + try { + lockProbe = await execHostCommand(conn, host, probeInstallLockExistsCommand(host, lockDir)) + } catch { + return false + } + const lockState = lockProbe.trim() + if (lockState !== 'OPEN' && lockState !== 'LOCKED') { + return false + } + const locked = lockState === 'LOCKED' + + if (locked) { + // Why: stale lock = crashed installer; finalize can leave a dir .install-complete yet locked (lock-rm failed), so it's reclaimable. + if (!(await isRelayInstallLockStale(conn, lockDir, host))) { + return false + } + process.stderr.write?.(`[${model.id}] GC: lock at ${lockDir} is stale; treating as recoverable\n`) + } + + // Legacy dirs predate .install-complete; skip the sentinel and rely on the live-socket probe alone. + if (!isLegacy) { + const completePath = joinRemotePath(host, dir, model.installCompleteFilename) + const completeProbe = await execHostCommand( + conn, + host, + probeFileExistsCommand(host, completePath) + ).catch(() => 'PARTIAL') + if (completeProbe.trim() !== 'COMPLETE') { + // Crashed-install partial; leave for the next deploy to recover. + return false + } + } + + return !(await options.isDirLive(dir)) +} + + +/** + * The relay's GC, bound to its own namespace and its own liveness probe (a live unix socket + * or Windows pipe inside the version dir). + */ +export async function gcOldRelayVersions( + conn: SshConnection, + remoteHome: string, + currentDirAbsPath: string, + host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, + options?: { + windowsNodePath?: string + windowsSockNames?: string[] + } +): Promise { + await gcOldRemoteInstallVersions(conn, RELAY_INSTALL_MODEL, remoteHome, currentDirAbsPath, host, { + ...options, + isDirLive: (dir) => hasLiveRelaySocket(conn, dir, host, options) + }) +} + +async function hasLiveRelaySocket( + conn: SshConnection, + dir: string, + host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, + options?: { + windowsNodePath?: string + windowsSockNames?: string[] + } +): Promise { + try { + // Why: `test -S` only — a connect-and-close probe would race with a daemon about to idle. + const windowsOptions = + isWindowsRemoteHost(host) && options?.windowsNodePath + ? { + nodePath: options.windowsNodePath, + pipePaths: (options.windowsSockNames ?? []).flatMap((sockName) => + windowsRelayPipePathsForSocketName(host, dir, sockName) + ) + } + : undefined + const out = await execHostCommand( + conn, + host, + relayLivenessProbeCommand(host, dir, windowsOptions) + ) + const state = out.trim() + return state !== 'DEAD' && state !== 'WAITING' + } catch { + // Why: an inconclusive liveness probe must never authorize deletion. + return true + } +} diff --git a/src/main/ssh/remote-install-model.test.ts b/src/main/ssh/remote-install-model.test.ts new file mode 100644 index 00000000000..5a80df8ff4f --- /dev/null +++ b/src/main/ssh/remote-install-model.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest' + +import { + inventoryRemoteInstallDirs, + ORCAD_INSTALL_MODEL, + RELAY_INSTALL_MODEL, + remoteInstallDirName, + remoteInstallDirOwner, + remoteInstallGcPermits, + remoteInstallListingRegexSource, + remoteInstallVersionDirRegex +} from './remote-install-model' + +const RELAY_DIRS = ['relay-0.1.0+abcdef123456', 'relay-v0.1.0', 'relay-1.2.3'] +const ORCAD_DIRS = ['orcad-0.1.0+abcdef123456', 'orcad-v0.1.0', 'orcad-1.2.3'] + +describe('remote install namespace', () => { + it('names each model its own version dir', () => { + expect(remoteInstallDirName(RELAY_INSTALL_MODEL, '0.1.0+aa')).toBe('relay-0.1.0+aa') + expect(remoteInstallDirName(ORCAD_INSTALL_MODEL, '0.1.0+aa')).toBe('orcad-0.1.0+aa') + }) + + it('keeps the relay listing pattern byte-identical to the one it shipped with', () => { + // The literal that was hardcoded in `listRelayBaseDirsCommand` before it was + // parameterized. A drift here changes what an existing host's GC can see. + expect(remoteInstallListingRegexSource(RELAY_INSTALL_MODEL)).toBe( + String.raw`^relay-(v?[0-9]+\.[0-9]+\.[0-9]+(\+[0-9a-f]+)?)(\.gc-tombstone\.[0-9]+\.[0-9]+)?$` + ) + }) + + it('refuses a dir prefix that could escape a remote glob or quote', () => { + const injected = { ...RELAY_INSTALL_MODEL, dirPrefix: "relay'; rm -rf ~" } + expect(() => remoteInstallVersionDirRegex(injected)).toThrow('Unsafe remote install dir prefix') + }) +}) + +describe('GC ownership — each model collects only its own namespace', () => { + it.each(ORCAD_DIRS)('the relay never permits GC of %s', (dirName) => { + expect(remoteInstallDirOwner(dirName)).toBe('orcad') + expect(remoteInstallGcPermits(RELAY_INSTALL_MODEL, dirName)).toBe(false) + }) + + it.each(RELAY_DIRS)('orcad never permits GC of %s', (dirName) => { + expect(remoteInstallDirOwner(dirName)).toBe('relay') + expect(remoteInstallGcPermits(ORCAD_INSTALL_MODEL, dirName)).toBe(false) + }) + + it('permits each model its own dirs and its own tombstones', () => { + expect(remoteInstallGcPermits(RELAY_INSTALL_MODEL, 'relay-0.1.0+aa')).toBe(true) + expect(remoteInstallGcPermits(ORCAD_INSTALL_MODEL, 'orcad-0.1.0+aa')).toBe(true) + expect(remoteInstallGcPermits(ORCAD_INSTALL_MODEL, 'orcad-0.1.0+aa.gc-tombstone.12.34')).toBe( + true + ) + }) + + it('claims nothing it did not create', () => { + for (const name of ['.orca-remote', 'orcad', 'relayish-0.1.0', 'orcad-notaversion', 'node']) { + expect(remoteInstallDirOwner(name)).toBeNull() + expect(remoteInstallGcPermits(RELAY_INSTALL_MODEL, name)).toBe(false) + expect(remoteInstallGcPermits(ORCAD_INSTALL_MODEL, name)).toBe(false) + } + }) + + it('groups a mixed listing without losing anything to the wrong owner', () => { + const inventory = inventoryRemoteInstallDirs([...RELAY_DIRS, ...ORCAD_DIRS, 'something-else']) + expect(inventory.relay).toEqual(RELAY_DIRS) + expect(inventory.orcad).toEqual(ORCAD_DIRS) + expect(inventory.unknown).toEqual(['something-else']) + }) +}) diff --git a/src/main/ssh/remote-install-model.ts b/src/main/ssh/remote-install-model.ts new file mode 100644 index 00000000000..54be330a55f --- /dev/null +++ b/src/main/ssh/remote-install-model.ts @@ -0,0 +1,143 @@ +/** + * The two things Orca installs into `~/.orca-remote/`, and the rules that keep them from + * touching each other. + * + * `docs/design/shipping-orcad.html` §06 settles that on-disk coexistence is permanent: the + * relay is the dumb execution host for SSH-target users, orcad is the peer for paired + * environments, and no plan item retires either. So `relay-/` and `orcad-/` + * sit side by side forever, and the namespace has to be a parameter rather than a literal. + * + * GC ownership is the trap that parameterization creates. Each model garbage-collects ONLY + * its own directories — see `remoteInstallDirOwner`. Relay's regex happened to be narrow + * enough already; making the prefix a parameter is exactly what could have widened it into + * deleting a live orcad tree, so the ownership rule is asserted here rather than left to + * whichever regex a caller passes. + */ +import { + relayArtifactFilenames, + RELAY_INSTALL_COMPLETE_FILENAME, + RELAY_VERSION_FILENAME +} from '../../shared/relay-artifacts' +import { + orcadArtifactFilenames, + ORCAD_INSTALL_COMPLETE_FILENAME, + ORCAD_VERSION_FILENAME +} from '../../shared/orcad-artifacts' + +export type RemoteInstallModelId = 'relay' | 'orcad' + +export type RemoteInstallModel = { + readonly id: RemoteInstallModelId + /** Leading segment of every version dir: `-`. */ + readonly dirPrefix: string + /** npm package name written into the remote `package.json` for native deps. */ + readonly nativeDepsPackageName: string + readonly versionFilename: string + readonly installCompleteFilename: string + /** Files whose absence means a torn install, so the probe forces a re-deploy. */ + requiredArtifacts(isWindows: boolean): string[] +} + +export const RELAY_INSTALL_MODEL: RemoteInstallModel = { + id: 'relay', + dirPrefix: 'relay', + nativeDepsPackageName: 'orca-relay', + versionFilename: RELAY_VERSION_FILENAME, + installCompleteFilename: RELAY_INSTALL_COMPLETE_FILENAME, + requiredArtifacts: (isWindows) => relayArtifactFilenames(isWindows) +} + +export const ORCAD_INSTALL_MODEL: RemoteInstallModel = { + id: 'orcad', + dirPrefix: 'orcad', + nativeDepsPackageName: 'orca-orcad', + versionFilename: ORCAD_VERSION_FILENAME, + installCompleteFilename: ORCAD_INSTALL_COMPLETE_FILENAME, + // Why the parameter is ignored: orcad's forked children are the same three .js files on + // every host. The Windows-only console-list agent patch is a relay/node-pty concern. + requiredArtifacts: () => orcadArtifactFilenames() +} + +export const REMOTE_INSTALL_MODELS: readonly RemoteInstallModel[] = [ + RELAY_INSTALL_MODEL, + ORCAD_INSTALL_MODEL +] + +/** + * The version half of a directory name, shared by both models. + * + * Why `[0-9]` and not `\d`: this exact source string is also embedded in an awk ERE and a + * PowerShell `-match` on the remote host. Those three dialects agree on `[0-9]`, `\.` and + * `\+`; only JavaScript understands `\d`. + */ +const VERSION_PATTERN = String.raw`v?[0-9]+\.[0-9]+\.[0-9]+(\+[0-9a-f]+)?` + +/** Suffix GC leaves behind mid-delete; the listing must surface these so they can be swept. */ +const TOMBSTONE_PATTERN = String.raw`\.gc-tombstone\.[0-9]+\.[0-9]+` + +/** + * Why validated and not merely typed: the prefix is interpolated into a remote `find -name` + * glob, an awk regex and a single-quoted PowerShell literal. A quote or a metacharacter + * here would be a remote-shell injection on the client's own connection. + */ +function assertSafeDirPrefix(dirPrefix: string): void { + if (!/^[a-z][a-z0-9-]*$/.test(dirPrefix)) { + throw new Error(`Unsafe remote install dir prefix: ${JSON.stringify(dirPrefix)}`) + } +} + +export function remoteInstallDirName(model: RemoteInstallModel, fullVersion: string): string { + assertSafeDirPrefix(model.dirPrefix) + return `${model.dirPrefix}-${fullVersion}` +} + +/** Matches a live version dir for exactly one model — never a tombstone, never a sibling model. */ +export function remoteInstallVersionDirRegex(model: RemoteInstallModel): RegExp { + assertSafeDirPrefix(model.dirPrefix) + return new RegExp(`^${model.dirPrefix}-(${VERSION_PATTERN})$`) +} + +/** What the remote listing is allowed to return: live dirs plus their tombstones. */ +export function remoteInstallListingRegexSource(model: RemoteInstallModel): string { + assertSafeDirPrefix(model.dirPrefix) + return `^${model.dirPrefix}-(${VERSION_PATTERN})(${TOMBSTONE_PATTERN})?$` +} + +/** + * Which model owns a directory found in `~/.orca-remote/`, or null for anything neither + * model created. + * + * This is the answer to §06 falsifier 1's first half: **the model that created a directory + * owns it, and nothing else may delete it.** A relay GC pass that saw `orcad-0.1.0+abc` + * would be looking at the live install of a peer whose lifecycle it has no view into — the + * SSH-execution-boundary collapse in directory form. + */ +export function remoteInstallDirOwner(dirName: string): RemoteInstallModelId | null { + for (const model of REMOTE_INSTALL_MODELS) { + if (new RegExp(remoteInstallListingRegexSource(model)).test(dirName)) { + return model.id + } + } + return null +} + +/** True when `model` is allowed to garbage-collect `dirName`. */ +export function remoteInstallGcPermits(model: RemoteInstallModel, dirName: string): boolean { + return remoteInstallDirOwner(dirName) === model.id +} + +export type RemoteInstallInventory = Record + +/** Group a raw `~/.orca-remote/` listing by owning model, for diagnostics and the client's choice. */ +export function inventoryRemoteInstallDirs(dirNames: readonly string[]): RemoteInstallInventory { + const inventory: RemoteInstallInventory = { relay: [], orcad: [], unknown: [] } + for (const name of dirNames) { + const owner = remoteInstallDirOwner(name) + if (owner) { + inventory[owner].push(name) + } else { + inventory.unknown.push(name) + } + } + return inventory +} diff --git a/src/main/ssh/ssh-relay-build-toolchain.ts b/src/main/ssh/ssh-relay-build-toolchain.ts index 608fca5c6eb..bcb64d3bdf9 100644 --- a/src/main/ssh/ssh-relay-build-toolchain.ts +++ b/src/main/ssh/ssh-relay-build-toolchain.ts @@ -1,164 +1,24 @@ import { execCommand } from './ssh-relay-deploy-helpers' import type { SshConnection } from './ssh-connection' import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform' +import { + buildToolchainProbeCommand, + parseBuildToolchainProbe, + type BuildToolchainStatus +} from './build-toolchain-diagnosis' -// Why: node-pty@1.1.0 ships no Linux prebuild, so the remote `npm install` falls -// back to `node-gyp rebuild` and needs a C/C++ toolchain. A missing toolchain is -// the dominant first-connect failure on Linux relays (#1693); node-gyp surfaces -// it as an opaque `not found: make`. We probe for the tools so we can replace -// that with an actionable "install build-essential" message. -const PROBED_TOOLS = [ - 'make', - 'gcc', - 'g++', - 'cc', - 'c++', - 'clang', - 'clang++', - 'python3', - 'python' -] as const - -// Package managers mapped to the one-liner that installs a C/C++ toolchain on -// the matching distro family. Ordered by detection priority. -const PACKAGE_MANAGER_HINTS: readonly { bin: string; install: string }[] = [ - { bin: 'apt-get', install: 'sudo apt-get install -y build-essential python3' }, - { bin: 'dnf', install: 'sudo dnf install -y make gcc gcc-c++ python3' }, - { bin: 'yum', install: 'sudo yum install -y make gcc gcc-c++ python3' }, - { bin: 'pacman', install: 'sudo pacman -S --needed base-devel python' }, - { bin: 'apk', install: 'sudo apk add build-base python3' }, - { bin: 'zypper', install: 'sudo zypper install -y gcc gcc-c++ make python3' } -] - -export type BuildToolchainStatus = { - present: string[] - packageManager: string | null - // node-gyp needs make, Python, and a C++ compiler. The caller only uses this - // verdict after npm/node-gyp output already points at a native-build failure, - // so custom Python paths do not make unrelated npm failures look toolchainy. - toolchainMissing: boolean -} - -function hasCxxCompiler(present: ReadonlySet): boolean { - return present.has('g++') || present.has('c++') || present.has('clang++') -} - -function hasPython(present: ReadonlySet): boolean { - return present.has('python3') || present.has('python') -} - -// POSIX-sh probe: echo a `HAVE ` line per resolvable build tool and a -// single `PKG ` line for the host's package manager. Runs under -// `/bin/sh -c` (see wrapRemoteCommandForPosixShell), so it stays portable. -export function buildToolchainProbeCommand(): string { - const toolLoop = `for t in ${PROBED_TOOLS.join( - ' ' - )}; do if command -v "$t" >/dev/null 2>&1; then echo "HAVE $t"; fi; done` - const pkgList = PACKAGE_MANAGER_HINTS.map((hint) => hint.bin).join(' ') - const pkgLoop = `for p in ${pkgList}; do if command -v "$p" >/dev/null 2>&1; then echo "PKG $p"; break; fi; done` - return `${toolLoop}; ${pkgLoop}` -} - -export function parseBuildToolchainProbe(output: string): BuildToolchainStatus { - const present = new Set() - let packageManager: string | null = null - for (const line of output.split('\n')) { - const haveMatch = line.trim().match(/^HAVE (\S+)$/) - if (haveMatch) { - present.add(haveMatch[1]) - continue - } - const pkgMatch = line.trim().match(/^PKG (\S+)$/) - if (pkgMatch && !packageManager) { - packageManager = pkgMatch[1] - } - } - return { - present: PROBED_TOOLS.filter((tool) => present.has(tool)), - packageManager, - toolchainMissing: !present.has('make') || !hasCxxCompiler(present) || !hasPython(present) - } -} - -export function shouldProbeBuildToolchainAfterNativeDepsFailure(message: string): boolean { - const lower = message.toLowerCase() - if (!lower.includes('gyp') && !lower.includes('node-gyp')) { - return false - } - return ( - /\bnot found:\s*(make|gmake|gcc|g\+\+|cc|c\+\+|clang|clang\+\+|python|python3)\b/i.test( - message - ) || - /\b(make|gmake|gcc|g\+\+|cc|c\+\+|clang|clang\+\+|python|python3)\b.*\bnot found\b/i.test( - message - ) || - lower.includes('could not find any python installation') || - lower.includes('no xcode or clt version detected') - ) -} - -function missingToolNames(status: BuildToolchainStatus): string[] { - const present = new Set(status.present) - const missing: string[] = [] - if (!present.has('make')) { - missing.push('make') - } - if (!hasCxxCompiler(present)) { - missing.push('a C++ compiler (g++ or clang++)') - } - if (!hasPython(present)) { - missing.push('python3') - } - return missing -} - -/** Install hint for the host's package manager, or the cross-distro list when it is unknown. */ -export function toolchainInstallHintLines(status: BuildToolchainStatus): string[] { - const tailored = status.packageManager - ? PACKAGE_MANAGER_HINTS.find((hint) => hint.bin === status.packageManager)?.install - : null - if (tailored) { - return [` ${tailored}`] - } - return [ - ' Debian/Ubuntu: sudo apt-get install -y build-essential python3', - ' Fedora/RHEL: sudo dnf install -y make gcc gcc-c++ python3', - ' Arch: sudo pacman -S --needed base-devel python', - ' Alpine: sudo apk add build-base python3' - ] -} - -/** One-line summary for the deploy log when node-pty is skipped rather than compiled. */ -export function formatSkippedNodePtyWarning(status: BuildToolchainStatus): string { - // Why: with no package manager detected the hint list is the cross-distro menu, whose first line - // is Debian's — quoting it alone would name the wrong distro, so stay neutral instead. - const hintLines = toolchainInstallHintLines(status) - const hint = - hintLines.length === 1 - ? hintLines[0].trim() - : 'install a C/C++ toolchain (make, a C++ compiler, python3)' - return ( - `missing build tools (${missingToolNames(status).join(', ')}); skipping node-pty so the ` + - `connection still serves files and git. Remote terminals need: ${hint}` - ) -} - -export function formatMissingToolchainError( - status: BuildToolchainStatus, - underlyingError: string -): string { - const lines = [ - `The remote host is missing the C/C++ build tools (${missingToolNames(status).join(', ')}) ` + - `needed to compile Orca's relay native modules (node-pty, @parcel/watcher). node-pty has no ` + - `prebuilt binary for Linux, so they must be compiled on the remote host.`, - '', - 'Install the build tools on the remote host, then reconnect:', - ...toolchainInstallHintLines(status), - '', - `Underlying install error: ${underlyingError}` - ] - return lines.join('\n') -} +// Why re-exported rather than moved outright: `ssh-relay-deploy.ts` and the relay tests +// import the whole diagnosis surface from here, and the split exists for bundle reasons, +// not to redraw the relay's own API. +export { + buildToolchainProbeCommand, + parseBuildToolchainProbe, + shouldProbeBuildToolchainAfterNativeDepsFailure, + toolchainInstallHintLines, + formatSkippedNodePtyWarning, + formatMissingToolchainError +} from './build-toolchain-diagnosis' +export type { BuildToolchainStatus } from './build-toolchain-diagnosis' // Best-effort: returns null on Windows hosts (node-pty ships win32 prebuilds, so // a missing toolchain isn't the failure there) or if the probe itself errors — diff --git a/src/main/ssh/ssh-relay-install-namespace.ts b/src/main/ssh/ssh-relay-install-namespace.ts index a75b8048841..6fd3c2aa89f 100644 --- a/src/main/ssh/ssh-relay-install-namespace.ts +++ b/src/main/ssh/ssh-relay-install-namespace.ts @@ -7,6 +7,11 @@ // See: docs/ssh-relay-sftp-namespace.md import { RELAY_REMOTE_DIR } from './relay-protocol' +import { + RELAY_INSTALL_MODEL, + remoteInstallDirName, + type RemoteInstallModel +} from './remote-install-model' import type { SftpNamespacePathMapping } from './sftp-namespace-resolution' import { shellEscape } from './ssh-connection-utils' import { RELAY_INSTALL_LOCK_NAME } from './ssh-relay-install-lock' @@ -37,7 +42,19 @@ export function relayRemoteDirSegments( fullVersion: string, pathFlavor: RemotePathFlavor ): string[] { - const segments = [RELAY_REMOTE_DIR, `relay-${fullVersion}`] + return remoteInstallDirSegments(RELAY_INSTALL_MODEL, fullVersion, pathFlavor) +} + +/** + * The model-parameterized form. `relay-` and `orcad-` are permanent siblings under + * one `.orca-remote/` (see remote-install-model.ts), so the prefix is an argument. + */ +export function remoteInstallDirSegments( + model: RemoteInstallModel, + fullVersion: string, + pathFlavor: RemotePathFlavor +): string[] { + const segments = [RELAY_REMOTE_DIR, remoteInstallDirName(model, fullVersion)] for (const segment of segments) { assertSafeRemotePathSegment(segment, pathFlavor) // Why: the version reaches logs and diagnostics, where an embedded CR/LF can forge lines. @@ -49,7 +66,14 @@ export function relayRemoteDirSegments( } export function relayHomeRelativeDir(fullVersion: string): string { - return relayRemoteDirSegments(fullVersion, 'posix').join('/') + return remoteInstallHomeRelativeDir(RELAY_INSTALL_MODEL, fullVersion) +} + +export function remoteInstallHomeRelativeDir( + model: RemoteInstallModel, + fullVersion: string +): string { + return remoteInstallDirSegments(model, fullVersion, 'posix').join('/') } export function createRelayInstallNamespace(homeRelativeRelayDir: string): RelayInstallNamespace { diff --git a/src/main/ssh/ssh-relay-versioned-install.ts b/src/main/ssh/ssh-relay-versioned-install.ts index e4380c319c3..d2ddc62e64b 100644 --- a/src/main/ssh/ssh-relay-versioned-install.ts +++ b/src/main/ssh/ssh-relay-versioned-install.ts @@ -7,24 +7,12 @@ import { join } from 'node:path' import { existsSync, readFileSync } from 'node:fs' import type { SshConnection } from './ssh-connection' -import { RELAY_REMOTE_DIR } from './relay-protocol' import { execCommand } from './ssh-relay-deploy-helpers' -import { probeInstallLockExistsCommand } from './ssh-relay-install-lock-commands' -import { isRelayInstallLockStale, RELAY_INSTALL_LOCK_NAME } from './ssh-relay-install-lock' -import { relayRemoteDirSegments } from './ssh-relay-install-namespace' +import { RELAY_INSTALL_LOCK_NAME } from './ssh-relay-install-lock' +import { remoteInstallDirSegments } from './ssh-relay-install-namespace' +import { RELAY_INSTALL_MODEL, type RemoteInstallModel } from './remote-install-model' import { - isRelayGcClaimOwned, - releaseRelayGcClaimWithRetry, - tryAcquireRelayGcClaim -} from './ssh-relay-gc-claim' -import { cleanupRelayGcTombstones } from './ssh-relay-gc-tombstone' -import { - listRelayBaseDirsCommand, - MAX_RELAY_GC_LISTING_ENTRIES, - moveRemoteTreeCommand, - probeFileExistsCommand, - probeRelayInstalledCommand, - relayLivenessProbeCommand, + probeRemoteInstallCompleteCommand, removeRemoteTreeCommand, writeRemoteEmptyFileCommand } from './ssh-remote-commands' @@ -32,20 +20,11 @@ import { getRemoteHostPlatform, isWindowsRemoteHost, joinRemotePath, - remoteBasename, type RemoteHostPlatform, type RemotePathFlavor } from './ssh-remote-platform' -import { windowsRelayPipePathsForSocketName } from './ssh-relay-endpoints' -import { isUnconfirmedSshCommandTermination } from './ssh-relay-exec-command' import { isSshSessionLimitError } from './ssh-session-limit-error' -// Single source of truth for GC and the version-dir parser; matches both the new and legacy relay-dir layouts. -const RELAY_VERSION_DIR_REGEX = /^relay-(v?\d+\.\d+\.\d+(\+[0-9a-f]+)?)$/ - -// Legacy dirs predate `.install-complete`; they need a liveness-only GC check so they eventually drain. -const LEGACY_RELAY_DIR_REGEX = /^relay-v\d+\.\d+\.\d+$/ - const INSTALL_COMPLETE_NAME = '.install-complete' const DEFAULT_REMOTE_HOST = getRemoteHostPlatform('linux-x64') @@ -97,13 +76,23 @@ export function computeRemoteRelayDir( remoteHome: string, fullVersion: string, pathFlavor: RemotePathFlavor = 'posix' +): string { + // Why: shell and SFTP-relative builders must derive the same validated segments or the namespaces diverge. + return computeRemoteInstallDir(RELAY_INSTALL_MODEL, remoteHome, fullVersion, pathFlavor) +} + +/** The model-parameterized form of `computeRemoteRelayDir`. */ +export function computeRemoteInstallDir( + model: RemoteInstallModel, + remoteHome: string, + fullVersion: string, + pathFlavor: RemotePathFlavor = 'posix' ): string { const host = pathFlavor === 'windows' ? getRemoteHostPlatform('win32-x64') : getRemoteHostPlatform('linux-x64') - // Why: shell and SFTP-relative builders must derive the same validated segments or the namespaces diverge. - return joinRemotePath(host, remoteHome, ...relayRemoteDirSegments(fullVersion, pathFlavor)) + return joinRemotePath(host, remoteHome, ...remoteInstallDirSegments(model, fullVersion, pathFlavor)) } /** @@ -116,11 +105,26 @@ export async function isRelayAlreadyInstalled( host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, options?: RelayInstalledProbeOptions ): Promise { + return isRemoteInstallComplete(conn, RELAY_INSTALL_MODEL, remoteRelayDir, host, options) +} + +/** The model-parameterized form: each model probes for its own artifact list. */ +export async function isRemoteInstallComplete( + conn: SshConnection, + model: RemoteInstallModel, + remoteInstallDir: string, + host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, + options?: RelayInstalledProbeOptions +): Promise { + const remoteRelayDir = remoteInstallDir try { const probe = await execHostCommand( conn, host, - probeRelayInstalledCommand(host, remoteRelayDir), + probeRemoteInstallCompleteCommand(host, remoteRelayDir, [ + ...model.requiredArtifacts(isWindowsRemoteHost(host)), + model.installCompleteFilename + ]), { signal: options?.signal } ) return probe.trim() === 'OK' @@ -175,188 +179,10 @@ export async function abandonInstall( * unlocked sibling version dir (never the current one). Best-effort — errors * are swallowed so GC never blocks the user from connecting. */ -export async function gcOldRelayVersions( - conn: SshConnection, - remoteHome: string, - currentDirAbsPath: string, - host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, - options?: { - windowsNodePath?: string - windowsSockNames?: string[] - } -): Promise { - const baseDir = joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR) - const currentDirName = remoteBasename(currentDirAbsPath, host) - let listing: string - try { - listing = await execHostCommand(conn, host, listRelayBaseDirsCommand(host, baseDir)) - } catch { - return - } - const entries = listing - .split('\n') - .map((s) => s.trim()) - .filter(Boolean) - .slice(0, MAX_RELAY_GC_LISTING_ENTRIES) - - await cleanupRelayGcTombstones(conn, baseDir, entries, host) - - const candidates = entries - .filter((name) => RELAY_VERSION_DIR_REGEX.test(name)) - .filter((name) => name !== currentDirName) - - if (candidates.length === 0) { - return - } - - const removed: string[] = [] - const kept: string[] = [] - for (const name of candidates) { - const dir = joinRemotePath(host, baseDir, name) - try { - const safe = await isCandidateSafeToRemove(conn, dir, name, host, options) - if (!safe) { - kept.push(name) - continue - } - // Why: the claim is a sibling, so it survives moving/deleting the candidate and lets installers back out first. - const gcClaimToken = await tryAcquireRelayGcClaim(conn, dir, host) - if (!gcClaimToken) { - kept.push(name) - continue - } - let preserveGcClaim = false - let gcClaimReleaseNeeded = true - try { - // Recheck under the stable claim; installers probe it before and after creating their lock, closing both orders. - if (!(await isCandidateSafeToRemove(conn, dir, name, host, options))) { - kept.push(name) - continue - } - if (!(await isRelayGcClaimOwned(conn, dir, gcClaimToken, host))) { - kept.push(name) - continue - } - const tombstone = `${dir}.gc-tombstone.${process.pid}.${Date.now()}` - const moved = await execHostCommand(conn, host, moveRemoteTreeCommand(host, dir, tombstone)) - if (moved.trim() !== 'MOVED') { - kept.push(name) - continue - } - // Once renamed, a fresh install at the original path is isolated from the tombstone's deletion, so release the claim. - const release = await releaseRelayGcClaimWithRetry(conn, dir, gcClaimToken, host) - gcClaimReleaseNeeded = release === 'unknown' - await execHostCommand(conn, host, removeRemoteTreeCommand(host, tombstone)) - } catch (err) { - if (isUnconfirmedSshCommandTermination(err)) { - preserveGcClaim = true - } - throw err - } finally { - if (!preserveGcClaim && gcClaimReleaseNeeded) { - await releaseRelayGcClaimWithRetry(conn, dir, gcClaimToken, host) - } - } - removed.push(name) - } catch (err) { - console.warn( - `[ssh-relay] GC failed for ${dir}: ${err instanceof Error ? err.message : String(err)}` - ) - kept.push(name) - } - } - - if (removed.length > 0) { - const keptSuffix = kept.length > 0 ? ` (kept: ${kept.join(', ')})` : '' - console.log( - `[ssh-relay] GC: removed ${removed.length} stale version dir(s): ${removed.join(', ')}${keptSuffix}` - ) - } -} - -async function isCandidateSafeToRemove( - conn: SshConnection, - dir: string, - name: string, - host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, - options?: { - windowsNodePath?: string - windowsSockNames?: string[] - } -): Promise { - const isLegacy = LEGACY_RELAY_DIR_REGEX.test(name) - - const lockDir = joinRemotePath(host, dir, RELAY_INSTALL_LOCK_NAME) - let lockProbe: string - try { - lockProbe = await execHostCommand(conn, host, probeInstallLockExistsCommand(host, lockDir)) - } catch { - return false - } - const lockState = lockProbe.trim() - if (lockState !== 'OPEN' && lockState !== 'LOCKED') { - return false - } - const locked = lockState === 'LOCKED' - - if (locked) { - // Why: stale lock = crashed installer; finalize can leave a dir .install-complete yet locked (lock-rm failed), so it's reclaimable. - if (!(await isRelayInstallLockStale(conn, lockDir, host))) { - return false - } - process.stderr.write?.(`[ssh-relay] GC: lock at ${lockDir} is stale; treating as recoverable\n`) - } - - // Legacy dirs predate .install-complete; skip the sentinel and rely on the live-socket probe alone. - if (!isLegacy) { - const completePath = joinRemotePath(host, dir, INSTALL_COMPLETE_NAME) - const completeProbe = await execHostCommand( - conn, - host, - probeFileExistsCommand(host, completePath) - ).catch(() => 'PARTIAL') - if (completeProbe.trim() !== 'COMPLETE') { - // Crashed-install partial; leave for the next deploy to recover. - return false - } - } - - const sockAlive = await hasLiveRelaySocket(conn, dir, host, options) - if (sockAlive) { - return false - } - return true -} - -async function hasLiveRelaySocket( - conn: SshConnection, - dir: string, - host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, - options?: { - windowsNodePath?: string - windowsSockNames?: string[] - } -): Promise { - try { - // Why: `test -S` only — a connect-and-close probe would race with a daemon about to idle. - const windowsOptions = - isWindowsRemoteHost(host) && options?.windowsNodePath - ? { - nodePath: options.windowsNodePath, - pipePaths: (options.windowsSockNames ?? []).flatMap((sockName) => - windowsRelayPipePathsForSocketName(host, dir, sockName) - ) - } - : undefined - const out = await execHostCommand( - conn, - host, - relayLivenessProbeCommand(host, dir, windowsOptions) - ) - const state = out.trim() - return state !== 'DEAD' && state !== 'WAITING' - } catch { - // Why: an inconclusive liveness probe must never authorize deletion. - return true - } -} +// Why re-exported rather than moved outright: deploy and the relay tests import the whole +// versioned-install surface from here, and the split exists for file size, not to redraw an API. +export { + gcOldRelayVersions, + gcOldRemoteInstallVersions, + type RemoteInstallGcOptions +} from './remote-install-gc' diff --git a/src/main/ssh/ssh-remote-commands.ts b/src/main/ssh/ssh-remote-commands.ts index 4fa1445969d..fc3e965193e 100644 --- a/src/main/ssh/ssh-remote-commands.ts +++ b/src/main/ssh/ssh-remote-commands.ts @@ -2,6 +2,11 @@ import { RELAY_INSTALL_COMPLETE_FILENAME, relayArtifactFilenames } from '../../shared/relay-artifacts' +import { + RELAY_INSTALL_MODEL, + remoteInstallListingRegexSource, + type RemoteInstallModel +} from './remote-install-model' import type { RemoteHostPlatform } from './ssh-remote-platform' import { isWindowsRemoteHost, joinRemotePath, remoteDirname } from './ssh-remote-platform' import { powerShellCommand, powerShellLiteral, powerShellNativeArg } from './ssh-remote-powershell' @@ -99,10 +104,28 @@ export function probeRelayInstalledCommand( host: RemoteHostPlatform, remoteRelayDir: string ): string { - const required = [ + return probeRemoteInstallCompleteCommand(host, remoteRelayDir, [ ...relayArtifactFilenames(isWindowsRemoteHost(host)), RELAY_INSTALL_COMPLETE_FILENAME - ].map((filename) => joinRemotePath(host, remoteRelayDir, filename)) + ]) +} + +/** + * The model-agnostic form: any install is complete when its directory exists and every + * named artifact is a regular file inside it. + * + * Why the caller passes the list: orcad and the relay ship different artifacts, and a probe + * that checked a shared subset would call a torn install complete. + */ +export function probeRemoteInstallCompleteCommand( + host: RemoteHostPlatform, + remoteInstallDir: string, + requiredFilenames: readonly string[] +): string { + const remoteRelayDir = remoteInstallDir + const required = requiredFilenames.map((filename) => + joinRemotePath(host, remoteRelayDir, filename) + ) if (!isWindowsRemoteHost(host)) { const fileTests = required.map((path) => `&& test -f ${shellEscape(path)} `).join('') return `test -d ${shellEscape(remoteRelayDir)} ${fileTests}&& echo OK || echo MISSING` @@ -121,12 +144,28 @@ export function probeRelayInstalledCommand( export const MAX_RELAY_GC_LISTING_ENTRIES = 64 export function listRelayBaseDirsCommand(host: RemoteHostPlatform, baseDir: string): string { + return listRemoteInstallBaseDirsCommand(host, baseDir, RELAY_INSTALL_MODEL) +} + +/** + * List one model's version dirs (and its own tombstones) under `~/.orca-remote/`. + * + * The model scopes BOTH the `find`/`Get-ChildItem` glob and the validating regex. That + * double filter is the on-the-wire half of the GC ownership rule: an orcad GC pass never + * even receives a relay directory name, so it cannot delete one through a later bug. + */ +export function listRemoteInstallBaseDirsCommand( + host: RemoteHostPlatform, + baseDir: string, + model: RemoteInstallModel +): string { + const namePattern = remoteInstallListingRegexSource(model) if (!isWindowsRemoteHost(host)) { const statusPrefix = '__ORCA_RELAY_GC_FIND_STATUS__' return [ `base=${shellEscape(baseDir)}; [ -d "$base" ] || exit 0;`, - `{ find "$base" -mindepth 1 -maxdepth 1 -type d -name 'relay-*' -print; status=$?; printf '\n${statusPrefix}%s\n' "$status"; } |`, - String.raw`awk 'BEGIN { count=0; status=-1 } /^${statusPrefix}[0-9]+$/ { status=substr($0, ${statusPrefix.length + 1}); next } { name=$0; sub(/^.*\//, "", name); if (name ~ /^relay-(v?[0-9]+\.[0-9]+\.[0-9]+(\+[0-9a-f]+)?)(\.gc-tombstone\.[0-9]+\.[0-9]+)?$/ && count < ${MAX_RELAY_GC_LISTING_ENTRIES}) { entries[count++]=name } } END { if (status != 0) exit 1; for (i=0; i !artifact.optional).map( + (artifact) => artifact.filename + ) +} diff --git a/src/shared/runtime-capability-degradation.ts b/src/shared/runtime-capability-degradation.ts new file mode 100644 index 00000000000..05620ac9942 --- /dev/null +++ b/src/shared/runtime-capability-degradation.ts @@ -0,0 +1,64 @@ +export const TERMINAL_UNAVAILABLE_ERROR_CODE = 'terminal_unavailable' as const + +export const TERMINAL_PTY_DEGRADATION_CAPABILITY = 'terminal.pty.v1' as const + +export type RuntimeBrowserUnavailableReason = + | 'unconfigured' + | 'driver_missing' + | 'executable_not_found' + | 'executable_not_executable' + | 'electron_start_failed' + | 'chromium_start_failed' + | 'provider_unhealthy' + | 'desktop_window_unavailable' + | 'unknown' + +/** + * Why this host cannot spawn PTYs. Members are opaque to clients: render `message`, + * never switch exhaustively. Dynamic-loader failures are proved out of process because + * an incompatible native binary can terminate the host before JavaScript can catch it. + */ +export type RuntimeTerminalUnavailableReason = + | 'dependency_missing' + | 'libc_floor' + | 'abi_mismatch' + | 'load_failed' + | 'load_crashed' + | 'spawn_helper_missing' + | 'unknown' + +export type RuntimeDegradation = { + /** + * Open vocabulary. New codes ship without a protocol bump, so clients must render + * `message` and must not switch exhaustively on this or the `reason` field. + */ + code: 'browser_unavailable' | typeof TERMINAL_UNAVAILABLE_ERROR_CODE + capability: 'browser.headless.v1' | typeof TERMINAL_PTY_DEGRADATION_CAPABILITY + message: string + reason?: RuntimeBrowserUnavailableReason | RuntimeTerminalUnavailableReason + /** Underlying error text when the host has one. Diagnostic only; never load-bearing. */ + detail?: string +} + +const TERMINAL_UNAVAILABLE_MESSAGES: Record = { + dependency_missing: + 'Terminals are unavailable on this host: node-pty has no native binary for this platform. Install or rebuild it, or deploy a build that ships a prebuilt binary for this platform.', + libc_floor: + "This host's node-pty binary was built against a newer C library than the host provides, so the dynamic loader refuses it. Rebuild node-pty on this host, or deploy a build whose prebuilt binary matches this platform's libc.", + abi_mismatch: + "This host's node-pty binary was built for a different Node ABI than the running Node, so it cannot be loaded. Rebuild node-pty against this Node version.", + load_failed: 'Terminals are unavailable on this host: node-pty failed to load.', + load_crashed: + 'Terminals are unavailable on this host: loading node-pty terminated the probe process, which means the binary is incompatible with this host rather than merely missing.', + spawn_helper_missing: + 'node-pty loaded, but its spawn-helper executable is missing or not executable, so every terminal spawn would fail. Reinstall node-pty on this host.', + unknown: 'Terminals are unavailable on this host, and the cause could not be determined.' +} + +export function terminalUnavailableMessage( + reason: RuntimeTerminalUnavailableReason, + detail?: string +): string { + const base = TERMINAL_UNAVAILABLE_MESSAGES[reason] + return detail ? `${base} (${detail})` : base +} diff --git a/src/shared/runtime-session-contracts.ts b/src/shared/runtime-session-contracts.ts index 9e96bf445bc..9d94980fcae 100644 --- a/src/shared/runtime-session-contracts.ts +++ b/src/shared/runtime-session-contracts.ts @@ -4,6 +4,10 @@ import type { RemoteServerUpdateSupport } from './remote-server-update' import type { RemoteRuntimeSharedConnectionDiagnostics } from './remote-runtime-shared-control-types' import type { RuntimeBrowserPlacement } from './runtime-browser-placement' import type { RuntimeCapability } from './protocol-version' +import type { + RuntimeBrowserUnavailableReason, + RuntimeDegradation +} from './runtime-capability-degradation' import type { TabGroupLayoutNode } from './tab-types' import type { TerminalColorOverrides } from './terminal-color-overrides' import type { TerminalLayoutSnapshot, TerminalPaneLayoutNode } from './terminal-tab-types' @@ -26,22 +30,6 @@ export type RuntimeBrowserDriverState = RuntimeTerminalDriverState export const BROWSER_UNAVAILABLE_ERROR_CODE = 'browser_unavailable' as const -/** - * Why a host declined browser automation. Members are opaque to clients: new ones - * ship without a protocol bump, so render `message` and never switch exhaustively - * (same contract as RuntimeTerminalWaitBlockedReason). - */ -export type RuntimeBrowserUnavailableReason = - | 'unconfigured' - | 'driver_missing' - | 'executable_not_found' - | 'executable_not_executable' - | 'electron_start_failed' - | 'chromium_start_failed' - | 'provider_unhealthy' - | 'desktop_window_unavailable' - | 'unknown' - // Why: one sentence per cause, each naming the thing the operator can change. The host // renders these so an older client still shows an accurate reason it cannot decode. const BROWSER_UNAVAILABLE_MESSAGES: Record = { @@ -69,19 +57,6 @@ export function browserUnavailableMessage( return detail ? `${base} (${detail})` : base } -export type RuntimeDegradation = { - code: typeof BROWSER_UNAVAILABLE_ERROR_CODE - capability: 'browser.headless.v1' - message: string - /** - * Machine-readable cause. Optional for mixed-version peers: absence means the host - * predates structured causes, NOT that the cause is 'unconfigured'. - */ - reason?: RuntimeBrowserUnavailableReason - /** Underlying error text when the host has one. Diagnostic only; never load-bearing. */ - detail?: string -} - export type RuntimeStatus = { runtimeId: string /** Authenticated requester identity. Missing for in-process callers and older hosts. */ diff --git a/src/shared/runtime-types.ts b/src/shared/runtime-types.ts index 762f4274e29..6ca9dd9bf58 100644 --- a/src/shared/runtime-types.ts +++ b/src/shared/runtime-types.ts @@ -97,13 +97,21 @@ export { HEADLESS_RUNTIME_WINDOW_ID, UNPUBLISHED_WORKTREE_PUBLICATION_EPOCH } from './runtime-session-contracts' +export { + TERMINAL_PTY_DEGRADATION_CAPABILITY, + TERMINAL_UNAVAILABLE_ERROR_CODE, + terminalUnavailableMessage +} from './runtime-capability-degradation' +export type { + RuntimeBrowserUnavailableReason, + RuntimeDegradation, + RuntimeTerminalUnavailableReason +} from './runtime-capability-degradation' export type { CliRuntimeState, CliStatusResult, DeviceScope, RuntimeBrowserDriverState, - RuntimeBrowserUnavailableReason, - RuntimeDegradation, RuntimeDesktopWindowStatus, RuntimeGraphStatus, RuntimeMobileSessionBrowserTab,