From 551879f9cb99e8524e97181a5b397fde53dc8ec2 Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Mon, 10 Aug 2026 01:06:17 -0400 Subject: [PATCH] fix(server): harden browserless package lifecycle --- config/docker/node-server/Dockerfile | 15 +- config/reliability-gates.jsonc | 80 +++++ config/scripts/build-node-server.mjs | 2 +- .../cli-runtime-client-deferral-benchmark.mjs | 8 +- ...li-runtime-client-deferral-equivalence.mjs | 8 +- .../node-server-installed-cli-oracle.ts | 279 ++++++++++++++++ ...e-server-installed-process-harness.test.ts | 24 ++ .../node-server-installed-process-harness.ts | 228 +++++++++++++ config/scripts/node-server-npm-invocation.mjs | 23 ++ ...-server-package-workflow-contract.test.mjs | 50 +++ .../scripts/node-server-runtime-verifier.ts | 84 ++--- .../node-server-verifier-host-path.test.ts | 36 ++ .../scripts/node-server-verifier-host-path.ts | 11 + config/scripts/verify-node-server-docker.mjs | 5 +- config/scripts/verify-node-server-package.mjs | 9 +- config/tsconfig.node.json | 1 + docs/reference/npm-server-design.md | 174 ++++++++-- resources/npm-server/package.json | 3 +- src/cli/cli-program.ts | 148 +++++++++ src/cli/index.ts | 163 +-------- src/cli/runtime-client-deferral.test.ts | 4 +- src/main/agent-hooks/server.test.ts | 23 ++ src/main/agent-hooks/server.ts | 23 +- .../headless-completion-settlement.ts | 50 +++ src/main/automations/headless-dispatch.ts | 1 + .../automations/headless-work-drain.test.ts | 43 +++ src/main/automations/headless-work-drain.ts | 28 ++ src/main/automations/precheck-result.ts | 61 ++++ src/main/automations/precheck-runner.test.ts | 151 ++++++++- src/main/automations/precheck-runner.ts | 248 +++++--------- src/main/automations/precheck-ssh-runner.ts | 212 ++++++++++++ src/main/automations/service-precheck.test.ts | 60 ++++ src/main/automations/service.test.ts | 45 +++ src/main/automations/service.ts | 84 ++--- src/main/runtime/orca-runtime.ts | 5 +- src/main/runtime/pairing-endpoint.test.ts | 12 + src/main/runtime/pairing-endpoint.ts | 4 + src/main/runtime/runtime-rpc.test.ts | 20 ++ src/main/runtime/runtime-rpc.ts | 4 +- .../browserless-automation-dispatcher.test.ts | 252 ++++++++++++++ .../browserless-automation-dispatcher.ts | 197 +++++++++++ .../browserless-runtime-composition.test.ts | 289 ++++++++++++++++ .../browserless-runtime-composition.ts | 233 +++++++++++++ src/node-server/index.ts | 95 ++++-- src/node-server/package-cli.ts | 19 ++ src/node-server/server-paths.test.ts | 7 +- src/node-server/server-paths.ts | 8 +- .../server-profile-process-lock.test.ts | 309 ++++++++++++++++++ .../server-profile-process-lock.ts | 190 +++++++++++ .../server-websocket-readiness.test.ts | 9 + src/node-server/server-websocket-readiness.ts | 6 + 51 files changed, 3552 insertions(+), 491 deletions(-) create mode 100644 config/scripts/node-server-installed-cli-oracle.ts create mode 100644 config/scripts/node-server-installed-process-harness.test.ts create mode 100644 config/scripts/node-server-installed-process-harness.ts create mode 100644 config/scripts/node-server-npm-invocation.mjs create mode 100644 config/scripts/node-server-package-workflow-contract.test.mjs create mode 100644 config/scripts/node-server-verifier-host-path.test.ts create mode 100644 config/scripts/node-server-verifier-host-path.ts create mode 100644 src/cli/cli-program.ts create mode 100644 src/main/automations/headless-completion-settlement.ts create mode 100644 src/main/automations/headless-work-drain.test.ts create mode 100644 src/main/automations/headless-work-drain.ts create mode 100644 src/main/automations/precheck-result.ts create mode 100644 src/main/automations/precheck-ssh-runner.ts create mode 100644 src/node-server/browserless-automation-dispatcher.test.ts create mode 100644 src/node-server/browserless-automation-dispatcher.ts create mode 100644 src/node-server/browserless-runtime-composition.test.ts create mode 100644 src/node-server/browserless-runtime-composition.ts create mode 100644 src/node-server/package-cli.ts create mode 100644 src/node-server/server-profile-process-lock.test.ts create mode 100644 src/node-server/server-profile-process-lock.ts create mode 100644 src/node-server/server-websocket-readiness.test.ts create mode 100644 src/node-server/server-websocket-readiness.ts diff --git a/config/docker/node-server/Dockerfile b/config/docker/node-server/Dockerfile index 4dd04ff8a72..b10fb2e6b6c 100644 --- a/config/docker/node-server/Dockerfile +++ b/config/docker/node-server/Dockerfile @@ -13,9 +13,14 @@ RUN case "$TARGETARCH" in \ arm64) node_arch=arm64 ;; \ *) echo "Unsupported architecture: $TARGETARCH" >&2; exit 1 ;; \ esac \ - && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${node_arch}.tar.xz" \ - && tar -xJf "node-v${NODE_VERSION}-linux-${node_arch}.tar.xz" -C /usr/local --strip-components=1 \ - && rm "node-v${NODE_VERSION}-linux-${node_arch}.tar.xz" \ + && archive="node-v${NODE_VERSION}-linux-${node_arch}.tar.xz" \ + && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/${archive}" \ + && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/SHASUMS256.txt" \ + && checksum_line="$(grep " ${archive}$" SHASUMS256.txt)" \ + && test -n "$checksum_line" \ + && printf '%s\n' "$checksum_line" | sha256sum -c - \ + && tar -xJf "$archive" -C /usr/local --strip-components=1 \ + && rm "$archive" SHASUMS256.txt \ && node --version \ && npm --version @@ -29,4 +34,6 @@ RUN npm init -y >/dev/null \ COPY verify-linux-glibc-floor.cjs runtime-verifier.cjs /opt/orca-test/ RUN node -e "const fs=require('node:fs'); const {ptyPath}=require('node-pty/lib/prebuild-file-path'); if(!ptyPath) throw new Error('active node-pty prebuild missing'); fs.mkdirSync('./active-native'); fs.copyFileSync(ptyPath,'./active-native/pty.node')" \ && node -e "require('./verify-linux-glibc-floor.cjs').verifyLinuxGlibcFloor('./active-native')" \ - && node ./runtime-verifier.cjs --cli ./node_modules/@stablyai/orca/dist/cli.js + && node ./runtime-verifier.cjs \ + --cli ./node_modules/@stablyai/orca/dist/cli.js \ + --expected-version "$(node -p "require('./node_modules/@stablyai/orca/package.json').version")" diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index c9bf94a1731..df6af123317 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -107,6 +107,86 @@ ], "demotionRule": "Keep experimental or demote if prompt fallback enters a normal, child, or line-init read; a non-shell executable triggers prompt fallback; the identified shell PID is not foreground; delivery duplicates; ordinary startup output spawns probes; probe attempts exceed four; or the focused gate flakes without an identified harness defect." }, + { + "id": "automation.headless-precheck-shutdown", + "title": "Headless shutdown cancels local and SSH automation prechecks", + "maturity": "experimental", + "protection": "partial", + "owner": "automations", + "layer": "main-process-lifecycle", + "surfaces": [ + "scheduled headless automation prechecks", + "local precheck process trees", + "SSH precheck channel acquisition and execution" + ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "ssh", "paired-runtime-host"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "ssh", "paired-runtime-host"], + "coverageNotes": "A real local child process and injected SSH acquisition/channel barriers cover cancellation, late fulfillment and rejection, process death, channel close, listener cleanup, and service drain. Physical Windows taskkill and live SSH transport remain uncollected.", + "motivatingLinks": ["https://github.com/stablyai/orca/pull/13452"], + "invariant": "Stopping a headless automation service must abort any scheduled precheck, terminate its local process tree or close its SSH channel, consume late SSH acquisition settlement, and drain without waiting for the configured precheck timeout or leaving an unobserved error event.", + "oracle": "Hold local execution, SSH channel execution, and SSH channel acquisition at deterministic barriers; abort the shared headless stop signal; require a cancelled result, local child death, acquired and late-acquired channel close, consumed late rejection, safe late channel and stderr errors, listener cleanup on close, no dispatcher call, and a settled service drain.", + "commands": [ + "pnpm exec vitest run --config config/vitest.config.ts src/main/automations/precheck-runner.test.ts src/main/automations/service-precheck.test.ts" + ], + "testFiles": [ + "src/main/automations/precheck-runner.test.ts", + "src/main/automations/service-precheck.test.ts" + ], + "assertionRefs": [ + { + "file": "src/main/automations/precheck-runner.test.ts", + "assertions": [ + "aborts a local precheck process tree and settles promptly", + "closes an acquired SSH channel and settles when aborted", + "bounds SSH channel acquisition and closes a channel that resolves after abort", + "consumes an SSH channel acquisition rejection after abort" + ] + }, + { + "file": "src/main/automations/service-precheck.test.ts", + "assertions": ["cancels and drains a scheduled headless precheck during shutdown"] + } + ], + "evidenceRuns": [ + { + "date": "2026-08-10", + "runner": "local", + "platform": "macos", + "result": "passed", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/automations/precheck-runner.test.ts src/main/automations/service-precheck.test.ts", + "durationSeconds": 5, + "summary": "Two deterministic files passed 12 precheck and service lifecycle tests; the broader four-file automation lifecycle selector passed 27 tests." + } + ], + "runtimeBudget": { + "p95Seconds": 8, + "scope": "focused main-process precheck and service lifecycle contracts" + }, + "flakeHistory": { + "status": "not-started", + "evidence": "New deterministic barriers and a real local child process; CI and soak history have not started." + }, + "redGreenEvidence": { + "status": "complete", + "evidence": "Before signal propagation, local and SSH cancellation tests waited for process or channel timeouts and the service drain could not settle. The candidate passes all cancellation, late-settlement, process-death, channel-close, and listener-cleanup assertions." + }, + "performanceBudget": { + "required": true, + "evidence": "Cancellation performs one process-tree termination or channel close, retains no recurring poll, removes abort and data listeners on settlement, and consumes late SSH promise settlement without retry or fanout." + }, + "promotionCriteria": [ + "Collect 100 consecutive focused CI passes or 14 days of soak history.", + "Collect physical Windows taskkill and live Linux or Windows SSH cancellation evidence.", + "Keep process death, channel close, late settlement, listener cleanup, and service drain assertions green." + ], + "knownGaps": [ + "Physical Windows process-tree cancellation is statically implemented but not exercised locally.", + "SSH behavior uses injected channels rather than a live remote transport." + ], + "demotionRule": "Demote if shutdown can wait for the precheck timeout, leave a child or channel live, leak listeners, surface an unhandled late error or rejection, or dispatch after cancellation." + }, { "id": "editor.restored-sibling-owner-reparent", "title": "Restored sibling tabs migrate filesystem authority before becoming editable", diff --git a/config/scripts/build-node-server.mjs b/config/scripts/build-node-server.mjs index a91b87b18c9..685a351389e 100644 --- a/config/scripts/build-node-server.mjs +++ b/config/scripts/build-node-server.mjs @@ -31,7 +31,7 @@ mkdirSync(distRoot, { recursive: true }) const serverResult = await build({ ...common, - entryPoints: [join(repoRoot, 'src', 'node-server', 'index.ts')], + entryPoints: [join(repoRoot, 'src', 'node-server', 'package-cli.ts')], outfile: join(distRoot, 'cli.js') }) const daemonResult = await build({ diff --git a/config/scripts/cli-runtime-client-deferral-benchmark.mjs b/config/scripts/cli-runtime-client-deferral-benchmark.mjs index c322410b253..693f14b6e5e 100644 --- a/config/scripts/cli-runtime-client-deferral-benchmark.mjs +++ b/config/scripts/cli-runtime-client-deferral-benchmark.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node // Benchmark: CLI process startup with the RuntimeClient module graph deferred. // -// src/cli/index.ts used to value-import RuntimeClient at module scope, and five +// The CLI program used to value-import RuntimeClient at module scope, and five // modules that load on every invocation (args, flags, dispatch, format, // selectors) pulled RuntimeClientError from the ./runtime-client barrel. Either // edge alone drags in the whole client graph: zod (via shared/pairing -> @@ -59,9 +59,9 @@ const TOUCHED = [ // comment that merely names the function cannot satisfy the check. function assertMarkersFresh() { const checks = [ - ['src/cli/index.ts', "await import('./runtime-client.js')"], - ['src/cli/index.ts', 'await loadRuntimeClientClass()'], - ['src/cli/index.ts', "import type { RuntimeClient } from './runtime-client'"], + ['src/cli/cli-program.ts', "await import('./runtime-client.js')"], + ['src/cli/cli-program.ts', 'await loadRuntimeClientClass()'], + ['src/cli/cli-program.ts', "import type { RuntimeClient } from './runtime-client'"], ['src/cli/runtime/client.ts', "await import('./websocket-transport.js')"], ['src/cli/runtime/client.ts', 'await loadSendWebSocketRequest()'], ['src/cli/args.ts', "import { RuntimeClientError } from './runtime/types'"], diff --git a/config/scripts/cli-runtime-client-deferral-equivalence.mjs b/config/scripts/cli-runtime-client-deferral-equivalence.mjs index f443bf3b9f9..03e8a4d7691 100644 --- a/config/scripts/cli-runtime-client-deferral-equivalence.mjs +++ b/config/scripts/cli-runtime-client-deferral-equivalence.mjs @@ -35,12 +35,12 @@ const TOUCHED = [ // silently become identical to the new one and every case would pass // vacuously. Re-read the real call form out of the source and fail loudly. function assertMarkersFresh() { - const index = readFileSync(join(REPO, 'src/cli/index.ts'), 'utf8') + const program = readFileSync(join(REPO, 'src/cli/cli-program.ts'), 'utf8') const client = readFileSync(join(REPO, 'src/cli/runtime/client.ts'), 'utf8') const checks = [ - ['src/cli/index.ts', index, 'await loadRuntimeClientClass()'], - ['src/cli/index.ts', index, "await import('./runtime-client.js')"], - ['src/cli/index.ts', index, "import type { RuntimeClient } from './runtime-client'"], + ['src/cli/cli-program.ts', program, 'await loadRuntimeClientClass()'], + ['src/cli/cli-program.ts', program, "await import('./runtime-client.js')"], + ['src/cli/cli-program.ts', program, "import type { RuntimeClient } from './runtime-client'"], ['src/cli/runtime/client.ts', client, 'await loadSendWebSocketRequest()'], ['src/cli/runtime/client.ts', client, "await import('./websocket-transport.js')"] ] diff --git a/config/scripts/node-server-installed-cli-oracle.ts b/config/scripts/node-server-installed-cli-oracle.ts new file mode 100644 index 00000000000..02d14def334 --- /dev/null +++ b/config/scripts/node-server-installed-cli-oracle.ts @@ -0,0 +1,279 @@ +import { existsSync, readFileSync } from 'node:fs' +import { join } from 'node:path' +import { + assertInstalledCliSuccess, + assertSingleInstalledReadiness, + formatInstalledExit, + installedReadinessEvents, + installedServeArguments, + runInstalledCli, + runInstalledCliJson, + startInstalledServer, + stopInstalledServer, + terminateInstalledServer, + waitForInstalledServerExit, + type RunningInstalledServer +} from './node-server-installed-process-harness' +import { stopNodeServerVerifierDaemons } from './node-server-verifier-daemon-cleanup' + +type RpcJson = { + ok: true + result: TResult + _meta: { runtimeId: string } +} + +export async function verifyInstalledCliProfileBehavior(options: { + cliPath: string + dataPath: string + expectedRuntimeId: string + expectedVersion: string + pairingUrl: string + terminalHandle: string + temporaryRoot: string +}): Promise { + await verifyLiveOwnerRefusal(options) + await verifyInstalledCommandSurface(options) + await verifyControlCliRoutes(options) + await verifyIndependentProfileLifecycle(options) + if (process.platform !== 'win32') { + await verifyCrashRecovery(options) + } + process.stdout.write('Installed CLI profile oracle passed\n') +} + +async function verifyInstalledCommandSurface(options: { + cliPath: string + dataPath: string + expectedVersion: string +}): Promise { + const controlHelp = await runInstalledCli(options.cliPath, options.dataPath, ['--help']) + assertInstalledCliSuccess(controlHelp, '--help') + assert(controlHelp.stdout.includes('Usage: orca [options]'), '--help omitted CLI help') + assert(installedReadinessEvents(controlHelp.stdout) === 0, '--help emitted server readiness') + + const serverHelp = await runInstalledCli(options.cliPath, options.dataPath, ['serve', '--help']) + assertInstalledCliSuccess(serverHelp, 'serve --help') + assert( + serverHelp.stdout.includes('Start a browserless Orca host'), + 'serve --help omitted server help' + ) + assert(installedReadinessEvents(serverHelp.stdout) === 0, 'serve --help emitted readiness') + + const version = await runInstalledCli(options.cliPath, options.dataPath, ['--version']) + assertInstalledCliSuccess(version, '--version') + assert(version.stdout.trim() === options.expectedVersion, '--version did not match the package') + assert(installedReadinessEvents(version.stdout) === 0, '--version emitted server readiness') +} + +async function verifyControlCliRoutes(options: { + cliPath: string + dataPath: string + expectedRuntimeId: string + expectedVersion: string + pairingUrl: string + terminalHandle: string +}): Promise { + const status = await runInstalledCliJson< + RpcJson<{ runtime: { runtimeId: string | null; appVersion?: string } }> + >(options, ['status', '--json']) + assert(status._meta.runtimeId === options.expectedRuntimeId, 'status used the wrong profile') + assert( + status.result.runtime.runtimeId === options.expectedRuntimeId, + 'status used the wrong runtime' + ) + assert( + status.result.runtime.appVersion === options.expectedVersion, + 'status used the wrong version' + ) + + const terminals = await runInstalledCliJson>( + options, + ['terminal', 'list', '--json'] + ) + assert( + terminals._meta.runtimeId === options.expectedRuntimeId, + 'terminal list used the wrong profile' + ) + assert( + terminals.result.terminals.some((terminal) => terminal.handle === options.terminalHandle), + 'terminal list omitted the npm profile terminal' + ) + + const runs = await runInstalledCliJson>(options, [ + 'orchestration', + 'run-list', + '--json' + ]) + assert(runs._meta.runtimeId === options.expectedRuntimeId, 'orchestration used the wrong profile') + assert(Array.isArray(runs.result.runs), 'orchestration read returned invalid JSON') + + const accounts = await runInstalledCliJson< + RpcJson<{ claude: { accounts: unknown[] }; codex: { accounts: unknown[] } }> + >(options, ['account', 'list', '--json']) + assert( + accounts._meta.runtimeId === options.expectedRuntimeId, + 'account list used the wrong profile' + ) + assert( + Array.isArray(accounts.result.claude.accounts) && Array.isArray(accounts.result.codex.accounts), + 'account list returned invalid JSON' + ) + + await runInstalledCliJson(options, [ + 'environment', + 'add', + '--name', + 'installed-profile-oracle', + '--pairing-code', + options.pairingUrl, + '--json' + ]) + const environments = await runInstalledCliJson<{ + result: { environments: { name: string }[] } + _meta: { runtimeId: string } + }>(options, ['environment', 'list', '--json']) + assert( + environments._meta.runtimeId === 'local', + 'environment list did not use local profile state' + ) + assert( + environments.result.environments.some( + (environment) => environment.name === 'installed-profile-oracle' + ), + 'environment list did not use the npm profile data directory' + ) + + const skill = await runInstalledCli(options.cliPath, options.dataPath, [ + 'skills', + 'get', + 'orchestration', + '--full' + ]) + assertInstalledCliSuccess(skill, 'skills get') + assert( + skill.stdout.includes('# Orca Inter-Agent Orchestration'), + 'skills get did not return the installed full guide' + ) + assert(installedReadinessEvents(skill.stdout) === 0, 'control command emitted server readiness') +} + +async function verifyLiveOwnerRefusal(options: { + cliPath: string + dataPath: string + expectedRuntimeId: string +}): Promise { + const before = snapshotProfileState(options.dataPath) + const duplicate = await runInstalledCli( + options.cliPath, + options.dataPath, + installedServeArguments() + ) + assert(duplicate.code === 3, `duplicate server exited ${formatInstalledExit(duplicate)}`) + assert(installedReadinessEvents(duplicate.stdout) === 0, 'duplicate server emitted readiness') + assertProfileStateUnchanged(options.dataPath, before) + + const status = await runInstalledCliJson>( + options, + ['status', '--json'] + ) + assert( + status.result.runtime.runtimeId === options.expectedRuntimeId, + 'live owner stopped responding' + ) +} + +async function verifyIndependentProfileLifecycle(options: { + cliPath: string + expectedRuntimeId: string + temporaryRoot: string +}): Promise { + const dataPath = join(options.temporaryRoot, 'independent-profile') + let server: RunningInstalledServer | null = null + try { + server = await startInstalledServer(options.cliPath, dataPath, []) + const status = await runInstalledCliJson>( + { cliPath: options.cliPath, dataPath }, + ['status', '--json'] + ) + assert( + status.result.runtime.runtimeId !== options.expectedRuntimeId, + 'profiles shared a runtime' + ) + await stopInstalledServer(server) + assertSingleInstalledReadiness(server, 'zero-argument server') + assertStoppedProfileLockState(dataPath) + server = null + + server = await startInstalledServer(options.cliPath, dataPath, installedServeArguments()) + await runInstalledCliJson({ cliPath: options.cliPath, dataPath }, ['status', '--json']) + await stopInstalledServer(server) + assertSingleInstalledReadiness(server, 'serve command') + assertStoppedProfileLockState(dataPath) + server = null + } finally { + await terminateInstalledServer(server) + await stopNodeServerVerifierDaemons(dataPath) + } +} + +async function verifyCrashRecovery(options: { + cliPath: string + temporaryRoot: string +}): Promise { + const dataPath = join(options.temporaryRoot, 'crash-profile') + let server: RunningInstalledServer | null = null + try { + server = await startInstalledServer(options.cliPath, dataPath, installedServeArguments()) + server.child.kill('SIGKILL') + await waitForInstalledServerExit(server.child) + assertSingleInstalledReadiness(server, 'crashed server') + server = null + + server = await startInstalledServer(options.cliPath, dataPath, installedServeArguments()) + await runInstalledCliJson({ cliPath: options.cliPath, dataPath }, ['status', '--json']) + await stopInstalledServer(server) + assertSingleInstalledReadiness(server, 'crash replacement') + server = null + } finally { + await terminateInstalledServer(server) + await stopNodeServerVerifierDaemons(dataPath) + } +} + +function snapshotProfileState(dataPath: string): Map { + return new Map( + ['orca-runtime.json', join('.server-profile-process-lock', 'managed-hook-install.lock')].map( + (name) => { + const path = join(dataPath, name) + return [name, existsSync(path) ? readFileSync(path) : null] + } + ) + ) +} + +function assertStoppedProfileLockState(dataPath: string): void { + const lockPath = join(dataPath, '.server-profile-process-lock', 'managed-hook-install.lock') + assert( + existsSync(lockPath) === (process.platform === 'win32'), + process.platform === 'win32' + ? 'Windows forced termination unexpectedly released the profile lock' + : 'graceful server shutdown retained the profile lock' + ) +} + +function assertProfileStateUnchanged(dataPath: string, before: Map): void { + const after = snapshotProfileState(dataPath) + for (const [name, prior] of before) { + const current = after.get(name) ?? null + assert( + prior === null ? current === null : current !== null && prior.equals(current), + `duplicate server mutated ${name}` + ) + } +} + +function assert(condition: unknown, message: string): asserts condition { + if (!condition) { + throw new Error(message) + } +} diff --git a/config/scripts/node-server-installed-process-harness.test.ts b/config/scripts/node-server-installed-process-harness.test.ts new file mode 100644 index 00000000000..fce657477b7 --- /dev/null +++ b/config/scripts/node-server-installed-process-harness.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from 'vitest' +import { isExpectedInstalledServerStopResult } from './node-server-installed-process-harness' + +describe('installed server stop result', () => { + it('accepts only a clean application exit on POSIX', () => { + expect(isExpectedInstalledServerStopResult({ code: 0, signal: null }, 'linux')).toBe(true) + expect(isExpectedInstalledServerStopResult({ code: 0, signal: null }, 'darwin')).toBe(true) + expect(isExpectedInstalledServerStopResult({ code: null, signal: 'SIGTERM' }, 'linux')).toBe( + false + ) + expect(isExpectedInstalledServerStopResult({ code: 1, signal: null }, 'linux')).toBe(false) + }) + + it('accepts only the Windows TerminateProcess result', () => { + expect(isExpectedInstalledServerStopResult({ code: null, signal: 'SIGTERM' }, 'win32')).toBe( + true + ) + expect(isExpectedInstalledServerStopResult({ code: 0, signal: null }, 'win32')).toBe(false) + expect(isExpectedInstalledServerStopResult({ code: 1, signal: null }, 'win32')).toBe(false) + expect(isExpectedInstalledServerStopResult({ code: null, signal: 'SIGKILL' }, 'win32')).toBe( + false + ) + }) +}) diff --git a/config/scripts/node-server-installed-process-harness.ts b/config/scripts/node-server-installed-process-harness.ts new file mode 100644 index 00000000000..cbc5e198266 --- /dev/null +++ b/config/scripts/node-server-installed-process-harness.ts @@ -0,0 +1,228 @@ +import { spawn, type ChildProcess } from 'node:child_process' + +export type RunningInstalledServer = { + child: ChildProcess + readinessCount: number + stderr: string[] +} + +export type InstalledCliResult = { + code: number | null + signal: NodeJS.Signals | null + stdout: string + stderr: string +} + +export type InstalledServerExitResult = { + code: number | null + signal: NodeJS.Signals | null +} + +export function installedServeArguments(): string[] { + return [ + 'serve', + '--port', + '0', + '--listen', + '127.0.0.1', + '--pairing-address', + '127.0.0.1', + '--json' + ] +} + +export function startInstalledServer( + cliPath: string, + dataPath: string, + args: string[] +): Promise { + return new Promise((resolveStart, rejectStart) => { + const server: RunningInstalledServer = { + child: spawn(process.execPath, [cliPath, ...args], { + env: installedProfileEnvironment(dataPath), + stdio: ['ignore', 'pipe', 'pipe'] + }), + readinessCount: 0, + stderr: [] + } + let stdout = '' + let settled = false + const timeout = setTimeout( + () => fail(new Error('installed server readiness timed out')), + 30_000 + ) + + server.child.stderr?.setEncoding('utf8') + server.child.stderr?.on('data', (chunk: string) => server.stderr.push(chunk)) + server.child.stdout?.setEncoding('utf8') + server.child.stdout?.on('data', (chunk: string) => { + stdout += chunk + const lines = stdout.split('\n') + stdout = lines.pop() ?? '' + for (const line of lines) { + if (isReadinessLine(line)) { + server.readinessCount += 1 + if (!settled) { + settled = true + clearTimeout(timeout) + resolveStart(server) + } + } + } + }) + server.child.once('error', fail) + server.child.once('exit', (code, signal) => { + if (!settled) { + fail(new Error(`installed server exited before readiness (${code ?? signal})`)) + } + }) + + function fail(error: unknown): void { + clearTimeout(timeout) + if (server.child.exitCode === null && server.child.signalCode === null) { + server.child.kill('SIGKILL') + } + if (!settled) { + settled = true + rejectStart(error) + } + } + }) +} + +export async function stopInstalledServer(server: RunningInstalledServer): Promise { + if (server.child.exitCode !== null || server.child.signalCode !== null) { + throw new Error('installed server exited unexpectedly') + } + if (!server.child.kill('SIGTERM')) { + throw new Error('installed server rejected the shutdown signal') + } + const result = await waitForInstalledServerExit(server.child) + if (!isExpectedInstalledServerStopResult(result, process.platform)) { + throw new Error(`installed server shutdown failed (${formatInstalledExit(result)})`) + } +} + +export function isExpectedInstalledServerStopResult( + result: InstalledServerExitResult, + platform: NodeJS.Platform +): boolean { + return platform === 'win32' + ? result.code === null && result.signal === 'SIGTERM' + : result.code === 0 && result.signal === null +} + +export async function terminateInstalledServer( + server: RunningInstalledServer | null +): Promise { + if (!server || server.child.exitCode !== null || server.child.signalCode !== null) { + return + } + server.child.kill('SIGKILL') + await waitForInstalledServerExit(server.child).catch(() => undefined) +} + +export function waitForInstalledServerExit( + child: ChildProcess +): Promise { + if (child.exitCode !== null || child.signalCode !== null) { + return Promise.resolve({ code: child.exitCode, signal: child.signalCode }) + } + return new Promise((resolveExit, rejectExit) => { + const timeout = setTimeout(() => rejectExit(new Error('installed server did not exit')), 10_000) + child.once('exit', (code, signal) => { + clearTimeout(timeout) + resolveExit({ code, signal }) + }) + }) +} + +export async function runInstalledCliJson( + options: { cliPath: string; dataPath: string }, + args: string[] +): Promise { + const result = await runInstalledCli(options.cliPath, options.dataPath, args) + assertInstalledCliSuccess(result, args.join(' ')) + try { + return JSON.parse(result.stdout) as TResult + } catch { + throw new Error(`${args.join(' ')} returned invalid JSON: ${result.stdout}`) + } +} + +export function runInstalledCli( + cliPath: string, + dataPath: string, + args: string[] +): Promise { + return new Promise((resolveRun, rejectRun) => { + const child = spawn(process.execPath, [cliPath, ...args], { + env: installedProfileEnvironment(dataPath), + stdio: ['ignore', 'pipe', 'pipe'] + }) + const stdout: Buffer[] = [] + const stderr: Buffer[] = [] + const timeout = setTimeout(() => { + child.kill('SIGKILL') + rejectRun(new Error(`${args.join(' ')} timed out`)) + }, 30_000) + child.stdout?.on('data', (chunk: Buffer) => stdout.push(chunk)) + child.stderr?.on('data', (chunk: Buffer) => stderr.push(chunk)) + child.once('error', (error) => { + clearTimeout(timeout) + rejectRun(error) + }) + child.once('exit', (code, signal) => { + clearTimeout(timeout) + resolveRun({ + code, + signal, + stdout: Buffer.concat(stdout).toString('utf8'), + stderr: Buffer.concat(stderr).toString('utf8') + }) + }) + }) +} + +export function installedReadinessEvents(stdout: string): number { + return stdout.split('\n').filter(isReadinessLine).length +} + +export function assertSingleInstalledReadiness( + server: RunningInstalledServer, + label: string +): void { + if (server.readinessCount !== 1) { + throw new Error(`${label} emitted ${server.readinessCount} readiness events`) + } +} + +export function assertInstalledCliSuccess(result: InstalledCliResult, label: string): void { + if (result.code !== 0) { + throw new Error(`${label} failed (${formatInstalledExit(result)}): ${result.stderr}`) + } +} + +export function formatInstalledExit(result: InstalledServerExitResult): string { + return result.code === null ? (result.signal ?? 'unknown') : String(result.code) +} + +function installedProfileEnvironment(dataPath: string): NodeJS.ProcessEnv { + const env = { ...process.env, ORCA_SERVER_DATA_DIR: dataPath } + delete env.ORCA_USER_DATA_PATH + delete env.ORCA_PAIRING_CODE + delete env.ORCA_REMOTE_PAIRING + delete env.ORCA_ENVIRONMENT + return env +} + +function isReadinessLine(line: string): boolean { + if (line === 'Orca server ready') { + return true + } + try { + return (JSON.parse(line) as { type?: unknown }).type === 'orca_server_ready' + } catch { + return false + } +} diff --git a/config/scripts/node-server-npm-invocation.mjs b/config/scripts/node-server-npm-invocation.mjs new file mode 100644 index 00000000000..477134a49b9 --- /dev/null +++ b/config/scripts/node-server-npm-invocation.mjs @@ -0,0 +1,23 @@ +import { existsSync } from 'node:fs' +import { win32 } from 'node:path' + +export function resolveNodeServerNpmInvocation({ + platform = process.platform, + executablePath = process.execPath, + pathExists = existsSync +} = {}) { + if (platform !== 'win32') { + return { command: 'npm', prefixArgs: [] } + } + const npmCliPath = win32.join( + win32.dirname(executablePath), + 'node_modules', + 'npm', + 'bin', + 'npm-cli.js' + ) + if (!pathExists(npmCliPath)) { + throw new Error(`npm CLI is missing beside Node: ${npmCliPath}`) + } + return { command: executablePath, prefixArgs: [npmCliPath] } +} diff --git a/config/scripts/node-server-package-workflow-contract.test.mjs b/config/scripts/node-server-package-workflow-contract.test.mjs new file mode 100644 index 00000000000..b1c3fcb05d1 --- /dev/null +++ b/config/scripts/node-server-package-workflow-contract.test.mjs @@ -0,0 +1,50 @@ +import { readFileSync } from 'node:fs' +import { win32 } from 'node:path' +import { describe, expect, it, vi } from 'vitest' +import { resolveNodeServerNpmInvocation } from './node-server-npm-invocation.mjs' + +const dockerfile = readFileSync('config/docker/node-server/Dockerfile', 'utf8') + +describe('Node server package workflow contract', () => { + it('verifies the selected Node archive before extracting it', () => { + const checksumDownload = dockerfile.indexOf('SHASUMS256.txt') + const checksumVerification = dockerfile.indexOf('sha256sum -c -') + const extraction = dockerfile.indexOf('tar -xJf "$archive"') + + expect(checksumDownload).toBeGreaterThanOrEqual(0) + expect(checksumVerification).toBeGreaterThan(checksumDownload) + expect(extraction).toBeGreaterThan(checksumVerification) + expect(dockerfile).toContain('grep " ${archive}$" SHASUMS256.txt') + }) + + it('invokes the adjacent npm CLI through Node on Windows', () => { + const executablePath = win32.join('C:\\', 'node', 'node.exe') + const invocation = resolveNodeServerNpmInvocation({ + platform: 'win32', + executablePath, + pathExists: vi.fn(() => true) + }) + + expect(invocation).toEqual({ + command: executablePath, + prefixArgs: [win32.join('C:\\', 'node', 'node_modules', 'npm', 'bin', 'npm-cli.js')] + }) + }) + + it('uses npm directly outside Windows', () => { + expect(resolveNodeServerNpmInvocation({ platform: 'linux' })).toEqual({ + command: 'npm', + prefixArgs: [] + }) + }) + + it('fails before packing when the adjacent Windows npm CLI is absent', () => { + expect(() => + resolveNodeServerNpmInvocation({ + platform: 'win32', + executablePath: win32.join('C:\\', 'node', 'node.exe'), + pathExists: () => false + }) + ).toThrow('npm CLI is missing beside Node') + }) +}) diff --git a/config/scripts/node-server-runtime-verifier.ts b/config/scripts/node-server-runtime-verifier.ts index 5c2b7cfce7b..8e72e1aa24b 100644 --- a/config/scripts/node-server-runtime-verifier.ts +++ b/config/scripts/node-server-runtime-verifier.ts @@ -3,11 +3,16 @@ import { mkdtempSync, realpathSync } from 'node:fs' import { tmpdir } from 'node:os' import { join, resolve } from 'node:path' import { removeHostTree } from '../../src/main/host-tree-removal' -import { normalizeRuntimePathForComparison } from '../../src/shared/cross-platform-path' import { decodePairingOffer, type PairingOffer } from '../../src/shared/pairing' import { sendRemoteRuntimeRequest } from '../../src/shared/remote-runtime-client' import type { RuntimeStatus, RuntimeTerminalRead } from '../../src/shared/runtime-types' +import { verifyInstalledCliProfileBehavior } from './node-server-installed-cli-oracle' +import { + isExpectedInstalledServerStopResult, + waitForInstalledServerExit +} from './node-server-installed-process-harness' import { stopNodeServerVerifierDaemons } from './node-server-verifier-daemon-cleanup' +import { isSameExistingHostPath } from './node-server-verifier-host-path' type ReadyPayload = { type: 'orca_server_ready' @@ -23,6 +28,7 @@ type ReadyPayload = { type RunningServer = { child: ChildProcess; ready: ReadyPayload; stderr: string[] } const cliPath = resolve(readArgument('--cli') ?? 'resources/npm-server/dist/cli.js') +const expectedVersion = readArgument('--expected-version') ?? packageVersionFromEnvironment() const shortTemporaryRoot = process.platform === 'win32' ? tmpdir() : '/tmp' const ownedRoot = realpathSync(mkdtempSync(join(shortTemporaryRoot, 'orca-nsv-'))) const dataPath = join(ownedRoot, 'state') @@ -42,10 +48,19 @@ async function main(): Promise { activeServer = first const pairing = decodePairingOffer(first.ready.pairing.url) await verifyWebClient(first.ready.pairing.webClientUrl) - await verifyStatus(pairing, first.ready.runtimeId) + await verifyStatus(pairing, first.ready.runtimeId, expectedVersion) await verifyBrowserUnavailable(pairing) await verifyGitWorkspace(pairing) const terminal = await verifyFolderWorkspaceAndTerminal(pairing) + await verifyInstalledCliProfileBehavior({ + cliPath, + dataPath, + expectedRuntimeId: first.ready.runtimeId, + expectedVersion, + pairingUrl: first.ready.pairing.url, + terminalHandle: terminal.handle, + temporaryRoot: ownedRoot + }) await stopServer(first) activeServer = null @@ -56,7 +71,7 @@ async function main(): Promise { 'runtime process identity did not rotate' ) const reconnectedPairing = { ...pairing, endpoint: second.ready.boundEndpoint } - await verifyStatus(reconnectedPairing, second.ready.runtimeId) + await verifyStatus(reconnectedPairing, second.ready.runtimeId, expectedVersion) const restored = await call<{ terminals: { handle: string; ptyId?: string }[] }>( reconnectedPairing, 'terminal.list', @@ -70,7 +85,7 @@ async function main(): Promise { activeServer = null process.stdout.write('Node server runtime verification passed\n') } finally { - if (activeServer?.child.exitCode === null) { + if (activeServer?.child.exitCode === null && activeServer.child.signalCode === null) { await stopServer(activeServer).catch(() => activeServer?.child.kill('SIGKILL')) } await stopNodeServerVerifierDaemons(dataPath) @@ -90,6 +105,10 @@ function readArgument(flag: string): string | undefined { return index === -1 ? undefined : process.argv[index + 1] } +function packageVersionFromEnvironment(): string { + return process.env.npm_package_version ?? '0.0.0-dev' +} + function startServer(): Promise { return new Promise((resolveStart, rejectStart) => { const stderr: string[] = [] @@ -139,7 +158,7 @@ function startServer(): Promise { function fail(error: unknown): void { clearTimeout(timeout) - if (child.exitCode === null) { + if (child.exitCode === null && child.signalCode === null) { child.kill('SIGTERM') } rejectStart(error) @@ -149,42 +168,34 @@ function startServer(): Promise { async function stopServer(server: RunningServer): Promise { const { child } = server - if (child.exitCode !== null) { - throw new Error(`server exited unexpectedly (${child.exitCode})`) + if (child.exitCode !== null || child.signalCode !== null) { + throw new Error(`server exited unexpectedly (${child.exitCode ?? child.signalCode})`) } - child.kill('SIGTERM') - const result = await waitForServerExit(child) - if (result.code !== 0) { + if (!child.kill('SIGTERM')) { + throw new Error('server rejected the shutdown signal') + } + const result = await waitForInstalledServerExit(child) + if (!isExpectedInstalledServerStopResult(result, process.platform)) { throw new Error( `server shutdown failed (${result.code ?? result.signal}): ${server.stderr.join('')}` ) } } -function waitForServerExit( - child: ChildProcess -): Promise<{ code: number | null; signal: NodeJS.Signals | null }> { - return new Promise((resolveExit, rejectExit) => { - const timeout = setTimeout( - () => rejectExit(new Error('server did not exit after SIGTERM')), - 10_000 - ) - child.once('exit', (code, signal) => { - clearTimeout(timeout) - resolveExit({ code, signal }) - }) - }) -} - async function verifyWebClient(url: string): Promise { const response = await fetch(url) assert(response.ok, `web client returned ${response.status}`) assert((await response.text()).includes(''), 'web client HTML was not served') } -async function verifyStatus(pairing: PairingOffer, expectedRuntimeId: string): Promise { +async function verifyStatus( + pairing: PairingOffer, + expectedRuntimeId: string, + expectedAppVersion: string +): Promise { const status = await call(pairing, 'status.get') assert(status.runtimeId === expectedRuntimeId, 'status returned the wrong runtime') + assert(status.appVersion === expectedAppVersion, 'status returned the wrong package version') assert(status.desktopWindowStatus === 'blocked', 'desktop status is not blocked') const capabilities = status.capabilities ?? [] for (const capability of [ @@ -225,17 +236,6 @@ async function verifyGitWorkspace(pairing: PairingOffer): Promise { ) } -function isSameExistingHostPath(left: string, right: string): boolean { - try { - return ( - normalizeRuntimePathForComparison(realpathSync.native(left)) === - normalizeRuntimePathForComparison(realpathSync.native(right)) - ) - } catch { - return false - } -} - async function verifyFolderWorkspaceAndTerminal(pairing: PairingOffer): Promise<{ handle: string marker: string @@ -257,12 +257,16 @@ async function verifyFolderWorkspaceAndTerminal(pairing: PairingOffer): Promise< const created = await call<{ terminal: { handle: string; ptyId?: string } }>( pairing, 'terminal.create', - { worktree: workspaceSelector, command: `printf '${marker}\\n'` } + { + worktree: workspaceSelector, + command: `node -p "'${marker}:'+process.env.TERM_PROGRAM_VERSION"` + } ) - await waitForTerminalMarker(pairing, created.terminal.handle, marker) + const versionedMarker = `${marker}:${expectedVersion}` + await waitForTerminalMarker(pairing, created.terminal.handle, versionedMarker) return { handle: created.terminal.handle, - marker, + marker: versionedMarker, ptyId: created.terminal.ptyId, workspaceSelector } diff --git a/config/scripts/node-server-verifier-host-path.test.ts b/config/scripts/node-server-verifier-host-path.test.ts new file mode 100644 index 00000000000..561a2f33232 --- /dev/null +++ b/config/scripts/node-server-verifier-host-path.test.ts @@ -0,0 +1,36 @@ +import { linkSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { isSameExistingHostPath } from './node-server-verifier-host-path' + +describe('node server verifier host path identity', () => { + it('recognizes different paths to the same filesystem object', () => { + const root = mkdtempSync(join(tmpdir(), 'orca-verifier-path-')) + const original = join(root, 'original') + const alias = join(root, 'alias') + try { + writeFileSync(original, 'same object') + linkSync(original, alias) + + expect(isSameExistingHostPath(original, alias)).toBe(true) + } finally { + rmSync(root, { force: true, recursive: true }) + } + }) + + it('rejects different or missing filesystem objects', () => { + const root = mkdtempSync(join(tmpdir(), 'orca-verifier-path-')) + const left = join(root, 'left') + const right = join(root, 'right') + try { + writeFileSync(left, 'left') + writeFileSync(right, 'right') + + expect(isSameExistingHostPath(left, right)).toBe(false) + expect(isSameExistingHostPath(left, join(root, 'missing'))).toBe(false) + } finally { + rmSync(root, { force: true, recursive: true }) + } + }) +}) diff --git a/config/scripts/node-server-verifier-host-path.ts b/config/scripts/node-server-verifier-host-path.ts new file mode 100644 index 00000000000..3927031f04f --- /dev/null +++ b/config/scripts/node-server-verifier-host-path.ts @@ -0,0 +1,11 @@ +import { statSync } from 'node:fs' + +export function isSameExistingHostPath(left: string, right: string): boolean { + try { + const leftStat = statSync(left, { bigint: true }) + const rightStat = statSync(right, { bigint: true }) + return leftStat.dev === rightStat.dev && leftStat.ino === rightStat.ino + } catch { + return false + } +} diff --git a/config/scripts/verify-node-server-docker.mjs b/config/scripts/verify-node-server-docker.mjs index 9d1598694d0..02a6933c917 100644 --- a/config/scripts/verify-node-server-docker.mjs +++ b/config/scripts/verify-node-server-docker.mjs @@ -3,13 +3,16 @@ import { copyFileSync, mkdtempSync, rmSync, renameSync } from 'node:fs' import { tmpdir } from 'node:os' import { join, resolve } from 'node:path' import { build } from 'esbuild' +import { resolveNodeServerNpmInvocation } from './node-server-npm-invocation.mjs' const repoRoot = resolve(import.meta.dirname, '..', '..') const contextRoot = mkdtempSync(join(tmpdir(), 'orca-server-docker-')) const versions = readVersions() +const npmInvocation = resolveNodeServerNpmInvocation() try { - const packOutput = await runCapture('npm', [ + const packOutput = await runCapture(npmInvocation.command, [ + ...npmInvocation.prefixArgs, 'pack', join(repoRoot, 'resources', 'npm-server'), '--json', diff --git a/config/scripts/verify-node-server-package.mjs b/config/scripts/verify-node-server-package.mjs index a64faad8fe4..5e982d49e50 100644 --- a/config/scripts/verify-node-server-package.mjs +++ b/config/scripts/verify-node-server-package.mjs @@ -53,7 +53,7 @@ try { 'orca', '--help' ]) - if (!help.includes('Usage: orca-ide')) { + if (!help.includes('Usage: orca [options]')) { throw new Error('installed CLI help is unavailable') } const version = ( @@ -73,7 +73,9 @@ try { await run(process.execPath, [ join(repoRoot, 'config', 'scripts', 'run-node-server-runtime-verifier.mjs'), '--cli', - cliPath + cliPath, + '--expected-version', + installedManifest.version ]) process.stdout.write( `Node server package verification passed (${packed.size} bytes packed, ${packed.unpackedSize} bytes unpacked)\n` @@ -117,6 +119,9 @@ function verifyManifest(manifest) { if (manifest.license !== 'MIT') { throw new Error('installed package license is inconsistent') } + if (manifest.publishConfig?.access !== 'public' || manifest.publishConfig?.tag !== 'rc') { + throw new Error('installed package publish configuration is inconsistent') + } if (manifest.bin?.orca !== 'dist/cli.js' || manifest.bin?.['orca-ide'] !== 'dist/cli.js') { throw new Error('installed package bins are inconsistent') } diff --git a/config/tsconfig.node.json b/config/tsconfig.node.json index 229c124cc1c..cb94a4d10e9 100644 --- a/config/tsconfig.node.json +++ b/config/tsconfig.node.json @@ -3,6 +3,7 @@ "include": [ "../electron.vite.config.*", "./build-plugins/**/*", + "../src/cli/**/*", "../src/main/**/*", "../src/node-server/**/*", "../src/renderer/src/lib/skill-freshness-display-status.ts", diff --git a/docs/reference/npm-server-design.md b/docs/reference/npm-server-design.md index 0b75a9eba4b..68cbf5e97b7 100644 --- a/docs/reference/npm-server-design.md +++ b/docs/reference/npm-server-design.md @@ -6,7 +6,7 @@ Orca should run as a remote host on a machine with Node.js without installing th desktop application. The npm distribution reuses the desktop host runtime, pairing protocol, terminal daemon, and web client while omitting Electron-only capabilities such as browser panes. -The first-run contract is: +The promoted stable first-run contract is: ```bash npx @stablyai/orca@latest @@ -16,6 +16,10 @@ On a headless machine this starts a foreground server, chooses a safe reachable prints a web-client URL and pairing URL. A non-interactive `serve` command retains explicit listener, advertised-address, port, and JSON controls. +The package is not published yet, so that command does not resolve during development. Release +candidate testing uses `npx @stablyai/orca@rc`; `@latest` becomes valid only after an exact +validated candidate is promoted. + ## Problem `orca serve` currently launches the packaged Electron executable. Linux hosts therefore need the @@ -53,10 +57,18 @@ agents, and orchestration. The missing boundary is the process environment aroun ### Guided foreground start +After stable promotion: + ```bash npx @stablyai/orca@latest ``` +Before promotion, release-candidate operators use the non-default dist-tag: + +```bash +npx @stablyai/orca@rc +``` + Startup chooses an advertised address in this order: 1. an explicit operator value; @@ -88,7 +100,8 @@ Existing serve flags and readiness schema version 1 remain valid. New fields mus The public package is `@stablyai/orca`. The durable executable is `orca-ide`, avoiding conflicts with the Linux screen reader. The package also exposes `orca` so npm can select the expected bin -for `npx @stablyai/orca`. +for `npx @stablyai/orca`. The package is currently unpublished; source documentation must not +describe `@latest` as available until registry promotion is complete. ## Architecture @@ -123,6 +136,23 @@ interfaces rather than adding facade surface. CI inspects the built package, dependency manifest, and bundle metadata so an Electron runtime dependency cannot enter transitively. +The package executable is the only process entry owner. Importing the reusable server composition +must never start a listener by itself; this keeps `--help`, version, and non-server control commands +from accidentally launching a second host. + +### Browserless service composition + +The Node composition root owns the non-Electron services that desktop runtime setup normally +wires. It installs Claude and Codex account services, rate-limit collection and settings sync, and +the account subscription backing exposed by runtime RPCs. A durable agent-session claim signer is +loaded from the server profile. + +AgentHook supplies lifecycle and status snapshots, compatibility authority, PTY environment, and +live Claude rate-limit observations. Headless PTY registration resolves managed Claude and Codex +launch environments. The composition also installs orchestration transport and startup recovery, +scheduled and manual headless automation dispatch, artifact cloud sharing, AI Vault Codex discovery +and resume resolution, and commit-message agent environment resolution. + ## Capability behavior The Node host publishes the existing runtime capabilities except those requiring a browser @@ -145,6 +175,13 @@ The server stores state beneath an OS-standard Orca data directory. The director owner-only permissions where the platform supports POSIX modes. Pairing identity and device files reuse the current secure-file handling and survive package updates. +The npm host deliberately uses a different default profile from the desktop app: `Orca Server` +under the platform application-data directory on macOS and Windows, and `orca` under the XDG state +directory on Linux. An explicit `--data-dir` or `ORCA_SERVER_DATA_DIR` still selects one npm profile +for service deployments. A process-incarnation-fenced lock prevents two npm processes from writing +that profile concurrently; different npm profiles and the desktop default profile can remain live +at the same time. + Foreground startup prints a pairing credential unless `--no-pairing` is set. Automation and service workflows that retain logs must either use that flag or treat readiness output as a secret. Pairing material stays in URL fragments for the web client so reverse proxies and referrer headers @@ -165,12 +202,14 @@ considered only after the server build and native dependency matrix pass there. ## Shutdown and persistence -SIGINT and SIGTERM stop listeners, flush the store, and detach from the terminal daemon. A normal -server shutdown does not kill daemon-owned terminals. Startup reconnects to the same daemon and -device identity. +SIGINT, SIGTERM, and startup failure join one sequential teardown: stop RPC listeners; cancel local +or SSH prechecks, abort and drain headless automations, and wait for AgentHook shutdown through the +composition root; disconnect from the terminal daemon; flush the store; then release the profile +lock. A failed step is recorded while later steps still run. A normal server shutdown does not kill +daemon-owned terminals. Startup reconnects to the same daemon and device identity. Every listener, socket, timer, and signal handler installed by the Node composition root has a -matching cleanup path. Startup failure runs the same cleanup before exiting nonzero. +matching joined cleanup path before the process exits. ## Desktop coexistence and handoff @@ -183,6 +222,10 @@ into Electron. Standalone-backend architectures keep the host process independen clients attach to or supervise it. Orca can move toward that model while keeping browser panes desktop-local. +The current npm profile lock is only a same-profile single-writer guard. It does not let desktop +discover the npm host, transfer ownership, reuse its identity, or reconnect clients, and therefore +must not be presented as desktop takeover support. + A safe handoff requires: 1. one cross-process profile lease with an owner identity and generation; @@ -228,13 +271,18 @@ the terminal marker. ## Rollout -1. Publish the package under a release-candidate tag. -2. Keep desktop `orca serve` as the documented stable path while collecting package telemetry and - install failures. -3. Promote npm/npx to the primary headless documentation after the Linux matrix is green. -4. Add service installation and readiness-based immutable updates. -5. Add desktop-managed SSH bootstrap using the same package. -6. Consider an opt-in browser sidecar only after the browserless server is stable. +1. Publish an immutable candidate with an explicit non-default tag, for example + `npm publish --tag rc`; never let the first publication implicitly claim `latest`. +2. Verify the registry dist-tags, install `@rc` on the supported package matrix, and retain the + exact version that passed. +3. Keep desktop `orca serve` as the documented stable path while collecting candidate install + failures. +4. Promote that exact version with `npm dist-tag add @stablyai/orca@ latest` only after + required gates pass. Ship UI and stable docs that recommend `@latest` no earlier than this step. +5. Move `rc` to later candidates independently; rollback `latest` to a previously validated exact + version rather than republishing mutable bytes. +6. Add service installation, readiness-based immutable updates, and desktop-managed SSH bootstrap. +7. Consider an opt-in browser sidecar only after the browserless server is stable. ## Implementation checklist @@ -245,18 +293,29 @@ the terminal marker. - [x] Add the Node process-environment facade. - [x] Keep the desktop composition root behavior unchanged. - [x] Prohibit Electron runtime dependencies in the server build. +- [x] Keep executable dispatch in the package entry so importing server composition has no startup + side effect. ### Runtime - [x] Initialize persistent state in an isolated server data directory. - [x] Initialize or adopt the durable terminal daemon. - [x] Register the headless PTY controller. +- [x] Wire Claude and Codex accounts, rate limits, subscriptions, and runtime-target settings sync. +- [x] Load the durable agent-session claim signer from the server profile. +- [x] Wire AgentHook lifecycle, snapshots, compatibility authority, PTY environment, and rate-limit + observations. +- [x] Resolve account-aware Claude and Codex PTY launches. +- [x] Wire orchestration transport plus prepare, refresh, and reconcile recovery. +- [x] Dispatch scheduled and manual automations without a renderer. +- [x] Wire artifact sharing, AI Vault Codex resume, and commit-message agent environments. - [x] Start local and WebSocket RPC transports. - [x] Serve the static web client. - [x] Publish readiness schema version 1. - [x] Generate runtime-scoped E2EE pairing offers. - [x] Omit browser capabilities and fail remote browser requests explicitly. -- [x] Flush state and detach cleanly on signals and startup errors. +- [x] Join RPC stop, composition drain, daemon disconnect, store flush, and lock release in order on + signals and startup errors. ### CLI and networking @@ -275,6 +334,8 @@ the terminal marker. - [x] Install native dependencies for the Node ABI. - [x] Add tarball inventory, size, executable, license, and dependency gates. - [x] Verify the tarball contains no Electron runtime. +- [ ] Publish the first immutable candidate under the non-default `rc` dist-tag. +- [ ] Promote the exact matrix-validated candidate to `latest` before stable onboarding ships. ### Product onboarding @@ -283,10 +344,14 @@ the terminal marker. - [x] Link the shorter npm-server documentation from the existing Linux guide. - [x] Capture visual proof for the UI change. -### Desktop coexistence and handoff +### Desktop coexistence - [x] Use separate npm and desktop profiles so both installations can run safely today. +- [x] Fence one npm profile to one live writer without blocking different profiles. - [x] Document the ownership, identity, daemon, and rollback contract for a future handoff. + +### Future desktop handoff follow-up (not part of the initial release) + - [ ] Add a cross-process profile lease with generation fencing. - [ ] Let desktop discover and authenticate to the local npm host. - [ ] Checkpoint and transfer ownership without two live profile writers. @@ -304,7 +369,17 @@ the terminal marker. - [x] Test missing browser capabilities, explicit rejection, and no local fallback in remote RPCs. - [x] Run clean Docker tests on Ubuntu 20.04, 22.04, and 24.04 amd64 and 20.04 arm64. - [x] Verify the glibc 2.31 floor and Node native ABI. -- [ ] Run a Windows package smoke test; macOS package verification passed locally. +- [ ] Pass the installed-package runtime oracle on Windows Server 2022 x64; macOS arm64 passed + locally. +- [x] Prove both installed bins route zero-argument, `serve`, help/version, and non-server control + commands exactly once without eagerly initializing the wrong runtime. +- [ ] Record installed CLI startup/readiness timing and subprocess counts on the final package. + +### Accepted platform follow-ups + +- [ ] Run the installed-package runtime oracle inside WSL. +- [ ] Run the installed-package runtime oracle on Windows arm64 hardware. +- [ ] Add and validate nested SSH connection-manager lifecycle parity for the browserless host. ### Release readiness @@ -313,19 +388,66 @@ the terminal marker. - [x] Audit security, package contents, scripts, and dependency changes. - [x] Audit persisted-state and mobile/old-client compatibility. - [x] Audit macOS, Linux, Windows, WSL, SSH, and path behavior; retain runtime gaps below. +- [x] Re-run the final-head focused selector and workflow contract suite and record their actual + counts below. - [ ] Attach package/Docker evidence and UI visual proof to the PR. - [ ] Complete the PR description with validation commands and remaining platform gaps. ## Validation record and residual gaps -The publishable tarball passed its inventory, executable, license, dependency, web-client, real -E2EE, workspace, Git, PTY, shutdown, and restart-continuity oracle on macOS. Docker passed the same -installed tarball and runtime oracle on Ubuntu 20.04, 22.04, and 24.04 amd64 plus Ubuntu 20.04 -arm64. The Ubuntu 20.04 runs used stock Git 2.25.1 and verified the active native PTY against glibc -2.31 without a compiler, Electron, Chromium, Xvfb, or FUSE installed. +The installed-tarball history covers its inventory, executable, license, dependency, web-client, +real E2EE, workspace, Git, PTY, shutdown, and restart-continuity oracle on macOS; Ubuntu 20.04, +22.04, and 24.04 amd64; and Ubuntu 20.04 arm64. The final package was rerun on macOS arm64 and +Ubuntu 20.04 amd64. The Ubuntu 20.04 runs used stock Git 2.25.1 and verified the active native PTY +against glibc 2.31 without a compiler, Electron, Chromium, Xvfb, or FUSE installed. -Windows and WSL package smoke tests remain release-CI gaps. The code uses Node path/process APIs and -the PTY dependency contains Windows x64 and arm64 prebuilds, but support should not be promoted -without a real Windows install and PTY run. The PTY package also retains a deprecated transitive -`prebuild-install` fallback; clean installs select the bundled prebuild before that downloader, but -the dependency should be replaced when an equally portable maintained package is available. +The current focused selector passed 521 tests across 21 server, RPC, cross-version wire, onboarding, +browserless composition, automation, precheck, profile-lock, readiness, installed-process, +AgentHook, and shutdown files. The two workflow contract files passed 17 tests: + +```bash +pnpm exec vitest run --config config/vitest.config.ts \ + config/scripts/node-server-installed-process-harness.test.ts \ + config/scripts/node-server-verifier-host-path.test.ts \ + src/cli/runtime-client-deferral.test.ts \ + src/main/agent-hooks/server.test.ts \ + src/main/automations/headless-work-drain.test.ts \ + src/main/automations/precheck-runner.test.ts \ + src/main/automations/service-precheck.test.ts \ + src/main/automations/service.test.ts \ + src/main/runtime/pairing-endpoint.test.ts \ + src/main/runtime/runtime-rpc.test.ts \ + src/node-server/browserless-automation-dispatcher.test.ts \ + src/node-server/browserless-runtime-composition.test.ts \ + src/node-server/server-address-discovery.test.ts \ + src/node-server/server-cli-arguments.test.ts \ + src/node-server/server-paths.test.ts \ + src/node-server/server-profile-process-lock.test.ts \ + src/node-server/server-websocket-readiness.test.ts \ + src/node-server/server-window-graph.test.ts \ + src/renderer/src/components/settings/RuntimeHostAccessForm.test.tsx \ + src/renderer/src/components/sidebar/AddRemoteHostFields.test.tsx \ + tests/e2e/cross-version-wire/cross-version-terminal-wire.unit.test.ts +pnpm exec vitest run --config config/vitest.config.ts \ + config/scripts/pr-workflow-parallelism.test.mjs \ + config/scripts/node-server-package-workflow-contract.test.mjs +``` + +The final macOS arm64 installed-tarball oracle passed with 18,114,920 packed bytes and 55,066,712 +unpacked bytes. Ubuntu 20.04 amd64 passed the same installed runtime oracle in Docker, including the +glibc 2.31 and `GLIBCXX_3.4.28` floors. + +As an overlapping final packaging subset, the installed-process harness passed 2 tests and the +package-workflow contract passed 4 tests. These six tests are already included in the focused and +workflow counts above rather than added to them. + +The required Windows Server 2022 x64 lane is pending on the current worktree, so Windows support is +not yet claimed. Once that lane passes the full clean-install, E2EE, workspace, native PTY, restart, +and cleanup journey, the explicit untested platform gaps remain WSL, Windows arm64, and nested SSH +connection-manager parity in the browserless host. The PTY dependency contains Windows x64 and +arm64 prebuilds, but a present binary is not runtime evidence for an untested target. + +The package remains unpublished: the registry currently has neither `rc` nor `latest`. The PTY +package also retains a deprecated transitive `prebuild-install` fallback; clean installs select the +bundled prebuild before that downloader, but the dependency should be replaced when an equally +portable maintained package is available. diff --git a/resources/npm-server/package.json b/resources/npm-server/package.json index a7946abba5f..e8f4fa48e2b 100644 --- a/resources/npm-server/package.json +++ b/resources/npm-server/package.json @@ -17,7 +17,8 @@ "README.md" ], "publishConfig": { - "access": "public" + "access": "public", + "tag": "rc" }, "dependencies": { "@parcel/watcher": "2.5.6", diff --git a/src/cli/cli-program.ts b/src/cli/cli-program.ts new file mode 100644 index 00000000000..7bf25a72c0a --- /dev/null +++ b/src/cli/cli-program.ts @@ -0,0 +1,148 @@ +import { + findCommandSpec, + isCommandGroup, + normalizeCommandPositionals, + parseArgs, + resolveHelpPath, + specPaths, + validateCommandAndFlags +} from './args' +import { dispatch } from './dispatch' +import { reportCliError } from './format' +import { printHelp } from './help' +import type { RuntimeClient } from './runtime-client' +import { COMMAND_SPECS } from './specs' + +export { COMMAND_SPECS } from './specs' +export { buildCurrentWorktreeSelector, normalizeWorktreeSelector } from './selectors' + +const COMMAND_PATHS = COMMAND_SPECS.flatMap((spec) => specPaths(spec)) + +function shouldIgnoreRemoteSelection(commandPath: string[]): boolean { + return ( + commandPath[0] === 'account' || + commandPath[0] === 'artifacts' || + commandPath[0] === 'environment' || + commandPath[0] === 'serve' || + commandPath[0] === 'agent' || + commandPath[0] === 'vm' || + commandPath[0] === 'agent-context' + ) +} + +// Why: loading the runtime client after parsing keeps help and syntax errors lightweight. +async function loadRuntimeClientClass(): Promise { + return (await import('./runtime-client.js')).RuntimeClient +} + +// Why: SSH relay invocations carry the remote cwd because the host process cannot chdir there. +function resolveInvocationCwd(): string { + const override = process.env.ORCA_CLI_CWD + return typeof override === 'string' && override.length > 0 ? override : process.cwd() +} + +export async function main( + argv = process.argv.slice(2), + cwd = resolveInvocationCwd() +): Promise { + if (argv[0] === 'agent-teams-tmux') { + await runAgentTeamsTmuxShim(argv.slice(1)) + return + } + if (argv[0] === 'claude-teams') { + await runClaudeTeams(argv.slice(1), cwd) + return + } + const parsed = normalizeCommandPositionals(COMMAND_SPECS, parseArgs(argv, COMMAND_PATHS)) + const helpPath = resolveHelpPath(parsed) + if (helpPath !== null) { + printHelp(COMMAND_SPECS, helpPath) + if ( + helpPath.length > 0 && + !findCommandSpec(COMMAND_SPECS, helpPath) && + !isCommandGroup(helpPath) + ) { + process.exitCode = 1 + } + return + } + if (parsed.commandPath.length === 0) { + printHelp(COMMAND_SPECS, []) + return + } + const json = parsed.flags.has('json') + + try { + // Why: syntax errors should not be masked by runtime lookup failures. + validateCommandAndFlags(COMMAND_SPECS, parsed) + const RuntimeClientClass = await loadRuntimeClientClass() + const ignoreRemoteSelection = shouldIgnoreRemoteSelection(parsed.commandPath) + const pairingCode = ignoreRemoteSelection ? null : parsed.flags.get('pairing-code') + const environmentSelector = ignoreRemoteSelection ? null : parsed.flags.get('environment') + // Why: null prevents environment fallbacks from reactivating remote selection for local commands. + let client: RuntimeClient | undefined + await dispatch(parsed.commandPath, { + flags: parsed.flags, + get client() { + client ??= new RuntimeClientClass( + undefined, + undefined, + typeof pairingCode === 'string' ? pairingCode : ignoreRemoteSelection ? null : undefined, + typeof environmentSelector === 'string' + ? environmentSelector + : ignoreRemoteSelection + ? null + : undefined + ) + return client + }, + cwd, + json + }) + } catch (error) { + reportCliError(error, json, { commandPath: parsed.commandPath }) + process.exitCode = 1 + } +} + +async function runClaudeTeams(argv: string[], cwd: string): Promise { + try { + // Why: Claude owns every argument following the compatibility command. + const client = new (await loadRuntimeClientClass())(undefined, undefined, null, null) + await dispatch(['claude-teams'], { + flags: new Map(), + client, + cwd, + json: false, + rawArgs: argv + }) + } catch (error) { + reportCliError(error, false, { commandPath: ['claude-teams'] }) + process.exitCode = 1 + } +} + +async function runAgentTeamsTmuxShim(argv: string[]): Promise { + try { + const client = new (await loadRuntimeClientClass())(undefined, 10_000) + const response = await client.call<{ + tmux: { stdout: string; stderr: string; exitCode: number } + }>( + 'agentTeams.tmuxCompat', + { + teamId: process.env.ORCA_AGENT_TEAMS_TEAM_ID, + token: process.env.ORCA_AGENT_TEAMS_TOKEN, + envPane: process.env.TMUX_PANE, + cwd: process.cwd(), + argv + }, + { timeoutMs: 10_000 } + ) + process.stdout.write(response.result.tmux.stdout) + process.stderr.write(response.result.tmux.stderr) + process.exitCode = response.result.tmux.exitCode + } catch (error) { + reportCliError(error, false, { commandPath: ['agent-teams-tmux'] }) + process.exitCode = 1 + } +} diff --git a/src/cli/index.ts b/src/cli/index.ts index 4d42adf30d2..9d2fa3f6fbf 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1,166 +1,7 @@ #!/usr/bin/env node -import { - findCommandSpec, - isCommandGroup, - normalizeCommandPositionals, - parseArgs, - resolveHelpPath, - specPaths, - validateCommandAndFlags -} from './args' -import { dispatch } from './dispatch' -import { reportCliError } from './format' -import { printHelp } from './help' -import type { RuntimeClient } from './runtime-client' -import { COMMAND_SPECS } from './specs' +import { main } from './cli-program' -export { COMMAND_SPECS } from './specs' -export { buildCurrentWorktreeSelector, normalizeWorktreeSelector } from './selectors' - -const COMMAND_PATHS = COMMAND_SPECS.flatMap((spec) => specPaths(spec)) - -function shouldIgnoreRemoteSelection(commandPath: string[]): boolean { - return ( - commandPath[0] === 'account' || - commandPath[0] === 'artifacts' || - commandPath[0] === 'environment' || - commandPath[0] === 'serve' || - commandPath[0] === 'agent' || - commandPath[0] === 'vm' || - commandPath[0] === 'agent-context' - ) -} - -// Why: the RuntimeClient graph is 153 of the CLI's 199 eager modules (zod via -// shared/pairing, ws + tweetnacl via websocket-transport). Loading it here -// rather than at module scope means --help, `help `, and command/flag -// errors — which all return before this call — never pay for it. Awaited -// before dispatch so `ctx.client` stays a synchronous getter. -async function loadRuntimeClientClass(): Promise { - return (await import('./runtime-client.js')).RuntimeClient -} - -// Why: the SSH relay bridge executes this CLI on the Orca host while the -// caller's shell cwd lives on the remote machine (which cannot be chdir'd -// into). ORCA_CLI_CWD carries that remote cwd so cwd-based selectors like -// `--worktree active` resolve against the caller's directory. -function resolveInvocationCwd(): string { - const override = process.env.ORCA_CLI_CWD - return typeof override === 'string' && override.length > 0 ? override : process.cwd() -} - -export async function main( - argv = process.argv.slice(2), - cwd = resolveInvocationCwd() -): Promise { - if (argv[0] === 'agent-teams-tmux') { - await runAgentTeamsTmuxShim(argv.slice(1)) - return - } - if (argv[0] === 'claude-teams') { - await runClaudeTeams(argv.slice(1), cwd) - return - } - const parsed = normalizeCommandPositionals(COMMAND_SPECS, parseArgs(argv, COMMAND_PATHS)) - const helpPath = resolveHelpPath(parsed) - if (helpPath !== null) { - printHelp(COMMAND_SPECS, helpPath) - if ( - helpPath.length > 0 && - !findCommandSpec(COMMAND_SPECS, helpPath) && - !isCommandGroup(helpPath) - ) { - process.exitCode = 1 - } - return - } - if (parsed.commandPath.length === 0) { - printHelp(COMMAND_SPECS, []) - return - } - const json = parsed.flags.has('json') - - try { - // Why: CLI syntax and flag errors should be reported before any runtime - // lookup so users do not get misleading "Orca is not running" failures for - // simple command typos or unsupported flags. - validateCommandAndFlags(COMMAND_SPECS, parsed) - const RuntimeClientClass = await loadRuntimeClientClass() - const ignoreRemoteSelection = shouldIgnoreRemoteSelection(parsed.commandPath) - const pairingCode = ignoreRemoteSelection ? null : parsed.flags.get('pairing-code') - const environmentSelector = ignoreRemoteSelection ? null : parsed.flags.get('environment') - // Why: pass `null` (not `undefined`) when remote selection is suppressed - // so the RuntimeClient default parameter does not re-activate the - // ORCA_PAIRING_CODE / ORCA_ENVIRONMENT env-var fallback for commands - // that must run locally (environment / serve). - let client: RuntimeClient | undefined - await dispatch(parsed.commandPath, { - flags: parsed.flags, - // Why: local-only handlers must not resolve runtime metadata just to dispatch. - get client() { - client ??= new RuntimeClientClass( - undefined, - undefined, - typeof pairingCode === 'string' ? pairingCode : ignoreRemoteSelection ? null : undefined, - typeof environmentSelector === 'string' - ? environmentSelector - : ignoreRemoteSelection - ? null - : undefined - ) - return client - }, - cwd, - json - }) - } catch (error) { - reportCliError(error, json, { commandPath: parsed.commandPath }) - process.exitCode = 1 - } -} - -async function runClaudeTeams(argv: string[], cwd: string): Promise { - try { - // Why: everything after `orca claude-teams` belongs to Claude Code, not - // Orca's own flag parser, so new Claude flags work without Orca changes. - const client = new (await loadRuntimeClientClass())(undefined, undefined, null, null) - await dispatch(['claude-teams'], { - flags: new Map(), - client, - cwd, - json: false, - rawArgs: argv - }) - } catch (error) { - reportCliError(error, false, { commandPath: ['claude-teams'] }) - process.exitCode = 1 - } -} - -async function runAgentTeamsTmuxShim(argv: string[]): Promise { - try { - const client = new (await loadRuntimeClientClass())(undefined, 10_000) - const response = await client.call<{ - tmux: { stdout: string; stderr: string; exitCode: number } - }>( - 'agentTeams.tmuxCompat', - { - teamId: process.env.ORCA_AGENT_TEAMS_TEAM_ID, - token: process.env.ORCA_AGENT_TEAMS_TOKEN, - envPane: process.env.TMUX_PANE, - cwd: process.cwd(), - argv - }, - { timeoutMs: 10_000 } - ) - process.stdout.write(response.result.tmux.stdout) - process.stderr.write(response.result.tmux.stderr) - process.exitCode = response.result.tmux.exitCode - } catch (error) { - reportCliError(error, false, { commandPath: ['agent-teams-tmux'] }) - process.exitCode = 1 - } -} +export * from './cli-program' if (require.main === module) { void main() diff --git a/src/cli/runtime-client-deferral.test.ts b/src/cli/runtime-client-deferral.test.ts index 396bc6656ec..62a4956b13d 100644 --- a/src/cli/runtime-client-deferral.test.ts +++ b/src/cli/runtime-client-deferral.test.ts @@ -65,8 +65,8 @@ describe('RuntimeClient module-graph deferral', () => { } ) - it('index.ts has no eager value-import of the runtime client', () => { - const source = readFileSync(join(CLI_DIR, 'index.ts'), 'utf8') + it('cli-program.ts has no eager value-import of the runtime client', () => { + const source = readFileSync(join(CLI_DIR, 'cli-program.ts'), 'utf8') expect(source).toContain("import type { RuntimeClient } from './runtime-client'") expect(source).not.toMatch(/^import \{[^}]*RuntimeClient[^}]*\} from '\.\/runtime-client'/m) expect(source).toContain("await import('./runtime-client.js')") diff --git a/src/main/agent-hooks/server.test.ts b/src/main/agent-hooks/server.test.ts index c1870d35a7c..543ca75ac65 100644 --- a/src/main/agent-hooks/server.test.ts +++ b/src/main/agent-hooks/server.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { execFileSync } from 'node:child_process' import { createHash } from 'node:crypto' +import { createConnection } from 'node:net' import { existsSync, mkdirSync, @@ -7936,6 +7937,28 @@ describe('Last-status persistence', () => { const parsed = JSON.parse(readFileSync(lastStatusPath(), 'utf8')) expect(parsed.entries[PANE]?.payload?.prompt).toBe('flush me') }) + + it('stopAndWait() joins open hook connections', async () => { + const server = new AgentHookServer() + await server.start({ env: 'production' }) + const port = Number(server.buildPtyEnv().ORCA_AGENT_HOOK_PORT) + const socket = createConnection({ host: '127.0.0.1', port }) + await new Promise((resolve, reject) => { + socket.once('connect', resolve) + socket.once('error', reject) + }) + + let stopped = false + const stop = server.stopAndWait().then(() => { + stopped = true + }) + await Promise.resolve() + expect(stopped).toBe(false) + + socket.destroy() + await stop + expect(stopped).toBe(true) + }) }) describe('AgentHookServer ingestRemote', () => { diff --git a/src/main/agent-hooks/server.ts b/src/main/agent-hooks/server.ts index 4dc64d523de..ed99d220a69 100644 --- a/src/main/agent-hooks/server.ts +++ b/src/main/agent-hooks/server.ts @@ -2200,9 +2200,29 @@ export class AgentHookServer { } stop(): void { + this.resetForStop()?.close() + } + + async stopAndWait(): Promise { + const server = this.resetForStop() + if (!server) { + return + } + await new Promise((resolve, reject) => { + server.close((error) => { + if (error) { + reject(error) + } else { + resolve() + } + }) + }) + } + + private resetForStop(): ReturnType | null { // Why: flush the pending debounced write before clearing the map, else a hook <250ms before quit is lost on relaunch. this.flushStatusPersistSync() - this.server?.close() + const server = this.server this.server = null this.port = 0 this.token = '' @@ -2236,6 +2256,7 @@ export class AgentHookServer { this.legacyPaneKeyAliases.clear() clearAllListenerCaches(this.state) this.notifyStatusChangeListeners() + return server } /** Drop only the status row (user dismissal); do NOT wipe prompt/tool caches since the pane's agent may still be alive. Use clearPaneState for PTY-teardown. */ diff --git a/src/main/automations/headless-completion-settlement.ts b/src/main/automations/headless-completion-settlement.ts new file mode 100644 index 00000000000..51a7e6aa4b6 --- /dev/null +++ b/src/main/automations/headless-completion-settlement.ts @@ -0,0 +1,50 @@ +import type { + AutomationDispatchResult, + AutomationPrecheckResult, + AutomationRun +} from '../../shared/automations-types' +import type { HeadlessAutomationDispatchLaunch } from './headless-dispatch' + +export function buildHeadlessAutomationRunTarget(launch: HeadlessAutomationDispatchLaunch): { + workspaceId: string + workspaceDisplayName: string | null + terminalSessionId: string | null + terminalPaneKey: string | null + terminalPtyId: string | null +} { + return { + workspaceId: launch.workspaceId, + workspaceDisplayName: launch.workspaceDisplayName ?? null, + terminalSessionId: launch.terminalSessionId, + terminalPaneKey: launch.terminalPaneKey ?? null, + terminalPtyId: launch.terminalPtyId ?? null + } +} + +export function settleHeadlessAutomationCompletion(args: { + completion: NonNullable + runId: string + target: ReturnType + precheckResult: AutomationPrecheckResult | null + mark: (result: AutomationDispatchResult) => Promise +}): Promise { + return args.completion + .then((completion) => + args.mark({ + runId: args.runId, + status: completion.status, + ...args.target, + precheckResult: args.precheckResult, + outputSnapshot: completion.outputSnapshot ?? null, + error: completion.error ?? null + }) + ) + .catch((error) => + args.mark({ + runId: args.runId, + status: 'dispatch_failed', + ...args.target, + error: error instanceof Error ? error.message : String(error) + }) + ) +} diff --git a/src/main/automations/headless-dispatch.ts b/src/main/automations/headless-dispatch.ts index 359673bc573..af360d589cd 100644 --- a/src/main/automations/headless-dispatch.ts +++ b/src/main/automations/headless-dispatch.ts @@ -24,6 +24,7 @@ export type HeadlessAutomationDispatcher = (request: { automation: Automation run: AutomationRun target: Extract + signal?: AbortSignal }) => Promise export function createHeadlessAutomationOutputSnapshotBuffer(): { diff --git a/src/main/automations/headless-work-drain.test.ts b/src/main/automations/headless-work-drain.test.ts new file mode 100644 index 00000000000..410fe89e5c4 --- /dev/null +++ b/src/main/automations/headless-work-drain.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest' +import { HeadlessAutomationWorkDrain } from './headless-work-drain' + +describe('HeadlessAutomationWorkDrain', () => { + it('waits for tracked work and preserves the caller promise', async () => { + let resolveWork!: () => void + const work = new Promise((resolve) => { + resolveWork = resolve + }) + const drain = new HeadlessAutomationWorkDrain(true) + + expect(drain.track(work)).toBe(work) + let drained = false + const draining = drain.drain().then(() => { + drained = true + }) + + await Promise.resolve() + expect(drained).toBe(false) + resolveWork() + await draining + expect(drained).toBe(true) + }) + + it('observes rejected fire-and-forget work without an unhandled rejection', async () => { + const unhandled: unknown[] = [] + const onUnhandled = (reason: unknown): void => { + unhandled.push(reason) + } + process.on('unhandledRejection', onUnhandled) + + try { + const drain = new HeadlessAutomationWorkDrain(true) + drain.track(Promise.reject(new Error('dispatch failed'))) + + await drain.drain() + await new Promise((resolve) => setImmediate(resolve)) + expect(unhandled).toEqual([]) + } finally { + process.off('unhandledRejection', onUnhandled) + } + }) +}) diff --git a/src/main/automations/headless-work-drain.ts b/src/main/automations/headless-work-drain.ts new file mode 100644 index 00000000000..c550bb9dcf7 --- /dev/null +++ b/src/main/automations/headless-work-drain.ts @@ -0,0 +1,28 @@ +export class HeadlessAutomationWorkDrain { + private readonly pending = new Set>() + + constructor(private readonly enabled: boolean) {} + + track(work: Promise): Promise { + if (!this.enabled) { + return work + } + let settlement: Promise + settlement = work.then( + () => { + this.pending.delete(settlement) + }, + () => { + this.pending.delete(settlement) + } + ) + this.pending.add(settlement) + return work + } + + async drain(): Promise { + while (this.pending.size > 0) { + await Promise.allSettled(this.pending) + } + } +} diff --git a/src/main/automations/precheck-result.ts b/src/main/automations/precheck-result.ts new file mode 100644 index 00000000000..54741d4b3bf --- /dev/null +++ b/src/main/automations/precheck-result.ts @@ -0,0 +1,61 @@ +import type { AutomationPrecheck, AutomationPrecheckResult } from '../../shared/automations-types' +import { MAX_AUTOMATION_PRECHECK_OUTPUT_CHARS } from '../../shared/automation-precheck' + +export type PrecheckTailBuffer = { + content: string + truncated: boolean +} + +export const PRECHECK_CANCELLED_ERROR = 'Precheck cancelled.' + +export function appendPrecheckTail(buffer: PrecheckTailBuffer, chunk: string): PrecheckTailBuffer { + const content = `${buffer.content}${chunk}` + if (content.length <= MAX_AUTOMATION_PRECHECK_OUTPUT_CHARS) { + return { ...buffer, content } + } + return { + content: content.slice(-MAX_AUTOMATION_PRECHECK_OUTPUT_CHARS), + truncated: true + } +} + +export function createPrecheckResult(args: { + precheck: AutomationPrecheck + startedAt: number + stdout: PrecheckTailBuffer + stderr: PrecheckTailBuffer + exitCode: number | null + timedOut: boolean + error: string | null +}): AutomationPrecheckResult { + const completedAt = Date.now() + return { + command: args.precheck.command, + exitCode: args.exitCode, + timedOut: args.timedOut, + durationMs: Math.max(0, completedAt - args.startedAt), + stdout: args.stdout.content, + stderr: args.stderr.content, + stdoutTruncated: args.stdout.truncated, + stderrTruncated: args.stderr.truncated, + error: args.error, + startedAt: args.startedAt, + completedAt + } +} + +export function failedPrecheckResult( + precheck: AutomationPrecheck, + startedAt: number, + error: string +): AutomationPrecheckResult { + return createPrecheckResult({ + precheck, + startedAt, + stdout: { content: '', truncated: false }, + stderr: { content: '', truncated: false }, + exitCode: null, + timedOut: false, + error + }) +} diff --git a/src/main/automations/precheck-runner.test.ts b/src/main/automations/precheck-runner.test.ts index bdcda1ba221..d93c54c1e27 100644 --- a/src/main/automations/precheck-runner.test.ts +++ b/src/main/automations/precheck-runner.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, rmSync } from 'node:fs' +import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { EventEmitter } from 'node:events' @@ -22,6 +22,15 @@ function nodeCommand(script: string): string { return `${node} -e ${JSON.stringify(script)}` } +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0) + return true + } catch { + return false + } +} + describe('runAutomationPrecheck', () => { let cwd = '' @@ -66,6 +75,39 @@ describe('runAutomationPrecheck', () => { expect(result.error).toBe('Precheck timed out after 1s.') }) + it('aborts a local precheck process tree and settles promptly', async () => { + const pidPath = join(cwd, 'child.pid') + const controller = new AbortController() + let childPid = 0 + try { + const resultPromise = runAutomationPrecheck({ + precheck: { + command: nodeCommand( + `require('node:fs').writeFileSync(${JSON.stringify(pidPath)}, String(process.pid)); setInterval(() => {}, 1000)` + ), + timeoutSeconds: 30 + }, + target: { type: 'local', cwd }, + signal: controller.signal + }) + await vi.waitFor(() => expect(existsSync(pidPath)).toBe(true)) + childPid = Number(readFileSync(pidPath, 'utf8')) + + controller.abort() + + await expect(resultPromise).resolves.toMatchObject({ + exitCode: null, + timedOut: false, + error: 'Precheck cancelled.' + }) + await vi.waitFor(() => expect(isProcessAlive(childPid)).toBe(false)) + } finally { + if (childPid && isProcessAlive(childPid)) { + process.kill(childPid, 'SIGKILL') + } + } + }) + it('uses the SSH channel exit event as the precheck exit code', async () => { const channel = Object.assign(new EventEmitter(), { stderr: new PassThrough(), @@ -89,7 +131,7 @@ describe('runAutomationPrecheck', () => { connectionId: 'ssh-1' } }) - await Promise.resolve() + await vi.waitFor(() => expect(channel.listenerCount('close')).toBe(1)) channel.emit('data', Buffer.from('ready\n')) channel.emit('exit', 0) channel.emit('close') @@ -99,4 +141,109 @@ describe('runAutomationPrecheck', () => { expect(result.stdout).toContain('ready') expect(result.error).toBeNull() }) + + it('closes an acquired SSH channel and settles when aborted', async () => { + const channel = Object.assign(new EventEmitter(), { + stderr: new PassThrough(), + close: vi.fn() + }) + const exec = vi.fn(async () => channel) + sshManagerState.manager = { + getConnection: vi.fn(() => ({ + getState: () => ({ status: 'connected' }), + exec + })) + } + const controller = new AbortController() + + const resultPromise = runAutomationPrecheck({ + precheck: { command: 'long-command', timeoutSeconds: 30 }, + target: { type: 'ssh', cwd: '/repo/path', connectionId: 'ssh-1' }, + signal: controller.signal + }) + await vi.waitFor(() => expect(exec).toHaveBeenCalledTimes(1)) + controller.abort() + + await expect(resultPromise).resolves.toMatchObject({ + exitCode: null, + timedOut: false, + error: 'Precheck cancelled.' + }) + expect(channel.close).toHaveBeenCalledTimes(1) + expect(channel.listenerCount('data')).toBe(0) + expect(channel.listenerCount('close')).toBe(1) + expect(channel.stderr.listenerCount('data')).toBe(0) + expect(() => channel.emit('error', new Error('late channel error'))).not.toThrow() + expect(() => channel.stderr.emit('error', new Error('late stderr error'))).not.toThrow() + channel.emit('close') + expect(channel.listenerCount('error')).toBe(0) + expect(channel.stderr.listenerCount('error')).toBe(0) + }) + + it('bounds SSH channel acquisition and closes a channel that resolves after abort', async () => { + let resolveExec!: ( + channel: EventEmitter & { stderr: PassThrough; close: ReturnType } + ) => void + const execPromise = new Promise< + EventEmitter & { stderr: PassThrough; close: ReturnType } + >((resolve) => { + resolveExec = resolve + }) + const exec = vi.fn(() => execPromise) + sshManagerState.manager = { + getConnection: vi.fn(() => ({ + getState: () => ({ status: 'connected' }), + exec + })) + } + const controller = new AbortController() + const resultPromise = runAutomationPrecheck({ + precheck: { command: 'long-command', timeoutSeconds: 30 }, + target: { type: 'ssh', cwd: '/repo/path', connectionId: 'ssh-1' }, + signal: controller.signal + }) + await vi.waitFor(() => expect(exec).toHaveBeenCalledTimes(1)) + + controller.abort() + + await expect(resultPromise).resolves.toMatchObject({ error: 'Precheck cancelled.' }) + const lateChannel = Object.assign(new EventEmitter(), { + stderr: new PassThrough(), + close: vi.fn() + }) + resolveExec(lateChannel) + await vi.waitFor(() => expect(lateChannel.close).toHaveBeenCalledTimes(1)) + expect(() => lateChannel.emit('error', new Error('late channel error'))).not.toThrow() + expect(() => lateChannel.stderr.emit('error', new Error('late stderr error'))).not.toThrow() + lateChannel.emit('close') + expect(lateChannel.listenerCount('error')).toBe(0) + expect(lateChannel.stderr.listenerCount('error')).toBe(0) + }) + + it('consumes an SSH channel acquisition rejection after abort', async () => { + let rejectExec!: (error: Error) => void + const execPromise = new Promise((_resolve, reject) => { + rejectExec = reject + }) + const exec = vi.fn(() => execPromise) + sshManagerState.manager = { + getConnection: vi.fn(() => ({ + getState: () => ({ status: 'connected' }), + exec + })) + } + const controller = new AbortController() + const resultPromise = runAutomationPrecheck({ + precheck: { command: 'long-command', timeoutSeconds: 30 }, + target: { type: 'ssh', cwd: '/repo/path', connectionId: 'ssh-1' }, + signal: controller.signal + }) + await vi.waitFor(() => expect(exec).toHaveBeenCalledTimes(1)) + + controller.abort() + + await expect(resultPromise).resolves.toMatchObject({ error: 'Precheck cancelled.' }) + rejectExec(new Error('late SSH acquisition failure')) + await new Promise((resolve) => setImmediate(resolve)) + }) }) diff --git a/src/main/automations/precheck-runner.ts b/src/main/automations/precheck-runner.ts index 753bd784b06..a60fbf9239a 100644 --- a/src/main/automations/precheck-runner.ts +++ b/src/main/automations/precheck-runner.ts @@ -1,9 +1,13 @@ import { spawn, type ChildProcess } from 'node:child_process' -import type { ClientChannel } from 'ssh2' import type { AutomationPrecheck, AutomationPrecheckResult } from '../../shared/automations-types' -import { MAX_AUTOMATION_PRECHECK_OUTPUT_CHARS } from '../../shared/automation-precheck' -import { getSshConnectionManager } from '../ipc/ssh' -import { shellEscape } from '../ssh/ssh-connection-utils' +import { + appendPrecheckTail, + createPrecheckResult, + failedPrecheckResult, + PRECHECK_CANCELLED_ERROR, + type PrecheckTailBuffer +} from './precheck-result' +import { runSshPrecheck } from './precheck-ssh-runner' type AutomationPrecheckExecutionTarget = | { @@ -16,64 +20,10 @@ type AutomationPrecheckExecutionTarget = connectionId: string } -type TailBuffer = { - content: string - truncated: boolean -} - -function appendTail(buffer: TailBuffer, chunk: string): TailBuffer { - const content = `${buffer.content}${chunk}` - if (content.length <= MAX_AUTOMATION_PRECHECK_OUTPUT_CHARS) { - return { ...buffer, content } - } - return { - content: content.slice(-MAX_AUTOMATION_PRECHECK_OUTPUT_CHARS), - truncated: true - } -} - -function createPrecheckResult(args: { - precheck: AutomationPrecheck - startedAt: number - stdout: TailBuffer - stderr: TailBuffer - exitCode: number | null - timedOut: boolean - error: string | null -}): AutomationPrecheckResult { - const completedAt = Date.now() - return { - command: args.precheck.command, - exitCode: args.exitCode, - timedOut: args.timedOut, - durationMs: Math.max(0, completedAt - args.startedAt), - stdout: args.stdout.content, - stderr: args.stderr.content, - stdoutTruncated: args.stdout.truncated, - stderrTruncated: args.stderr.truncated, - error: args.error, - startedAt: args.startedAt, - completedAt - } -} - -function failedPrecheckResult( - precheck: AutomationPrecheck, - startedAt: number, - error: string -): AutomationPrecheckResult { - return createPrecheckResult({ - precheck, - startedAt, - stdout: { content: '', truncated: false }, - stderr: { content: '', truncated: false }, - exitCode: null, - timedOut: false, - error - }) -} - -function killLocalPrecheckProcessTree(child: ChildProcess): ReturnType | null { +function killLocalPrecheckProcessTree( + child: ChildProcess, + forceImmediately = false +): ReturnType | null { const pid = child.pid if (!pid) { child.kill() @@ -97,9 +47,12 @@ function killLocalPrecheckProcessTree(child: ChildProcess): ReturnType { @@ -113,16 +66,30 @@ function killLocalPrecheckProcessTree(child: ChildProcess): ReturnType {} + const stopObserving = (): void => { + child.off('error', consumeError) + } + child.on('error', consumeError) + child.once('close', stopObserving) +} + function runLocalPrecheck( precheck: AutomationPrecheck, - target: Extract + target: Extract, + signal?: AbortSignal ): Promise { const startedAt = Date.now() + if (signal?.aborted) { + return Promise.resolve(failedPrecheckResult(precheck, startedAt, PRECHECK_CANCELLED_ERROR)) + } const timeoutMs = precheck.timeoutSeconds * 1000 return new Promise((resolve) => { - let stdout: TailBuffer = { content: '', truncated: false } - let stderr: TailBuffer = { content: '', truncated: false } + let stdout: PrecheckTailBuffer = { content: '', truncated: false } + let stderr: PrecheckTailBuffer = { content: '', truncated: false } let timedOut = false + let cancelled = false let settled = false let timeout: ReturnType | null = null let forceKillTimer: ReturnType | null = null @@ -135,6 +102,38 @@ function runLocalPrecheck( windowsHide: true }) + const collectStdout = (chunk: string): void => { + stdout = appendPrecheckTail(stdout, chunk) + } + const collectStderr = (chunk: string): void => { + stderr = appendPrecheckTail(stderr, chunk) + } + const onError = (error: Error): void => { + settle(null, error.message) + } + const onClose = (code: number | null): void => { + settle( + timedOut || cancelled || typeof code !== 'number' ? null : code, + cancelled + ? PRECHECK_CANCELLED_ERROR + : timedOut + ? `Precheck timed out after ${precheck.timeoutSeconds}s.` + : null + ) + } + const onAbort = (): void => { + cancelled = true + observeLocalPrecheckTeardown(child) + forceKillTimer = killLocalPrecheckProcessTree(child, true) + settle(null, PRECHECK_CANCELLED_ERROR) + } + const cleanup = (): void => { + signal?.removeEventListener('abort', onAbort) + child.stdout?.off('data', collectStdout) + child.stderr?.off('data', collectStderr) + child.off('error', onError) + child.off('close', onClose) + } const settle = (exitCode: number | null, error: string | null): void => { if (settled) { return @@ -148,122 +147,37 @@ function runLocalPrecheck( clearTimeout(forceKillTimer) forceKillTimer = null } + cleanup() resolve( createPrecheckResult({ precheck, startedAt, stdout, stderr, exitCode, timedOut, error }) ) } + child.stdout?.setEncoding('utf8') + child.stderr?.setEncoding('utf8') + child.stdout?.on('data', collectStdout) + child.stderr?.on('data', collectStderr) + child.on('error', onError) + child.on('close', onClose) + signal?.addEventListener('abort', onAbort, { once: true }) + if (signal?.aborted) { + onAbort() + return + } timeout = setTimeout(() => { timedOut = true forceKillTimer = killLocalPrecheckProcessTree(child) }, timeoutMs) - - child.stdout?.setEncoding('utf8') - child.stderr?.setEncoding('utf8') - child.stdout?.on('data', (chunk: string) => { - stdout = appendTail(stdout, chunk) - }) - child.stderr?.on('data', (chunk: string) => { - stderr = appendTail(stderr, chunk) - }) - child.on('error', (error) => { - settle(null, error.message) - }) - child.on('close', (code) => { - settle( - timedOut || typeof code !== 'number' ? null : code, - timedOut ? `Precheck timed out after ${precheck.timeoutSeconds}s.` : null - ) - }) }) } -function runSshChannelPrecheck(args: { - precheck: AutomationPrecheck - channel: ClientChannel - startedAt: number -}): Promise { - const { precheck, channel, startedAt } = args - const timeoutMs = precheck.timeoutSeconds * 1000 - return new Promise((resolve) => { - let stdout: TailBuffer = { content: '', truncated: false } - let stderr: TailBuffer = { content: '', truncated: false } - let exitCode: number | null = null - let timedOut = false - let settled = false - let timeout: ReturnType | null = null - - const settle = (exitCode: number | null, error: string | null): void => { - if (settled) { - return - } - settled = true - if (timeout) { - clearTimeout(timeout) - timeout = null - } - resolve( - createPrecheckResult({ precheck, startedAt, stdout, stderr, exitCode, timedOut, error }) - ) - } - - timeout = setTimeout(() => { - timedOut = true - channel.close() - }, timeoutMs) - - const fail = (error: Error): void => { - settle(null, error.message) - } - channel.on('error', fail) - channel.stderr.on('error', fail) - channel.on('data', (data: Buffer | string) => { - stdout = appendTail(stdout, data.toString()) - }) - channel.stderr.on('data', (data: Buffer | string) => { - stderr = appendTail(stderr, data.toString()) - }) - channel.on('exit', (code: number | null) => { - exitCode = typeof code === 'number' ? code : null - }) - channel.on('close', (code?: number | null) => { - if (typeof code === 'number') { - exitCode = code - } - settle(exitCode, timedOut ? `Precheck timed out after ${precheck.timeoutSeconds}s.` : null) - }) - }) -} - -async function runSshPrecheck( - precheck: AutomationPrecheck, - target: Extract -): Promise { - const startedAt = Date.now() - const manager = getSshConnectionManager() - const connection = manager?.getConnection(target.connectionId) - if (!connection || connection.getState().status !== 'connected') { - return failedPrecheckResult(precheck, startedAt, 'SSH target is not connected.') - } - try { - const remoteCommand = `cd ${shellEscape(target.cwd)} && ${precheck.command}` - const channel = await connection.exec(remoteCommand) - return await runSshChannelPrecheck({ precheck, channel, startedAt }) - } catch (error) { - return failedPrecheckResult( - precheck, - startedAt, - error instanceof Error ? error.message : String(error) - ) - } -} - export async function runAutomationPrecheck(args: { precheck: AutomationPrecheck target: AutomationPrecheckExecutionTarget + signal?: AbortSignal }): Promise { if (args.target.type === 'ssh') { - return await runSshPrecheck(args.precheck, args.target) + return await runSshPrecheck(args.precheck, args.target, args.signal) } - return await runLocalPrecheck(args.precheck, args.target) + return await runLocalPrecheck(args.precheck, args.target, args.signal) } diff --git a/src/main/automations/precheck-ssh-runner.ts b/src/main/automations/precheck-ssh-runner.ts new file mode 100644 index 00000000000..b0a35e320f8 --- /dev/null +++ b/src/main/automations/precheck-ssh-runner.ts @@ -0,0 +1,212 @@ +import type { ClientChannel } from 'ssh2' +import type { AutomationPrecheck, AutomationPrecheckResult } from '../../shared/automations-types' +import { getSshConnectionManager } from '../ipc/ssh' +import { shellEscape } from '../ssh/ssh-connection-utils' +import { + appendPrecheckTail, + createPrecheckResult, + failedPrecheckResult, + PRECHECK_CANCELLED_ERROR, + type PrecheckTailBuffer +} from './precheck-result' + +type SshPrecheckTarget = { + cwd: string + connectionId: string +} + +function closeSshPrecheckChannel(channel: ClientChannel): void { + const consumeError = (): void => {} + const stopObserving = (): void => { + channel.off('error', consumeError) + channel.stderr.off('error', consumeError) + } + channel.on('error', consumeError) + channel.stderr.on('error', consumeError) + channel.once('close', stopObserving) + try { + channel.close() + } catch { + stopObserving() + } +} + +function runSshChannelPrecheck(args: { + precheck: AutomationPrecheck + channel: ClientChannel + startedAt: number + signal?: AbortSignal +}): Promise { + const { precheck, channel, startedAt, signal } = args + if (signal?.aborted) { + closeSshPrecheckChannel(channel) + return Promise.resolve(failedPrecheckResult(precheck, startedAt, PRECHECK_CANCELLED_ERROR)) + } + const timeoutMs = precheck.timeoutSeconds * 1000 + return new Promise((resolve) => { + let stdout: PrecheckTailBuffer = { content: '', truncated: false } + let stderr: PrecheckTailBuffer = { content: '', truncated: false } + let exitCode: number | null = null + let timedOut = false + let cancelled = false + let settled = false + let timeout: ReturnType | null = null + + const collectStdout = (data: Buffer | string): void => { + stdout = appendPrecheckTail(stdout, data.toString()) + } + const collectStderr = (data: Buffer | string): void => { + stderr = appendPrecheckTail(stderr, data.toString()) + } + const fail = (error: Error): void => { + settle(null, error.message) + } + const onExit = (code: number | null): void => { + exitCode = typeof code === 'number' ? code : null + } + const onClose = (code?: number | null): void => { + if (typeof code === 'number') { + exitCode = code + } + settle( + cancelled ? null : exitCode, + cancelled + ? PRECHECK_CANCELLED_ERROR + : timedOut + ? `Precheck timed out after ${precheck.timeoutSeconds}s.` + : null + ) + } + const onAbort = (): void => { + cancelled = true + closeSshPrecheckChannel(channel) + settle(null, PRECHECK_CANCELLED_ERROR) + } + const cleanup = (): void => { + signal?.removeEventListener('abort', onAbort) + channel.off('error', fail) + channel.stderr.off('error', fail) + channel.off('data', collectStdout) + channel.stderr.off('data', collectStderr) + channel.off('exit', onExit) + channel.off('close', onClose) + } + const settle = (resultExitCode: number | null, error: string | null): void => { + if (settled) { + return + } + settled = true + if (timeout) { + clearTimeout(timeout) + timeout = null + } + cleanup() + resolve( + createPrecheckResult({ + precheck, + startedAt, + stdout, + stderr, + exitCode: resultExitCode, + timedOut, + error + }) + ) + } + + timeout = setTimeout(() => { + timedOut = true + closeSshPrecheckChannel(channel) + }, timeoutMs) + channel.on('error', fail) + channel.stderr.on('error', fail) + channel.on('data', collectStdout) + channel.stderr.on('data', collectStderr) + channel.on('exit', onExit) + channel.on('close', onClose) + signal?.addEventListener('abort', onAbort, { once: true }) + if (signal?.aborted) { + onAbort() + } + }) +} + +type SshChannelAcquisition = { channel: ClientChannel } | { aborted: true } + +function acquireSshPrecheckChannel( + channelPromise: Promise, + signal: AbortSignal +): Promise { + return new Promise((resolve, reject) => { + let settled = false + const onAbort = (): void => { + if (settled) { + return + } + settled = true + signal.removeEventListener('abort', onAbort) + resolve({ aborted: true }) + } + signal.addEventListener('abort', onAbort, { once: true }) + channelPromise.then( + (channel) => { + if (settled) { + closeSshPrecheckChannel(channel) + return + } + settled = true + signal.removeEventListener('abort', onAbort) + resolve({ channel }) + }, + (error: unknown) => { + if (settled) { + return + } + settled = true + signal.removeEventListener('abort', onAbort) + reject(error) + } + ) + if (signal.aborted) { + onAbort() + } + }) +} + +export async function runSshPrecheck( + precheck: AutomationPrecheck, + target: SshPrecheckTarget, + signal?: AbortSignal +): Promise { + const startedAt = Date.now() + if (signal?.aborted) { + return failedPrecheckResult(precheck, startedAt, PRECHECK_CANCELLED_ERROR) + } + const manager = getSshConnectionManager() + const connection = manager?.getConnection(target.connectionId) + if (!connection || connection.getState().status !== 'connected') { + return failedPrecheckResult(precheck, startedAt, 'SSH target is not connected.') + } + try { + const remoteCommand = `cd ${shellEscape(target.cwd)} && ${precheck.command}` + const channelPromise = connection.exec(remoteCommand) + const acquisition = signal + ? await acquireSshPrecheckChannel(channelPromise, signal) + : { channel: await channelPromise } + if ('aborted' in acquisition) { + return failedPrecheckResult(precheck, startedAt, PRECHECK_CANCELLED_ERROR) + } + return await runSshChannelPrecheck({ + precheck, + channel: acquisition.channel, + startedAt, + ...(signal ? { signal } : {}) + }) + } catch (error) { + return failedPrecheckResult( + precheck, + startedAt, + error instanceof Error ? error.message : String(error) + ) + } +} diff --git a/src/main/automations/service-precheck.test.ts b/src/main/automations/service-precheck.test.ts index 49122511184..ad69871bc9a 100644 --- a/src/main/automations/service-precheck.test.ts +++ b/src/main/automations/service-precheck.test.ts @@ -226,4 +226,64 @@ describe('AutomationService prechecks', () => { error: 'Precheck exited with code 1.' }) }) + + it('cancels and drains a scheduled headless precheck during shutdown', async () => { + const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone + vi.setSystemTime(new Date(2026, 4, 13, 8, 59)) + const store = await createStore() + store.addRepo(makeRepo()) + const automation = store.createAutomation({ + name: 'Conditional headless check', + prompt: 'Check the repo', + precheck: { command: 'long-command', timeoutSeconds: 30 }, + agentId: 'claude', + projectId: 'r1', + workspaceMode: 'new_per_run', + timezone, + rrule: 'FREQ=DAILY;BYHOUR=9;BYMINUTE=0', + dtstart: new Date(2026, 4, 12, 0, 0).getTime() + }) + let precheckSignal: AbortSignal | undefined + runAutomationPrecheckMock.mockImplementation( + ({ signal }: { signal?: AbortSignal }) => + new Promise((resolve) => { + precheckSignal = signal + signal?.addEventListener( + 'abort', + () => + resolve({ + command: 'long-command', + exitCode: null, + timedOut: false, + durationMs: 1, + stdout: '', + stderr: '', + stdoutTruncated: false, + stderrTruncated: false, + error: 'Precheck cancelled.', + startedAt: Date.now(), + completedAt: Date.now() + }), + { once: true } + ) + }) + ) + const headlessDispatcher = vi.fn() + const service = new AutomationService(store, { + allowRemoteHostScheduling: true, + headlessDispatcher + }) + vi.setSystemTime(new Date(2026, 4, 13, 9, 1)) + service.start() + await vi.waitFor(() => expect(runAutomationPrecheckMock).toHaveBeenCalledTimes(1)) + + await service.stopAndDrain() + + expect(precheckSignal?.aborted).toBe(true) + expect(headlessDispatcher).not.toHaveBeenCalled() + expect(store.listAutomationRuns(automation.id)[0]).toMatchObject({ + status: 'skipped_precheck', + precheckResult: { error: 'Precheck cancelled.' } + }) + }) }) diff --git a/src/main/automations/service.test.ts b/src/main/automations/service.test.ts index 1c9c314b41b..dab39792584 100644 --- a/src/main/automations/service.test.ts +++ b/src/main/automations/service.test.ts @@ -412,6 +412,51 @@ describe('AutomationService', () => { } }) + it('drains a headless completion before shutdown returns', async () => { + const store = await createStore() + store.addRepo(makeRepo()) + const automation = store.createAutomation({ + name: 'Drain completion', + prompt: 'Finish before shutdown', + agentId: 'claude', + projectId: 'r1', + workspaceMode: 'existing', + workspaceId: 'wt1', + timezone: 'UTC', + rrule: 'FREQ=DAILY;BYHOUR=9;BYMINUTE=0', + dtstart: Date.now() + 60_000 + }) + let finish!: (result: { status: 'completed'; outputSnapshot: null; error: null }) => void + const completion = new Promise<{ + status: 'completed' + outputSnapshot: null + error: null + }>((resolve) => { + finish = resolve + }) + let dispatchSignal: AbortSignal | undefined + const service = new AutomationService(store, { + allowRemoteHostScheduling: true, + headlessDispatcher: vi.fn().mockImplementation(async ({ signal }) => { + dispatchSignal = signal + return { workspaceId: 'wt1', terminalSessionId: 'tab-1', completion } + }) + }) + + await service.runNow(automation.id) + let drained = false + const drain = service.stopAndDrain().then(() => { + drained = true + }) + await Promise.resolve() + expect(drained).toBe(false) + expect(dispatchSignal?.aborted).toBe(true) + + finish({ status: 'completed', outputSnapshot: null, error: null }) + await drain + expect(store.listAutomationRuns(automation.id)[0]?.status).toBe('completed') + }) + it('attaches provider usage when a completed run can be attributed', async () => { vi.setSystemTime(new Date('2026-05-13T10:00:00')) const store = await createStore() diff --git a/src/main/automations/service.ts b/src/main/automations/service.ts index 91b70ff6088..8fea664f0b7 100644 --- a/src/main/automations/service.ts +++ b/src/main/automations/service.ts @@ -14,6 +14,11 @@ import { runAutomationPrecheck } from './precheck-runner' import { resolveAutomationRunTarget, type AutomationRunTargetResult } from './run-target-resolution' import { collectAutomationRunUsage } from './run-usage-collection' import type { HeadlessAutomationDispatcher } from './headless-dispatch' +import { HeadlessAutomationWorkDrain } from './headless-work-drain' +import { + buildHeadlessAutomationRunTarget, + settleHeadlessAutomationCompletion +} from './headless-completion-settlement' import { clearAutomationDispatchTokens, createAutomationDispatchToken } from './dispatch-tokens' import { didAutomationPrecheckPass, @@ -29,10 +34,12 @@ export class AutomationService { private webContents: WebContents | null = null private rendererReady = false private evaluating = false + private readonly headlessWork: HeadlessAutomationWorkDrain private readonly claudeUsage: ClaudeUsageStore | null private readonly codexUsage: CodexUsageStore | null private readonly allowRemoteHostScheduling: boolean private readonly headlessDispatcher: HeadlessAutomationDispatcher | null + private readonly headlessStopController: AbortController | null constructor( store: Store, @@ -50,6 +57,8 @@ export class AutomationService { this.codexUsage = opts.codexUsage ?? null this.allowRemoteHostScheduling = opts.allowRemoteHostScheduling ?? false this.headlessDispatcher = opts.headlessDispatcher ?? null + this.headlessWork = new HeadlessAutomationWorkDrain(this.headlessDispatcher !== null) + this.headlessStopController = this.headlessDispatcher ? new AbortController() : null } setWebContents(webContents: WebContents | null): void { @@ -59,7 +68,7 @@ export class AutomationService { setRendererReady(): void { this.rendererReady = true - void this.evaluateDueRuns() + void this.headlessWork.track(this.evaluateDueRuns()) } start(): void { @@ -67,21 +76,26 @@ export class AutomationService { return } this.timer = setInterval(() => { - void this.evaluateDueRuns() + void this.headlessWork.track(this.evaluateDueRuns()) }, this.tickMs) // Why: headless serve never gets a renderer-ready IPC, but due runs still // need the same startup catch-up pass desktop gets after renderer attach. if (this.rendererReady || this.headlessDispatcher) { - void this.evaluateDueRuns() + void this.headlessWork.track(this.evaluateDueRuns()) } } stop(): void { - if (!this.timer) { - return + if (this.timer) { + clearInterval(this.timer) + this.timer = null } - clearInterval(this.timer) - this.timer = null + } + + async stopAndDrain(): Promise { + this.stop() + this.headlessStopController?.abort() + await this.headlessWork.drain() } async runNow(automationId: string): Promise { @@ -90,10 +104,14 @@ export class AutomationService { throw new Error('Automation not found.') } const run = this.store.createAutomationRun(automation, Date.now(), 'manual') - return await this.requestDispatch(automation, run) + return await this.headlessWork.track(this.requestDispatch(automation, run)) } - async runPrecheck(automationId: string, runId: string): Promise { + async runPrecheck( + automationId: string, + runId: string, + signal?: AbortSignal + ): Promise { const automation = this.store.listAutomations().find((entry) => entry.id === automationId) if (!automation) { throw new Error('Automation not found.') @@ -128,7 +146,8 @@ export class AutomationService { target: automation.executionTargetType === 'ssh' ? { type: 'ssh', cwd: target.cwd, connectionId: automation.executionTargetId } - : { type: 'local', cwd: target.cwd } + : { type: 'local', cwd: target.cwd }, + ...(signal ? { signal } : {}) }) } @@ -256,7 +275,7 @@ export class AutomationService { ): Promise { const precheckResult = run.trigger === 'scheduled' && automation.precheck - ? await this.runPrecheck(automation.id, run.id) + ? await this.runPrecheck(automation.id, run.id, this.headlessStopController?.signal) : null if (precheckResult && !didAutomationPrecheckPass(precheckResult)) { return this.store.updateAutomationRun({ @@ -268,14 +287,13 @@ export class AutomationService { }) } try { - const launch = await this.headlessDispatcher!({ automation, run, target }) - const launchRunTarget = { - workspaceId: launch.workspaceId, - workspaceDisplayName: launch.workspaceDisplayName ?? null, - terminalSessionId: launch.terminalSessionId, - terminalPaneKey: launch.terminalPaneKey ?? null, - terminalPtyId: launch.terminalPtyId ?? null - } + const launch = await this.headlessDispatcher!({ + automation, + run, + target, + signal: this.headlessStopController?.signal + }) + const launchRunTarget = buildHeadlessAutomationRunTarget(launch) const updated = this.store.updateAutomationRun({ runId: run.id, status: 'dispatched', @@ -283,25 +301,15 @@ export class AutomationService { error: null }) if (launch.completion) { - void launch.completion - .then((completion) => - this.markDispatchResult({ - runId: run.id, - status: completion.status, - ...launchRunTarget, - precheckResult, - outputSnapshot: completion.outputSnapshot ?? null, - error: completion.error ?? null - }) - ) - .catch((error) => - this.markDispatchResult({ - runId: run.id, - status: 'dispatch_failed', - ...launchRunTarget, - error: error instanceof Error ? error.message : String(error) - }) - ) + void this.headlessWork.track( + settleHeadlessAutomationCompletion({ + completion: launch.completion, + runId: run.id, + target: launchRunTarget, + precheckResult, + mark: (result) => this.markDispatchResult(result) + }) + ) } return updated } catch (error) { diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 7cad4ad94bc..51fec15f378 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -26052,7 +26052,7 @@ export class OrcaRuntimeService { async launchAgentTerminal( worktreeSelector: string, - opts: { agent: TuiAgent; prompt: string; title?: string } + opts: { agent: TuiAgent; prompt: string; title?: string; signal?: AbortSignal } ): Promise { const worktree = await this.resolveWorktreeSelector(worktreeSelector) const repo = this.store?.getRepo(worktree.repoId) @@ -26072,7 +26072,8 @@ export class OrcaRuntimeService { launchAgent: startup.agent, startupCommandDelivery: startup.startup.startupCommandDelivery, telemetry: startup.startup.telemetry, - title: opts.title + title: opts.title, + signal: opts.signal }) } diff --git a/src/main/runtime/pairing-endpoint.test.ts b/src/main/runtime/pairing-endpoint.test.ts index 5fcebe0afd8..d102a0b2bf0 100644 --- a/src/main/runtime/pairing-endpoint.test.ts +++ b/src/main/runtime/pairing-endpoint.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest' import { + formatBoundWebSocketEndpoint, resolveAdvertisedPairingEndpoint, resolveAdvertisedPairingHostname } from './pairing-endpoint' @@ -7,6 +8,17 @@ import { PAIRING_OFFER_VERSION, PairingOfferSchema } from '../../shared/mobile-r import { PAIRING_ENDPOINT_MAX_CHARACTERS } from '../../shared/mobile-pairing-protocol-limits' import { parseManualNetworkAddress } from '../../shared/network/manual-address' +describe('formatBoundWebSocketEndpoint', () => { + it.each([ + ['127.0.0.1', 'ws://127.0.0.1:6768'], + ['::1', 'ws://[::1]:6768'], + ['[::1]', 'ws://[::1]:6768'] + ])('formats %s as a valid WebSocket URL', (host, endpoint) => { + expect(formatBoundWebSocketEndpoint(host, 6768)).toBe(endpoint) + expect(new URL(endpoint).protocol).toBe('ws:') + }) +}) + describe('resolveAdvertisedPairingEndpoint', () => { const bound = 'ws://0.0.0.0:6768' diff --git a/src/main/runtime/pairing-endpoint.ts b/src/main/runtime/pairing-endpoint.ts index 75ef7ae68c6..6fe4d01fd1d 100644 --- a/src/main/runtime/pairing-endpoint.ts +++ b/src/main/runtime/pairing-endpoint.ts @@ -13,6 +13,10 @@ export type PairingEndpointResolution = guidance: string } +export function formatBoundWebSocketEndpoint(host: string, port: number): string { + return `ws://${bracketIpv6(host)}:${port}` +} + export function resolveAdvertisedPairingEndpoint( boundEndpoint: string, advertisedAddress: string | null | undefined diff --git a/src/main/runtime/runtime-rpc.test.ts b/src/main/runtime/runtime-rpc.test.ts index f8e9ef77471..56b21e03b2f 100644 --- a/src/main/runtime/runtime-rpc.test.ts +++ b/src/main/runtime/runtime-rpc.test.ts @@ -5661,6 +5661,26 @@ describe('OrcaRuntimeRpcServer WebSocket bind host (STA-2370)', () => { } }) + it('publishes a valid endpoint when the standalone host binds IPv6 loopback', async () => { + const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-rpc-')) + const server = new OrcaRuntimeRpcServer({ + runtime: new OrcaRuntimeService(), + userDataPath, + enableWebSocket: true, + wsPort: 0, + wsBindHost: '::1' + }) + + await server.start() + try { + expect(wsTransportOf(server)?.resolvedHost).toBe('::1') + expect(server.getWebSocketEndpoint()).toMatch(/^ws:\/\/\[::1\]:\d+$/) + expect(new URL(server.getWebSocketEndpoint()!).hostname).toBe('[::1]') + } finally { + await server.stop() + } + }) + it('binds all interfaces at startup when a previously-connected device can reconnect', async () => { const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-rpc-')) // Why: a device that has actually connected (lastSeenAt > 0) may reconnect, so the listener must be diff --git a/src/main/runtime/runtime-rpc.ts b/src/main/runtime/runtime-rpc.ts index 5651d700318..6e62d53a3c2 100644 --- a/src/main/runtime/runtime-rpc.ts +++ b/src/main/runtime/runtime-rpc.ts @@ -47,7 +47,7 @@ import type { PairingProvisionRelayParams } from '../../shared/mobile-relay-credential-contract' import { encodePairingOffer, PAIRING_OFFER_VERSION } from '../../shared/pairing' -import { resolveAdvertisedPairingEndpoint } from './pairing-endpoint' +import { formatBoundWebSocketEndpoint, resolveAdvertisedPairingEndpoint } from './pairing-endpoint' import { decodeTerminalStreamFrame, type TerminalStreamFrame @@ -1295,7 +1295,7 @@ export class OrcaRuntimeRpcServer { this.wsBoundHost = options.host return { transport: wsTransport, - endpoint: `ws://${options.host}:${wsTransport.resolvedPort}` + endpoint: formatBoundWebSocketEndpoint(options.host, wsTransport.resolvedPort) } } diff --git a/src/node-server/browserless-automation-dispatcher.test.ts b/src/node-server/browserless-automation-dispatcher.test.ts new file mode 100644 index 00000000000..e89a40542de --- /dev/null +++ b/src/node-server/browserless-automation-dispatcher.test.ts @@ -0,0 +1,252 @@ +import { describe, expect, it, vi } from 'vitest' +import type { OrcaRuntimeService } from '../main/runtime/orca-runtime' +import { createBrowserlessAutomationDispatcher } from './browserless-automation-dispatcher' + +describe('browserless automation dispatcher', () => { + it('cancels the terminal wait during server shutdown', async () => { + const waitForTerminal = vi.fn( + async (_handle: string, options: { signal?: AbortSignal }) => + await new Promise((_resolve, reject) => { + options.signal?.addEventListener('abort', () => reject(new Error('request_aborted')), { + once: true + }) + }) + ) + const runtime = { + launchAgentTerminal: vi.fn().mockResolvedValue({ + handle: 'terminal-1', + tabId: 'tab-1', + paneKey: 'pane-1', + ptyId: 'pty-1', + worktreeId: 'workspace-1' + }), + showManagedWorktree: vi.fn().mockResolvedValue({ displayName: 'Workspace' }), + waitForTerminal, + readTerminal: vi.fn() + } as unknown as OrcaRuntimeService + const controller = new AbortController() + const dispatch = createBrowserlessAutomationDispatcher(runtime) + + const launch = await dispatch({ + automation: { + workspaceMode: 'existing', + workspaceId: 'workspace-1', + agentId: 'claude', + prompt: 'Run checks' + }, + run: { title: 'Checks' }, + target: {}, + signal: controller.signal + } as never) + controller.abort() + + await expect(launch.completion).resolves.toEqual({ + status: 'dispatch_failed', + outputSnapshot: null, + error: 'Automation stopped during server shutdown.' + }) + expect(runtime.readTerminal).not.toHaveBeenCalled() + }) + + it('bounds shutdown when existing-workspace terminal launch never settles', async () => { + vi.useFakeTimers() + try { + let launchSignal: AbortSignal | undefined + const runtime = { + launchAgentTerminal: vi.fn(async (_selector: string, options: { signal?: AbortSignal }) => { + launchSignal = options.signal + return await new Promise(() => undefined) + }), + showManagedWorktree: vi.fn(), + waitForTerminal: vi.fn(), + readTerminal: vi.fn() + } as unknown as OrcaRuntimeService + const controller = new AbortController() + const dispatch = createBrowserlessAutomationDispatcher(runtime) + const launched = dispatch({ + automation: { + workspaceMode: 'existing', + workspaceId: 'workspace-1', + agentId: 'claude', + prompt: 'Run checks' + }, + run: { title: 'Checks' }, + target: {}, + signal: controller.signal + } as never) + expect(runtime.launchAgentTerminal).toHaveBeenCalledTimes(1) + const rejection = expect(launched).rejects.toThrow( + 'Automation stopped during server shutdown.' + ) + + controller.abort() + await vi.runAllTimersAsync() + + await rejection + expect(launchSignal?.aborted).toBe(true) + expect(runtime.showManagedWorktree).not.toHaveBeenCalled() + expect(runtime.waitForTerminal).not.toHaveBeenCalled() + } finally { + vi.useRealTimers() + } + }) + + it('does not start workspace lookup after launch settles into an abort', async () => { + const controller = new AbortController() + const terminal = Promise.resolve({ + handle: 'terminal-1', + worktreeId: 'workspace-1' + }) + void terminal.then(() => controller.abort()) + const runtime = { + launchAgentTerminal: vi.fn(() => terminal), + showManagedWorktree: vi.fn(), + waitForTerminal: vi.fn(), + readTerminal: vi.fn() + } as unknown as OrcaRuntimeService + const dispatch = createBrowserlessAutomationDispatcher(runtime) + + await expect( + dispatch({ + automation: { + workspaceMode: 'existing', + workspaceId: 'workspace-1', + agentId: 'claude', + prompt: 'Run checks' + }, + run: { title: 'Checks' }, + target: {}, + signal: controller.signal + } as never) + ).rejects.toThrow('Automation stopped during server shutdown.') + expect(runtime.showManagedWorktree).not.toHaveBeenCalled() + expect(runtime.waitForTerminal).not.toHaveBeenCalled() + }) + + it('consumes a late worktree-create rejection without continuing dispatch', async () => { + vi.useFakeTimers() + try { + let rejectCreate!: (error: Error) => void + const create = new Promise((_resolve, reject) => { + rejectCreate = reject + }) + const runtime = { + createManagedWorktree: vi.fn(() => create), + launchAgentTerminal: vi.fn(), + showManagedWorktree: vi.fn(), + waitForTerminal: vi.fn(), + readTerminal: vi.fn() + } as unknown as OrcaRuntimeService + const controller = new AbortController() + const dispatch = createBrowserlessAutomationDispatcher(runtime) + const launched = dispatch({ + automation: { + workspaceMode: 'new_per_run', + agentId: 'claude', + prompt: 'Run checks' + }, + run: { id: 'run-1', title: 'Checks', scheduledFor: Date.now() }, + target: { repo: { id: 'repo-1' } }, + signal: controller.signal + } as never) + expect(runtime.createManagedWorktree).toHaveBeenCalledTimes(1) + const rejection = expect(launched).rejects.toThrow( + 'Automation stopped during server shutdown.' + ) + + controller.abort() + await vi.runAllTimersAsync() + await rejection + + rejectCreate(new Error('late create failure')) + await Promise.resolve() + expect(runtime.launchAgentTerminal).not.toHaveBeenCalled() + expect(runtime.waitForTerminal).not.toHaveBeenCalled() + } finally { + vi.useRealTimers() + } + }) + + it('bounds shutdown while resolving the launched workspace display', async () => { + vi.useFakeTimers() + try { + const runtime = { + launchAgentTerminal: vi.fn().mockResolvedValue({ + handle: 'terminal-1', + worktreeId: 'workspace-1' + }), + showManagedWorktree: vi.fn(async () => await new Promise(() => undefined)), + waitForTerminal: vi.fn(), + readTerminal: vi.fn() + } as unknown as OrcaRuntimeService + const controller = new AbortController() + const dispatch = createBrowserlessAutomationDispatcher(runtime) + const launched = dispatch({ + automation: { + workspaceMode: 'existing', + workspaceId: 'workspace-1', + agentId: 'claude', + prompt: 'Run checks' + }, + run: { title: 'Checks' }, + target: {}, + signal: controller.signal + } as never) + await vi.advanceTimersByTimeAsync(0) + expect(runtime.showManagedWorktree).toHaveBeenCalledTimes(1) + const rejection = expect(launched).rejects.toThrow( + 'Automation stopped during server shutdown.' + ) + + controller.abort() + await vi.runAllTimersAsync() + + await rejection + expect(runtime.waitForTerminal).not.toHaveBeenCalled() + } finally { + vi.useRealTimers() + } + }) + + it('bounds shutdown while reading the completed terminal', async () => { + vi.useFakeTimers() + try { + const runtime = { + launchAgentTerminal: vi.fn().mockResolvedValue({ + handle: 'terminal-1', + worktreeId: 'workspace-1' + }), + showManagedWorktree: vi.fn().mockResolvedValue({ displayName: 'Workspace' }), + waitForTerminal: vi.fn().mockResolvedValue({ satisfied: true }), + readTerminal: vi.fn(async () => await new Promise(() => undefined)) + } as unknown as OrcaRuntimeService + const controller = new AbortController() + const dispatch = createBrowserlessAutomationDispatcher(runtime) + const launch = await dispatch({ + automation: { + workspaceMode: 'existing', + workspaceId: 'workspace-1', + agentId: 'claude', + prompt: 'Run checks' + }, + run: { title: 'Checks' }, + target: {}, + signal: controller.signal + } as never) + await vi.advanceTimersByTimeAsync(0) + expect(runtime.readTerminal).toHaveBeenCalledTimes(1) + const completion = expect(launch.completion).resolves.toEqual({ + status: 'dispatch_failed', + outputSnapshot: null, + error: 'Automation stopped during server shutdown.' + }) + + controller.abort() + await vi.runAllTimersAsync() + + await completion + } finally { + vi.useRealTimers() + } + }) +}) diff --git a/src/node-server/browserless-automation-dispatcher.ts b/src/node-server/browserless-automation-dispatcher.ts new file mode 100644 index 00000000000..0352030f46d --- /dev/null +++ b/src/node-server/browserless-automation-dispatcher.ts @@ -0,0 +1,197 @@ +import { createHeadlessAutomationOutputSnapshotBuffer } from '../main/automations/headless-dispatch' +import type { HeadlessAutomationDispatcher } from '../main/automations/headless-dispatch' +import { buildHeadlessAutomationWorktreeCreateArgs } from '../main/automations/headless-workspace-create' +import type { OrcaRuntimeService } from '../main/runtime/orca-runtime' + +const TERMINAL_SNAPSHOT_LIMIT = 2_000 +const RUNTIME_OPERATION_SHUTDOWN_GRACE_MS = 1_000 +const SHUTDOWN_ERROR = 'Automation stopped during server shutdown.' + +type RuntimeOperationSettlement = + | { status: 'fulfilled'; value: T } + | { status: 'rejected'; reason: unknown } + +export function createBrowserlessAutomationDispatcher( + runtime: OrcaRuntimeService +): HeadlessAutomationDispatcher { + return async ({ automation, run, target, signal }) => { + if (signal?.aborted) { + throw new Error(SHUTDOWN_ERROR) + } + let terminalHandle: string + let terminalSessionId: string | null = null + let terminalPaneKey: string | null = null + let terminalPtyId: string | null = null + let workspaceId: string + let workspaceDisplayName: string | null = null + + if (automation.workspaceMode === 'new_per_run') { + const created = await awaitRuntimeOperation( + () => + runtime.createManagedWorktree( + buildHeadlessAutomationWorktreeCreateArgs({ automation, run, repo: target.repo }) + ), + signal + ) + terminalHandle = created.startupTerminal?.handle ?? '' + terminalSessionId = created.startupTerminal?.tabId ?? null + terminalPaneKey = created.startupTerminal?.paneKey ?? null + terminalPtyId = created.startupTerminal?.ptyId ?? null + workspaceId = created.worktree.id + workspaceDisplayName = created.worktree.displayName ?? null + if (!terminalHandle) { + throw new Error( + created.warning || 'Automation workspace was created, but no agent terminal started.' + ) + } + } else { + if (!automation.workspaceId) { + throw new Error('The target workspace is no longer available.') + } + const terminal = await awaitRuntimeOperation( + () => + runtime.launchAgentTerminal(`id:${automation.workspaceId}`, { + agent: automation.agentId, + prompt: automation.prompt, + title: run.title, + signal + }), + signal + ) + terminalHandle = terminal.handle + terminalSessionId = terminal.tabId ?? null + terminalPaneKey = terminal.paneKey ?? null + terminalPtyId = terminal.ptyId ?? null + workspaceId = terminal.worktreeId + const worktree = await awaitRuntimeOperation( + () => runtime.showManagedWorktree(`id:${workspaceId}`), + signal + ) + workspaceDisplayName = worktree.displayName ?? null + } + + return { + workspaceId, + workspaceDisplayName, + terminalSessionId, + terminalPaneKey, + terminalPtyId, + completion: captureCompletion(runtime, terminalHandle, signal) + } + } +} + +async function captureCompletion( + runtime: OrcaRuntimeService, + terminalHandle: string, + signal?: AbortSignal +): Promise<{ + status: 'completed' | 'dispatch_failed' + outputSnapshot: ReturnType< + ReturnType['snapshot'] + > + error: string | null +}> { + try { + const wait = await awaitRuntimeOperation( + () => runtime.waitForTerminal(terminalHandle, { condition: 'tui-idle', signal }), + signal + ) + if (signal?.aborted) { + return shutdownCompletion() + } + const read = await awaitRuntimeOperation( + () => runtime.readTerminal(terminalHandle, { limit: TERMINAL_SNAPSHOT_LIMIT }), + signal + ) + const snapshotBuffer = createHeadlessAutomationOutputSnapshotBuffer() + snapshotBuffer.append(read.tail.join('\n')) + if (wait.satisfied) { + return { + status: 'completed', + outputSnapshot: snapshotBuffer.snapshot(), + error: null + } + } + return { + status: 'dispatch_failed', + outputSnapshot: snapshotBuffer.snapshot(), + error: wait.blockedReason + ? `Automation agent is blocked: ${wait.blockedReason}.` + : 'Automation agent did not report completion.' + } + } catch (error) { + if (signal?.aborted) { + return shutdownCompletion() + } + throw error + } +} + +function shutdownCompletion(): { + status: 'dispatch_failed' + outputSnapshot: null + error: string +} { + return { + status: 'dispatch_failed', + outputSnapshot: null, + error: SHUTDOWN_ERROR + } +} + +async function awaitRuntimeOperation( + operation: () => Promise, + signal?: AbortSignal +): Promise { + if (signal?.aborted) { + throw new Error(SHUTDOWN_ERROR) + } + const settlement = operation().then, RuntimeOperationSettlement>( + (value) => ({ status: 'fulfilled', value }), + (reason) => ({ status: 'rejected', reason }) + ) + if (!signal) { + return unwrapRuntimeOperationSettlement(await settlement) + } + let onAbort!: () => void + const aborted = new Promise<{ status: 'aborted' }>((resolve) => { + onAbort = () => resolve({ status: 'aborted' }) + signal.addEventListener('abort', onAbort, { once: true }) + if (signal.aborted) { + onAbort() + } + }) + const first = await Promise.race([settlement, aborted]) + signal.removeEventListener('abort', onAbort) + if (first.status !== 'aborted') { + if (signal.aborted) { + throw new Error(SHUTDOWN_ERROR) + } + return unwrapRuntimeOperationSettlement(first) + } + + // Why: some runtime operations have no cancellation contract. Observe their + // eventual outcome, but bound shutdown and never continue after the abort. + await waitForRuntimeOperationShutdownGrace(settlement) + throw new Error(SHUTDOWN_ERROR) +} + +function waitForRuntimeOperationShutdownGrace( + settlement: Promise> +): Promise { + return new Promise((resolve) => { + const timer = setTimeout(resolve, RUNTIME_OPERATION_SHUTDOWN_GRACE_MS) + void settlement.then(() => { + clearTimeout(timer) + resolve() + }) + }) +} + +function unwrapRuntimeOperationSettlement(settlement: RuntimeOperationSettlement): T { + if (settlement.status === 'rejected') { + throw settlement.reason + } + return settlement.value +} diff --git a/src/node-server/browserless-runtime-composition.test.ts b/src/node-server/browserless-runtime-composition.test.ts new file mode 100644 index 00000000000..db5880bec92 --- /dev/null +++ b/src/node-server/browserless-runtime-composition.test.ts @@ -0,0 +1,289 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { Store } from '../main/persistence' + +const state = vi.hoisted(() => { + const calls: string[] = [] + const signerKeyIds: string[] = [] + + class Runtime { + private accountServices: { + claudeAccounts: { listAccounts(): unknown } + codexAccounts: { listAccounts(): unknown } + rateLimits: { getState(): unknown } + } | null = null + private automationService: { runNow(id: string): Promise } | null = null + private artifactService: { list(options: unknown): Promise } | null = null + + constructor( + _store: unknown, + _stats: unknown, + deps: { agentSessionClaimSigner: { keyId: string } } + ) { + signerKeyIds.push(deps.agentSessionClaimSigner.keyId) + } + + prepareLegacyWorkerTerminalRecovery(): void { + calls.push('prepare') + } + setAutomationService(service: NonNullable): void { + this.automationService = service + } + setArtifactService(service: NonNullable): void { + this.artifactService = service + } + setCommitMessageAgentEnvironmentResolvers(): void {} + setAccountServices(services: NonNullable): void { + this.accountServices = services + } + getAccountsSnapshot(): unknown { + if (!this.accountServices) { + throw new Error('missing accounts') + } + return { + claude: this.accountServices.claudeAccounts.listAccounts(), + codex: this.accountServices.codexAccounts.listAccounts(), + rateLimits: this.accountServices.rateLimits.getState() + } + } + onAccountsChanged(): () => void { + if (!this.accountServices) { + throw new Error('missing accounts') + } + return () => {} + } + async runAutomationNow(id: string): Promise { + return this.automationService?.runNow(id) + } + async listArtifacts(options: unknown): Promise { + return this.artifactService?.list(options) + } + async refreshRestoredOrchestrationAuthority(): Promise { + calls.push('refresh') + } + async reconcileLegacyWorkerTerminals(): Promise { + calls.push('reconcile') + } + } + + class RateLimits { + setCodexHomePathResolver(): void {} + setClaudeAuthPreparationResolver(): void {} + setCodexFetchTarget(): void {} + setClaudeFetchTarget(): void {} + setInactiveClaudeAccountsResolver(): void {} + setInactiveCodexAccountsResolver(): void {} + ingestLiveClaudeRateLimits(): void {} + getState(): unknown { + return { codex: null, claude: null } + } + start(): void { + calls.push('rate-start') + } + stop(): void { + calls.push('rate-stop') + } + } + + class Automation { + async runNow(id: string): Promise { + return { id } + } + start(): void { + calls.push('automation-start') + } + stop(): void { + calls.push('automation-stop') + } + async stopAndDrain(): Promise { + this.stop() + } + } + + return { Automation, calls, RateLimits, Runtime, signerKeyIds } +}) + +vi.mock('../main/runtime/orca-runtime', () => ({ OrcaRuntimeService: state.Runtime })) +vi.mock('../main/rate-limits/service', () => ({ RateLimitService: state.RateLimits })) +vi.mock('../main/automations/service', () => ({ AutomationService: state.Automation })) +vi.mock('../main/automations/headless-dispatch', () => ({ + createHeadlessAutomationOutputSnapshotBuffer: vi.fn() +})) +vi.mock('../main/automations/headless-workspace-create', () => ({ + buildHeadlessAutomationWorktreeCreateArgs: vi.fn() +})) +vi.mock('../main/artifacts/artifact-cloud-service', () => ({ + ArtifactCloudService: class { + async list(options: unknown): Promise { + return { options } + } + } +})) +vi.mock('../main/codex-accounts/service', () => ({ + CodexAccountService: class { + listAccounts(): unknown { + return { accounts: [], activeAccountId: null } + } + } +})) +vi.mock('../main/claude-accounts/service', () => ({ + ClaudeAccountService: class { + listAccounts(): unknown { + return { accounts: [], activeAccountId: null } + } + } +})) +vi.mock('../main/codex-accounts/runtime-home-service', () => ({ + CodexRuntimeHomeService: class { + prepareForRateLimitFetch(): null { + return null + } + prepareForCodexLaunch(): null { + return null + } + getHostCodexHomePathsForSessionDiscovery(): string[] { + return [] + } + isHostSystemDefaultRealHome(): boolean { + return false + } + getSelectedHostAccountCodexHomePath(): null { + return null + } + } +})) +vi.mock('../main/claude-accounts/runtime-auth-service', () => ({ + ClaudeRuntimeAuthService: class { + async prepareForRateLimitFetch(): Promise { + return {} + } + async prepareForClaudeLaunch(): Promise { + return {} + } + } +})) +vi.mock('../main/rate-limits/codex-rate-limit-target', () => ({ + getInitialCodexRateLimitTarget: () => ({ runtime: 'host' }) +})) +vi.mock('../main/rate-limits/claude-rate-limit-target', () => ({ + getInitialClaudeRateLimitTarget: () => ({ runtime: 'host' }) +})) +vi.mock('../main/rate-limits/account-runtime-target-sync', () => ({ + createAccountRuntimeTargetSettingsSync: () => async () => {} +})) +vi.mock('../main/codex-accounts/runtime-selection', () => ({ + normalizeCodexRuntimeSelection: () => ({ host: null, wsl: {} }) +})) +vi.mock('../main/claude-accounts/runtime-selection', () => ({ + normalizeClaudeRuntimeSelection: () => ({ host: null, wsl: {} }) +})) + +const hookServer = vi.hoisted(() => ({ + start: vi.fn(async () => state.calls.push('hook-start')), + stop: vi.fn(() => state.calls.push('hook-stop')), + stopAndWait: vi.fn(async () => state.calls.push('hook-stop')), + setClaudeStatusLineListener: vi.fn(), + ingestTerminalStatus: vi.fn(), + getStatusSnapshot: vi.fn(() => []), + getStatusSnapshotForPane: vi.fn(() => []), + attestCompatibilityAuthority: vi.fn(() => null), + retirePaneAuthority: vi.fn(), + buildPtyEnv: vi.fn(() => ({})) +})) +vi.mock('../main/agent-hooks/server', () => ({ agentHookServer: hookServer })) +vi.mock('../main/agent-hooks/managed-agent-hook-controls', () => ({ + isAgentStatusHooksEnabled: () => true +})) +vi.mock('../main/daemon/daemon-init', () => ({ getDaemonProvider: () => ({}) })) +vi.mock('../main/ipc/pty', () => ({ + clearProviderPtyState: vi.fn(), + getLocalPtyProvider: () => ({}), + getSshPtyProvider: () => undefined, + registerHeadlessPtyRuntime: () => state.calls.push('register-pty') +})) +vi.mock('../main/ipc/runtime-environment-transport-routing', () => ({ + callRuntimeEnvironment: vi.fn() +})) + +import { createBrowserlessRuntimeComposition } from './browserless-runtime-composition' + +const temporaryPaths: string[] = [] + +afterEach(() => { + state.calls.length = 0 + state.signerKeyIds.length = 0 + vi.clearAllMocks() + for (const path of temporaryPaths.splice(0)) { + rmSync(path, { force: true, recursive: true }) + } +}) + +describe('browserless runtime composition', () => { + it('provides browserless services and keeps authority stable across restarts', async () => { + const dataPath = mkdtempSync(join(tmpdir(), 'orca-browserless-composition-')) + temporaryPaths.push(dataPath) + const store = createStore() + + const first = createBrowserlessRuntimeComposition({ store, dataPath }) + createBrowserlessRuntimeComposition({ store, dataPath }) + + expect(first.runtime.getAccountsSnapshot()).toMatchObject({ + claude: { accounts: [] }, + codex: { accounts: [] }, + rateLimits: { codex: null, claude: null } + }) + expect(() => first.runtime.onAccountsChanged(() => {})).not.toThrow() + await expect(first.runtime.runAutomationNow('automation-1')).resolves.toEqual({ + id: 'automation-1' + }) + await expect(first.runtime.listArtifacts({})).resolves.toEqual({ options: {} }) + expect(state.signerKeyIds).toHaveLength(2) + expect(state.signerKeyIds[0]).toBe(state.signerKeyIds[1]) + }) + + it('orders startup recovery and joins owned lifecycle shutdown', async () => { + const dataPath = mkdtempSync(join(tmpdir(), 'orca-browserless-composition-')) + temporaryPaths.push(dataPath) + const composition = createBrowserlessRuntimeComposition({ store: createStore(), dataPath }) + + await composition.startAfterDaemon() + await composition.stop() + await composition.stop() + + expect(state.calls).toEqual([ + 'prepare', + 'hook-start', + 'register-pty', + 'refresh', + 'reconcile', + 'rate-start', + 'automation-start', + 'automation-stop', + 'rate-stop', + 'hook-stop' + ]) + }) + + it('cleans up once when startup fails', async () => { + const dataPath = mkdtempSync(join(tmpdir(), 'orca-browserless-composition-')) + temporaryPaths.push(dataPath) + const composition = createBrowserlessRuntimeComposition({ store: createStore(), dataPath }) + hookServer.start.mockRejectedValueOnce(new Error('hook bind failed')) + + await expect(composition.startAfterDaemon()).rejects.toThrow('hook bind failed') + await composition.stop() + + expect(state.calls).toEqual(['prepare', 'automation-stop', 'rate-stop', 'hook-stop']) + await expect(composition.startAfterDaemon()).rejects.toThrow('already stopped') + }) +}) + +function createStore(): Store { + const settings = { claudeManagedAccounts: [], codexManagedAccounts: [] } + return { + getSettings: () => settings, + onSettingsChanged: () => () => {} + } as unknown as Store +} diff --git a/src/node-server/browserless-runtime-composition.ts b/src/node-server/browserless-runtime-composition.ts new file mode 100644 index 00000000000..550b419fa01 --- /dev/null +++ b/src/node-server/browserless-runtime-composition.ts @@ -0,0 +1,233 @@ +import { isArtifactSharingEnabled } from '../shared/artifact-sharing-gate' +import { getPreferredPairingOffer } from '../shared/runtime-environments' +import { resolveEnvironment } from '../shared/runtime-environment-store' +import { agentHookServer } from '../main/agent-hooks/server' +import { isAgentStatusHooksEnabled } from '../main/agent-hooks/managed-agent-hook-controls' +import { ArtifactCloudService } from '../main/artifacts/artifact-cloud-service' +import { AutomationService } from '../main/automations/service' +import { ClaudeAccountService } from '../main/claude-accounts/service' +import { normalizeClaudeRuntimeSelection } from '../main/claude-accounts/runtime-selection' +import { ClaudeRuntimeAuthService } from '../main/claude-accounts/runtime-auth-service' +import { prepareLegacySharedCodexSessionResume } from '../main/codex/codex-legacy-session-resume' +import { resolveHostCodexSessionSourceHome } from '../main/codex/codex-session-source-home' +import { CodexAccountService } from '../main/codex-accounts/service' +import { normalizeCodexRuntimeSelection } from '../main/codex-accounts/runtime-selection' +import { CodexRuntimeHomeService } from '../main/codex-accounts/runtime-home-service' +import { getDaemonProvider } from '../main/daemon/daemon-init' +import { callRuntimeEnvironment } from '../main/ipc/runtime-environment-transport-routing' +import { + clearProviderPtyState, + getLocalPtyProvider, + getSshPtyProvider, + registerHeadlessPtyRuntime +} from '../main/ipc/pty' +import type { Store } from '../main/persistence' +import { createAccountRuntimeTargetSettingsSync } from '../main/rate-limits/account-runtime-target-sync' +import { getInitialClaudeRateLimitTarget } from '../main/rate-limits/claude-rate-limit-target' +import { getInitialCodexRateLimitTarget } from '../main/rate-limits/codex-rate-limit-target' +import { RateLimitService } from '../main/rate-limits/service' +import { loadAgentSessionClaimSigner } from '../main/runtime/agent-session-claim-identity' +import { + fingerprintOrchestrationPeer, + type OrchestrationEnvironmentTransport +} from '../main/runtime/orchestration/environment-transport' +import { OrcaRuntimeService } from '../main/runtime/orca-runtime' +import { createBrowserlessAutomationDispatcher } from './browserless-automation-dispatcher' + +export type BrowserlessRuntimeComposition = { + runtime: OrcaRuntimeService + startAfterDaemon(): Promise + stop(): Promise +} + +export function createBrowserlessRuntimeComposition(args: { + store: Store + dataPath: string +}): BrowserlessRuntimeComposition { + const { store, dataPath } = args + const rateLimits = new RateLimitService() + const codexRuntimeHome = new CodexRuntimeHomeService(store) + const claudeRuntimeAuth = new ClaudeRuntimeAuthService(store) + const codexAccounts = new CodexAccountService(store, rateLimits, codexRuntimeHome) + const claudeAccounts = new ClaudeAccountService(store, rateLimits, claudeRuntimeAuth) + const stopAccountRuntimeTargetSync = configureAccountServices( + store, + rateLimits, + codexRuntimeHome, + claudeRuntimeAuth + ) + + const runtime = new OrcaRuntimeService(store, undefined, { + agentSessionClaimSigner: loadAgentSessionClaimSigner(dataPath, dataPath), + canRecoverPersistentLocalPtys: () => getDaemonProvider() !== null, + getLocalProvider: () => getLocalPtyProvider(), + getSshProvider: (connectionId) => getSshPtyProvider(connectionId), + onPtyStopped: clearProviderPtyState, + getDesktopWindowStatus: () => 'blocked', + onTerminalAgentStatus: (event) => agentHookServer.ingestTerminalStatus(event), + getAgentStatusSnapshot: () => + agentHookServer.getStatusSnapshot().filter((entry) => entry.providerSessionOnly !== true), + getAgentProviderSessionSnapshot: () => agentHookServer.getStatusSnapshot(), + getAgentProviderSessionRowsForPane: (paneKey) => + agentHookServer.getStatusSnapshotForPane(paneKey), + attestAgentHookCompatibilityAuthority: (candidate) => + agentHookServer.attestCompatibilityAuthority(candidate), + retireAgentHookCompatibilityAuthority: (paneKey) => + agentHookServer.retirePaneAuthority(paneKey), + getAdditionalAiVaultCodexHomePaths: () => + codexRuntimeHome.getHostCodexHomePathsForSessionDiscovery(), + prepareAiVaultSessionResume: (resumeArgs) => + prepareLegacySharedCodexSessionResume(resumeArgs, { + isHostSystemDefaultRealHome: () => codexRuntimeHome.isHostSystemDefaultRealHome(), + getSelectedHostAccountCodexHomePath: () => + codexRuntimeHome.getSelectedHostAccountCodexHomePath(), + systemCodexHomePath: resolveHostCodexSessionSourceHome(store.getSettings()) + }), + buildAgentHookPtyEnv: () => + isAgentStatusHooksEnabled(store.getSettings()) ? agentHookServer.buildPtyEnv() : {}, + orchestrationEnvironmentTransport: createOrchestrationEnvironmentTransport(dataPath) + }) + runtime.prepareLegacyWorkerTerminalRecovery() + + const automations = new AutomationService(store, { + allowRemoteHostScheduling: true, + headlessDispatcher: createBrowserlessAutomationDispatcher(runtime) + }) + runtime.setAutomationService(automations) + runtime.setArtifactService( + new ArtifactCloudService(dataPath, () => isArtifactSharingEnabled(store.getSettings())) + ) + runtime.setAccountServices({ claudeAccounts, codexAccounts, rateLimits }) + runtime.setCommitMessageAgentEnvironmentResolvers({ + prepareForCodexLaunch: (target) => codexRuntimeHome.prepareForCodexLaunch(target), + prepareForClaudeLaunch: (target) => claudeRuntimeAuth.prepareForClaudeLaunch(target) + }) + + let started = false + let stopped = false + const stopOwnedServices = async (): Promise => { + if (stopped) { + return + } + stopped = true + await automations.stopAndDrain() + rateLimits.stop() + stopAccountRuntimeTargetSync() + agentHookServer.setClaudeStatusLineListener(null) + await agentHookServer.stopAndWait() + } + return { + runtime, + async startAfterDaemon(): Promise { + if (started) { + return + } + if (stopped) { + throw new Error('The browserless runtime composition is already stopped.') + } + started = true + try { + if (isAgentStatusHooksEnabled(store.getSettings())) { + await agentHookServer.start({ env: 'production', userDataPath: dataPath }) + } + registerHeadlessPtyRuntime( + runtime, + (target, launchEnv, context) => + codexRuntimeHome.prepareForCodexLaunch(target, launchEnv, { + unavailableManagedHomePath: context?.unavailableManagedHomePath + }), + () => store.getSettings(), + (target) => claudeRuntimeAuth.prepareForClaudeLaunch(target), + store + ) + await runtime.refreshRestoredOrchestrationAuthority() + await runtime.reconcileLegacyWorkerTerminals() + rateLimits.start({ fetchImmediately: false }) + automations.start() + } catch (error) { + started = false + await stopOwnedServices() + throw error + } + }, + async stop(): Promise { + await stopOwnedServices() + } + } +} + +function configureAccountServices( + store: Store, + rateLimits: RateLimitService, + codexRuntimeHome: CodexRuntimeHomeService, + claudeRuntimeAuth: ClaudeRuntimeAuthService +): () => void { + rateLimits.setCodexHomePathResolver((target) => codexRuntimeHome.prepareForRateLimitFetch(target)) + rateLimits.setClaudeAuthPreparationResolver((target) => + claudeRuntimeAuth.prepareForRateLimitFetch(target) + ) + rateLimits.setCodexFetchTarget(getInitialCodexRateLimitTarget(store.getSettings())) + rateLimits.setClaudeFetchTarget(getInitialClaudeRateLimitTarget(store.getSettings())) + configureInactiveAccountResolvers(store, rateLimits) + const syncTargets = createAccountRuntimeTargetSettingsSync(rateLimits, store.getSettings()) + const stopTargetSync = store.onSettingsChanged((updates, settings) => { + void syncTargets(updates, settings).catch((error) => + console.warn('[rate-limits] Failed to apply account runtime target:', error) + ) + }) + agentHookServer.setClaudeStatusLineListener((event) => { + rateLimits.ingestLiveClaudeRateLimits(event) + }) + return stopTargetSync +} + +function configureInactiveAccountResolvers(store: Store, rateLimits: RateLimitService): void { + rateLimits.setInactiveClaudeAccountsResolver(() => { + const settings = store.getSettings() + const activeIds = new Set( + [ + normalizeClaudeRuntimeSelection(settings).host, + ...Object.values(normalizeClaudeRuntimeSelection(settings).wsl) + ].filter(Boolean) + ) + return settings.claudeManagedAccounts + .filter((account) => !activeIds.has(account.id)) + .map((account) => ({ + id: account.id, + managedAuthPath: account.managedAuthPath, + managedAuthRuntime: account.managedAuthRuntime, + wslDistro: account.wslDistro, + wslLinuxAuthPath: account.wslLinuxAuthPath + })) + }) + rateLimits.setInactiveCodexAccountsResolver(() => { + const settings = store.getSettings() + const activeIds = new Set( + [ + normalizeCodexRuntimeSelection(settings).host, + ...Object.values(normalizeCodexRuntimeSelection(settings).wsl) + ].filter(Boolean) + ) + return settings.codexManagedAccounts + .filter((account) => !activeIds.has(account.id)) + .map((account) => ({ id: account.id, managedHomePath: account.managedHomePath })) + }) +} + +function createOrchestrationEnvironmentTransport( + dataPath: string +): OrchestrationEnvironmentTransport { + return { + resolve: (selector) => { + const environment = resolveEnvironment(dataPath, selector) + const pairing = getPreferredPairingOffer(environment) + return { + environmentId: environment.id, + name: environment.name, + peerFingerprint: fingerprintOrchestrationPeer(pairing.publicKeyB64) + } + }, + call: (selector, method, params, timeoutMs, envelope) => + callRuntimeEnvironment(dataPath, selector, method, params, timeoutMs, undefined, envelope) + } +} diff --git a/src/node-server/index.ts b/src/node-server/index.ts index 81ea64146e7..d6da3effe9b 100644 --- a/src/node-server/index.ts +++ b/src/node-server/index.ts @@ -2,26 +2,27 @@ import { existsSync } from 'node:fs' import { join } from 'node:path' import packageJson from '../../package.json' with { type: 'json' } -import { - disconnectDaemon, - getDaemonProvider, - initDaemonPtyProvider -} from '../main/daemon/daemon-init' -import { getLocalPtyProvider, registerHeadlessPtyRuntime } from '../main/ipc/pty' +import { disconnectDaemon, initDaemonPtyProvider } from '../main/daemon/daemon-init' import { initDataPath, Store } from '../main/persistence' -import { OrcaRuntimeService } from '../main/runtime/orca-runtime' +import type { OrcaRuntimeService } from '../main/runtime/orca-runtime' import { resolveAdvertisedPairingEndpoint } from '../main/runtime/pairing-endpoint' import { OrcaRuntimeRpcServer } from '../main/runtime/runtime-rpc' import { ServeReadinessPublisher } from '../main/server/serve-readiness' import { HEADLESS_RUNTIME_WINDOW_ID } from '../shared/runtime-types' import { configureNodeHostEnvironment } from './node-host-electron-facade' +import { createBrowserlessRuntimeComposition } from './browserless-runtime-composition' import { discoverServerPairingAddress } from './server-address-discovery' import { parseServerCliArguments, renderServerHelp } from './server-cli-arguments' import { ensureServerDataPath, resolveServerDataPath } from './server-paths' +import { + acquireServerProfileProcessLock, + ServerProfileProcessLockError +} from './server-profile-process-lock' import { createServerWindowGraph } from './server-window-graph' +import { requireServerWebSocketEndpoint } from './server-websocket-readiness' -async function main(): Promise { - const args = parseServerCliArguments(process.argv.slice(2)) +export async function runNodeServer(argv = process.argv.slice(2)): Promise { + const args = parseServerCliArguments(argv) if (args.command === 'help') { process.stdout.write(`${renderServerHelp()}\n`) return @@ -35,14 +36,18 @@ async function main(): Promise { const dataPath = resolveServerDataPath(args.dataPath) ensureServerDataPath(dataPath) configureNodeHostEnvironment({ appPath, dataPath, version: packageJson.version }) + process.env.ORCA_APP_VERSION = packageJson.version + const profileLock = await acquireServerProfileProcessLock(dataPath).catch((error: unknown) => { + if (error instanceof ServerProfileProcessLockError && error.reason === 'already_owned') { + process.exitCode = 3 + } + throw error + }) initDataPath() const store = new Store({ dataFile: join(dataPath, 'orca-data.json') }) - const runtime = new OrcaRuntimeService(store, undefined, { - canRecoverPersistentLocalPtys: () => getDaemonProvider() !== null, - getDesktopWindowStatus: () => 'blocked', - getLocalProvider: () => getLocalPtyProvider() - }) + const composition = createBrowserlessRuntimeComposition({ store, dataPath }) + const { runtime } = composition let rpc: OrcaRuntimeRpcServer | null = null let shuttingDown = false @@ -51,17 +56,28 @@ async function main(): Promise { return } shuttingDown = true - await Promise.allSettled([ - rpc?.stop() ?? Promise.resolve(), - disconnectDaemon(), - store.flushAsync() + const lifecycleResults = await settleLifecycleSteps([ + () => rpc?.stop() ?? Promise.resolve(), + () => composition.stop(), + () => disconnectDaemon(), + () => store.flushAsync() ]) - process.exitCode = exitCode + const lifecycleFailed = lifecycleResults.some((result) => result.status === 'rejected') + const lockReleased = await profileLock.release().then( + () => true, + (error) => { + process.stderr.write( + `Orca server failed to release its profile lock: ${error instanceof Error ? error.message : String(error)}\n` + ) + return false + } + ) + process.exitCode = lifecycleFailed || !lockReleased ? 1 : exitCode } try { await initDaemonPtyProvider(undefined, { macosLoginSessionWatch: false }) - registerHeadlessPtyRuntime(runtime, undefined, () => store.getSettings(), undefined, store) + await composition.startAfterDaemon() runtime.syncWindowGraph(HEADLESS_RUNTIME_WINDOW_ID, createServerWindowGraph(store)) const pairingAddress = await discoverServerPairingAddress(args.pairingAddress) validatePairingAddress(pairingAddress.address) @@ -81,6 +97,7 @@ async function main(): Promise { webClientRoot: resolveWebClientRoot(appPath) }) await rpc.start() + requireServerWebSocketEndpoint(rpc.getWebSocketEndpoint()) await publishReadiness(runtime, rpc, args, pairingAddress.address) if (!args.json && pairingAddress.source === 'loopback' && process.env.SSH_CONNECTION) { const resolvedPort = new URL(rpc.getWebSocketEndpoint()!).port @@ -88,14 +105,34 @@ async function main(): Promise { `SSH tunnel: ssh -L ${resolvedPort}:127.0.0.1:${resolvedPort} ${process.env.USER ?? 'user'}@\n` ) } - process.once('SIGINT', () => void shutdown(0).then(() => process.exit(0))) - process.once('SIGTERM', () => void shutdown(0).then(() => process.exit(0))) + const stopForSignal = (): void => { + void shutdown(0).then(() => + process.exit(typeof process.exitCode === 'number' ? process.exitCode : 0) + ) + } + process.once('SIGINT', stopForSignal) + process.once('SIGTERM', stopForSignal) } catch (error) { await shutdown(1) throw error } } +async function settleLifecycleSteps( + steps: readonly (() => Promise)[] +): Promise[]> { + const results: PromiseSettledResult[] = [] + for (const step of steps) { + try { + await step() + results.push({ status: 'fulfilled', value: undefined }) + } catch (reason) { + results.push({ status: 'rejected', reason }) + } + } + return results +} + function validatePairingAddress(address: string): void { const result = resolveAdvertisedPairingEndpoint('ws://127.0.0.1:6768', address) if (!result.ok) { @@ -114,10 +151,8 @@ async function publishReadiness( args: ReturnType, pairingAddress: string ): Promise { - const boundEndpoint = rpc.getWebSocketEndpoint() - const advertised = boundEndpoint - ? resolveAdvertisedPairingEndpoint(boundEndpoint, pairingAddress) - : null + const boundEndpoint = requireServerWebSocketEndpoint(rpc.getWebSocketEndpoint()) + const advertised = resolveAdvertisedPairingEndpoint(boundEndpoint, pairingAddress) const pairing = args.noPairing ? { available: false as const, @@ -129,7 +164,7 @@ async function publishReadiness( { runtimeId: runtime.getRuntimeId(), boundEndpoint, - advertisedEndpoint: advertised?.ok ? advertised.endpoint : null, + advertisedEndpoint: advertised.ok ? advertised.endpoint : null, managedWslCliReconciliation: 'settled', pairing: pairing.available ? { @@ -147,9 +182,9 @@ async function publishReadiness( ) } -void main().catch((error) => { +export function reportNodeServerFailure(error: unknown): void { process.stderr.write( `Orca server failed: ${error instanceof Error ? (error.stack ?? error.message) : String(error)}\n` ) - process.exitCode = 1 -}) + process.exitCode ??= 1 +} diff --git a/src/node-server/package-cli.ts b/src/node-server/package-cli.ts new file mode 100644 index 00000000000..38bb8368560 --- /dev/null +++ b/src/node-server/package-cli.ts @@ -0,0 +1,19 @@ +#!/usr/bin/env node +import { main as runControlCli } from '../cli/cli-program' +import { reportNodeServerFailure, runNodeServer } from './index' +import { resolveServerDataPath } from './server-paths' + +const argv = process.argv.slice(2) +const serverCommand = + argv.length === 0 || + argv[0] === 'serve' || + argv[0] === 'version' || + argv[0] === '--version' || + argv[0] === '-v' + +if (serverCommand) { + void runNodeServer(argv).catch(reportNodeServerFailure) +} else { + process.env.ORCA_USER_DATA_PATH ??= resolveServerDataPath() + void runControlCli(argv) +} diff --git a/src/node-server/server-paths.test.ts b/src/node-server/server-paths.test.ts index 5dcd6e0ae4b..42b3c90a418 100644 --- a/src/node-server/server-paths.test.ts +++ b/src/node-server/server-paths.test.ts @@ -2,7 +2,7 @@ import { mkdtempSync, rmSync, statSync } from 'node:fs' import { tmpdir } from 'node:os' import { join, resolve } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' -import { ensureServerDataPath, resolveServerDataPath } from './server-paths' +import { ensureServerDataPath, resolveLinuxStateHome, resolveServerDataPath } from './server-paths' const originalDataDirectory = process.env.ORCA_SERVER_DATA_DIR const createdPaths: string[] = [] @@ -29,6 +29,11 @@ describe('server data paths', () => { expect(resolveServerDataPath()).toBe(resolve('./environment-state')) }) + it('ignores a relative XDG state directory', () => { + expect(resolveLinuxStateHome('relative-state', '/home/ada')).toBe('/home/ada/.local/state') + expect(resolveLinuxStateHome('/var/lib/ada', '/home/ada')).toBe('/var/lib/ada') + }) + it('creates an owner-only state directory on POSIX', () => { const parent = mkdtempSync(join(tmpdir(), 'orca-server-paths-')) const dataPath = join(parent, 'state') diff --git a/src/node-server/server-paths.ts b/src/node-server/server-paths.ts index 534c01db2b1..208c9778e0f 100644 --- a/src/node-server/server-paths.ts +++ b/src/node-server/server-paths.ts @@ -1,6 +1,6 @@ import { chmodSync, mkdirSync } from 'node:fs' import { homedir } from 'node:os' -import { join, resolve } from 'node:path' +import { isAbsolute, join, resolve } from 'node:path' export function resolveServerDataPath(explicitPath?: string): string { if (explicitPath) { @@ -15,7 +15,11 @@ export function resolveServerDataPath(explicitPath?: string): string { if (process.platform === 'darwin') { return join(homedir(), 'Library', 'Application Support', 'Orca Server') } - return join(process.env.XDG_STATE_HOME ?? join(homedir(), '.local', 'state'), 'orca') + return join(resolveLinuxStateHome(process.env.XDG_STATE_HOME, homedir()), 'orca') +} + +export function resolveLinuxStateHome(xdgStateHome: string | undefined, homePath: string): string { + return xdgStateHome && isAbsolute(xdgStateHome) ? xdgStateHome : join(homePath, '.local', 'state') } export function ensureServerDataPath(path: string): void { diff --git a/src/node-server/server-profile-process-lock.test.ts b/src/node-server/server-profile-process-lock.test.ts new file mode 100644 index 00000000000..eabe2456424 --- /dev/null +++ b/src/node-server/server-profile-process-lock.test.ts @@ -0,0 +1,309 @@ +import { randomUUID } from 'node:crypto' +import { + link, + lstat, + mkdir, + mkdtemp, + readFile, + readdir, + rename, + rm, + writeFile +} from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + readManagedHookHostIdentity, + readManagedHookProcessIdentity +} from '../main/agent-hooks/managed-hook-owner-identity' +import { + acquireServerProfileProcessLock, + getServerProfileProcessLockDirectory, + ServerProfileProcessLockError +} from './server-profile-process-lock' + +const roots: string[] = [] +const staleToken = '00000000-0000-4000-8000-000000000000' + +async function createRoot(): Promise { + const root = await mkdtemp(join(tmpdir(), 'orca-server-profile-lock-')) + roots.push(root) + return root +} + +async function currentIdentities(): Promise<{ host: string; process: string }> { + const [host, processIdentity] = await Promise.all([ + readManagedHookHostIdentity(), + readManagedHookProcessIdentity(process.pid) + ]) + if (!processIdentity) { + throw new Error('test process identity is unavailable') + } + return { host, process: processIdentity } +} + +function paths(root: string): { directory: string; lock: string; owner: string } { + const directory = getServerProfileProcessLockDirectory(root) + return { + directory, + lock: join(directory, 'managed-hook-install.lock'), + owner: join(directory, `managed-hook-install.owner-${staleToken}.json`) + } +} + +async function seedOwner( + root: string, + owner: { hostIdentity: string; pid: number; processIdentity: string; token?: string } +): Promise { + const lockPaths = paths(root) + const token = owner.token ?? staleToken + const ownerPath = join(lockPaths.directory, `managed-hook-install.owner-${token}.json`) + await mkdir(lockPaths.directory, { recursive: true }) + await writeFile(ownerPath, JSON.stringify({ ...owner, token })) + await link(ownerPath, lockPaths.lock) +} + +afterEach(async () => { + vi.restoreAllMocks() + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))) +}) + +describe('server profile process lock', () => { + it('publishes a complete hard-linked owner record and releases it', async () => { + const root = await createRoot() + const lock = await acquireServerProfileProcessLock(root) + const lockPaths = paths(root) + const entries = await readdir(lockPaths.directory) + const ownerName = entries.find((entry) => entry.startsWith('managed-hook-install.owner-')) + expect(ownerName).toBeDefined() + if (!ownerName) { + throw new Error('owner record was not published') + } + const ownerPath = join(lockPaths.directory, ownerName) + const owner = JSON.parse(await readFile(lockPaths.lock, 'utf8')) as { + pid: number + token: string + } + const [lockStats, ownerStats] = await Promise.all([lstat(lockPaths.lock), lstat(ownerPath)]) + + expect(owner).toMatchObject({ pid: process.pid }) + expect(owner.token).toMatch(/^[\da-f-]{36}$/) + expect(lockStats.ino).toBe(ownerStats.ino) + expect(lockStats.nlink).toBe(2) + + await lock.release() + expect(await readdir(lockPaths.directory)).toEqual([]) + }) + + it('fails fast for a live owner without stealing it', async () => { + const root = await createRoot() + const first = await acquireServerProfileProcessLock(root) + const before = await readFile(paths(root).lock, 'utf8') + + await expect(acquireServerProfileProcessLock(root)).rejects.toMatchObject({ + reason: 'already_owned', + ownerPid: process.pid + }) + expect(await readFile(paths(root).lock, 'utf8')).toBe(before) + await first.release() + }) + + it('does not expire a live owner based on wall-clock age', async () => { + const root = await createRoot() + const first = await acquireServerProfileProcessLock(root) + vi.spyOn(Date, 'now').mockReturnValue(Number.MAX_SAFE_INTEGER) + + await expect(acquireServerProfileProcessLock(root)).rejects.toMatchObject({ + reason: 'already_owned' + }) + await first.release() + }) + + it('lets different profiles coexist', async () => { + const firstRoot = await createRoot() + const secondRoot = await createRoot() + const [first, second] = await Promise.all([ + acquireServerProfileProcessLock(firstRoot), + acquireServerProfileProcessLock(secondRoot) + ]) + + await Promise.all([first.release(), second.release()]) + }) + + it('recovers a dead owner and a reused pid with a different incarnation', async () => { + const deadRoot = await createRoot() + const reusedRoot = await createRoot() + const identities = await currentIdentities() + await seedOwner(deadRoot, { + hostIdentity: identities.host, + pid: 2_000_000_000, + processIdentity: 'dead' + }) + await seedOwner(reusedRoot, { + hostIdentity: identities.host, + pid: process.pid, + processIdentity: 'prior-incarnation' + }) + const probes = { + readHostIdentity: async () => identities.host, + readProcessIdentity: async (pid: number) => + pid === 2_000_000_000 ? null : identities.process + } + + const [deadRecovered, reusedRecovered] = await Promise.all([ + acquireServerProfileProcessLock(deadRoot, probes), + acquireServerProfileProcessLock(reusedRoot, probes) + ]) + await Promise.all([deadRecovered.release(), reusedRecovered.release()]) + }) + + it('refuses foreign-host, unavailable, and malformed owners', async () => { + const foreignRoot = await createRoot() + const unavailableRoot = await createRoot() + const malformedRoot = await createRoot() + const identities = await currentIdentities() + const unavailablePid = 2_000_000_003 + await seedOwner(foreignRoot, { + hostIdentity: 'another-host', + pid: process.pid, + processIdentity: identities.process + }) + await seedOwner(unavailableRoot, { + hostIdentity: identities.host, + pid: unavailablePid, + processIdentity: 'unavailable-incarnation' + }) + await mkdir(paths(malformedRoot).directory, { recursive: true }) + await writeFile(paths(malformedRoot).lock, '{') + + await expect(acquireServerProfileProcessLock(foreignRoot)).rejects.toMatchObject({ + reason: 'foreign_host' + }) + await expect( + acquireServerProfileProcessLock(unavailableRoot, { + readHostIdentity: async () => identities.host, + readProcessIdentity: async (pid) => + pid === unavailablePid ? undefined : identities.process + }) + ).rejects.toMatchObject({ reason: 'unverifiable' }) + expect(JSON.parse(await readFile(paths(unavailableRoot).lock, 'utf8'))).toMatchObject({ + pid: unavailablePid, + processIdentity: 'unavailable-incarnation' + }) + await expect(acquireServerProfileProcessLock(malformedRoot)).rejects.toMatchObject({ + reason: 'unverifiable' + }) + }) + + it('allows only one contender to replace a stale generation', async () => { + const root = await createRoot() + const identities = await currentIdentities() + const stalePid = 2_000_000_001 + await seedOwner(root, { + hostIdentity: identities.host, + pid: stalePid, + processIdentity: 'dead' + }) + const probes = { + readHostIdentity: async () => identities.host, + readProcessIdentity: async (pid: number) => (pid === stalePid ? null : identities.process) + } + + const attempts = await Promise.allSettled([ + acquireServerProfileProcessLock(root, probes), + acquireServerProfileProcessLock(root, probes) + ]) + const acquired = attempts.flatMap((result) => + result.status === 'fulfilled' ? [result.value] : [] + ) + const rejected = attempts.flatMap((result) => + result.status === 'rejected' ? [result.reason] : [] + ) + + expect(acquired).toHaveLength(1) + expect(rejected).toHaveLength(1) + expect(rejected[0]).toMatchObject({ reason: 'already_owned' }) + const winner = acquired[0] + if (!winner) { + throw new Error('stale generation had no winner') + } + await winner.release() + }) + + it('recovers when a previous recovery claimant crashed', async () => { + const root = await createRoot() + const identities = await currentIdentities() + const lockPaths = paths(root) + const claimToken = randomUUID() + const stalePid = 2_000_000_002 + await seedOwner(root, { + hostIdentity: identities.host, + pid: stalePid, + processIdentity: 'dead' + }) + const claimedOwner = join( + lockPaths.directory, + `managed-hook-install.claimed-${staleToken}-${claimToken}.json` + ) + await writeFile( + join(lockPaths.directory, `managed-hook-install.claim-${staleToken}-${claimToken}.json`), + JSON.stringify({ + ownerToken: staleToken, + claimToken, + pid: stalePid, + hostIdentity: identities.host, + processIdentity: 'dead-claimant' + }) + ) + await rename(lockPaths.owner, claimedOwner) + + const acquired = await acquireServerProfileProcessLock(root, { + readHostIdentity: async () => identities.host, + readProcessIdentity: async (pid) => (pid === stalePid ? null : identities.process) + }) + await acquired.release() + expect(await readdir(lockPaths.directory)).toEqual([]) + }) + + it('does not let a late release remove a replacement generation', async () => { + const root = await createRoot() + const first = await acquireServerProfileProcessLock(root) + const lockPaths = paths(root) + await rename(lockPaths.lock, `${lockPaths.lock}.displaced`) + const second = await acquireServerProfileProcessLock(root) + const secondOwner = await readFile(lockPaths.lock, 'utf8') + + await first.release() + expect(await readFile(lockPaths.lock, 'utf8')).toBe(secondOwner) + await second.release() + }) + + it('makes release idempotent and recovers an abandoned same-process lock', async () => { + const firstRoot = await createRoot() + const abandonedRoot = await createRoot() + const identities = await currentIdentities() + const first = await acquireServerProfileProcessLock(firstRoot) + await first.release() + await expect(first.release()).resolves.toBeUndefined() + + await seedOwner(abandonedRoot, { + hostIdentity: identities.host, + pid: process.pid, + processIdentity: identities.process + }) + const recovered = await acquireServerProfileProcessLock(abandonedRoot) + await recovered.release() + }) + + it('surfaces a typed error when the current process cannot be identified', async () => { + const root = await createRoot() + const error = await acquireServerProfileProcessLock(root, { + readHostIdentity: async () => 'host', + readProcessIdentity: async () => undefined + }).catch((reason: unknown) => reason) + + expect(error).toBeInstanceOf(ServerProfileProcessLockError) + expect(error).toMatchObject({ reason: 'unverifiable' }) + }) +}) diff --git a/src/node-server/server-profile-process-lock.ts b/src/node-server/server-profile-process-lock.ts new file mode 100644 index 00000000000..d4405678d27 --- /dev/null +++ b/src/node-server/server-profile-process-lock.ts @@ -0,0 +1,190 @@ +import { mkdir } from 'node:fs/promises' +import { join } from 'node:path' +import { removeManagedHookLock } from '../main/agent-hooks/managed-hook-lock-claims' +import { + deactivateManagedHookLockOwner, + inspectManagedHookLock, + isManagedHookLockOwnerActive, + tryCreateManagedHookLock, + type ManagedHookLockOwner +} from '../main/agent-hooks/managed-hook-lock-records' +import { + readManagedHookHostIdentity, + readManagedHookProcessIdentity +} from '../main/agent-hooks/managed-hook-owner-identity' + +const MAX_RECOVERY_ATTEMPTS = 32 + +export type ServerProfileProcessLock = { + ownerPid: number + release: () => Promise +} + +export type ServerProfileProcessLockFailure = + | 'already_owned' + | 'foreign_host' + | 'recovery_contended' + | 'release_failed' + | 'unverifiable' + +export class ServerProfileProcessLockError extends Error { + constructor( + readonly reason: ServerProfileProcessLockFailure, + message: string, + readonly ownerPid?: number + ) { + super(message) + this.name = 'ServerProfileProcessLockError' + } +} + +type ServerProfileProcessLockProbes = { + readHostIdentity: () => Promise + readProcessIdentity: (pid: number) => Promise +} + +const defaultProbes: ServerProfileProcessLockProbes = { + readHostIdentity: readManagedHookHostIdentity, + readProcessIdentity: readManagedHookProcessIdentity +} + +export function getServerProfileProcessLockDirectory(dataPath: string): string { + return join(dataPath, '.server-profile-process-lock') +} + +export async function acquireServerProfileProcessLock( + dataPath: string, + probes: ServerProfileProcessLockProbes = defaultProbes +): Promise { + const lockParent = getServerProfileProcessLockDirectory(dataPath) + const lockPath = join(lockParent, 'managed-hook-install.lock') + await mkdir(lockParent, { recursive: true, mode: 0o700 }) + const [hostIdentity, processIdentity] = await Promise.all([ + probes.readHostIdentity(), + probes.readProcessIdentity(process.pid) + ]) + if (!processIdentity) { + throw new ServerProfileProcessLockError( + 'unverifiable', + 'Could not identify the Orca server process' + ) + } + + for (let attempt = 0; attempt < MAX_RECOVERY_ATTEMPTS; attempt += 1) { + const owner = await tryCreateManagedHookLock( + lockParent, + lockPath, + hostIdentity, + processIdentity + ) + if (owner) { + return createOwnedLock(lockPath, lockParent, owner, hostIdentity, processIdentity) + } + + const state = await inspectManagedHookLock(lockPath) + if (state.kind === 'missing') { + continue + } + if (state.kind === 'unknown') { + throw new ServerProfileProcessLockError( + 'unverifiable', + 'The Orca server profile lock has an unverifiable owner' + ) + } + if (state.owner.hostIdentity !== hostIdentity) { + throw new ServerProfileProcessLockError( + 'foreign_host', + 'The Orca server profile lock belongs to another host', + state.owner.pid + ) + } + + const currentIdentity = await probes.readProcessIdentity(state.owner.pid) + if (currentIdentity === undefined) { + throw new ServerProfileProcessLockError( + 'unverifiable', + 'Could not verify the Orca server profile lock owner', + state.owner.pid + ) + } + const abandonedOwnLock = + state.owner.pid === process.pid && + currentIdentity === processIdentity && + !isManagedHookLockOwnerActive(state.owner.token) + if ( + currentIdentity !== null && + currentIdentity === state.owner.processIdentity && + !abandonedOwnLock + ) { + throw new ServerProfileProcessLockError( + 'already_owned', + `Another Orca server process (${state.owner.pid}) owns this profile`, + state.owner.pid + ) + } + + const removal = await removeManagedHookLock( + lockPath, + lockParent, + state.owner, + hostIdentity, + processIdentity + ) + if (removal === 'removed' || removal === 'active') { + continue + } + throw removal === 'foreign' + ? new ServerProfileProcessLockError( + 'foreign_host', + 'Orca server profile lock recovery belongs to another host', + state.owner.pid + ) + : new ServerProfileProcessLockError( + 'unverifiable', + 'Orca server profile lock recovery could not verify ownership', + state.owner.pid + ) + } + + throw new ServerProfileProcessLockError( + 'recovery_contended', + 'Orca server profile lock recovery remained contended' + ) +} + +function createOwnedLock( + lockPath: string, + lockParent: string, + owner: ManagedHookLockOwner, + hostIdentity: string, + processIdentity: string +): ServerProfileProcessLock { + let released = false + return { + ownerPid: owner.pid, + release: async () => { + if (released) { + return + } + released = true + try { + const removal = await removeManagedHookLock( + lockPath, + lockParent, + owner, + hostIdentity, + processIdentity + ) + if (removal !== 'removed') { + throw new ServerProfileProcessLockError( + 'release_failed', + `Could not release the Orca server profile lock: ${removal}`, + owner.pid + ) + } + } finally { + deactivateManagedHookLockOwner(owner.token) + } + } + } +} diff --git a/src/node-server/server-websocket-readiness.test.ts b/src/node-server/server-websocket-readiness.test.ts new file mode 100644 index 00000000000..7444aba45cb --- /dev/null +++ b/src/node-server/server-websocket-readiness.test.ts @@ -0,0 +1,9 @@ +import { describe, expect, it } from 'vitest' +import { requireServerWebSocketEndpoint } from './server-websocket-readiness' + +describe('requireServerWebSocketEndpoint', () => { + it('rejects readiness without a remote listener', () => { + expect(() => requireServerWebSocketEndpoint(null)).toThrow('WebSocket listener failed') + expect(requireServerWebSocketEndpoint('ws://127.0.0.1:6768')).toBe('ws://127.0.0.1:6768') + }) +}) diff --git a/src/node-server/server-websocket-readiness.ts b/src/node-server/server-websocket-readiness.ts new file mode 100644 index 00000000000..2bf32db55ec --- /dev/null +++ b/src/node-server/server-websocket-readiness.ts @@ -0,0 +1,6 @@ +export function requireServerWebSocketEndpoint(endpoint: string | null): string { + if (!endpoint) { + throw new Error('The remote WebSocket listener failed to start; no readiness was published.') + } + return endpoint +}