diff --git a/config/build-plugins/plain-node-entry-guard.ts b/config/build-plugins/plain-node-entry-guard.ts index 650a0f58b1b..6fe5d0a2e2f 100644 --- a/config/build-plugins/plain-node-entry-guard.ts +++ b/config/build-plugins/plain-node-entry-guard.ts @@ -25,6 +25,7 @@ export const CLI_MAIN_ENTRY_NAMES = [ 'claude-accounts/keychain', 'skills/bundled-skill-install/cli-entry', 'macos-tree-clone', + 'orcad/orcad-stale-lock-reclaim', ...[ 'access', 'active-location', diff --git a/config/electron-builder.config.cjs b/config/electron-builder.config.cjs index a9513c7acfc..bb1f6d9618b 100644 --- a/config/electron-builder.config.cjs +++ b/config/electron-builder.config.cjs @@ -393,6 +393,7 @@ module.exports = { 'out/main/computer-sidecar.js', 'out/main/parcel-watcher-process-entry.js', 'out/main/orcad/orcad-local-serve-selection-entry.js', + 'out/main/orcad/orcad-stale-lock-reclaim.js', 'out/main/chunks/**', 'out/renderer/**', 'out/mobile-web/**', diff --git a/src/cli/handlers/core.test.ts b/src/cli/handlers/core.test.ts index 4d673a1c51c..9cf4e790fe2 100644 --- a/src/cli/handlers/core.test.ts +++ b/src/cli/handlers/core.test.ts @@ -168,4 +168,20 @@ describe('orca claude-teams CLI handler', () => { const spawnEnv = spawnMock.mock.calls.at(-1)?.[2].env as SpawnEnv expect(spawnEnv.ANTHROPIC_API_KEY).toBe('sk-ant-system') }) + + // #32177: an empty --data-root must never fall back to the default profile's lock. + it('refuses `orca serve --reclaim-stale-lock` with a --data-root that names no directory', async () => { + const ctx: HandlerContext = { + flags: new Map([ + ['reclaim-stale-lock', true], + ['data-root', true], + ['yes', true] + ]), + client, + cwd: '/tmp', + json: true, + rawArgs: [] + } + await expect(CORE_HANDLERS.serve(ctx)).rejects.toMatchObject({ code: 'invalid_argument' }) + }) }) diff --git a/src/cli/handlers/core.ts b/src/cli/handlers/core.ts index 33a764135ef..f0031819bfb 100644 --- a/src/cli/handlers/core.ts +++ b/src/cli/handlers/core.ts @@ -1,4 +1,5 @@ import { spawn } from 'node:child_process' +import { resolve } from 'node:path' import type { CommandHandler } from '../dispatch' import { formatCliStatus, formatStatus, printResult } from '../format' import { RuntimeClientError, serveOrcaApp } from '../runtime-client' @@ -6,6 +7,7 @@ import { stripElectronRunAsNode } from '../runtime/orca-app-executable' import { resolveCliStatusCaller } from '../runtime/status-caller' import { getServeSuperviseValidationError } from '../runtime/serve-supervised-service' import { getOptionalServePort } from '../runtime/serve-port-flag' +import { resolveLaunchUserDataPath } from '../runtime/launch-user-data-path' import { getServeOptionValidationError } from '../../shared/serve-option-validation' function envRecord(): Record { @@ -82,6 +84,27 @@ export const CORE_HANDLERS: Record = { printResult(result, json, formatCliStatus) }, serve: async ({ flags, json }) => { + if (flags.get('reclaim-stale-lock') === true) { + const dataRoot = flags.get('data-root') + // Why refuse rather than default: a missing value would point a destructive command elsewhere. + if (flags.has('data-root') && (typeof dataRoot !== 'string' || !dataRoot.trim())) { + throw new RuntimeClientError( + 'invalid_argument', + '--data-root expects a directory. Usage: orca serve --reclaim-stale-lock [--data-root ] [--yes] [--json]' + ) + } + // Why lazy: only this mode needs orcad's lock code, and every other command skips loading it. + const { printOrcadStaleLockReclaimResult, reclaimStaleOrcadInstanceLock } = + await import('../../main/orcad/orcad-stale-lock-reclaim.js') + const result = await reclaimStaleOrcadInstanceLock( + typeof dataRoot === 'string' ? resolve(dataRoot) : resolveLaunchUserDataPath(), + // This CLI's own binary is no command to give a user; `orca serve` is. + { yes: flags.get('yes') === true, hintRuntime: null } + ) + printOrcadStaleLockReclaimResult(result, json) + process.exitCode = result.exitCode + return + } const projectRootValue = flags.get('project-root') const projectRoot = typeof projectRootValue === 'string' ? projectRootValue : null const noPairing = flags.get('no-pairing') === true diff --git a/src/cli/runtime/serve-supervised-service.test.ts b/src/cli/runtime/serve-supervised-service.test.ts index 3a16826a7f8..b79399417c2 100644 --- a/src/cli/runtime/serve-supervised-service.test.ts +++ b/src/cli/runtime/serve-supervised-service.test.ts @@ -19,6 +19,8 @@ import { type ServeSupervisorBrake } from './serve-supervisor-brake' import type { ServeInstallEvidence } from './serve-install-hold' +import { resolveOrcadExitCode } from '../../main/orcad/orcad-exit-code' +import { OrcadInstanceLockError } from '../../main/orcad/orcad-data-root-privacy' /** Virtual clock: timers fire in order as soon as the loop awaits them. */ function harness(overrides: Partial & { userDataPath: string }) { @@ -215,6 +217,27 @@ describe('superviseServeService', () => { expect(Math.max(...h.sleeps)).toBe(60_000) }) + // #32177: a lock this machine cannot judge (a shared home) is a peer, not a fault to give up on. + it('waits on a lock holder it cannot judge, without giving up', async () => { + const holderUnverifiable = resolveOrcadExitCode( + new OrcadInstanceLockError('orcad_instance_lock_holder_unverifiable', 'shared home') + ) + let runs = 0 + const h = harness({ + userDataPath, + runOnce: async () => { + runs++ + if (runs === 10) { + h.stop() + } + return holderUnverifiable + } + }) + await expect(superviseServeService(h.options)).resolves.toBe(0) + expect(runs).toBe(10) + expect(readServeSupervisorBrake(userDataPath)).toBeNull() + }) + it('counts a thrown child failure and keeps supervising', async () => { let runs = 0 const h = harness({ diff --git a/src/cli/runtime/serve-supervised-service.ts b/src/cli/runtime/serve-supervised-service.ts index 8d59b578051..e43712cee84 100644 --- a/src/cli/runtime/serve-supervised-service.ts +++ b/src/cli/runtime/serve-supervised-service.ts @@ -1,4 +1,5 @@ import { SINGLE_INSTANCE_ALREADY_RUNNING_EXIT_CODE } from '../../shared/single-instance-exit-code' +import { orcadStaleLockReclaimCommand } from '../../shared/orcad-stale-lock-reclaim' import { brakeForBuild, clearServeSupervisorBrake, @@ -271,7 +272,7 @@ export async function superviseServeService(options: SupervisedServeOptions): Pr // An occupied profile warrants standby, not a crash-brake failure. const delayMs = standbyDelayMs(standbyAttempt++, random) log( - `[serve] Another Orca serves this profile; checking again in ${Math.round(delayMs / 1000)}s.` + `[serve] Another Orca serves this profile, or holds it and cannot be shown to have stopped; checking again in ${Math.round(delayMs / 1000)}s.${process.platform === 'linux' ? ` If its message says it may have stopped, \`${orcadStaleLockReclaimCommand(options.userDataPath, null)}\` checks it.` : ''}` ) await sleep(delayMs) continue diff --git a/src/cli/serve-electron-flag-parity.test.ts b/src/cli/serve-electron-flag-parity.test.ts index d6d70b0a414..77040924d7f 100644 --- a/src/cli/serve-electron-flag-parity.test.ts +++ b/src/cli/serve-electron-flag-parity.test.ts @@ -5,13 +5,17 @@ import { SERVE_COMMAND_SPECS } from './specs/serve' // The main tsconfig cannot import CLI specs, so test the duplicated serve flag lists from here. -// Help is refused; pairing selectors pass through; force and supervise apply only to the CLI parent. +// Help is refused; pairing selectors pass through; force, supervise and the stale-lock +// reclaim flags apply only to the CLI parent, which starts no runtime for a reclaim. const UNTRANSLATED_GLOBAL_FLAGS = new Set([ 'help', 'pairing-code', 'environment', 'force', - 'supervise' + 'supervise', + 'reclaim-stale-lock', + 'data-root', + 'yes' ]) const serveSpec = SERVE_COMMAND_SPECS.find((spec) => spec.path.join(' ') === 'serve') diff --git a/src/cli/specs/serve.ts b/src/cli/specs/serve.ts index 1cdc21db249..373655695dc 100644 --- a/src/cli/specs/serve.ts +++ b/src/cli/specs/serve.ts @@ -6,7 +6,7 @@ export const SERVE_COMMAND_SPECS: CommandSpec[] = [ path: ['serve'], summary: 'Start an Orca runtime server without opening a desktop window', usage: - 'orca serve [--port ] [--pairing-address ] [--mobile-pairing] [--no-pairing] [--grant-desktop-control] [--project-root ] [--recipe-json] [--supervise] [--force] [--json]', + 'orca serve [--port ] [--pairing-address ] [--mobile-pairing] [--no-pairing] [--grant-desktop-control] [--project-root ] [--recipe-json] [--supervise] [--force] [--reclaim-stale-lock [--data-root ] [--yes]] [--json]', allowedFlags: [ ...GLOBAL_FLAGS, 'port', @@ -17,7 +17,10 @@ export const SERVE_COMMAND_SPECS: CommandSpec[] = [ 'project-root', 'recipe-json', 'supervise', - 'force' + 'force', + 'reclaim-stale-lock', + 'data-root', + 'yes' ], notes: [ 'Runs in the foreground and prints the bound endpoint, advertised endpoint, and pairing status. Stop it with Ctrl+C.', @@ -28,6 +31,7 @@ export const SERVE_COMMAND_SPECS: CommandSpec[] = [ "A paired client cannot drive this machine's desktop (orca computer) unless the offer was created with --grant-desktop-control.", 'When the web client bundle is available, the server also prints a browser URL with the pairing data embedded.', 'Exits with code 3, without starting anything, when another Orca already serves this profile or cannot be shown to have stopped.', + "--reclaim-stale-lock starts nothing. When a start refuses because this profile's lock may belong to an Orca on another machine sharing this home (for example after a reboot, on a network home), it checks that the lock was written on a host with this name in an earlier boot, is at least 2 minutes old, and that nothing answers on this machine at its holder's endpoints. Those checks cannot rule out another machine with the same host name, so it then only prints what it found (exit 2); add --yes, once you are sure no other machine sharing this home runs Orca for this profile, to clear it (exit 0). It exits 1 when a check fails and 4 when there is no lock. --data-root names the data root the refusal printed. --json prints the status (none, eligible, refused or reclaimed). Linux only.", 'On macOS, after Orca serve aborts at startup (it could not register with Launch Services), later runs from the same environment refuse for 30 minutes with exit code 78. Use --force to try anyway.', '--supervise is for service managers (launchd, systemd). Orca serve then restarts the server itself with growing delays instead of exiting when it stops, waits while an update replaces the app, and waits while another Orca serves this profile. After 6 failures in 15 minutes it gives up and exits with code 0 so the manager stops too, and later supervised runs refuse until a new build is installed or --force is passed. Not available on Windows yet.', 'Experimental and off by default: set ORCA_TERMINAL_PROCESS_CEILING= (64 or more) before the terminal host starts to have the OS cap how many processes each terminal may run (Linux, through a systemd user scope per terminal, which needs a user manager that outlives logout; Windows, through a job per terminal). `orca serve` reports the ceiling as on or off, with the reason.', @@ -38,7 +42,8 @@ export const SERVE_COMMAND_SPECS: CommandSpec[] = [ 'orca serve --json', 'orca serve --project-root /workspace/repo --pairing-address wss://sandbox.example.com --recipe-json', 'orca serve --port 6768 --pairing-address 100.64.1.20', - 'orca serve --pairing-address 100.64.1.20 --mobile-pairing' + 'orca serve --pairing-address 100.64.1.20 --mobile-pairing', + 'orca serve --reclaim-stale-lock' ] }, { diff --git a/src/main/orcad/main.ts b/src/main/orcad/main.ts index ca28cea4174..8d09e01914f 100644 --- a/src/main/orcad/main.ts +++ b/src/main/orcad/main.ts @@ -24,6 +24,7 @@ import { } from '../../shared/windows-breakaway-launch' import { runWindowsBreakawayLaunchIfRequested } from '../../shared/windows-breakaway-launcher' import { parseArgs } from './orcad-command-arguments' +import { ORCAD_RECLAIM_STALE_LOCK_FLAG } from '../../shared/orcad-stale-lock-reclaim' // Why exit before the preflight: reaching this line means the whole module graph resolved // under plain Node, which is all the build guard needs to prove. Probing natives or @@ -56,6 +57,10 @@ if (process.argv[2] === '--orcad-structured-provider-load-check') { void import('./orcad-managed-stop-command').then(({ runOrcadManagedStopCommandAndExit }) => runOrcadManagedStopCommandAndExit(process.argv.slice(2)) ) +} else if (process.argv[2] === ORCAD_RECLAIM_STALE_LOCK_FLAG) { + void import('./orcad-stale-lock-reclaim').then(({ runOrcadStaleLockReclaimCommandAndExit }) => + runOrcadStaleLockReclaimCommandAndExit(process.argv.slice(2)) + ) } else { startOrcadProcess() } diff --git a/src/main/orcad/orcad-artifact-cache-retention.ts b/src/main/orcad/orcad-artifact-cache-retention.ts index 450f9acbf03..517556d5b0d 100644 --- a/src/main/orcad/orcad-artifact-cache-retention.ts +++ b/src/main/orcad/orcad-artifact-cache-retention.ts @@ -10,11 +10,8 @@ import { readdir, rm, stat } from 'node:fs/promises' import { join } from 'node:path' import { nodeRuntimeAsset } from '../../shared/node-runtime-pin' -import { - ORCAD_LOCK_FILE_NAME, - orcadPeerPidIsJudgeableHere, - readOrcadInstanceLockRecord -} from './orcad-instance-lock' +import { ORCAD_LOCK_FILE_NAME, readOrcadInstanceLockRecord } from './orcad-instance-lock' +import { orcadPeerPidIsJudgeableHere } from './orcad-lock-holder-locality' import { materializedOrcadArtifactVersions } from './orcad-artifact-materializer' import { isProcessAlive } from '../daemon/daemon-process-inspection' import { diff --git a/src/main/orcad/orcad-command-arguments.ts b/src/main/orcad/orcad-command-arguments.ts index 51431de4d74..ac236962a50 100644 --- a/src/main/orcad/orcad-command-arguments.ts +++ b/src/main/orcad/orcad-command-arguments.ts @@ -26,6 +26,10 @@ Options: --project-root Existing absolute project directory for --recipe-json. --web-client-root Serve the browser client from this directory (holds web-index.html). --help, -h Print this text and exit without starting the server. + --reclaim-stale-lock [--yes] + Without starting the server, check whether this data root's + instance lock belongs to an orcad that stopped (same host, + an earlier boot, nothing answering) and, with --yes, clear it. Environment: ORCA_USER_DATA Data directory for this instance's profile and state. diff --git a/src/main/orcad/orcad-daemon-slot-pins.ts b/src/main/orcad/orcad-daemon-slot-pins.ts index 31eac0e32db..42f234a942f 100644 --- a/src/main/orcad/orcad-daemon-slot-pins.ts +++ b/src/main/orcad/orcad-daemon-slot-pins.ts @@ -10,7 +10,7 @@ import { currentOrcadPeerMachine, orcadPeerPidIsJudgeableHere, type OrcadPeerMachine -} from './orcad-instance-lock' +} from './orcad-lock-holder-locality' import { orcadDataRootIsOnLocalDisk } from './orcad-data-root-locality' /** This machine, for telling a peer record's PID apart from one another machine issued. */ diff --git a/src/main/orcad/orcad-data-root-privacy.ts b/src/main/orcad/orcad-data-root-privacy.ts index 1de644ffac2..95398cc1e88 100644 --- a/src/main/orcad/orcad-data-root-privacy.ts +++ b/src/main/orcad/orcad-data-root-privacy.ts @@ -7,6 +7,8 @@ type OrcadInstanceLockCode = | 'orcad_data_root_wrong_owner' | 'orcad_data_root_shared' | 'orcad_instance_lock_held' + /** A holder this machine cannot judge, such as one on a shared home; it may still be running. */ + | 'orcad_instance_lock_holder_unverifiable' | 'orcad_instance_lock_foreign_identity' | 'orcad_instance_lock_unreadable' diff --git a/src/main/orcad/orcad-exit-code.ts b/src/main/orcad/orcad-exit-code.ts index 8259c5f7f5f..5c24cc8ae14 100644 --- a/src/main/orcad/orcad-exit-code.ts +++ b/src/main/orcad/orcad-exit-code.ts @@ -3,11 +3,19 @@ import { OrcadBundledRuntimeError } from './orcad-bundled-runtime' import { OrcadInstanceLockError } from './orcad-data-root-privacy' import { ProfileStateAccessError } from '../persistence/profile-state/profile-state-access' import { ORCAD_EXIT_CONFIGURATION } from './orcad-configuration-exit' +import { SINGLE_INSTANCE_ALREADY_RUNNING_EXIT_CODE } from '../../shared/single-instance-exit-code' export const ORCAD_EXIT_FAILED = 1 /** Configuration faults cannot be repaired by a supervisor restart. */ export function resolveOrcadExitCode(error: unknown): number { + // Why the already-running code: a supervisor waits for that holder instead of giving up on it. + if ( + error instanceof OrcadInstanceLockError && + error.code === 'orcad_instance_lock_holder_unverifiable' + ) { + return SINGLE_INSTANCE_ALREADY_RUNNING_EXIT_CODE + } return error instanceof OrcadInstanceLockError || error instanceof OrcadBindAddressError || error instanceof OrcadBundledRuntimeError || diff --git a/src/main/orcad/orcad-instance-lock.test.ts b/src/main/orcad/orcad-instance-lock.test.ts index 305cf0bd5bc..068a11af4cc 100644 --- a/src/main/orcad/orcad-instance-lock.test.ts +++ b/src/main/orcad/orcad-instance-lock.test.ts @@ -495,7 +495,7 @@ describe('acquireOrcadInstanceLock', () => { for (const role of ['orcad', 'desktop'] as const) { expect(() => acquireOrcadInstanceLock(root, hooks({ role }))).toThrow( expect.objectContaining({ - code: 'orcad_instance_lock_held', + code: 'orcad_instance_lock_holder_unverifiable', message: expect.stringContaining('from another machine') }) ) @@ -512,6 +512,16 @@ describe('acquireOrcadInstanceLock', () => { }) }) + it('never lets a confirmed reclaim take a lock another machine id holds', () => { + const root = makeRoot() + const lockPath = join(root, ORCAD_LOCK_FILE_NAME) + writeFileSync(lockPath, JSON.stringify(persistedRecord({ machineIdentity: 'machine-b' }))) + expect(() => acquireOrcadInstanceLock(root, hooks({ reclaimConfirmed: () => true }))).toThrow( + expect.objectContaining({ code: 'orcad_instance_lock_holder_unverifiable' }) + ) + expect(readOrcadInstanceLockRecord(lockPath)?.nonce).toBe('stale') + }) + it('reclaims a record naming no machine only when it is provably from this one', () => { const root = makeRoot() const lockPath = join(root, ORCAD_LOCK_FILE_NAME) @@ -519,11 +529,11 @@ describe('acquireOrcadInstanceLock', () => { writeFileSync(lockPath, JSON.stringify(persistedRecord({ bootId: 'boot-1' }))) const shared = { dataRootIsOnLocalDisk: () => false } expect(() => acquireOrcadInstanceLock(root, hooks(shared))).toThrow( - expect.objectContaining({ code: 'orcad_instance_lock_held' }) + expect.objectContaining({ code: 'orcad_instance_lock_holder_unverifiable' }) ) expect(() => acquireOrcadInstanceLock(root, hooks({ ...shared, machineIdentity: () => null })) - ).toThrow(expect.objectContaining({ code: 'orcad_instance_lock_held' })) + ).toThrow(expect.objectContaining({ code: 'orcad_instance_lock_holder_unverifiable' })) expect(readOrcadInstanceLockRecord(lockPath)?.nonce).toBe('stale') // The same kernel boot is this machine. diff --git a/src/main/orcad/orcad-instance-lock.ts b/src/main/orcad/orcad-instance-lock.ts index c83f0acab6e..9890cc663ce 100644 --- a/src/main/orcad/orcad-instance-lock.ts +++ b/src/main/orcad/orcad-instance-lock.ts @@ -14,7 +14,7 @@ */ import { randomUUID } from 'node:crypto' import { linkSync, mkdirSync, statSync, unlinkSync, writeFileSync } from 'node:fs' -import { userInfo } from 'node:os' +import { hostname, userInfo } from 'node:os' import { join } from 'node:path' import process from 'node:process' import { z } from 'zod' @@ -34,6 +34,11 @@ import { import { hasErrorCode, isProcessAlive } from '../daemon/daemon-process-inspection' import { profileStateAccessMachineIdentity } from '../persistence/profile-state/profile-state-access-identity' import { orcadDataRootIsOnLocalDisk } from './orcad-data-root-locality' +import type { OrcadReclaimHintRuntime } from '../../shared/orcad-stale-lock-reclaim' +import { + orcadLockHolderLocality, + orcadLockHolderUnverifiableError +} from './orcad-lock-holder-locality' import { MAX_ORCAD_LOCK_BYTES, readBoundedLockFile, @@ -68,7 +73,9 @@ const OrcadLockRecordSchema = z.object({ * The holder's machine (as profile-state owners record it). Absent from older writers. A PID is * evidence only on the machine that issued it, and a shared home can put two on one data root. */ - machineIdentity: z.string().min(1).max(255).optional().catch(undefined) + machineIdentity: z.string().min(1).max(255).optional().catch(undefined), + /** The holder's hostname, absent from older writers; evidence for `--reclaim-stale-lock`. */ + host: z.string().min(1).max(255).optional().catch(undefined) }) export type OrcadLockRecord = z.infer @@ -104,6 +111,13 @@ export type OrcadInstanceLockHooks = OrcadDataRootPrivacyHooks & { machineIdentity?: () => string | null /** Whether no other machine can reach the data root, for records that name no machine. */ dataRootIsOnLocalDisk?: (dataRoot: string) => boolean + /** + * `--reclaim-stale-lock` only: the user confirmed this unverifiable record (the lock's holder, or + * an earlier reclaimer cut short) is stale. Its PID is then judged as this machine's. + */ + reclaimConfirmed?: (record: OrcadLockRecord, role: 'holder' | 'recovery-owner') => boolean + /** The program a refusal names; defaults to this orcad's own runtime, or `orca serve`. */ + reclaimHintRuntime?: OrcadReclaimHintRuntime | null } function defaultIdentity(): string { @@ -172,7 +186,8 @@ export function acquireOrcadInstanceLock( acquiredAt: (hooks.now ?? (() => new Date()))().toISOString(), nonce: randomUUID(), role: hooks.role ?? 'orcad', - ...(machineIdentity ? { machineIdentity } : {}) + ...(machineIdentity ? { machineIdentity } : {}), + host: hostname() } const serialized = JSON.stringify(record) @@ -202,6 +217,12 @@ export function acquireOrcadInstanceLock( const dataRootIsLocal = (): boolean => (hooks.dataRootIsOnLocalDisk ?? orcadDataRootIsOnLocalDisk)(dataRoot) + const judgedHere = (holder: OrcadLockRecord, role: 'holder' | 'recovery-owner'): boolean => { + const locality = orcadLockHolderLocality(holder, record, dataRootIsLocal) + return ( + locality === 'local' || (locality === 'unknown' && !!hooks.reclaimConfirmed?.(holder, role)) + ) + } // A recovery lock is ours by the same record, and broken only on the same proof of exit. const recoveryOwner: OrcadLockRecoveryOwner = { record: serialized, @@ -210,7 +231,7 @@ export function acquireOrcadInstanceLock( return Boolean( owner && owner.identity === identity && - orcadLockHolderLocality(owner, record, dataRootIsLocal) === 'local' && + judgedHere(owner, 'recovery-owner') && (!isAlive(owner.pid) || holderIdentity(owner) === 'differs') ) } @@ -247,17 +268,20 @@ export function acquireOrcadInstanceLock( 'root corrupts it. Give each its own ORCA_USER_DATA.' ) } - const locality = orcadLockHolderLocality(existing, record, dataRootIsLocal) - if (locality !== 'local') { + if (!judgedHere(existing, 'holder')) { // Its PID means nothing here, so neither a missing nor a reused one proves it exited. - throw new OrcadInstanceLockError( - 'orcad_instance_lock_held', - `${describeLockHolder(existing)} (pid ${existing.pid}, started ${existing.acquiredAt}) ` + - `holds the data root ${dataRoot}${locality === 'foreign' ? ' from another machine' : ''}. ` + - 'Orca cannot check from here whether it is still running, because this data root may be ' + - `shared with other machines. Stop it there, or if it is not running anywhere, delete ` + - `${lockPath} and try again.` - ) + throw orcadLockHolderUnverifiableError({ + holder: `${describeLockHolder(existing)} (pid ${existing.pid}, started ${existing.acquiredAt})`, + foreign: orcadLockHolderLocality(existing, record, dataRootIsLocal) === 'foreign', + dataRoot, + lockPath, + hintRuntime: + hooks.reclaimHintRuntime !== undefined + ? hooks.reclaimHintRuntime + : hooks.role === 'desktop' + ? null + : { execPath: process.execPath, entry: process.argv[1] } + }) } // The desktop takes this lock only after Electron's single-instance lock, which already proves // no other desktop runs; a desktop record is then stale even when its PID was reused. @@ -278,47 +302,6 @@ export function acquireOrcadInstanceLock( return makeLock(lockPath, record) } -/** - * Same machine: the same machine identity, or (records without one) the same kernel boot. A record - * that names neither is local only when no other machine can reach the data root. - */ -function orcadLockHolderLocality( - holder: OrcadPeerMachine, - self: OrcadPeerMachine, - dataRootIsOnLocalDisk: () => boolean -): 'local' | 'foreign' | 'unknown' { - if (holder.machineIdentity && self.machineIdentity) { - return holder.machineIdentity === self.machineIdentity ? 'local' : 'foreign' - } - if (holder.bootId && holder.bootId === self.bootId) { - return 'local' - } - return dataRootIsOnLocalDisk() ? 'local' : 'unknown' -} - -export type OrcadPeerMachine = Pick - -export function currentOrcadPeerMachine(): OrcadPeerMachine { - const machineIdentity = profileStateAccessMachineIdentity() - const bootId = readOrcadProcessIncarnation(process.pid)?.bootId - return { ...(machineIdentity ? { machineIdentity } : {}), ...(bootId ? { bootId } : {}) } -} - -/** - * Whether a peer's recorded PID means anything here: same machine and kernel boot. A different or - * unprovable one leaves the peer unverifiable, never exited. - */ -export function orcadPeerPidIsJudgeableHere( - peer: OrcadPeerMachine, - self: OrcadPeerMachine, - dataRootIsOnLocalDisk: () => boolean -): boolean { - if (peer.bootId && self.bootId && peer.bootId !== self.bootId) { - return false - } - return orcadLockHolderLocality(peer, self, dataRootIsOnLocalDisk) === 'local' -} - /** A record without a boot id cannot tell its holder from a reused PID; say so. */ function possiblyReusedPidError( lockPath: string, diff --git a/src/main/orcad/orcad-lock-holder-locality.ts b/src/main/orcad/orcad-lock-holder-locality.ts new file mode 100644 index 00000000000..0f95b9bc34a --- /dev/null +++ b/src/main/orcad/orcad-lock-holder-locality.ts @@ -0,0 +1,73 @@ +/** Whether an orcad lock or peer record was written on this machine, where its PID means something. */ +import process from 'node:process' +import { profileStateAccessMachineIdentity } from '../persistence/profile-state/profile-state-access-identity' +import { readOrcadProcessIncarnation } from './orcad-process-start-time' +import { OrcadInstanceLockError } from './orcad-data-root-privacy' +import { + orcadStaleLockReclaimCommand, + type OrcadReclaimHintRuntime +} from '../../shared/orcad-stale-lock-reclaim' + +/** + * Same machine: the same machine identity, or (records without one) the same kernel boot. A record + * that names neither is local only when no other machine can reach the data root. + */ +export function orcadLockHolderLocality( + holder: OrcadPeerMachine, + self: OrcadPeerMachine, + dataRootIsOnLocalDisk: () => boolean +): 'local' | 'foreign' | 'unknown' { + if (holder.machineIdentity && self.machineIdentity) { + return holder.machineIdentity === self.machineIdentity ? 'local' : 'foreign' + } + if (holder.bootId && holder.bootId === self.bootId) { + return 'local' + } + return dataRootIsOnLocalDisk() ? 'local' : 'unknown' +} + +export type OrcadPeerMachine = { machineIdentity?: string; bootId?: string } + +export function currentOrcadPeerMachine(): OrcadPeerMachine { + const machineIdentity = profileStateAccessMachineIdentity() + const bootId = readOrcadProcessIncarnation(process.pid)?.bootId + return { ...(machineIdentity ? { machineIdentity } : {}), ...(bootId ? { bootId } : {}) } +} + +/** + * Whether a peer's recorded PID means anything here: same machine and kernel boot. A different or + * unprovable one leaves the peer unverifiable, never exited. + */ +export function orcadPeerPidIsJudgeableHere( + peer: OrcadPeerMachine, + self: OrcadPeerMachine, + dataRootIsOnLocalDisk: () => boolean +): boolean { + if (peer.bootId && self.bootId && peer.bootId !== self.bootId) { + return false + } + return orcadLockHolderLocality(peer, self, dataRootIsOnLocalDisk) === 'local' +} + +/** A lock this machine cannot judge; on Linux it names the command that checks and clears it. */ +export function orcadLockHolderUnverifiableError(lock: { + holder: string + foreign: boolean + dataRoot: string + lockPath: string + /** An orcad names its own runtime and entry: managed hosts have no `orca` or `orcad` on PATH. */ + hintRuntime: OrcadReclaimHintRuntime | null +}): OrcadInstanceLockError { + const command = orcadStaleLockReclaimCommand(lock.dataRoot, lock.hintRuntime) + const reclaim = + process.platform === 'linux' + ? `run \`${command}\` on the machine it ran on to check and clear it, or ` + : '' + return new OrcadInstanceLockError( + 'orcad_instance_lock_holder_unverifiable', + `${lock.holder} holds the data root ${lock.dataRoot}${lock.foreign ? ' from another machine' : ''}. ` + + 'Orca cannot check from here whether it is still running, because this data root may be ' + + `shared with other machines. Stop it there. If it is not running anywhere, ${reclaim}` + + `delete ${lock.lockPath} once you are sure.` + ) +} diff --git a/src/main/orcad/orcad-stale-lock-reclaim.test.ts b/src/main/orcad/orcad-stale-lock-reclaim.test.ts new file mode 100644 index 00000000000..eb357eba2a3 --- /dev/null +++ b/src/main/orcad/orcad-stale-lock-reclaim.test.ts @@ -0,0 +1,303 @@ +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + utimesSync, + writeFileSync +} from 'node:fs' +import { createServer, type Server } from 'node:net' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { ORCAD_LOCK_FILE_NAME, readOrcadInstanceLockRecord } from './orcad-instance-lock' +import type { OrcadLockRecord } from './orcad-instance-lock' +import { orcadLockRecoveryPrefix } from './orcad-instance-lock-reclaim' +import { + probeOrcadEndpoint, + reclaimStaleOrcadInstanceLock, + type OrcadStaleLockReclaimHooks +} from './orcad-stale-lock-reclaim' + +const roots: string[] = [] +const servers: Server[] = [] +afterEach(() => { + for (const server of servers.splice(0)) { + server.close() + } + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +// Why /tmp: a Unix socket path under macOS's long tmpdir exceeds the 104-byte limit. +function makeRoot(): string { + const root = mkdtempSync('/tmp/orcad-reclaim-') + roots.push(root) + return root +} + +const HOLDER_PID = 15698 + +/** The issue's record: written on host `box` in boot-1, ten minutes ago, naming no machine id. */ +function holderRecord(overrides: Partial = {}): OrcadLockRecord { + return { + pid: HOLDER_PID, + startedAtMs: 1, + linuxStartTicks: '77', + bootId: 'boot-1', + identity: '1000', + version: '1.0.0', + acquiredAt: new Date(Date.now() - 10 * 60_000).toISOString(), + nonce: 'stale-holder', + role: 'orcad', + host: 'box', + ...overrides + } +} + +/** Host `box` after a reboot, with an empty machine id and its home on a network mount. */ +function rebootedBox(overrides: OrcadStaleLockReclaimHooks = {}): OrcadStaleLockReclaimHooks { + return { + hostname: () => 'box', + bootId: () => 'boot-2', + machineIdentity: () => null, + dataRootIsOnLocalDisk: () => false, + lockHooks: { + identity: () => '1000', + machineIdentity: () => null, + dataRootIsOnLocalDisk: () => false, + processIsAlive: () => false, + incarnation: () => ({ linuxStartTicks: '9', bootId: 'boot-2' }) + }, + ...overrides + } +} + +function writeLock(root: string, record: OrcadLockRecord): string { + const lockPath = join(root, ORCAD_LOCK_FILE_NAME) + writeFileSync(lockPath, JSON.stringify(record), { mode: 0o600 }) + const writtenAt = new Date(record.acquiredAt) + utimesSync(lockPath, writtenAt, writtenAt) + return lockPath +} + +// POSIX ownership, modes and Unix sockets; the command only ever applies to Linux hosts. +describe.skipIf(process.platform === 'win32')('reclaimStaleOrcadInstanceLock (#32177)', () => { + it('clears a lock from an earlier boot on this host once every check passes and --yes', async () => { + const root = makeRoot() + const lockPath = writeLock(root, holderRecord()) + const result = await reclaimStaleOrcadInstanceLock( + root, + { yes: true, hintRuntime: null }, + rebootedBox() + ) + expect(result).toMatchObject({ status: 'reclaimed', exitCode: 0 }) + expect(existsSync(lockPath)).toBe(false) + }) + + it('never reports an empty data root as a recovery', async () => { + const result = await reclaimStaleOrcadInstanceLock( + makeRoot(), + { yes: true, hintRuntime: null }, + rebootedBox() + ) + expect(result).toMatchObject({ + status: 'none', + exitCode: 4, + message: expect.stringContaining('--data-root') + }) + }) + + it('without --yes, names what it found and warns about other machines, then refuses', async () => { + const root = makeRoot() + writeLock(root, holderRecord()) + const result = await reclaimStaleOrcadInstanceLock( + root, + { yes: false, hintRuntime: null }, + rebootedBox() + ) + expect(result).toMatchObject({ + status: 'eligible', + exitCode: 2, + evidence: { pid: HOLDER_PID, host: 'box', bootId: 'boot-1', currentBootId: 'boot-2' } + }) + for (const evidence of [`pid ${HOLDER_PID}`, 'host box', 'boot boot-1', '10 minutes ago']) { + expect(result.message).toContain(evidence) + } + expect(result.message).toContain('nothing answers on this machine') + expect(result.message).toContain('cannot rule out another machine with the same host name') + expect(result.message).toContain('no other machine sharing this home runs Orca') + expect(readOrcadInstanceLockRecord(join(root, ORCAD_LOCK_FILE_NAME))?.nonce).toBe( + 'stale-holder' + ) + }) + + it('refuses while the holder still answers at its endpoint', async () => { + const root = makeRoot() + writeLock(root, holderRecord()) + const server = createServer() + servers.push(server) + await new Promise((resolve) => + server.listen(join(root, `o-${HOLDER_PID}-rt.sock`), resolve) + ) + const result = await reclaimStaleOrcadInstanceLock( + root, + { yes: true, hintRuntime: null }, + rebootedBox() + ) + expect(result).toMatchObject({ + status: 'refused', + exitCode: 1, + message: expect.stringContaining('Something on this machine answers') + }) + expect(readOrcadInstanceLockRecord(join(root, ORCAD_LOCK_FILE_NAME))?.nonce).toBe( + 'stale-holder' + ) + }) + + it('also clears a reclaimer from the same earlier boot that the reboot cut short', async () => { + const root = makeRoot() + const lockPath = writeLock(root, holderRecord()) + const generation = `${orcadLockRecoveryPrefix(lockPath, readFileSync(lockPath, 'utf8'))}0` + mkdirSync(generation) + writeFileSync( + join(generation, 'owner'), + JSON.stringify(holderRecord({ pid: 16000, nonce: 'cut-short-reclaimer' })) + ) + const result = await reclaimStaleOrcadInstanceLock( + root, + { yes: true, hintRuntime: null }, + rebootedBox() + ) + expect(result.exitCode).toBe(0) + expect(existsSync(lockPath)).toBe(false) + }) + + // Root reads through mode 000, so the case only exists for an ordinary user. + it.skipIf(process.getuid?.() === 0)( + 'refuses, rather than reporting nothing to do, when the lock cannot be reached', + async () => { + const root = makeRoot() + writeLock(root, holderRecord()) + chmodSync(root, 0o000) + try { + const result = await reclaimStaleOrcadInstanceLock( + root, + { yes: true, hintRuntime: null }, + rebootedBox() + ) + expect(result).toMatchObject({ + exitCode: 1, + message: expect.stringContaining('cannot be read') + }) + } finally { + chmodSync(root, 0o700) + } + } + ) + + it('probes an IPv6 WebSocket endpoint by its bare address', async () => { + const server = createServer() + servers.push(server) + await new Promise((resolve) => server.listen(0, '::1', resolve)) + const address = server.address() + const port = typeof address === 'object' && address ? address.port : 0 + await expect(probeOrcadEndpoint(`ws://[::1]:${port}`)).resolves.toBe('answers') + }) + + it('reclaims a holder whose PID was reused after the reboot, but not one it cannot identify', async () => { + const reused = makeRoot() + const reusedLock = writeLock(reused, holderRecord()) + const alive = (incarnation: { linuxStartTicks: string; bootId: string } | null) => + rebootedBox({ + lockHooks: { + ...rebootedBox().lockHooks, + processIsAlive: () => true, + incarnation: () => incarnation + } + }) + // Another process now has that PID: its incarnation names this boot, not the record's. + const reclaimed = await reclaimStaleOrcadInstanceLock( + reused, + { yes: true, hintRuntime: null }, + alive({ linuxStartTicks: '9', bootId: 'boot-2' }) + ) + expect(reclaimed.status).toBe('reclaimed') + expect(existsSync(reusedLock)).toBe(false) + + const unreadable = makeRoot() + writeLock(unreadable, holderRecord()) + const refused = await reclaimStaleOrcadInstanceLock( + unreadable, + { yes: true, hintRuntime: null }, + alive(null) + ) + expect(refused.status).toBe('refused') + expect(readOrcadInstanceLockRecord(join(unreadable, ORCAD_LOCK_FILE_NAME))?.nonce).toBe( + 'stale-holder' + ) + }) + + it('never clears a record other than the one it checked', async () => { + const root = makeRoot() + const lockPath = writeLock(root, holderRecord()) + // A different holder's record lands after the checks and before the reclaim. + const swap = async (): Promise<'silent'> => { + writeLock(root, holderRecord({ nonce: 'newer-holder' })) + return 'silent' + } + writeFileSync(join(root, `o-${HOLDER_PID}-rt.sock`), '') + const result = await reclaimStaleOrcadInstanceLock( + root, + { yes: true, hintRuntime: null }, + rebootedBox({ probeEndpoint: swap }) + ) + expect(result.status).toBe('refused') + expect(readOrcadInstanceLockRecord(lockPath)?.nonce).toBe('newer-holder') + // The refusal names `orca serve`, never the binary this command happens to run under. + expect(result.message).not.toContain(process.execPath) + if (process.platform === 'linux') { + expect(result.message).toContain(`orca serve --reclaim-stale-lock --data-root ${root}`) + } + }) + + it('answers a TCP probe only for a listener, and anything it cannot check as unknown', async () => { + const server = createServer() + servers.push(server) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const address = server.address() + const port = typeof address === 'object' && address ? address.port : 0 + await expect(probeOrcadEndpoint(`ws://127.0.0.1:${port}`)).resolves.toBe('answers') + await expect(probeOrcadEndpoint('ws://[::1')).resolves.toBe('unknown') + }) + + it.each([ + ['another host', holderRecord({ host: 'other-box' }), 'not this host'], + // The same boot is this machine's to judge: a normal start does, by its PID. + ['the same boot', holderRecord({ bootId: 'boot-2' }), 'judge this lock itself'], + ['an older orcad that recorded no host', holderRecord({ host: undefined }), 'which host'], + [ + 'a lock taken a moment ago', + holderRecord({ acquiredAt: new Date().toISOString() }), + 'minutes old' + ] + ])('refuses a lock from %s', async (_case, record, reason) => { + const root = makeRoot() + writeLock(root, record) + const result = await reclaimStaleOrcadInstanceLock( + root, + { yes: true, hintRuntime: null }, + rebootedBox() + ) + expect(result).toMatchObject({ + status: 'refused', + exitCode: 1, + message: expect.stringContaining(reason) + }) + expect(readOrcadInstanceLockRecord(join(root, ORCAD_LOCK_FILE_NAME))?.nonce).toBe( + 'stale-holder' + ) + }) +}) diff --git a/src/main/orcad/orcad-stale-lock-reclaim.ts b/src/main/orcad/orcad-stale-lock-reclaim.ts new file mode 100644 index 00000000000..cec8c59aeee --- /dev/null +++ b/src/main/orcad/orcad-stale-lock-reclaim.ts @@ -0,0 +1,318 @@ +/** + * `--reclaim-stale-lock`: clears an orcad instance lock that no automatic start may judge (#32177). + * + * A lock on a home other machines may share names a PID that is evidence only on the machine that + * issued it, so a start after a reboot refuses it rather than guess. This command gathers what this + * machine can check (same host name, an earlier boot, an old record, nothing answering on this + * machine at the holder's endpoints), shows it, and clears the lock only when the user confirms + * with --yes that no other machine sharing the home runs Orca. Losing contact is never treated as + * exit without that confirmation. + */ +import { readdirSync, statSync } from 'node:fs' +import { createConnection } from 'node:net' +import { hostname } from 'node:os' +import { join } from 'node:path' +import process from 'node:process' +import { + acquireOrcadInstanceLock, + ORCAD_LOCK_FILE_NAME, + parseOrcadInstanceLockRecord, + type OrcadInstanceLockHooks, + type OrcadLockRecord +} from './orcad-instance-lock' +import { readBoundedLockFile } from './orcad-instance-lock-reclaim' +import { orcadLockHolderLocality } from './orcad-lock-holder-locality' +import { orcadDataRootIsOnLocalDisk } from './orcad-data-root-locality' +import { readOrcadProcessIncarnation } from './orcad-process-start-time' +import { hasErrorCode } from '../daemon/daemon-process-inspection' +import { resolveUserDataPath } from './orcad-app-paths' +import { profileStateAccessMachineIdentity } from '../persistence/profile-state/profile-state-access-identity' +import { readRuntimeMetadata } from '../runtime/runtime-metadata' +import { RUNTIME_SOCKET_NAME_REGEX } from '../runtime/runtime-rpc/runtime-rpc-socket-metadata' +import { + ORCAD_RECLAIM_STALE_LOCK_FLAG, + ORCAD_STALE_LOCK_DATA_ROOT_FLAG, + ORCAD_STALE_LOCK_RECLAIM_EXIT_CODES, + type OrcadReclaimHintRuntime, + type OrcadStaleLockReclaimStatus +} from '../../shared/orcad-stale-lock-reclaim' + +// A holder that just took the lock may not be listening yet; its silence would prove nothing. +export const ORCAD_STALE_LOCK_MIN_AGE_MS = 2 * 60_000 +const ENDPOINT_PROBE_TIMEOUT_MS = 1_500 + +/** `silent` only on a refusal the OS gives for an endpoint nobody listens on; anything else is not. */ +export type OrcadEndpointProbe = 'answers' | 'silent' | 'unknown' + +export type OrcadStaleLockReclaimHooks = { + now?: () => number + hostname?: () => string + bootId?: () => string | null + machineIdentity?: () => string | null + dataRootIsOnLocalDisk?: (dataRoot: string) => boolean + probeEndpoint?: (endpoint: string) => Promise + lockHooks?: OrcadInstanceLockHooks +} + +export type OrcadStaleLockEvidence = { + pid: number + host: string + bootId: string + currentBootId: string + ageMinutes: number +} + +export type OrcadStaleLockReclaimResult = { + status: OrcadStaleLockReclaimStatus + exitCode: number + lockPath: string + message: string + evidence?: OrcadStaleLockEvidence +} + +export async function reclaimStaleOrcadInstanceLock( + dataRoot: string, + options: { + yes: boolean + /** Who to name in a refusal's command: an orcad names its own runtime, the CLI `orca serve`. */ + hintRuntime: OrcadReclaimHintRuntime | null + }, + hooks: OrcadStaleLockReclaimHooks = {} +): Promise { + const lockPath = join(dataRoot, ORCAD_LOCK_FILE_NAME) + const result = ( + status: OrcadStaleLockReclaimStatus, + message: string, + evidence?: OrcadStaleLockEvidence + ): OrcadStaleLockReclaimResult => ({ + status, + exitCode: ORCAD_STALE_LOCK_RECLAIM_EXIT_CODES[status], + lockPath, + message, + ...(evidence ? { evidence } : {}) + }) + const contents = readBoundedLockFile(lockPath) + if (contents === null) { + // Only a confirmed absence is "none"; a permission or I/O error hides a lock. + return lockIsAbsent(lockPath) + ? result( + 'none', + `No orcad instance lock at ${lockPath}. If a start reported a lock in another data ` + + `root, re-run with ${ORCAD_STALE_LOCK_DATA_ROOT_FLAG} and the directory it named.` + ) + : result('refused', `The lock ${lockPath} cannot be read. Check it by hand.`) + } + const holder = parseOrcadInstanceLockRecord(contents) + if (!holder) { + return result('refused', `The lock ${lockPath} is not a complete record. Check it by hand.`) + } + const currentBoot = (hooks.bootId ?? currentBootId)() + const checked = await checkStaleLock(dataRoot, lockPath, holder, currentBoot, hooks) + if ('refusal' in checked) { + return result('refused', `${checked.refusal} Not reclaiming ${lockPath}.`) + } + const { evidence } = checked + const found = + `${holder.role === 'desktop' ? 'The Orca desktop app' : 'orcad'} pid ${evidence.pid} on ` + + `host ${evidence.host} took ${lockPath} ${evidence.ageMinutes} minutes ago, in boot ` + + `${evidence.bootId} (this machine is now in boot ${evidence.currentBootId}), and nothing ` + + 'answers on this machine at its endpoints. The host name and boot checks cannot rule out ' + + 'another machine with the same host name that shares this home.' + if (!options.yes) { + return result( + 'eligible', + `${found}\nMake sure no other machine sharing this home runs Orca for this profile, then ` + + 'run the same command again with --yes added.', + evidence + ) + } + try { + // Why through the normal acquire: it reclaims only this exact record, through the same + // race-safe rename, and still refuses if that PID is running here under the same identity. + acquireOrcadInstanceLock(dataRoot, { + ...hooks.lockHooks, + reclaimHintRuntime: options.hintRuntime, + // Only the record shown above; a reclaimer cut short by the same reboot is just as stale. + reclaimConfirmed: (record, role) => + role === 'holder' + ? record.nonce === holder.nonce + : record.host === holder.host && !!record.bootId && record.bootId !== currentBoot + }).release() + } catch (error) { + return result('refused', error instanceof Error ? error.message : String(error), evidence) + } + return result('reclaimed', `${found}\nReclaimed ${lockPath}. Orca can start now.`, evidence) +} + +async function checkStaleLock( + dataRoot: string, + lockPath: string, + holder: OrcadLockRecord, + bootId: string | null, + hooks: OrcadStaleLockReclaimHooks +): Promise<{ refusal: string } | { evidence: OrcadStaleLockEvidence }> { + const machineIdentity = (hooks.machineIdentity ?? profileStateAccessMachineIdentity)() + const self = { ...(machineIdentity ? { machineIdentity } : {}), ...(bootId ? { bootId } : {}) } + const locality = orcadLockHolderLocality(holder, self, () => + (hooks.dataRootIsOnLocalDisk ?? orcadDataRootIsOnLocalDisk)(dataRoot) + ) + // Covers a record from this boot too: the same boot proves the same machine. + if (locality === 'local') { + return { + refusal: + 'This machine can judge this lock itself, so a normal start reclaims it if its holder exited.' + } + } + if (locality === 'foreign') { + return { refusal: 'The lock was written by another machine (a different machine id).' } + } + const host = (hooks.hostname ?? hostname)() + if (!holder.host) { + return { + refusal: 'The lock does not record which host wrote it (an older orcad wrote it).' + } + } + if (holder.host !== host) { + return { refusal: `The lock was written on host ${holder.host}, not this host (${host}).` } + } + if (!holder.bootId || !bootId) { + return { refusal: 'The lock or this machine has no boot id to compare.' } + } + const now = (hooks.now ?? Date.now)() + const writtenMs = Math.max(Date.parse(holder.acquiredAt), statOrNull(lockPath)?.mtimeMs ?? now) + if (!(now - writtenMs >= ORCAD_STALE_LOCK_MIN_AGE_MS)) { + return { refusal: `The lock is less than ${ORCAD_STALE_LOCK_MIN_AGE_MS / 60_000} minutes old.` } + } + const endpoints = holderEndpoints(dataRoot, holder.pid) + if (!endpoints) { + return { refusal: 'The runtime record in the data root cannot be read.' } + } + const probe = hooks.probeEndpoint ?? probeOrcadEndpoint + for (const endpoint of endpoints) { + const answer = await probe(endpoint) + if (answer === 'answers') { + return { + refusal: `Something on this machine answers at ${endpoint}, so it may still be running.` + } + } + if (answer === 'unknown') { + return { refusal: `Whether anything answers at ${endpoint} could not be checked.` } + } + } + return { + evidence: { + pid: holder.pid, + host: holder.host, + bootId: holder.bootId, + currentBootId: bootId, + ageMinutes: Math.round((now - Date.parse(holder.acquiredAt)) / 60_000) + } + } +} + +/** The holder's sockets and recorded transports; null when its runtime record is unreadable. */ +function holderEndpoints(dataRoot: string, pid: number): string[] | null { + const endpoints = new Set() + try { + const metadata = readRuntimeMetadata(dataRoot) + if (metadata?.pid === pid) { + for (const transport of metadata.transports ?? []) { + endpoints.add(transport.endpoint) + } + } + for (const entry of readdirSync(dataRoot)) { + if (RUNTIME_SOCKET_NAME_REGEX.exec(entry)?.[1] === String(pid)) { + endpoints.add(join(dataRoot, entry)) + } + } + } catch { + return null + } + return [...endpoints] +} + +export function probeOrcadEndpoint(endpoint: string): Promise { + let target: { path: string } | { host: string; port: number } + try { + const url = /^wss?:\/\//u.test(endpoint) ? new URL(endpoint) : null + const defaultPort = url?.protocol === 'wss:' ? 443 : 80 + // Why strip brackets: URL keeps them around an IPv6 host, which the socket API rejects. + target = url + ? { host: url.hostname.replace(/^\[|\]$/gu, ''), port: Number(url.port || defaultPort) } + : { path: endpoint } + } catch { + return Promise.resolve('unknown') + } + return new Promise((resolve) => { + const socket = createConnection(target) + const finish = (answer: OrcadEndpointProbe): void => { + clearTimeout(timer) + socket.destroy() + resolve(answer) + } + const timer = setTimeout(() => finish('unknown'), ENDPOINT_PROBE_TIMEOUT_MS) + socket.once('connect', () => finish('answers')) + socket.once('error', (error: NodeJS.ErrnoException) => + finish(error.code === 'ECONNREFUSED' || error.code === 'ENOENT' ? 'silent' : 'unknown') + ) + }) +} + +function currentBootId(): string | null { + return readOrcadProcessIncarnation(process.pid)?.bootId ?? null +} + +function lockIsAbsent(path: string): boolean { + try { + statSync(path) + return false + } catch (error) { + return hasErrorCode(error, 'ENOENT') + } +} + +function statOrNull(path: string): { mtimeMs: number } | null { + try { + return statSync(path) + } catch { + return null + } +} + +/** Prints a result as one line of JSON or as text, the way both entry points report it. */ +export function printOrcadStaleLockReclaimResult( + result: OrcadStaleLockReclaimResult, + json: boolean +): void { + if (json) { + console.log(JSON.stringify(result)) + return + } + ;(result.status === 'reclaimed' ? console.log : console.error)(result.message) +} + +/** `orcad --reclaim-stale-lock [--data-root ] [--yes] [--json]`. */ +export async function runOrcadStaleLockReclaimCommandAndExit(argv: string[]): Promise { + const args = argv.slice(1) + const json = args.includes('--json') + const dataRootIndex = args.indexOf(ORCAD_STALE_LOCK_DATA_ROOT_FLAG) + const dataRoot = dataRootIndex === -1 ? resolveUserDataPath() : args[dataRootIndex + 1] + const unknown = args.filter( + (arg, index) => + !['--yes', '--json', ORCAD_STALE_LOCK_DATA_ROOT_FLAG].includes(arg) && + (dataRootIndex === -1 || index !== dataRootIndex + 1) + ) + // A missing value must never fall through to another data root, nor read a flag as a path. + if (!dataRoot?.trim() || dataRoot.startsWith('-') || unknown.length) { + console.error( + `Usage: orcad ${ORCAD_RECLAIM_STALE_LOCK_FLAG} [${ORCAD_STALE_LOCK_DATA_ROOT_FLAG} ] [--yes] [--json]` + ) + process.exit(ORCAD_STALE_LOCK_RECLAIM_EXIT_CODES.refused) + } + const result = await reclaimStaleOrcadInstanceLock(dataRoot, { + yes: args.includes('--yes'), + hintRuntime: { execPath: process.execPath, entry: process.argv[1] } + }) + printOrcadStaleLockReclaimResult(result, json) + process.exit(result.exitCode) +} diff --git a/src/main/startup/desktop-profile-instance-lock.test.ts b/src/main/startup/desktop-profile-instance-lock.test.ts index 938ff74d174..5ff32b03339 100644 --- a/src/main/startup/desktop-profile-instance-lock.test.ts +++ b/src/main/startup/desktop-profile-instance-lock.test.ts @@ -14,21 +14,25 @@ describe('the desktop share of the profile instance lock', () => { expect(acquire).toHaveBeenCalledWith('/profile', { role: 'desktop' }) }) - it('refuses while a live orcad serves the profile, and says so', () => { - const write = vi.fn() - const result = acquireDesktopProfileInstanceLock( - '/profile', - () => { - throw new OrcadInstanceLockError('orcad_instance_lock_held', 'Another orcad (pid 7) ...') - }, - write - ) - expect(result).toEqual({ - state: 'held', - message: 'Another orcad (pid 7) ...' - }) - expect(write).toHaveBeenCalledWith(2, '[single-instance] Another orcad (pid 7) ...\n') - }) + // #32177: a holder this machine cannot judge (a shared home) may be running, so it is held too. + it.each(['orcad_instance_lock_held', 'orcad_instance_lock_holder_unverifiable'] as const)( + 'refuses while an orcad may serve the profile (%s), and says so', + (code) => { + const write = vi.fn() + const result = acquireDesktopProfileInstanceLock( + '/profile', + () => { + throw new OrcadInstanceLockError(code, 'Another orcad (pid 7) ...') + }, + write + ) + expect(result).toEqual({ + state: 'held', + message: 'Another orcad (pid 7) ...' + }) + expect(write).toHaveBeenCalledWith(2, '[single-instance] Another orcad (pid 7) ...\n') + } + ) it('still starts when the lock cannot be taken for any other reason', () => { vi.spyOn(console, 'warn').mockImplementation(() => {}) diff --git a/src/main/startup/desktop-profile-instance-lock.ts b/src/main/startup/desktop-profile-instance-lock.ts index b16c5c4563a..845bfe63cec 100644 --- a/src/main/startup/desktop-profile-instance-lock.ts +++ b/src/main/startup/desktop-profile-instance-lock.ts @@ -26,7 +26,12 @@ export function acquireDesktopProfileInstanceLock( process.once('exit', () => lock.release()) return { state: 'acquired', lock } } catch (error) { - if (error instanceof OrcadInstanceLockError && error.code === 'orcad_instance_lock_held') { + // A holder this machine cannot judge may still be running: refusing keeps one writer. + if ( + error instanceof OrcadInstanceLockError && + (error.code === 'orcad_instance_lock_held' || + error.code === 'orcad_instance_lock_holder_unverifiable') + ) { writeStartupDiagnosticLine(`[single-instance] ${error.message}`, write) return { state: 'held', message: error.message } } diff --git a/src/shared/cli-argument-boundary.ts b/src/shared/cli-argument-boundary.ts index da732cab23f..795958c38d2 100644 --- a/src/shared/cli-argument-boundary.ts +++ b/src/shared/cli-argument-boundary.ts @@ -40,6 +40,7 @@ export const CLI_BOOLEAN_FLAGS = new Set([ 'provision', 'ready', 'recipe-json', + 'reclaim-stale-lock', 'references', 'relations', 'reinstall', diff --git a/src/shared/orcad-runtime-capabilities.ts b/src/shared/orcad-runtime-capabilities.ts index d732e437a49..b7f13542e46 100644 --- a/src/shared/orcad-runtime-capabilities.ts +++ b/src/shared/orcad-runtime-capabilities.ts @@ -25,7 +25,12 @@ export const ORCAD_MIGRATION_BLOB_CHANNEL_RUNTIME_CAPABILITY = 'orcad.migration- // WSL host's) pushes them only to a host advertising this. export const HOST_ACCOUNT_LAUNCH_ENV_RUNTIME_CAPABILITY = 'accounts.host-launch-env.v1' as const +// Why: older orcads reject `--reclaim-stale-lock` as an unknown argument; a client offers to run it +// on a host only when this is advertised, and otherwise shows the manual instruction. +export const ORCAD_STALE_LOCK_RECLAIM_RUNTIME_CAPABILITY = 'orcad.stale-lock-reclaim.v1' as const + export const ORCAD_RUNTIME_CAPABILITIES = [ + ORCAD_STALE_LOCK_RECLAIM_RUNTIME_CAPABILITY, ORCAD_TERMINAL_CENSUS_RUNTIME_CAPABILITY, ORCAD_MIGRATION_CATALOG_RUNTIME_CAPABILITY, ORCAD_MIGRATION_CATALOG_ABORT_BY_REFERENCE_RUNTIME_CAPABILITY, diff --git a/src/shared/orcad-stale-lock-reclaim.ts b/src/shared/orcad-stale-lock-reclaim.ts new file mode 100644 index 00000000000..fbe22abb04f --- /dev/null +++ b/src/shared/orcad-stale-lock-reclaim.ts @@ -0,0 +1,36 @@ +/** + * `--reclaim-stale-lock`: the explicit recovery for an orcad instance lock that no automatic start + * may judge, such as one on a shared home after a reboot (#32177). Dependency-free so the CLI, + * orcad's launcher and clients can all name it and read its result. + */ +import { quotePosixShellUnlessPlain } from './shell-argument-quoting' + +export const ORCAD_RECLAIM_STALE_LOCK_FLAG = '--reclaim-stale-lock' +export const ORCAD_STALE_LOCK_DATA_ROOT_FLAG = '--data-root' + +/** `--json` prints one of these; the exit code names the status too. */ +export type OrcadStaleLockReclaimStatus = 'none' | 'eligible' | 'refused' | 'reclaimed' + +// Why `none` is not 0: a command pointed at the wrong data root must never read as a recovery. +export const ORCAD_STALE_LOCK_RECLAIM_EXIT_CODES: Record = { + reclaimed: 0, + refused: 1, + eligible: 2, + none: 4 +} + +/** + * A command the user can paste on the holder's machine, naming this data root. An orcad names its + * own runtime and entry, since managed hosts have neither `orca` nor `orcad` on PATH. + */ +export type OrcadReclaimHintRuntime = { execPath: string; entry: string | undefined } + +export function orcadStaleLockReclaimCommand( + dataRoot: string, + runtime: OrcadReclaimHintRuntime | null +): string { + const program = runtime?.entry + ? `${quotePosixShellUnlessPlain(runtime.execPath)} ${quotePosixShellUnlessPlain(runtime.entry)}` + : 'orca serve' + return `${program} ${ORCAD_RECLAIM_STALE_LOCK_FLAG} ${ORCAD_STALE_LOCK_DATA_ROOT_FLAG} ${quotePosixShellUnlessPlain(dataRoot)}` +}