diff --git a/config/flaky-test-pattern-baseline.txt b/config/flaky-test-pattern-baseline.txt index 9fd13acadd1..e604c236e9a 100644 --- a/config/flaky-test-pattern-baseline.txt +++ b/config/flaky-test-pattern-baseline.txt @@ -866,7 +866,6 @@ 1 test-no-wall-clock:src/main/orca-profiles/profile-cloud-service.test.ts 4 test-no-wall-clock:src/main/orcad/orcad-artifact-cache-retention.test.ts 2 test-no-wall-clock:src/main/orcad/orcad-artifact-materializer.test.ts -2 test-no-wall-clock:src/main/orcad/orcad-instance-lock.test.ts 3 test-no-wall-clock:src/main/orcad/orcad-node-launcher.integration.test.ts 1 test-no-wall-clock:src/main/orcad/orcad-push-startup.test.ts 2 test-no-wall-clock:src/main/orcad/orcad-secret-store.test.ts diff --git a/src/cli/specs/serve.ts b/src/cli/specs/serve.ts index 373655695dc..a5dde0121a7 100644 --- a/src/cli/specs/serve.ts +++ b/src/cli/specs/serve.ts @@ -31,7 +31,7 @@ export const SERVE_COMMAND_SPECS: CommandSpec[] = [ "A paired client cannot drive this machine's desktop (orca computer) unless the offer was created with --grant-desktop-control.", 'When the web client bundle is available, the server also prints a browser URL with the pairing data embedded.', 'Exits with code 3, without starting anything, when another Orca already serves this profile or cannot be shown to have stopped.', - "--reclaim-stale-lock starts nothing. When a start refuses because this profile's lock may belong to an Orca on another machine sharing this home (for example after a reboot, on a network home), it checks that the lock was written on a host with this name in an earlier boot, is at least 2 minutes old, and that nothing answers on this machine at its holder's endpoints. Those checks cannot rule out another machine with the same host name, so it then only prints what it found (exit 2); add --yes, once you are sure no other machine sharing this home runs Orca for this profile, to clear it (exit 0). It exits 1 when a check fails and 4 when there is no lock. --data-root names the data root the refusal printed. --json prints the status (none, eligible, refused or reclaimed). Linux only.", + "--reclaim-stale-lock starts nothing. When a start refuses because this profile's lock may belong to an Orca on another machine sharing this home (for example after a reboot, on a network home), it checks that the lock was written on a host with this name in an earlier boot, was last written or refreshed at least 5 minutes ago, and that nothing answers on this machine at its holder's endpoints. Those checks cannot rule out another machine with the same host name, so it then only prints what it found (exit 2); add --yes, once you are sure no other machine sharing this home runs Orca for this profile, to clear it (exit 0). It exits 1 when a check fails and 4 when there is no lock. --data-root names the data root the refusal printed. --json prints the status (none, eligible, refused or reclaimed). Linux only.", 'On macOS, after Orca serve aborts at startup (it could not register with Launch Services), later runs from the same environment refuse for 30 minutes with exit code 78. Use --force to try anyway.', '--supervise is for service managers (launchd, systemd). Orca serve then restarts the server itself with growing delays instead of exiting when it stops, waits while an update replaces the app, and waits while another Orca serves this profile. After 6 failures in 15 minutes it gives up and exits with code 0 so the manager stops too, and later supervised runs refuse until a new build is installed or --force is passed. Not available on Windows yet.', 'Experimental and off by default: set ORCA_TERMINAL_PROCESS_CEILING= (64 or more) before the terminal host starts to have the OS cap how many processes each terminal may run (Linux, through a systemd user scope per terminal, which needs a user manager that outlives logout; Windows, through a job per terminal). `orca serve` reports the ceiling as on or off, with the reason.', diff --git a/src/main/managed-server/host-state/orcad-machine-home.test.ts b/src/main/managed-server/host-state/orcad-machine-home.test.ts index 401e086a94c..44379aafb68 100644 --- a/src/main/managed-server/host-state/orcad-machine-home.test.ts +++ b/src/main/managed-server/host-state/orcad-machine-home.test.ts @@ -97,6 +97,7 @@ type MachineOptions = { claimMoveError?: string /** Simulate stderr merged into stdout by an SSH gateway. */ catStderrNoise?: string + machineName?: string } /** Model macOS machines sharing one NFS home. */ @@ -157,7 +158,8 @@ function machine(name: string, hostname = name, options: MachineOptions = {}) { ...process.env, HOME: home, PATH: `${bin}:${process.env.PATH ?? ''}`, - ORCA_TEST_MERGE_STDERR: options.catStderrNoise === undefined ? '0' : '1' + ORCA_TEST_MERGE_STDERR: options.catStderrNoise === undefined ? '0' : '1', + ORCA_MACHINE_NAME: options.machineName ?? '' } // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the mocked execCommand reads only env. const conn = { @@ -347,6 +349,34 @@ describe.runIf(POSIX)('resolveOrcadMachineHome on a home several machines share' expect(await resolveOrcadMachineHome(a, true)).toEqual(loginHomeResult()) }) + it('gives a machine named in its environment its own home, even a clone of the claimer (#32918)', async () => { + // Clones copy their machine identity; only an explicit name separates them. + expect(await resolveOrcadMachineHome(machine('node-a', 'dev-a'), true)).toEqual( + loginHomeResult() + ) + expect(await resolveOrcadMachineHome(machine('node-a', 'dev-b'), true)).toEqual( + loginHomeResult() + ) + const named = machine('node-a', 'dev-b', { machineName: 'dev-b' }) + const namedHome = await resolveOrcadMachineHome(named, true) + expect(namedHome.userDataDir).toMatch(new RegExp(`^${home}/\\.orca-[a-f0-9]{8}$`)) + expect(await resolveOrcadMachineHome(named, true)).toEqual(namedHome) + const other = machine('node-a', 'dev-c', { machineName: 'dev-c' }) + expect((await resolveOrcadMachineHome(other, true)).userDataDir).not.toBe(namedHome.userDataDir) + }) + + it('refuses a machine name it cannot use instead of ignoring it', async () => { + await expect( + resolveOrcadMachineHome(machine('node-a', 'dev-a', { machineName: 'dev a!' }), true) + ).rejects.toThrow(/ORCA_MACHINE_NAME is set to "dev a!"/) + await expect( + resolveOrcadMachineHome(machine('node-a', 'dev-a', { machineName: 'dev\nb' }), true) + ).rejects.toThrow(/ORCA_MACHINE_NAME is set to "dev b"/) + await expect( + resolveOrcadMachineHome(machine('node-a', 'dev-a', { machineName: ' ' }), true) + ).rejects.toThrow(/ORCA_MACHINE_NAME is set to " {3}"/) + }) + it('keeps a renamed machine on its own install', async () => { expect(await resolveOrcadMachineHome(machine('node-a'), true)).toEqual(loginHomeResult()) expect(await resolveOrcadMachineHome(machine('node-a', 'renamed'), true)).toEqual( @@ -472,6 +502,7 @@ function probe(host: Host, overrides: Partial = {}): Probe { machineHomeKeys: [], loginHomeInstalled: false, homeOnLocalDisk: false, + machineName: null, ...overrides } } diff --git a/src/main/managed-server/host-state/orcad-machine-home.ts b/src/main/managed-server/host-state/orcad-machine-home.ts index 37cc8a5b646..b53abb7d112 100644 --- a/src/main/managed-server/host-state/orcad-machine-home.ts +++ b/src/main/managed-server/host-state/orcad-machine-home.ts @@ -1,7 +1,12 @@ // Shared-home hosts need per-machine state and runtime GC: a PID can prove ownership only on the machine that issued it (#29059). -import { createHash } from 'node:crypto' import { quotePosixShell } from '../../../shared/shell-argument-quoting' import { RELAY_REMOTE_DIR } from '../../ssh/remote-exec/remote-frame-protocol' +import { + machineKeyOf, + namedMachineHome, + ORCAD_MACHINE_NAME_PROBE, + parseOrcadMachineNameProbe +} from './orcad-machine-name-home' import { ORCAD_ACTIVATION_FILENAME } from '../activation/orcad-activation-record' import { parseDarwinPlatformUuid } from '../../persistence/profile-state/profile-state-access-identity' import { @@ -69,9 +74,11 @@ export type OrcadMachineHomeProbe = { loginHomeInstalled: boolean /** Local disks cannot belong to another machine's live server. */ homeOnLocalDisk: boolean + /** Raw ORCA_MACHINE_NAME; invalid names refuse rather than fall back silently. */ + machineName: string | null } -type OrcadMachineHomeDecision = +export type OrcadMachineHomeDecision = | { kind: 'login-home' } | { kind: 'claim'; machineKey: string } | { kind: 'machine-home'; machineKey: string } @@ -93,7 +100,8 @@ export function orcadMachineHomeProbeCommand(host: RemoteHostPlatform): string { '{ ioreg -rd1 -c IOPlatformExpertDevice 2>/dev/null ||' + ' /usr/sbin/ioreg -rd1 -c IOPlatformExpertDevice 2>/dev/null; } | grep IOPlatformUUID;' : 'cat /etc/machine-id 2>/dev/null;', - `echo; echo ${SECTION}; uname -n 2>/dev/null; echo; echo ${SECTION};`, + `echo; echo ${SECTION}; uname -n 2>/dev/null; echo;`, + `${ORCAD_MACHINE_NAME_PROBE} echo ${SECTION};`, // Full disks can leave an empty claim; publication can replace it. `if { [ ! -e ${claim} ] && [ ! -L ${claim} ]; } || { [ -f ${claim} ] && [ ! -s ${claim} ]; };`, `then echo ${CLAIM_ABSENT};`, @@ -120,13 +128,18 @@ export function parseOrcadMachineHomeProbe( const [homeRaw, machineRaw, hostnameRaw, claimRaw, ownersRaw, homesRaw, installedRaw] = sections const filesystemType = host.os === 'darwin' ? '' : sections[7].trim() const dfSource = sections.at(-1)?.trim().split(/\s+/u)[0] ?? '' + const hostnameLines = hostnameRaw.split('\n').map((line) => line.trim()) const loginHome = normalizeRemoteHome(homeRaw, host) if (!validateRemoteHome(loginHome, host)) { throw new Error(`Remote home is not a valid path: ${loginHome.slice(0, 100)}`) } return { loginHome, - identity: orcadMachineIdentity(host, machineRaw.trim(), hostnameRaw.trim()), + identity: orcadMachineIdentity( + host, + machineRaw.trim(), + hostnameLines.find((line) => line && !line.startsWith('ORCA_')) ?? '' + ), claim: parseClaimSection(claimRaw.trim()), dataRootOwners: ownersRaw .split('\n') @@ -140,7 +153,8 @@ export function parseOrcadMachineHomeProbe( // Without a filesystem type, only a device source proves local storage. homeOnLocalDisk: filesystemType ? LOCAL_FILESYSTEM_TYPES.has(filesystemType) - : dfSource.startsWith('/dev/') + : dfSource.startsWith('/dev/'), + ...parseOrcadMachineNameProbe(hostnameLines) } } @@ -159,10 +173,7 @@ export function orcadMachineIdentity( if (!machineIdentity) { return null } - const key = createHash('sha256') - .update(`${host.os}:${machineIdentity}`) - .digest('hex') - .slice(0, 16) + const key = machineKeyOf(`${host.os}:${machineIdentity}`) const hostname = /^[\x21-\x7e]{1,255}$/u.test(hostnameRaw) ? hostnameRaw : null return { key, machineIdentity, hostname } } @@ -245,6 +256,10 @@ export function decideOrcadMachineHome( host: RemoteHostPlatform ): OrcadMachineHomeDecision { const { identity, claim } = probe + // An explicit name never shares state by machine identity, which clones copy. + if (probe.machineName) { + return namedMachineHome(host, probe.machineName) + } if (!identity) { return { kind: 'login-home' } } diff --git a/src/main/managed-server/host-state/orcad-machine-name-home.ts b/src/main/managed-server/host-state/orcad-machine-name-home.ts new file mode 100644 index 00000000000..27145ff82d4 --- /dev/null +++ b/src/main/managed-server/host-state/orcad-machine-name-home.ts @@ -0,0 +1,38 @@ +// ORCA_MACHINE_NAME separates hosts that share a home and a cloned machine-id (#32918). +import { createHash } from 'node:crypto' +import { ORCA_MACHINE_NAME_ENV } from '../../../shared/orcad-managed-launch-env' +import type { RemoteHostPlatform } from '../../ssh/ssh-remote-platform' +import type { OrcadMachineHomeDecision } from './orcad-machine-home' + +const MACHINE_NAME_PREFIX = `${ORCA_MACHINE_NAME_ENV}=` +const MACHINE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/u + +/** Probe line for the hostname section; brackets keep spaces and newlines become spaces, so both are refused. */ +export const ORCAD_MACHINE_NAME_PROBE = `printf '${MACHINE_NAME_PREFIX}[%s]\\n' "$(printf %s "\${${ORCA_MACHINE_NAME_ENV}-}" | tr '\\n' ' ')";` + +export function parseOrcadMachineNameProbe(lines: readonly string[]): { + machineName: string | null +} { + const value = lines + .find((candidate) => candidate.startsWith(MACHINE_NAME_PREFIX)) + ?.slice(MACHINE_NAME_PREFIX.length) + // An unset or empty name means none; anything else must validate. + return { + machineName: value?.startsWith('[') && value.endsWith(']') ? value.slice(1, -1) || null : null + } +} + +export function machineKeyOf(input: string): string { + return createHash('sha256').update(input).digest('hex').slice(0, 16) +} + +/** A name always selects the same home, so it never flips with whatever happens to be running. */ +export function namedMachineHome(host: RemoteHostPlatform, name: string): OrcadMachineHomeDecision { + if (!MACHINE_NAME.test(name)) { + throw new Error( + `${ORCA_MACHINE_NAME_ENV} is set to "${name.slice(0, 80)}", which Orca can't use. Use 1-64 ` + + "letters, digits, '.', '_' or '-', starting with a letter or digit, then reconnect." + ) + } + return { kind: 'machine-home', machineKey: machineKeyOf(`${host.os}:name:${name}`) } +} diff --git a/src/main/orcad/orcad-instance-lock.test.ts b/src/main/orcad/orcad-instance-lock.test.ts index 66985534061..df987859520 100644 --- a/src/main/orcad/orcad-instance-lock.test.ts +++ b/src/main/orcad/orcad-instance-lock.test.ts @@ -10,7 +10,7 @@ import { utimesSync, writeFileSync } from 'node:fs' -import { tmpdir } from 'node:os' +import { hostname, tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, describe, expect, it, vi } from 'vitest' import type * as NodeFs from 'node:fs' @@ -498,6 +498,77 @@ describe('acquireOrcadInstanceLock', () => { }) }) + describe.skipIf(process.platform === 'win32')( + 'a machine cloned with this machine-id (#32918)', + () => { + const HOUR = 3_600_000 + const sharedLinux = (overrides: OrcadInstanceLockHooks = {}) => + hooks({ + platform: 'linux', + dataRootIsOnLocalDisk: () => false, + incarnation: () => ({ linuxStartTicks: '1', bootId: 'boot-self' }), + now: () => new Date(), + bootStartedAtMs: () => Date.now() - HOUR, + ...overrides + }) + const writeLock = (root: string, writtenAtMs: number, host: string): string => { + const lockPath = join(root, ORCAD_LOCK_FILE_NAME) + const acquiredAt = new Date(writtenAtMs).toISOString() + const clone = { machineIdentity: 'machine-a', bootId: 'boot-clone', host, acquiredAt } + writeFileSync(lockPath, JSON.stringify(persistedRecord(clone))) + utimesSync(lockPath, writtenAtMs / 1000, writtenAtMs / 1000) + return lockPath + } + + it.each(['dev-clone', hostname()])( + 'refuses its live lock, written since this boot, whatever its hostname (%s)', + (host) => { + const root = makeRoot() + const lockPath = writeLock(root, Date.now(), host) + expect(() => acquireOrcadInstanceLock(root, sharedLinux())).toThrow( + expect.objectContaining({ + code: 'orcad_instance_lock_holder_unverifiable', + message: expect.stringContaining('systemd-machine-id-setup') + }) + ) + expect(readOrcadInstanceLockRecord(lockPath)?.nonce).toBe('stale') + } + ) + + it('reclaims a lock this machine wrote before its boot, though since renamed', () => { + const root = makeRoot() + writeLock(root, Date.now() - 2 * HOUR, 'old-name') + acquireOrcadInstanceLock(root, sharedLinux()).release() + }) + + it('reclaims a crashed clone’s lock once its heartbeat is stale', () => { + const root = makeRoot() + writeLock(root, Date.now() - 10 * 60_000, 'dev-clone') + acquireOrcadInstanceLock(root, sharedLinux()).release() + }) + + it('reclaims on a local disk, which only one machine can use', () => { + const root = makeRoot() + writeLock(root, Date.now(), 'dev-clone') + acquireOrcadInstanceLock(root, sharedLinux({ dataRootIsOnLocalDisk: () => true })).release() + }) + + it('keeps the lock’s mtime fresh while held', () => { + vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval', 'Date'] }) + try { + const root = makeRoot() + const lock = acquireOrcadInstanceLock(root, hooks()) + const before = statSync(lock.path).mtimeMs + vi.advanceTimersByTime(2 * 60_000) + expect(statSync(lock.path).mtimeMs).toBeGreaterThan(before) + lock.release() + } finally { + vi.useRealTimers() + } + }) + } + ) + it('never lets a confirmed reclaim take a lock another machine id holds', () => { const root = makeRoot() const lockPath = join(root, ORCAD_LOCK_FILE_NAME) diff --git a/src/main/orcad/orcad-instance-lock.ts b/src/main/orcad/orcad-instance-lock.ts index aad860b3d85..87409e65983 100644 --- a/src/main/orcad/orcad-instance-lock.ts +++ b/src/main/orcad/orcad-instance-lock.ts @@ -27,6 +27,12 @@ import { orcadLockHolderLocality, orcadLockHolderUnverifiableError } from './orcad-lock-holder-locality' +import { + orcadLockPeer, + orcadLockSelf, + orcadPeerIsLiveCloneMachine, + startOrcadLockHeartbeat +} from './orcad-lock-clone-machine' import { MAX_ORCAD_LOCK_BYTES, readBoundedLockFile, @@ -91,6 +97,7 @@ export type OrcadInstanceLockHooks = OrcadDataRootPrivacyHooks & { /** Explicit stale-reclaim confirmation permits judging otherwise unverifiable PIDs locally. */ reclaimConfirmed?: (record: OrcadLockRecord, role: 'holder' | 'recovery-owner') => boolean reclaimHintRuntime?: OrcadReclaimHintRuntime | null + bootStartedAtMs?: () => number | null } function defaultIdentity(): string { @@ -156,6 +163,8 @@ export function acquireOrcadInstanceLock( host: hostname() } const serialized = JSON.stringify(record) + const platform = hooks.platform ?? process.platform + const self = orcadLockSelf(record, Date.parse(record.acquiredAt), hooks.bootStartedAtMs?.()) // Stage then hard-link so canonical records are always complete. const publish = (): boolean => { @@ -184,7 +193,8 @@ export function acquireOrcadInstanceLock( const dataRootIsLocal = (): boolean => (hooks.dataRootIsOnLocalDisk ?? orcadDataRootIsOnLocalDisk)(dataRoot) const judgedHere = (holder: OrcadLockRecord, role: 'holder' | 'recovery-owner'): boolean => { - const locality = orcadLockHolderLocality(holder, record, dataRootIsLocal) + const peer = orcadLockPeer(holder, role === 'holder' ? lockPath : null) + const locality = orcadLockHolderLocality(peer, self, dataRootIsLocal, platform) return ( locality === 'local' || (locality === 'unknown' && !!hooks.reclaimConfirmed?.(holder, role)) ) @@ -232,11 +242,15 @@ export function acquireOrcadInstanceLock( 'root corrupts it. Give each its own ORCA_USER_DATA.' ) } + const holder = orcadLockPeer(existing, lockPath) if (!judgedHere(existing, 'holder')) { // Foreign/unverifiable PIDs prove neither absence nor reuse here. throw orcadLockHolderUnverifiableError({ holder: `${describeLockHolder(existing)} (pid ${existing.pid}, started ${existing.acquiredAt})`, - foreign: orcadLockHolderLocality(existing, record, dataRootIsLocal) === 'foreign', + foreign: orcadLockHolderLocality(holder, self, dataRootIsLocal, platform) === 'foreign', + ...(orcadPeerIsLiveCloneMachine(holder, self, dataRootIsLocal, platform) + ? { cloneMachine: { role: hooks.role ?? 'orcad' } } + : {}), dataRoot, lockPath, hintRuntime: @@ -297,6 +311,7 @@ function describeLockHolder(record: OrcadLockRecord): string { function makeLock(lockPath: string, record: OrcadLockRecord): OrcadInstanceLock { let released = false + const stopHeartbeat = startOrcadLockHeartbeat(lockPath) return { path: lockPath, record, @@ -305,6 +320,7 @@ function makeLock(lockPath: string, record: OrcadLockRecord): OrcadInstanceLock return } released = true + stopHeartbeat() // Re-read ownership before unlinking so a successor’s live runtime cannot be unlocked. const current = parseOrcadInstanceLockRecord(readBoundedLockFile(lockPath) ?? '') if (!current || current.nonce !== record.nonce) { diff --git a/src/main/orcad/orcad-lock-clone-machine.ts b/src/main/orcad/orcad-lock-clone-machine.ts new file mode 100644 index 00000000000..de69750b8bd --- /dev/null +++ b/src/main/orcad/orcad-lock-clone-machine.ts @@ -0,0 +1,94 @@ +// Images cloned with one /etc/machine-id look like one machine to every PID check (#32918). +import { statSync, utimesSync } from 'node:fs' +import process from 'node:process' +import { ORCA_MACHINE_NAME_ENV } from '../../shared/orcad-managed-launch-env' +import type { OrcadPeerMachine } from './orcad-lock-holder-locality' +import { currentLinuxBootStartedAtMs } from '../persistence/profile-state/profile-state-access-identity' + +const CLONE_MACHINE_ID_HINT = + 'Machines cloned from one image share /etc/machine-id, so Orca cannot tell them apart. If the Orca ' + + 'on the clone crashed, a normal start takes over once its lock goes 5 minutes without a heartbeat. ' + + 'On the clone, run `sudo rm /etc/machine-id && sudo systemd-machine-id-setup` and reboot it.' +const CLONE_OWN_DATA_HINT = + 'Or, on that machine only, give Orca its own data: for Orca-managed SSH servers set ' + + `${ORCA_MACHINE_NAME_ENV} where its non-interactive SSH logins read it (~/.zshenv, or above the ` + + 'interactive check in ~/.bashrc), guarded by a test that matches only that machine, since a ' + + 'shared home applies it everywhere; for `orca serve` set ORCA_USER_DATA.' + +/** Lock heartbeats refresh the record's mtime well inside this window. */ +export const ORCAD_LOCK_HEARTBEAT_MS = 60_000 +export const ORCAD_LOCK_FRESH_MS = 5 * ORCAD_LOCK_HEARTBEAT_MS + +/** + * Images cloned with one /etc/machine-id (and often one hostname) differ only by boot. A record from + * another boot is this machine's own unless it was written after this boot began; staleness lets a + * crashed clone's record go. Hostnames are ignored: renames change them and clones repeat them. + */ +export function orcadPeerIsLiveCloneMachine( + peer: OrcadPeerMachine, + self: OrcadPeerMachine, + dataRootIsOnLocalDisk: () => boolean, + platform: NodeJS.Platform = process.platform +): boolean { + const { writtenAtMs } = peer + const { bootStartedAtMs, nowMs } = self + return ( + platform === 'linux' && + !!peer.machineIdentity && + peer.machineIdentity === self.machineIdentity && + !!peer.bootId && + !!self.bootId && + peer.bootId !== self.bootId && + writtenAtMs !== undefined && + bootStartedAtMs !== undefined && + nowMs !== undefined && + writtenAtMs > bootStartedAtMs && + nowMs - writtenAtMs < ORCAD_LOCK_FRESH_MS && + !dataRootIsOnLocalDisk() + ) +} + +/** A heartbeat refreshes the lock file's mtime; the record's own time covers older versions. */ +export function orcadLockPeer( + record: OrcadPeerMachine & { acquiredAt: string }, + lockPath: string | null +): OrcadPeerMachine { + let mtimeMs = 0 + try { + mtimeMs = lockPath === null ? 0 : statSync(lockPath).mtimeMs + } catch { + // A vanished lock leaves the record's own time. + } + const acquiredAtMs = Date.parse(record.acquiredAt) + const writtenAtMs = Math.max(Number.isFinite(acquiredAtMs) ? acquiredAtMs : 0, mtimeMs) + return { ...record, ...(writtenAtMs > 0 ? { writtenAtMs } : {}) } +} + +/** This machine as judged against peers: its boot start dates records from earlier boots. */ +export function orcadLockSelf( + self: OrcadPeerMachine, + nowMs: number, + bootStartedAtMs: number | null = currentLinuxBootStartedAtMs() +): OrcadPeerMachine { + return { ...self, nowMs, ...(bootStartedAtMs === null ? {} : { bootStartedAtMs }) } +} + +/** Keep the lock's mtime fresh so a machine sharing this machine-id can see the holder lives. */ +export function startOrcadLockHeartbeat(lockPath: string): () => void { + const heartbeat = setInterval(() => { + try { + const now = new Date() + utimesSync(lockPath, now, now) + } catch (error) { + // A stalled heartbeat lets a machine sharing this machine-id take the lock. + console.warn('[orcad] could not refresh the instance lock heartbeat:', error) + } + }, ORCAD_LOCK_HEARTBEAT_MS) + heartbeat.unref() + return () => clearInterval(heartbeat) +} + +export function orcadCloneMachineHint(role: 'orcad' | 'desktop'): string { + const ownData = role === 'desktop' ? '' : ` ${CLONE_OWN_DATA_HINT}` + return `${CLONE_MACHINE_ID_HINT}${ownData}` +} diff --git a/src/main/orcad/orcad-lock-holder-locality.ts b/src/main/orcad/orcad-lock-holder-locality.ts index a6d885413b9..12934a7612b 100644 --- a/src/main/orcad/orcad-lock-holder-locality.ts +++ b/src/main/orcad/orcad-lock-holder-locality.ts @@ -2,6 +2,7 @@ import process from 'node:process' import { profileStateAccessMachineIdentity } from '../persistence/profile-state/profile-state-access-identity' import { readOrcadProcessIncarnation } from './orcad-process-start-time' import { OrcadInstanceLockError } from './orcad-data-root-privacy' +import { orcadCloneMachineHint, orcadPeerIsLiveCloneMachine } from './orcad-lock-clone-machine' import { orcadStaleLockReclaimCommand, type OrcadReclaimHintRuntime @@ -11,10 +12,16 @@ import { export function orcadLockHolderLocality( holder: OrcadPeerMachine, self: OrcadPeerMachine, - dataRootIsOnLocalDisk: () => boolean + dataRootIsOnLocalDisk: () => boolean, + platform: NodeJS.Platform = process.platform ): 'local' | 'foreign' | 'unknown' { if (holder.machineIdentity && self.machineIdentity) { - return holder.machineIdentity === self.machineIdentity ? 'local' : 'foreign' + if (holder.machineIdentity !== self.machineIdentity) { + return 'foreign' + } + return orcadPeerIsLiveCloneMachine(holder, self, dataRootIsOnLocalDisk, platform) + ? 'unknown' + : 'local' } if (holder.bootId && holder.bootId === self.bootId) { return 'local' @@ -22,7 +29,14 @@ export function orcadLockHolderLocality( return dataRootIsOnLocalDisk() ? 'local' : 'unknown' } -export type OrcadPeerMachine = { machineIdentity?: string; bootId?: string } +export type OrcadPeerMachine = { + machineIdentity?: string + bootId?: string + /** Latest of the record's own timestamp and its heartbeat mtime. */ + writtenAtMs?: number + bootStartedAtMs?: number + nowMs?: number +} export function currentOrcadPeerMachine(): OrcadPeerMachine { const machineIdentity = profileStateAccessMachineIdentity() @@ -45,6 +59,8 @@ export function orcadPeerPidIsJudgeableHere( export function orcadLockHolderUnverifiableError(lock: { holder: string foreign: boolean + /** The holder runs on another machine that reports this machine-id. */ + cloneMachine?: { role: 'orcad' | 'desktop' } dataRoot: string lockPath: string /** Managed hosts may lack launchers on PATH, so name this runtime/entry in reclaim hints. */ @@ -55,11 +71,12 @@ export function orcadLockHolderUnverifiableError(lock: { process.platform === 'linux' ? `run \`${command}\` on the machine it ran on to check and clear it, or ` : '' + const cloneHint = lock.cloneMachine ? ` ${orcadCloneMachineHint(lock.cloneMachine.role)}` : '' return new OrcadInstanceLockError( 'orcad_instance_lock_holder_unverifiable', - `${lock.holder} holds the data root ${lock.dataRoot}${lock.foreign ? ' from another machine' : ''}. ` + + `${lock.holder} holds the data root ${lock.dataRoot}${lock.foreign || lock.cloneMachine ? ' from another machine' : ''}. ` + 'Orca cannot check from here whether it is still running, because this data root may be ' + `shared with other machines. Stop it there. If it is not running anywhere, ${reclaim}` + - `delete ${lock.lockPath} once you are sure.` + `delete ${lock.lockPath} once you are sure.${cloneHint}` ) } diff --git a/src/main/orcad/orcad-stale-lock-reclaim.test.ts b/src/main/orcad/orcad-stale-lock-reclaim.test.ts index 8e93676adb3..37a6119a348 100644 --- a/src/main/orcad/orcad-stale-lock-reclaim.test.ts +++ b/src/main/orcad/orcad-stale-lock-reclaim.test.ts @@ -290,6 +290,22 @@ describe.skipIf(process.platform === 'win32')('reclaimStaleOrcadInstanceLock (#3 await expect(probeOrcadEndpoint('ws://[::1')).resolves.toBe('unknown') }) + it('refuses a lock refreshed three minutes ago, which NFS caching can show for a live clone (#32918)', async () => { + const root = makeRoot() + const acquiredAt = '2026-01-01T00:00:00.000Z' + writeLock(root, holderRecord({ acquiredAt })) + const now = () => Date.parse(acquiredAt) + 3 * 60_000 + const result = await reclaimStaleOrcadInstanceLock( + root, + { yes: true, hintRuntime: null }, + rebootedBox({ now, bootStartedAtMs: () => Date.parse(acquiredAt) - 60_000 }) + ) + expect(result).toMatchObject({ + status: 'refused', + message: expect.stringContaining('minutes old') + }) + }) + it.each([ ['another host', holderRecord({ host: 'other-box' }), 'not this host'], ['the same boot', holderRecord({ bootId: 'boot-2' }), 'judge this lock itself'], diff --git a/src/main/orcad/orcad-stale-lock-reclaim.ts b/src/main/orcad/orcad-stale-lock-reclaim.ts index cf92414b3a0..fc511dc3fc2 100644 --- a/src/main/orcad/orcad-stale-lock-reclaim.ts +++ b/src/main/orcad/orcad-stale-lock-reclaim.ts @@ -13,6 +13,7 @@ import { } from './orcad-instance-lock' import { readBoundedLockFile } from './orcad-instance-lock-reclaim' import { orcadLockHolderLocality } from './orcad-lock-holder-locality' +import { ORCAD_LOCK_FRESH_MS, orcadLockPeer, orcadLockSelf } from './orcad-lock-clone-machine' import { orcadDataRootIsOnLocalDisk } from './orcad-data-root-locality' import { readOrcadProcessIncarnation } from './orcad-process-start-time' import { hasErrorCode } from '../daemon/daemon-process-inspection' @@ -30,8 +31,8 @@ import { type OrcadStaleLockReclaimStatus } from '../../shared/orcad-stale-lock-reclaim' -// A holder that just took the lock may not be listening yet; its silence would prove nothing. -export const ORCAD_STALE_LOCK_MIN_AGE_MS = 2 * 60_000 +// A new holder may not listen yet, and NFS attribute caching can age a live heartbeat this much. +export const ORCAD_STALE_LOCK_MIN_AGE_MS = ORCAD_LOCK_FRESH_MS const ENDPOINT_PROBE_TIMEOUT_MS = 1_500 /** Silent requires OS-confirmed absence of a listener; errors leave ownership unknown. */ @@ -41,6 +42,7 @@ export type OrcadStaleLockReclaimHooks = { now?: () => number hostname?: () => string bootId?: () => string | null + bootStartedAtMs?: () => number | null machineIdentity?: () => string | null dataRootIsOnLocalDisk?: (dataRoot: string) => boolean probeEndpoint?: (endpoint: string) => Promise @@ -139,8 +141,13 @@ async function checkStaleLock( hooks: OrcadStaleLockReclaimHooks ): Promise<{ refusal: string } | { evidence: OrcadStaleLockEvidence }> { const machineIdentity = (hooks.machineIdentity ?? profileStateAccessMachineIdentity)() - const self = { ...(machineIdentity ? { machineIdentity } : {}), ...(bootId ? { bootId } : {}) } - const locality = orcadLockHolderLocality(holder, self, () => + // Judge as a normal start does, so "a normal start reclaims it" stays true. + const self = orcadLockSelf( + { ...(machineIdentity ? { machineIdentity } : {}), ...(bootId ? { bootId } : {}) }, + (hooks.now ?? Date.now)(), + hooks.bootStartedAtMs?.() + ) + const locality = orcadLockHolderLocality(orcadLockPeer(holder, lockPath), self, () => (hooks.dataRootIsOnLocalDisk ?? orcadDataRootIsOnLocalDisk)(dataRoot) ) if (locality === 'local') { @@ -191,7 +198,7 @@ async function checkStaleLock( host: holder.host, bootId: holder.bootId, currentBootId: bootId, - ageMinutes: Math.round((now - Date.parse(holder.acquiredAt)) / 60_000), + ageMinutes: Math.round((now - writtenMs) / 60_000), nonce: holder.nonce } } diff --git a/src/main/persistence/profile-state/profile-state-access-identity.ts b/src/main/persistence/profile-state/profile-state-access-identity.ts index bb393fe096a..4dd7c3b4027 100644 --- a/src/main/persistence/profile-state/profile-state-access-identity.ts +++ b/src/main/persistence/profile-state/profile-state-access-identity.ts @@ -1,6 +1,9 @@ -import { readFileSync } from 'node:fs' +import { readFileSync, statSync } from 'node:fs' import { runProcessSync } from '@orca/process-host' -import { parseLinuxProcStartTicks } from '../../daemon/daemon-process-start-time' +import { + parseLinuxBootTimeSeconds, + parseLinuxProcStartTicks +} from '../../daemon/daemon-process-start-time' import { getPsProcessIdentity } from '../../daemon/daemon-process-identity-query' import { loadWindowsNativeRegistry, WINDOWS_REG_SZ } from '../../windows-native-registry' import { readWindowsProcessCreationTime } from '../../windows/windows-process-table' @@ -114,3 +117,23 @@ function readProcessIdentity(pid: number): string | null { // Local wall times are ambiguous during daylight-saving transitions. return startedAtMs == null ? null : `darwin-utc-start-ms:${startedAtMs}` } + +/** Null where boot start is unknown, which keeps every record judged as before. */ +export function currentLinuxBootStartedAtMs(): number | null { + try { + const seconds = parseLinuxBootTimeSeconds(readFileSync('/proc/stat', 'utf8')) + return Number.isFinite(seconds) && seconds > 0 ? seconds * 1000 : null + } catch { + return null + } +} + +/** A record changed after this boot began cannot be this machine's from an earlier boot. */ +export function profileStateAccessWrittenThisBoot(path: string): boolean { + const bootStartedAtMs = currentLinuxBootStartedAtMs() + try { + return bootStartedAtMs !== null && statSync(path).mtimeMs > bootStartedAtMs + } catch { + return false + } +} diff --git a/src/main/persistence/profile-state/profile-state-access-owner.ts b/src/main/persistence/profile-state/profile-state-access-owner.ts index d8af17fa855..a2d19ac22a4 100644 --- a/src/main/persistence/profile-state/profile-state-access-owner.ts +++ b/src/main/persistence/profile-state/profile-state-access-owner.ts @@ -21,7 +21,8 @@ import { import { profileStateAccessBootIdentity, profileStateAccessMachineIdentity, - profileStateAccessProcessIdentity + profileStateAccessProcessIdentity, + profileStateAccessWrittenThisBoot } from './profile-state-access-identity' export class ProfileStateAccessError extends Error { @@ -132,7 +133,7 @@ function readOwner(path: string): AccessOwner | undefined { throw new ProfileStateAccessError(`Profile state ownership is malformed: ${path}`) } -function ownerExited(owner: AccessOwner): boolean { +function ownerExited(owner: AccessOwner, path: string): boolean { const currentBoot = profileStateAccessBootIdentity() const currentMachine = profileStateAccessMachineIdentity() const sameBoot = Boolean(owner.bootIdentity && owner.bootIdentity === currentBoot) @@ -154,7 +155,8 @@ function ownerExited(owner: AccessOwner): boolean { currentBoot && owner.bootIdentity !== currentBoot ) { - return true + // Clones repeat machine-id and hostname; only a record older than this boot is this machine's. + return !profileStateAccessWrittenThisBoot(path) } // Windows/WSL and Linux PID namespaces cannot establish each other's process absence. if ( @@ -234,7 +236,7 @@ export function reclaimExitedOwner(path: string): void { if (owner === undefined) { continue } - if (owner.token !== token || !ownerExited(owner)) { + if (owner.token !== token || !ownerExited(owner, join(path, entry))) { throw new ProfileStateAccessError( `Profile state is in use or its owner is unverifiable: ${path}. Stop Orca and orcad on every host using this profile, then retry. If this remains, verify PID ${owner.pid} on ${owner.host} has exited before removing its owner entry ${join(path, entry)}.` ) diff --git a/src/main/persistence/profile-state/profile-state-access.test.ts b/src/main/persistence/profile-state/profile-state-access.test.ts index dda93626013..b9ac901a173 100644 --- a/src/main/persistence/profile-state/profile-state-access.test.ts +++ b/src/main/persistence/profile-state/profile-state-access.test.ts @@ -212,6 +212,7 @@ describe('profile state owner reclamation', () => { const path = root() vi.spyOn(identity, 'profileStateAccessBootIdentity').mockReturnValue('current-boot') vi.spyOn(identity, 'profileStateAccessMachineIdentity').mockReturnValue('same-machine') + vi.spyOn(identity, 'profileStateAccessWrittenThisBoot').mockReturnValue(false) const owner = staleGate(path, process.pid, hostname(), { bootIdentity: 'previous-boot', machineIdentity: 'same-machine' @@ -222,6 +223,19 @@ describe('profile state owner reclamation', () => { expect(fs.existsSync(owner)).toBe(false) }) + it('keeps an owner written since this boot by a clone sharing machine-id and hostname (#32918)', () => { + const path = root() + vi.spyOn(identity, 'profileStateAccessBootIdentity').mockReturnValue('current-boot') + vi.spyOn(identity, 'profileStateAccessMachineIdentity').mockReturnValue('cloned-machine') + vi.spyOn(identity, 'profileStateAccessWrittenThisBoot').mockReturnValue(true) + const owner = staleGate(path, 12345, hostname(), { + bootIdentity: 'clone-boot', + machineIdentity: 'cloned-machine' + }) + expect(() => acquireProfileStateMaintenance(path)).toThrow('in use') + expect(fs.existsSync(owner)).toBe(true) + }) + it('keeps a differently booted machine with the same hostname unverifiable', () => { const path = root() vi.spyOn(identity, 'profileStateAccessBootIdentity').mockReturnValue('current-boot') diff --git a/src/shared/orcad-managed-launch-env.ts b/src/shared/orcad-managed-launch-env.ts index ee53f1c82bf..7e12e6ebaf8 100644 --- a/src/shared/orcad-managed-launch-env.ts +++ b/src/shared/orcad-managed-launch-env.ts @@ -1,3 +1,6 @@ /** Managed launches alone enable idle exit and name the activation fence that keeps updates/rollbacks alive. * Dependency-free so the orcad launcher can read it. */ export const ORCAD_MANAGED_ACTIVATION_ROOT_ENV = 'ORCA_ORCAD_MANAGED_ACTIVATION_ROOT' + +/** Host login env naming a machine's own managed-server data on a shared home; clones copy /etc/machine-id. */ +export const ORCA_MACHINE_NAME_ENV = 'ORCA_MACHINE_NAME'