From bcad291f355af71ea7a0a153a655f8eff8944f81 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Mon, 12 Oct 2026 18:21:06 -0700 Subject: [PATCH] fix(orcad): identify instances by kernel start ticks, not drifting wall-clock start time (#28125) * fix(orcad): identify instances by kernel start ticks, not drifting wall-clock start time On Linux the start time is derived from /proc/stat btime, which drifts (WSL2, NTP), so a live orcad was read as a reused PID and its managed stop reported exited without ever writing the request. The lock and stop request now carry optional linuxStartTicks + bootId; readers prefer them, and a wall-clock mismatch on Linux for records without them is unverifiable, never proof of exit. Co-Authored-By: Claude * fix(orcad): match stop receipts without the incarnation an older client drops A request recovered by an older client loses linuxStartTicks/bootId, which made the newer host reject its own receipts and decision records as mismatched. Also exercise the incarnation reader against a real Linux /proc. --------- Co-authored-by: Claude --- .../orcad/orcad-completed-stop-receipt.ts | 4 +- src/main/orcad/orcad-instance-lock.test.ts | 89 ++++++++++- src/main/orcad/orcad-instance-lock.ts | 33 ++-- .../orcad/orcad-managed-stop-completion.ts | 57 ++++--- .../orcad/orcad-managed-stop-identity.test.ts | 145 ++++++++++++++++++ src/main/orcad/orcad-managed-stop-request.ts | 13 +- src/main/orcad/orcad-managed-stop.test.ts | 14 +- .../orcad/orcad-process-start-time.test.ts | 94 +++++++++++- src/main/orcad/orcad-process-start-time.ts | 83 +++++++++- src/main/ssh/orcad-managed-remote-stop.ts | 11 +- src/shared/orcad-stop-request.ts | 10 ++ 11 files changed, 497 insertions(+), 56 deletions(-) create mode 100644 src/main/orcad/orcad-managed-stop-identity.test.ts diff --git a/src/main/orcad/orcad-completed-stop-receipt.ts b/src/main/orcad/orcad-completed-stop-receipt.ts index 7f155df696b..6612c311f38 100644 --- a/src/main/orcad/orcad-completed-stop-receipt.ts +++ b/src/main/orcad/orcad-completed-stop-receipt.ts @@ -10,6 +10,7 @@ import { OrcadCompletedStopReceiptSchema, OrcadManagedStopRequestSchema, OrcadDaemonRetirementRecordSchema, + orcadManagedStopRequestKey, type OrcadCompletedStopReceipt, type OrcadDaemonRetirementRecord, type OrcadManagedStopRequest @@ -49,8 +50,7 @@ export function readOrcadStopReceipt 'uid-1000', version: () => '1.0.0-test', startedAtMs: () => 1_000, - startTimeMatches: () => true, + incarnation: () => null, + holderIdentity: () => 'matches', processIsAlive: () => false, ...overrides } @@ -94,11 +97,93 @@ describe('acquireOrcadInstanceLock', () => { writeFileSync(join(root, ORCAD_LOCK_FILE_NAME), JSON.stringify(persistedRecord())) const lock = acquireOrcadInstanceLock( root, - hooks({ processIsAlive: () => true, startTimeMatches: () => false }) + hooks({ processIsAlive: () => true, holderIdentity: () => 'differs' }) ) expect(JSON.parse(readFileSync(lock.path, 'utf8')).pid).toBe(process.pid) }) + it('keeps a live holder whose ticks match though its btime-derived start time drifted', () => { + const root = makeRoot() + const incarnation = { linuxStartTicks: '1234', bootId: 'boot-a' } + writeFileSync( + join(root, ORCAD_LOCK_FILE_NAME), + JSON.stringify(persistedRecord({ startedAtMs: 1_000, ...incarnation })) + ) + const drifted = { + processIsAlive: () => true, + holderIdentity: undefined, + startedAtMs: () => 158_000, + incarnation: () => incarnation + } + expect(() => acquireOrcadInstanceLock(root, hooks(drifted))).toThrow( + expect.objectContaining({ code: 'orcad_instance_lock_held' }) + ) + const reused = { ...drifted, incarnation: () => ({ ...incarnation, linuxStartTicks: '9' }) } + expect(acquireOrcadInstanceLock(root, hooks(reused)).record.nonce).not.toBe('stale') + }) + + it.runIf(process.platform === 'linux')( + 'keeps an older Linux record whose wall-clock start time disagrees, since that proves nothing', + () => { + const root = makeRoot() + writeFileSync(join(root, ORCAD_LOCK_FILE_NAME), JSON.stringify(persistedRecord())) + expect(() => + acquireOrcadInstanceLock( + root, + hooks({ processIsAlive: () => true, holderIdentity: undefined, startedAtMs: () => 9e9 }) + ) + ).toThrow(expect.objectContaining({ code: 'orcad_instance_lock_held' })) + } + ) + + it('records the incarnation in a lock an older reader still parses', () => { + const root = makeRoot() + const incarnation = { linuxStartTicks: '1234', bootId: 'boot-a' } + const lock = acquireOrcadInstanceLock(root, hooks({ incarnation: () => incarnation })) + const written: unknown = JSON.parse(readFileSync(lock.path, 'utf8')) + expect(written).toMatchObject({ startedAtMs: 1_000, ...incarnation }) + // The shape desktops before this change parse with: unknown keys are dropped, not refused. + const olderReader = z.object({ + pid: z.number().int().positive(), + startedAtMs: z.number().finite().nonnegative().nullable(), + identity: z.string().min(1), + version: z.string().min(1), + acquiredAt: z.iso.datetime({ offset: true }), + nonce: z.string().min(1), + role: z.enum(['orcad', 'desktop']).optional() + }) + expect(olderReader.parse(written)).toEqual({ + pid: process.pid, + startedAtMs: 1_000, + identity: 'uid-1000', + version: '1.0.0-test', + acquiredAt: lock.record.acquiredAt, + nonce: lock.record.nonce, + role: 'orcad' + }) + lock.release() + }) + + it('reads an older lock without an incarnation, and salvages a malformed one', () => { + const root = makeRoot() + writeFileSync( + join(root, ORCAD_LOCK_FILE_NAME), + JSON.stringify(persistedRecord({ linuxStartTicks: undefined })) + ) + expect(readOrcadInstanceLockRecord(join(root, ORCAD_LOCK_FILE_NAME))).not.toHaveProperty( + 'linuxStartTicks' + ) + writeFileSync( + join(root, ORCAD_LOCK_FILE_NAME), + JSON.stringify({ ...persistedRecord(), linuxStartTicks: 12, bootId: '' }) + ) + expect(readOrcadInstanceLockRecord(join(root, ORCAD_LOCK_FILE_NAME))).toMatchObject({ + nonce: 'stale', + linuxStartTicks: undefined, + bootId: undefined + }) + }) + it('does not displace a successor published between stale inspection and reclaim', () => { const root = makeRoot() const lockPath = join(root, ORCAD_LOCK_FILE_NAME) diff --git a/src/main/orcad/orcad-instance-lock.ts b/src/main/orcad/orcad-instance-lock.ts index ab152d2a2d8..b20b9b85e0b 100644 --- a/src/main/orcad/orcad-instance-lock.ts +++ b/src/main/orcad/orcad-instance-lock.ts @@ -19,8 +19,11 @@ import { join } from 'node:path' import process from 'node:process' import { z } from 'zod' import { - orcadProcessStartTimeMatches, - readOrcadProcessStartedAtMs + compareOrcadProcessIdentity, + readOrcadProcessIncarnation, + readOrcadProcessStartedAtMs, + type OrcadProcessIdentityVerdict, + type OrcadProcessIncarnation } from './orcad-process-start-time' import { assertOrcadDataRootIsPrivate, @@ -39,6 +42,12 @@ const OrcadLockRecordSchema = z.object({ pid: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), /** Null where the platform cannot read it; PID alone is then the (weaker) fence. */ startedAtMs: z.number().finite().nonnegative().nullable(), + /** + * Linux only, absent from older writers: drift-free identity preferred over `startedAtMs`. + * Salvaged rather than refused, so a future encoding never makes the whole lock unreadable. + */ + linuxStartTicks: z.string().regex(/^\d+$/).max(32).optional().catch(undefined), + bootId: z.string().min(1).max(255).optional().catch(undefined), /** POSIX uid, or the Windows username. Compared as an opaque string. */ identity: z.string().min(1).max(1_024), version: z.string().min(1).max(255), @@ -69,7 +78,9 @@ export type OrcadInstanceLockHooks = OrcadDataRootPrivacyHooks & { /** Whether a PID is running. EPERM counts as alive: it proves the process exists. */ processIsAlive?: (pid: number) => boolean startedAtMs?: (pid: number) => number | null - startTimeMatches?: (pid: number, expected: number | null) => boolean + incarnation?: (pid: number) => OrcadProcessIncarnation | null + /** Only `differs` frees a live holder's PID; anything less proves nothing. */ + holderIdentity?: (record: OrcadLockRecord) => OrcadProcessIdentityVerdict /** * Who takes the profile. The desktop app takes it too, so the two refuse each other; it must * hold Electron's single-instance lock first, which is what lets it reclaim a desktop record. @@ -108,7 +119,14 @@ export function acquireOrcadInstanceLock( const identity = (hooks.identity ?? defaultIdentity)() const isAlive = hooks.processIsAlive ?? isProcessAlive const readStartedAt = hooks.startedAtMs ?? readOrcadProcessStartedAtMs - const matchesStartTime = hooks.startTimeMatches ?? orcadProcessStartTimeMatches + const readIncarnation = hooks.incarnation ?? readOrcadProcessIncarnation + const holderIdentity = + hooks.holderIdentity ?? + ((holder: OrcadLockRecord) => + compareOrcadProcessIdentity(holder.pid, holder, { + startedAtMs: readStartedAt, + incarnation: readIncarnation + })) try { mkdirSync(dataRoot, { recursive: true, mode: 0o700 }) @@ -127,6 +145,7 @@ export function acquireOrcadInstanceLock( const record: OrcadLockRecord = { pid: process.pid, startedAtMs: readStartedAt(process.pid), + ...readIncarnation(process.pid), identity, version: (hooks.version ?? (() => process.env.ORCA_VERSION ?? 'unknown'))(), acquiredAt: (hooks.now ?? (() => new Date()))().toISOString(), @@ -192,11 +211,7 @@ export function acquireOrcadInstanceLock( // The desktop takes this lock only after Electron's single-instance lock, which already proves // no other desktop runs; a desktop record is then stale even when its PID was reused. const staleDesktopRecord = hooks.role === 'desktop' && existing.role === 'desktop' - if ( - !staleDesktopRecord && - isAlive(existing.pid) && - matchesStartTime(existing.pid, existing.startedAtMs) - ) { + if (!staleDesktopRecord && isAlive(existing.pid) && holderIdentity(existing) !== 'differs') { throw new OrcadInstanceLockError( 'orcad_instance_lock_held', `${describeLockHolder(existing)} (pid ${existing.pid}, started ` + diff --git a/src/main/orcad/orcad-managed-stop-completion.ts b/src/main/orcad/orcad-managed-stop-completion.ts index 7f3e8015056..f9a28829303 100644 --- a/src/main/orcad/orcad-managed-stop-completion.ts +++ b/src/main/orcad/orcad-managed-stop-completion.ts @@ -3,22 +3,25 @@ * * Runs as a short-lived command on the execution host. It never signals: it writes the * instance-bound request the running orcad watches for, then observes until the instance is - * gone. `exited` needs proof — no process with that PID, or a PID whose start time shows it now - * belongs to another process. Anything the host cannot answer is `unverifiable`, never exit. + * gone. `exited` needs proof — no process with that PID, or a PID whose kernel identity shows it + * now belongs to another process. A wall-clock disagreement, or anything the host cannot answer, + * is `unverifiable`, never exit. */ import { setTimeout as delay } from 'node:timers/promises' import { writeDurableSecureJsonFile } from '../../shared/secure-file' import { OrcadManagedStopRequestSchema, + orcadManagedStopRequestKey, type OrcadManagedStopInstance, type OrcadManagedStopRequest, type OrcadManagedStopVerdict } from '../../shared/orcad-stop-request' import { - START_TIME_TOLERANCE_MS, - startTimesWithinTolerance -} from '../daemon/daemon-process-start-time' -import { readOrcadProcessStartedAtMs } from './orcad-process-start-time' + compareOrcadProcessIdentity, + recordedOrcadProcessIncarnation, + type OrcadProcessIdentityReaders, + type OrcadRecordedIdentity +} from './orcad-process-start-time' import { hasErrorCode, inspectProcessSignal } from '../daemon/daemon-process-inspection' import { persistOrcadCompletedStopReceipt } from './orcad-completed-stop-receipt' import { readOrcadManagedStopDecision } from './orcad-managed-stop-decision' @@ -26,6 +29,7 @@ import { withdrawOrcadManagedStopRequest } from './orcad-managed-stop-cancellati import { orcadInstanceLockNames, orcadManagedStopRequestPath, + readOrcadInstanceLockNaming, readOrcadManagedStopRequest } from './orcad-managed-stop-request' import { @@ -35,9 +39,8 @@ import { export type OrcadProcessProbe = (pid: number) => 'alive' | 'missing' | 'unverifiable' -export type OrcadManagedStopCompletionOptions = { +export type OrcadManagedStopCompletionOptions = OrcadProcessIdentityReaders & { probeProcess?: OrcadProcessProbe - startedAtMs?: (pid: number) => number | null sleep?: () => Promise attempts?: number now?: () => Date @@ -51,23 +54,31 @@ function defaultProbe(pid: number): ReturnType { function observeInstance( instance: OrcadManagedStopInstance, + recorded: OrcadRecordedIdentity, options: OrcadManagedStopCompletionOptions ): OrcadManagedStopVerdict { const probe = (options.probeProcess ?? defaultProbe)(instance.pid) if (probe !== 'alive') { return probe === 'missing' ? 'exited' : 'unverifiable' } - const actual = (options.startedAtMs ?? readOrcadProcessStartedAtMs)(instance.pid) - // A reused PID is proof of exit only when both start times are known and disagree; on Windows - // without the addon both stay null, so a live PID is never read as exited. - if ( - actual !== null && - instance.startedAtMs !== null && - !startTimesWithinTolerance(actual, instance.startedAtMs, START_TIME_TOLERANCE_MS) - ) { - return 'exited' + // Only a kernel identity (or a stored creation time) proves reuse; an unknown one stays live. + const identity = compareOrcadProcessIdentity(instance.pid, recorded, options) + return identity === 'differs' ? 'exited' : identity === 'unproven' ? 'unverifiable' : 'live' +} + +/** An older client's request lacks the incarnation; the lock still naming the instance has it. */ +function recordedIdentity(instance: OrcadManagedStopInstance): OrcadRecordedIdentity { + if (recordedOrcadProcessIncarnation(instance)) { + return instance } - return 'live' + let record: OrcadRecordedIdentity | null = null + try { + record = readOrcadInstanceLockNaming(instance) + } catch { + return instance + } + const incarnation = record ? recordedOrcadProcessIncarnation(record) : null + return incarnation ? { ...instance, ...incarnation } : instance } /** Writes the request only while the lock still names this instance, then waits for exit. */ @@ -84,7 +95,8 @@ export async function completeOrcadManagedStop( persistOrcadCompletedStopReceipt(request, (options.now ?? (() => new Date()))()) return 'exited' } - let verdict = observeInstance(request.instance, options) + const recorded = recordedIdentity(request.instance) + let verdict = observeInstance(request.instance, recorded, options) if (verdict !== 'live') { return verdict === 'exited' ? completed() : verdict } @@ -110,7 +122,7 @@ export async function completeOrcadManagedStop( } for (let attempt = 0; attempt < attempts; attempt++) { await (options.sleep ?? (() => delay(ORCAD_STOP_COMPLETION_POLL_MS)))() - verdict = observeInstance(request.instance, options) + verdict = observeInstance(request.instance, recorded, options) if (verdict !== 'live') { return verdict === 'exited' ? completed() : verdict } @@ -129,7 +141,10 @@ function lockStillNamesInstance(instance: OrcadManagedStopInstance): boolean { /** A request already present must be this one; another transaction's request is not ours. */ function existingRequestMatches(path: string, request: OrcadManagedStopRequest): boolean { try { - return JSON.stringify(readOrcadManagedStopRequest(path)) === JSON.stringify(request) + return ( + orcadManagedStopRequestKey(readOrcadManagedStopRequest(path)) === + orcadManagedStopRequestKey(request) + ) } catch (error) { return hasErrorCode(error, 'ENOENT') } diff --git a/src/main/orcad/orcad-managed-stop-identity.test.ts b/src/main/orcad/orcad-managed-stop-identity.test.ts new file mode 100644 index 00000000000..2bd3a7fc991 --- /dev/null +++ b/src/main/orcad/orcad-managed-stop-identity.test.ts @@ -0,0 +1,145 @@ +import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { z } from 'zod' +import { + OrcadManagedStopRequestSchema, + type OrcadManagedStopRequest +} from '../../shared/orcad-stop-request' +import { acquireOrcadInstanceLock } from './orcad-instance-lock' +import { orcadManagedStopRequestPath } from './orcad-managed-stop-request' +import { + completeOrcadManagedStop, + type OrcadManagedStopCompletionOptions +} from './orcad-managed-stop-completion' +import { orcadStopReceiptPath } from './orcad-completed-stop-receipt' +import type { OrcadProcessIncarnation } from './orcad-process-start-time' + +const roots: string[] = [] +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +const RECORDED: OrcadProcessIncarnation = { + linuxStartTicks: '4242', + bootId: 'boot-a' +} +// The WSL2 smoke saw btime move by 157 s against a 1.5 s tolerance. +const DRIFTED_STARTED_AT_MS = 5_000 + 157_000 + +function running( + recorded: OrcadProcessIncarnation | null, + { requestCarriesIncarnation = false } = {} +): OrcadManagedStopRequest { + const root = mkdtempSync(join(tmpdir(), 'orcad-stop-identity-')) + roots.push(root) + const lock = acquireOrcadInstanceLock(root, { + identity: () => 'uid-1000', + startedAtMs: () => 5_000, + incarnation: () => recorded + }) + const { pid, startedAtMs, nonce } = lock.record + return { + schemaVersion: 1, + transactionId: '0b9f6a3e-9e2c-4c8e-8f58-4c0f6b1d2e3a', + version: '1.0.0', + runtimeId: 'runtime-1', + instance: { + pid, + startedAtMs, + ...(requestCarriesIncarnation && recorded ? recorded : {}), + nonce, + lockPath: lock.path + } + } +} + +/** Alive for `aliveFor` observations, then gone. */ +function linuxHost( + actual: OrcadProcessIncarnation | null, + aliveFor = Number.POSITIVE_INFINITY +): OrcadManagedStopCompletionOptions { + let observations = 0 + return { + platform: 'linux', + probeProcess: () => (observations++ < aliveFor ? 'alive' : 'missing'), + startedAtMs: () => DRIFTED_STARTED_AT_MS, + incarnation: () => actual, + sleep: async () => {}, + attempts: 3, + now: () => new Date('2026-10-01T00:00:00.000Z') + } +} + +const requestWritten = (request: OrcadManagedStopRequest): boolean => + existsSync(orcadManagedStopRequestPath(request.instance)) + +describe('managed stop instance identity', () => { + it.each([ + ['the lock', false], + ['the request', true] + ])( + 'keeps a drifted-btime instance live by its ticks from %s, writes the request, then proves exit', + async (_source, requestCarriesIncarnation) => { + const request = running(RECORDED, { requestCarriesIncarnation }) + expect(await completeOrcadManagedStop(request, linuxHost(RECORDED, 2))).toBe('exited') + expect(requestWritten(request)).toBe(true) + expect(existsSync(orcadStopReceiptPath(request, 'completed'))).toBe(true) + } + ) + + it.each([ + ['different start ticks', { ...RECORDED, linuxStartTicks: '9999' }], + ['a different boot', { ...RECORDED, bootId: 'boot-b' }] + ])('proves exit for a PID now naming a process with %s', async (_name, actual) => { + const request = running(RECORDED) + expect(await completeOrcadManagedStop(request, linuxHost(actual))).toBe('exited') + expect(requestWritten(request)).toBe(false) + }) + + it('stays live, never exited, while the incarnation cannot be read', async () => { + const request = running(RECORDED) + expect(await completeOrcadManagedStop(request, linuxHost(null))).toBe('live') + expect(requestWritten(request)).toBe(true) + }) + + it('answers unverifiable, not exited, for an older lock whose wall-clock time drifted', async () => { + const request = running(null) + expect(await completeOrcadManagedStop(request, linuxHost(RECORDED))).toBe('unverifiable') + expect(existsSync(orcadStopReceiptPath(request, 'completed'))).toBe(false) + }) + + it('stays readable by peers on either side of the change', async () => { + const request = running(RECORDED, { requestCarriesIncarnation: true }) + // The instance shape slots before this change parse: the incarnation is dropped, not refused. + const olderInstance = z.object({ + pid: z.number().int().positive(), + startedAtMs: z.number().finite().nonnegative().nullable(), + nonce: z.string().min(1), + lockPath: z.string().min(1) + }) + const { linuxStartTicks: _ticks, bootId: _boot, ...withoutIncarnation } = request.instance + expect(olderInstance.parse(request.instance)).toEqual(withoutIncarnation) + // And a newer slot reads what an older client wrote, taking the incarnation from the lock. + const fromOlderClient = OrcadManagedStopRequestSchema.parse({ + ...request, + instance: withoutIncarnation + }) + expect(fromOlderClient.instance).not.toHaveProperty('linuxStartTicks') + expect(await completeOrcadManagedStop(fromOlderClient, linuxHost(RECORDED, 2))).toBe('exited') + expect(JSON.parse(readFileSync(orcadManagedStopRequestPath(request.instance), 'utf8'))).toEqual( + fromOlderClient + ) + }) + + it('matches its own receipt when an older client retries without the incarnation', async () => { + const request = running(RECORDED, { requestCarriesIncarnation: true }) + expect(await completeOrcadManagedStop(request, linuxHost(RECORDED, 2))).toBe('exited') + const { linuxStartTicks: _ticks, bootId: _boot, ...withoutIncarnation } = request.instance + const retried = { ...request, instance: withoutIncarnation } + expect(await completeOrcadManagedStop(retried, linuxHost(RECORDED, 0))).toBe('exited') + }) +}) diff --git a/src/main/orcad/orcad-managed-stop-request.ts b/src/main/orcad/orcad-managed-stop-request.ts index f7211dd7d7c..78b8a84188f 100644 --- a/src/main/orcad/orcad-managed-stop-request.ts +++ b/src/main/orcad/orcad-managed-stop-request.ts @@ -11,7 +11,7 @@ import { type OrcadManagedStopInstance, type OrcadManagedStopRequest } from '../../shared/orcad-stop-request' -import { readOrcadInstanceLockRecord } from './orcad-instance-lock' +import { readOrcadInstanceLockRecord, type OrcadLockRecord } from './orcad-instance-lock' /** Keyed by the lock nonce, so a request for a previous instance is never this one's. */ export function orcadManagedStopRequestPath(instance: OrcadManagedStopInstance): string { @@ -38,11 +38,18 @@ export function sameOrcadManagedStopInstance( /** Whether the instance lock still names exactly this instance. */ export function orcadInstanceLockNames(instance: OrcadManagedStopInstance): boolean { + return readOrcadInstanceLockNaming(instance) !== null +} + +/** The lock record, only while it still names exactly this instance. */ +export function readOrcadInstanceLockNaming( + instance: OrcadManagedStopInstance +): OrcadLockRecord | null { if (!lstatSync(instance.lockPath).isFile()) { - return false + return null } const record = readOrcadInstanceLockRecord(instance.lockPath) - return record !== null && sameOrcadManagedStopInstance(instance, record) + return record !== null && sameOrcadManagedStopInstance(instance, record) ? record : null } /** Throws unless the request names this running instance and its lock is still ours. */ diff --git a/src/main/orcad/orcad-managed-stop.test.ts b/src/main/orcad/orcad-managed-stop.test.ts index 5334bec5f94..f331054faf3 100644 --- a/src/main/orcad/orcad-managed-stop.test.ts +++ b/src/main/orcad/orcad-managed-stop.test.ts @@ -73,7 +73,8 @@ function running(): { lock: OrcadInstanceLock; request: OrcadManagedStopRequest const lock = acquireOrcadInstanceLock(root, { identity: () => 'uid-1000', version: () => '1.0.0', - startedAtMs: () => 5_000 + startedAtMs: () => 5_000, + incarnation: () => null }) const { pid, startedAtMs, nonce } = lock.record return { @@ -189,11 +190,14 @@ describe('completing a managed stop', () => { }) }) - it('treats a reused PID with a different start time as proof of exit', async () => { + it('treats a reused PID with a different stored creation time as proof of exit', async () => { const { request } = running() - expect(await completeOrcadManagedStop(request, options({ startedAtMs: () => 99_000 }))).toBe( - 'exited' - ) + expect( + await completeOrcadManagedStop( + request, + options({ startedAtMs: () => 99_000, platform: 'win32' }) + ) + ).toBe('exited') }) it.each([ diff --git a/src/main/orcad/orcad-process-start-time.test.ts b/src/main/orcad/orcad-process-start-time.test.ts index 50f6df6ff09..b5b178248e9 100644 --- a/src/main/orcad/orcad-process-start-time.test.ts +++ b/src/main/orcad/orcad-process-start-time.test.ts @@ -1,3 +1,7 @@ +import { spawn } from 'node:child_process' +import { mkdtempSync, readFileSync, rmSync, symlinkSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import { afterEach, describe, expect, it, vi } from 'vitest' import type * as DaemonProcessStartTime from '../daemon/daemon-process-start-time' @@ -5,14 +9,17 @@ const { readWindowsProcessCreationTime, getProcessStartedAtMs } = vi.hoisted(() readWindowsProcessCreationTime: vi.fn<(pid: number) => number | null>(), getProcessStartedAtMs: vi.fn<(pid: number) => number | null>() })) -vi.mock('../windows/windows-process-table', () => ({ readWindowsProcessCreationTime })) +vi.mock('../windows/windows-process-table', () => ({ + readWindowsProcessCreationTime +})) vi.mock('../daemon/daemon-process-start-time', async (importOriginal) => ({ ...(await importOriginal()), getProcessStartedAtMs })) import { - orcadProcessStartTimeMatches, + compareOrcadProcessIdentity, + readOrcadProcessIncarnation, readOrcadProcessStartedAtMs } from './orcad-process-start-time' @@ -45,9 +52,86 @@ describe('orcad process start time', () => { it('tells a reused Windows PID apart, and fails open when the addon cannot answer', () => { onPlatform('win32') readWindowsProcessCreationTime.mockReturnValue(9_000_000) - expect(orcadProcessStartTimeMatches(42, 1_000)).toBe(false) - expect(orcadProcessStartTimeMatches(42, 9_000_100)).toBe(true) + expect(compareOrcadProcessIdentity(42, { startedAtMs: 1_000 })).toBe('differs') + expect(compareOrcadProcessIdentity(42, { startedAtMs: 9_000_100 })).toBe('matches') readWindowsProcessCreationTime.mockReturnValue(null) - expect(orcadProcessStartTimeMatches(42, 1_000)).toBe(true) + expect(compareOrcadProcessIdentity(42, { startedAtMs: 1_000 })).toBe('unknown') + }) +}) + +describe('orcad process identity', () => { + const incarnation = { linuxStartTicks: '1234', bootId: 'boot-a' } + const recorded = { startedAtMs: 1_000, ...incarnation } + + it('matches on ticks and boot id even when the btime-derived start time drifted', () => { + expect( + compareOrcadProcessIdentity(42, recorded, { + startedAtMs: () => 158_000, + incarnation: () => incarnation, + platform: 'linux' + }) + ).toBe('matches') + }) + + it.each([ + ['different start ticks', { ...incarnation, linuxStartTicks: '9999' }], + ['a different boot', { ...incarnation, bootId: 'boot-b' }] + ])('proves reuse on %s', (_name, actual) => { + expect(compareOrcadProcessIdentity(42, recorded, { incarnation: () => actual })).toBe('differs') + }) + + it('is unknown, never different, when the incarnation cannot be read', () => { + expect(compareOrcadProcessIdentity(42, recorded, { incarnation: () => null })).toBe('unknown') + }) + + it('reads a Linux wall-clock mismatch on a record without ticks as unproven', () => { + expect( + compareOrcadProcessIdentity( + 42, + { startedAtMs: 1_000 }, + { + startedAtMs: () => 158_000, + incarnation: () => incarnation, + platform: 'linux' + } + ) + ).toBe('unproven') + }) +}) + +describe.runIf(platform === 'linux')('orcad process incarnation on a real /proc', () => { + it('reads ticks and the boot id, even for a command name with spaces and parens', async () => { + const dir = mkdtempSync(join(tmpdir(), 'orcad-incarnation-')) + const odd = join(dir, 'x) y (z') + symlinkSync('/bin/sleep', odd) + const child = spawn(odd, ['30'], { stdio: 'ignore' }) + try { + await new Promise((resolve, reject) => { + child.once('spawn', resolve) + child.once('error', reject) + }) + const pid = child.pid ?? 0 + const bootId = readFileSync('/proc/sys/kernel/random/boot_id', 'utf8').trim() + const own = readOrcadProcessIncarnation(process.pid) + const other = readOrcadProcessIncarnation(pid) + expect(own).toEqual({ + linuxStartTicks: expect.stringMatching(/^\d+$/), + bootId + }) + expect(other).toEqual({ + linuxStartTicks: expect.stringMatching(/^\d+$/), + bootId + }) + expect(compareOrcadProcessIdentity(pid, { startedAtMs: null, ...other })).toBe('matches') + // This test process started well before the child, so its ticks name another process. + expect(compareOrcadProcessIdentity(pid, { startedAtMs: null, ...own })).toBe('differs') + } finally { + child.kill('SIGKILL') + rmSync(dir, { recursive: true, force: true }) + } + }) + + it('answers null for a PID that does not exist', () => { + expect(readOrcadProcessIncarnation(2 ** 30)).toBeNull() }) }) diff --git a/src/main/orcad/orcad-process-start-time.ts b/src/main/orcad/orcad-process-start-time.ts index 9952d8485bb..01637f4e380 100644 --- a/src/main/orcad/orcad-process-start-time.ts +++ b/src/main/orcad/orcad-process-start-time.ts @@ -1,11 +1,15 @@ /** - * A process's start time as orcad's identity records carry it (instance lock, managed stop). + * A process's identity as orcad's records carry it (instance lock, managed stop). * * Windows reuses PIDs aggressively and `getProcessStartedAtMs` answers null there, so a PID * alone would be the whole identity. The process-tree addon orcad's Windows slot stages reads * the kernel creation time for one PID without a table snapshot or a PowerShell spawn. + * Linux derives wall-clock start time from `/proc/stat` btime, which drifts (WSL2, NTP slews), + * so there the kernel start ticks plus boot id are the identity and wall time is only a hint. * Null still means "cannot say": callers must never read it as proof of exit. */ +import { readFileSync } from 'node:fs' +import { parseLinuxStartTicks } from '../agent-hooks/managed-hook-owner-identity' import { getProcessStartedAtMs, START_TIME_TOLERANCE_MS, @@ -13,17 +17,80 @@ import { } from '../daemon/daemon-process-start-time' import { readWindowsProcessCreationTime } from '../windows/windows-process-table' +export type OrcadProcessIncarnation = { linuxStartTicks: string; bootId: string } + +/** What a lock or stop request recorded; the incarnation is absent from older writers. */ +export type OrcadRecordedIdentity = { + startedAtMs: number | null + linuxStartTicks?: string + bootId?: string +} + +/** + * `differs` proves the PID now names another process. `unproven` is a wall-clock disagreement + * that proves nothing, because the clock it was derived from can drift. + */ +export type OrcadProcessIdentityVerdict = 'matches' | 'differs' | 'unknown' | 'unproven' + +export type OrcadProcessIdentityReaders = { + startedAtMs?: (pid: number) => number | null + incarnation?: (pid: number) => OrcadProcessIncarnation | null + platform?: NodeJS.Platform +} + export function readOrcadProcessStartedAtMs(pid: number): number | null { return process.platform === 'win32' ? readWindowsProcessCreationTime(pid) : getProcessStartedAtMs(pid) } -/** Fails open on an unknown time on either side: an unprovable mismatch keeps the holder. */ -export function orcadProcessStartTimeMatches(pid: number, expected: number | null): boolean { - return startTimesWithinTolerance( - readOrcadProcessStartedAtMs(pid), - expected, - START_TIME_TOLERANCE_MS - ) +/** Linux only: `/proc//stat` field 22 and the kernel boot id, neither tied to a clock. */ +export function readOrcadProcessIncarnation(pid: number): OrcadProcessIncarnation | null { + if (process.platform !== 'linux') { + return null + } + try { + const linuxStartTicks = parseLinuxStartTicks(readFileSync(`/proc/${pid}/stat`, 'utf8')) + const bootId = readFileSync('/proc/sys/kernel/random/boot_id', 'utf8').trim() + return linuxStartTicks && /^\d+$/.test(linuxStartTicks) && bootId + ? { linuxStartTicks, bootId } + : null + } catch { + return null + } +} + +export function recordedOrcadProcessIncarnation( + recorded: OrcadRecordedIdentity +): OrcadProcessIncarnation | null { + return recorded.linuxStartTicks && recorded.bootId + ? { linuxStartTicks: recorded.linuxStartTicks, bootId: recorded.bootId } + : null +} + +/** Prefers the recorded incarnation; wall time decides only for records that lack one. */ +export function compareOrcadProcessIdentity( + pid: number, + recorded: OrcadRecordedIdentity, + readers: OrcadProcessIdentityReaders = {} +): OrcadProcessIdentityVerdict { + const expected = recordedOrcadProcessIncarnation(recorded) + if (expected) { + const actual = (readers.incarnation ?? readOrcadProcessIncarnation)(pid) + if (!actual) { + return 'unknown' + } + return actual.bootId === expected.bootId && actual.linuxStartTicks === expected.linuxStartTicks + ? 'matches' + : 'differs' + } + const actual = (readers.startedAtMs ?? readOrcadProcessStartedAtMs)(pid) + if (actual === null || recorded.startedAtMs === null) { + return 'unknown' + } + if (startTimesWithinTolerance(actual, recorded.startedAtMs, START_TIME_TOLERANCE_MS)) { + return 'matches' + } + // Windows and macOS report a stored creation time; Linux recomputes it from a drifting btime. + return (readers.platform ?? process.platform) === 'linux' ? 'unproven' : 'differs' } diff --git a/src/main/ssh/orcad-managed-remote-stop.ts b/src/main/ssh/orcad-managed-remote-stop.ts index 9e9fe969169..87c639d01fb 100644 --- a/src/main/ssh/orcad-managed-remote-stop.ts +++ b/src/main/ssh/orcad-managed-remote-stop.ts @@ -84,7 +84,16 @@ export async function readRemoteOrcadManagedStopTarget( context: { version, runtimeId, - instance: { pid: lock.pid, startedAtMs: lock.startedAtMs, nonce: lock.nonce, lockPath } + instance: { + pid: lock.pid, + startedAtMs: lock.startedAtMs, + // Older slots drop these; newer ones prove identity without the drifting wall clock. + ...(lock.linuxStartTicks && lock.bootId + ? { linuxStartTicks: lock.linuxStartTicks, bootId: lock.bootId } + : {}), + nonce: lock.nonce, + lockPath + } } } } diff --git a/src/shared/orcad-stop-request.ts b/src/shared/orcad-stop-request.ts index f6e4dc243d4..cc14bc1c051 100644 --- a/src/shared/orcad-stop-request.ts +++ b/src/shared/orcad-stop-request.ts @@ -22,6 +22,9 @@ export const ORCAD_MANAGED_STOP_REQUEST_FILE_FLAG = '--request-file' export const OrcadManagedStopInstanceSchema = z.object({ pid: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), startedAtMs: z.number().finite().nonnegative().nullable(), + /** Linux only, copied from the lock; older clients omit it and older slots drop it. */ + linuxStartTicks: z.string().regex(/^\d+$/).max(32).optional().catch(undefined), + bootId: z.string().min(1).max(255).optional().catch(undefined), nonce: z.string().min(1).max(255), lockPath: z.string().min(1).max(4096) }) @@ -40,6 +43,13 @@ export const OrcadManagedStopRequestSchema = z.object({ export type OrcadManagedStopInstance = z.infer export type OrcadManagedStopRequest = z.infer + +/** Equality key for a request; omits the incarnation, which an older client recovering it drops. */ +export function orcadManagedStopRequestKey(request: OrcadManagedStopRequest): string { + const { instance, ...rest } = OrcadManagedStopRequestSchema.parse(request) + const { linuxStartTicks: _ticks, bootId: _bootId, ...bound } = instance + return JSON.stringify({ ...rest, instance: bound }) +} /** What a stopping orcad must match before a managed request may stop it. */ export type OrcadManagedStopContext = Omit< OrcadManagedStopRequest,