mirror of
https://github.com/stablyai/orca.git
synced 2026-10-31 00:57:45 +00:00
fix(orcad): identify instances by kernel start ticks, not drifting wall-clock start time (#28125)
* fix(orcad): identify instances by kernel start ticks, not drifting wall-clock start time On Linux the start time is derived from /proc/stat btime, which drifts (WSL2, NTP), so a live orcad was read as a reused PID and its managed stop reported exited without ever writing the request. The lock and stop request now carry optional linuxStartTicks + bootId; readers prefer them, and a wall-clock mismatch on Linux for records without them is unverifiable, never proof of exit. Co-Authored-By: Claude <noreply@anthropic.com> * fix(orcad): match stop receipts without the incarnation an older client drops A request recovered by an older client loses linuxStartTicks/bootId, which made the newer host reject its own receipts and decision records as mismatched. Also exercise the incarnation reader against a real Linux /proc. --------- Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -10,6 +10,7 @@ import {
|
||||
OrcadCompletedStopReceiptSchema,
|
||||
OrcadManagedStopRequestSchema,
|
||||
OrcadDaemonRetirementRecordSchema,
|
||||
orcadManagedStopRequestKey,
|
||||
type OrcadCompletedStopReceipt,
|
||||
type OrcadDaemonRetirementRecord,
|
||||
type OrcadManagedStopRequest
|
||||
@@ -49,8 +50,7 @@ export function readOrcadStopReceipt<T extends { request: OrcadManagedStopReques
|
||||
const receipt = schema.parse(
|
||||
JSON.parse(readNodeFileSyncWithinLimit(path, RECEIPT_MAX_BYTES).buffer.toString('utf8'))
|
||||
)
|
||||
const expected = OrcadManagedStopRequestSchema.parse(request)
|
||||
if (JSON.stringify(receipt.request) !== JSON.stringify(expected)) {
|
||||
if (orcadManagedStopRequestKey(receipt.request) !== orcadManagedStopRequestKey(request)) {
|
||||
throw new Error('orcad_stop_receipt_mismatch')
|
||||
}
|
||||
return receipt
|
||||
|
||||
@@ -13,9 +13,11 @@ import {
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { z } from 'zod'
|
||||
import {
|
||||
acquireOrcadInstanceLock,
|
||||
ORCAD_LOCK_FILE_NAME,
|
||||
readOrcadInstanceLockRecord,
|
||||
type OrcadInstanceLockHooks,
|
||||
type OrcadLockRecord
|
||||
} from './orcad-instance-lock'
|
||||
@@ -35,7 +37,8 @@ function hooks(overrides: OrcadInstanceLockHooks = {}): OrcadInstanceLockHooks {
|
||||
identity: () => 'uid-1000',
|
||||
version: () => '1.0.0-test',
|
||||
startedAtMs: () => 1_000,
|
||||
startTimeMatches: () => true,
|
||||
incarnation: () => null,
|
||||
holderIdentity: () => 'matches',
|
||||
processIsAlive: () => false,
|
||||
...overrides
|
||||
}
|
||||
@@ -94,11 +97,93 @@ describe('acquireOrcadInstanceLock', () => {
|
||||
writeFileSync(join(root, ORCAD_LOCK_FILE_NAME), JSON.stringify(persistedRecord()))
|
||||
const lock = acquireOrcadInstanceLock(
|
||||
root,
|
||||
hooks({ processIsAlive: () => true, startTimeMatches: () => false })
|
||||
hooks({ processIsAlive: () => true, holderIdentity: () => 'differs' })
|
||||
)
|
||||
expect(JSON.parse(readFileSync(lock.path, 'utf8')).pid).toBe(process.pid)
|
||||
})
|
||||
|
||||
it('keeps a live holder whose ticks match though its btime-derived start time drifted', () => {
|
||||
const root = makeRoot()
|
||||
const incarnation = { linuxStartTicks: '1234', bootId: 'boot-a' }
|
||||
writeFileSync(
|
||||
join(root, ORCAD_LOCK_FILE_NAME),
|
||||
JSON.stringify(persistedRecord({ startedAtMs: 1_000, ...incarnation }))
|
||||
)
|
||||
const drifted = {
|
||||
processIsAlive: () => true,
|
||||
holderIdentity: undefined,
|
||||
startedAtMs: () => 158_000,
|
||||
incarnation: () => incarnation
|
||||
}
|
||||
expect(() => acquireOrcadInstanceLock(root, hooks(drifted))).toThrow(
|
||||
expect.objectContaining({ code: 'orcad_instance_lock_held' })
|
||||
)
|
||||
const reused = { ...drifted, incarnation: () => ({ ...incarnation, linuxStartTicks: '9' }) }
|
||||
expect(acquireOrcadInstanceLock(root, hooks(reused)).record.nonce).not.toBe('stale')
|
||||
})
|
||||
|
||||
it.runIf(process.platform === 'linux')(
|
||||
'keeps an older Linux record whose wall-clock start time disagrees, since that proves nothing',
|
||||
() => {
|
||||
const root = makeRoot()
|
||||
writeFileSync(join(root, ORCAD_LOCK_FILE_NAME), JSON.stringify(persistedRecord()))
|
||||
expect(() =>
|
||||
acquireOrcadInstanceLock(
|
||||
root,
|
||||
hooks({ processIsAlive: () => true, holderIdentity: undefined, startedAtMs: () => 9e9 })
|
||||
)
|
||||
).toThrow(expect.objectContaining({ code: 'orcad_instance_lock_held' }))
|
||||
}
|
||||
)
|
||||
|
||||
it('records the incarnation in a lock an older reader still parses', () => {
|
||||
const root = makeRoot()
|
||||
const incarnation = { linuxStartTicks: '1234', bootId: 'boot-a' }
|
||||
const lock = acquireOrcadInstanceLock(root, hooks({ incarnation: () => incarnation }))
|
||||
const written: unknown = JSON.parse(readFileSync(lock.path, 'utf8'))
|
||||
expect(written).toMatchObject({ startedAtMs: 1_000, ...incarnation })
|
||||
// The shape desktops before this change parse with: unknown keys are dropped, not refused.
|
||||
const olderReader = z.object({
|
||||
pid: z.number().int().positive(),
|
||||
startedAtMs: z.number().finite().nonnegative().nullable(),
|
||||
identity: z.string().min(1),
|
||||
version: z.string().min(1),
|
||||
acquiredAt: z.iso.datetime({ offset: true }),
|
||||
nonce: z.string().min(1),
|
||||
role: z.enum(['orcad', 'desktop']).optional()
|
||||
})
|
||||
expect(olderReader.parse(written)).toEqual({
|
||||
pid: process.pid,
|
||||
startedAtMs: 1_000,
|
||||
identity: 'uid-1000',
|
||||
version: '1.0.0-test',
|
||||
acquiredAt: lock.record.acquiredAt,
|
||||
nonce: lock.record.nonce,
|
||||
role: 'orcad'
|
||||
})
|
||||
lock.release()
|
||||
})
|
||||
|
||||
it('reads an older lock without an incarnation, and salvages a malformed one', () => {
|
||||
const root = makeRoot()
|
||||
writeFileSync(
|
||||
join(root, ORCAD_LOCK_FILE_NAME),
|
||||
JSON.stringify(persistedRecord({ linuxStartTicks: undefined }))
|
||||
)
|
||||
expect(readOrcadInstanceLockRecord(join(root, ORCAD_LOCK_FILE_NAME))).not.toHaveProperty(
|
||||
'linuxStartTicks'
|
||||
)
|
||||
writeFileSync(
|
||||
join(root, ORCAD_LOCK_FILE_NAME),
|
||||
JSON.stringify({ ...persistedRecord(), linuxStartTicks: 12, bootId: '' })
|
||||
)
|
||||
expect(readOrcadInstanceLockRecord(join(root, ORCAD_LOCK_FILE_NAME))).toMatchObject({
|
||||
nonce: 'stale',
|
||||
linuxStartTicks: undefined,
|
||||
bootId: undefined
|
||||
})
|
||||
})
|
||||
|
||||
it('does not displace a successor published between stale inspection and reclaim', () => {
|
||||
const root = makeRoot()
|
||||
const lockPath = join(root, ORCAD_LOCK_FILE_NAME)
|
||||
|
||||
@@ -19,8 +19,11 @@ import { join } from 'node:path'
|
||||
import process from 'node:process'
|
||||
import { z } from 'zod'
|
||||
import {
|
||||
orcadProcessStartTimeMatches,
|
||||
readOrcadProcessStartedAtMs
|
||||
compareOrcadProcessIdentity,
|
||||
readOrcadProcessIncarnation,
|
||||
readOrcadProcessStartedAtMs,
|
||||
type OrcadProcessIdentityVerdict,
|
||||
type OrcadProcessIncarnation
|
||||
} from './orcad-process-start-time'
|
||||
import {
|
||||
assertOrcadDataRootIsPrivate,
|
||||
@@ -39,6 +42,12 @@ const OrcadLockRecordSchema = z.object({
|
||||
pid: z.number().int().positive().max(Number.MAX_SAFE_INTEGER),
|
||||
/** Null where the platform cannot read it; PID alone is then the (weaker) fence. */
|
||||
startedAtMs: z.number().finite().nonnegative().nullable(),
|
||||
/**
|
||||
* Linux only, absent from older writers: drift-free identity preferred over `startedAtMs`.
|
||||
* Salvaged rather than refused, so a future encoding never makes the whole lock unreadable.
|
||||
*/
|
||||
linuxStartTicks: z.string().regex(/^\d+$/).max(32).optional().catch(undefined),
|
||||
bootId: z.string().min(1).max(255).optional().catch(undefined),
|
||||
/** POSIX uid, or the Windows username. Compared as an opaque string. */
|
||||
identity: z.string().min(1).max(1_024),
|
||||
version: z.string().min(1).max(255),
|
||||
@@ -69,7 +78,9 @@ export type OrcadInstanceLockHooks = OrcadDataRootPrivacyHooks & {
|
||||
/** Whether a PID is running. EPERM counts as alive: it proves the process exists. */
|
||||
processIsAlive?: (pid: number) => boolean
|
||||
startedAtMs?: (pid: number) => number | null
|
||||
startTimeMatches?: (pid: number, expected: number | null) => boolean
|
||||
incarnation?: (pid: number) => OrcadProcessIncarnation | null
|
||||
/** Only `differs` frees a live holder's PID; anything less proves nothing. */
|
||||
holderIdentity?: (record: OrcadLockRecord) => OrcadProcessIdentityVerdict
|
||||
/**
|
||||
* Who takes the profile. The desktop app takes it too, so the two refuse each other; it must
|
||||
* hold Electron's single-instance lock first, which is what lets it reclaim a desktop record.
|
||||
@@ -108,7 +119,14 @@ export function acquireOrcadInstanceLock(
|
||||
const identity = (hooks.identity ?? defaultIdentity)()
|
||||
const isAlive = hooks.processIsAlive ?? isProcessAlive
|
||||
const readStartedAt = hooks.startedAtMs ?? readOrcadProcessStartedAtMs
|
||||
const matchesStartTime = hooks.startTimeMatches ?? orcadProcessStartTimeMatches
|
||||
const readIncarnation = hooks.incarnation ?? readOrcadProcessIncarnation
|
||||
const holderIdentity =
|
||||
hooks.holderIdentity ??
|
||||
((holder: OrcadLockRecord) =>
|
||||
compareOrcadProcessIdentity(holder.pid, holder, {
|
||||
startedAtMs: readStartedAt,
|
||||
incarnation: readIncarnation
|
||||
}))
|
||||
|
||||
try {
|
||||
mkdirSync(dataRoot, { recursive: true, mode: 0o700 })
|
||||
@@ -127,6 +145,7 @@ export function acquireOrcadInstanceLock(
|
||||
const record: OrcadLockRecord = {
|
||||
pid: process.pid,
|
||||
startedAtMs: readStartedAt(process.pid),
|
||||
...readIncarnation(process.pid),
|
||||
identity,
|
||||
version: (hooks.version ?? (() => process.env.ORCA_VERSION ?? 'unknown'))(),
|
||||
acquiredAt: (hooks.now ?? (() => new Date()))().toISOString(),
|
||||
@@ -192,11 +211,7 @@ export function acquireOrcadInstanceLock(
|
||||
// The desktop takes this lock only after Electron's single-instance lock, which already proves
|
||||
// no other desktop runs; a desktop record is then stale even when its PID was reused.
|
||||
const staleDesktopRecord = hooks.role === 'desktop' && existing.role === 'desktop'
|
||||
if (
|
||||
!staleDesktopRecord &&
|
||||
isAlive(existing.pid) &&
|
||||
matchesStartTime(existing.pid, existing.startedAtMs)
|
||||
) {
|
||||
if (!staleDesktopRecord && isAlive(existing.pid) && holderIdentity(existing) !== 'differs') {
|
||||
throw new OrcadInstanceLockError(
|
||||
'orcad_instance_lock_held',
|
||||
`${describeLockHolder(existing)} (pid ${existing.pid}, started ` +
|
||||
|
||||
@@ -3,22 +3,25 @@
|
||||
*
|
||||
* Runs as a short-lived command on the execution host. It never signals: it writes the
|
||||
* instance-bound request the running orcad watches for, then observes until the instance is
|
||||
* gone. `exited` needs proof — no process with that PID, or a PID whose start time shows it now
|
||||
* belongs to another process. Anything the host cannot answer is `unverifiable`, never exit.
|
||||
* gone. `exited` needs proof — no process with that PID, or a PID whose kernel identity shows it
|
||||
* now belongs to another process. A wall-clock disagreement, or anything the host cannot answer,
|
||||
* is `unverifiable`, never exit.
|
||||
*/
|
||||
import { setTimeout as delay } from 'node:timers/promises'
|
||||
import { writeDurableSecureJsonFile } from '../../shared/secure-file'
|
||||
import {
|
||||
OrcadManagedStopRequestSchema,
|
||||
orcadManagedStopRequestKey,
|
||||
type OrcadManagedStopInstance,
|
||||
type OrcadManagedStopRequest,
|
||||
type OrcadManagedStopVerdict
|
||||
} from '../../shared/orcad-stop-request'
|
||||
import {
|
||||
START_TIME_TOLERANCE_MS,
|
||||
startTimesWithinTolerance
|
||||
} from '../daemon/daemon-process-start-time'
|
||||
import { readOrcadProcessStartedAtMs } from './orcad-process-start-time'
|
||||
compareOrcadProcessIdentity,
|
||||
recordedOrcadProcessIncarnation,
|
||||
type OrcadProcessIdentityReaders,
|
||||
type OrcadRecordedIdentity
|
||||
} from './orcad-process-start-time'
|
||||
import { hasErrorCode, inspectProcessSignal } from '../daemon/daemon-process-inspection'
|
||||
import { persistOrcadCompletedStopReceipt } from './orcad-completed-stop-receipt'
|
||||
import { readOrcadManagedStopDecision } from './orcad-managed-stop-decision'
|
||||
@@ -26,6 +29,7 @@ import { withdrawOrcadManagedStopRequest } from './orcad-managed-stop-cancellati
|
||||
import {
|
||||
orcadInstanceLockNames,
|
||||
orcadManagedStopRequestPath,
|
||||
readOrcadInstanceLockNaming,
|
||||
readOrcadManagedStopRequest
|
||||
} from './orcad-managed-stop-request'
|
||||
import {
|
||||
@@ -35,9 +39,8 @@ import {
|
||||
|
||||
export type OrcadProcessProbe = (pid: number) => 'alive' | 'missing' | 'unverifiable'
|
||||
|
||||
export type OrcadManagedStopCompletionOptions = {
|
||||
export type OrcadManagedStopCompletionOptions = OrcadProcessIdentityReaders & {
|
||||
probeProcess?: OrcadProcessProbe
|
||||
startedAtMs?: (pid: number) => number | null
|
||||
sleep?: () => Promise<void>
|
||||
attempts?: number
|
||||
now?: () => Date
|
||||
@@ -51,23 +54,31 @@ function defaultProbe(pid: number): ReturnType<OrcadProcessProbe> {
|
||||
|
||||
function observeInstance(
|
||||
instance: OrcadManagedStopInstance,
|
||||
recorded: OrcadRecordedIdentity,
|
||||
options: OrcadManagedStopCompletionOptions
|
||||
): OrcadManagedStopVerdict {
|
||||
const probe = (options.probeProcess ?? defaultProbe)(instance.pid)
|
||||
if (probe !== 'alive') {
|
||||
return probe === 'missing' ? 'exited' : 'unverifiable'
|
||||
}
|
||||
const actual = (options.startedAtMs ?? readOrcadProcessStartedAtMs)(instance.pid)
|
||||
// A reused PID is proof of exit only when both start times are known and disagree; on Windows
|
||||
// without the addon both stay null, so a live PID is never read as exited.
|
||||
if (
|
||||
actual !== null &&
|
||||
instance.startedAtMs !== null &&
|
||||
!startTimesWithinTolerance(actual, instance.startedAtMs, START_TIME_TOLERANCE_MS)
|
||||
) {
|
||||
return 'exited'
|
||||
// Only a kernel identity (or a stored creation time) proves reuse; an unknown one stays live.
|
||||
const identity = compareOrcadProcessIdentity(instance.pid, recorded, options)
|
||||
return identity === 'differs' ? 'exited' : identity === 'unproven' ? 'unverifiable' : 'live'
|
||||
}
|
||||
|
||||
/** An older client's request lacks the incarnation; the lock still naming the instance has it. */
|
||||
function recordedIdentity(instance: OrcadManagedStopInstance): OrcadRecordedIdentity {
|
||||
if (recordedOrcadProcessIncarnation(instance)) {
|
||||
return instance
|
||||
}
|
||||
return 'live'
|
||||
let record: OrcadRecordedIdentity | null = null
|
||||
try {
|
||||
record = readOrcadInstanceLockNaming(instance)
|
||||
} catch {
|
||||
return instance
|
||||
}
|
||||
const incarnation = record ? recordedOrcadProcessIncarnation(record) : null
|
||||
return incarnation ? { ...instance, ...incarnation } : instance
|
||||
}
|
||||
|
||||
/** Writes the request only while the lock still names this instance, then waits for exit. */
|
||||
@@ -84,7 +95,8 @@ export async function completeOrcadManagedStop(
|
||||
persistOrcadCompletedStopReceipt(request, (options.now ?? (() => new Date()))())
|
||||
return 'exited'
|
||||
}
|
||||
let verdict = observeInstance(request.instance, options)
|
||||
const recorded = recordedIdentity(request.instance)
|
||||
let verdict = observeInstance(request.instance, recorded, options)
|
||||
if (verdict !== 'live') {
|
||||
return verdict === 'exited' ? completed() : verdict
|
||||
}
|
||||
@@ -110,7 +122,7 @@ export async function completeOrcadManagedStop(
|
||||
}
|
||||
for (let attempt = 0; attempt < attempts; attempt++) {
|
||||
await (options.sleep ?? (() => delay(ORCAD_STOP_COMPLETION_POLL_MS)))()
|
||||
verdict = observeInstance(request.instance, options)
|
||||
verdict = observeInstance(request.instance, recorded, options)
|
||||
if (verdict !== 'live') {
|
||||
return verdict === 'exited' ? completed() : verdict
|
||||
}
|
||||
@@ -129,7 +141,10 @@ function lockStillNamesInstance(instance: OrcadManagedStopInstance): boolean {
|
||||
/** A request already present must be this one; another transaction's request is not ours. */
|
||||
function existingRequestMatches(path: string, request: OrcadManagedStopRequest): boolean {
|
||||
try {
|
||||
return JSON.stringify(readOrcadManagedStopRequest(path)) === JSON.stringify(request)
|
||||
return (
|
||||
orcadManagedStopRequestKey(readOrcadManagedStopRequest(path)) ===
|
||||
orcadManagedStopRequestKey(request)
|
||||
)
|
||||
} catch (error) {
|
||||
return hasErrorCode(error, 'ENOENT')
|
||||
}
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { z } from 'zod'
|
||||
import {
|
||||
OrcadManagedStopRequestSchema,
|
||||
type OrcadManagedStopRequest
|
||||
} from '../../shared/orcad-stop-request'
|
||||
import { acquireOrcadInstanceLock } from './orcad-instance-lock'
|
||||
import { orcadManagedStopRequestPath } from './orcad-managed-stop-request'
|
||||
import {
|
||||
completeOrcadManagedStop,
|
||||
type OrcadManagedStopCompletionOptions
|
||||
} from './orcad-managed-stop-completion'
|
||||
import { orcadStopReceiptPath } from './orcad-completed-stop-receipt'
|
||||
import type { OrcadProcessIncarnation } from './orcad-process-start-time'
|
||||
|
||||
const roots: string[] = []
|
||||
afterEach(() => {
|
||||
for (const root of roots.splice(0)) {
|
||||
rmSync(root, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
const RECORDED: OrcadProcessIncarnation = {
|
||||
linuxStartTicks: '4242',
|
||||
bootId: 'boot-a'
|
||||
}
|
||||
// The WSL2 smoke saw btime move by 157 s against a 1.5 s tolerance.
|
||||
const DRIFTED_STARTED_AT_MS = 5_000 + 157_000
|
||||
|
||||
function running(
|
||||
recorded: OrcadProcessIncarnation | null,
|
||||
{ requestCarriesIncarnation = false } = {}
|
||||
): OrcadManagedStopRequest {
|
||||
const root = mkdtempSync(join(tmpdir(), 'orcad-stop-identity-'))
|
||||
roots.push(root)
|
||||
const lock = acquireOrcadInstanceLock(root, {
|
||||
identity: () => 'uid-1000',
|
||||
startedAtMs: () => 5_000,
|
||||
incarnation: () => recorded
|
||||
})
|
||||
const { pid, startedAtMs, nonce } = lock.record
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
transactionId: '0b9f6a3e-9e2c-4c8e-8f58-4c0f6b1d2e3a',
|
||||
version: '1.0.0',
|
||||
runtimeId: 'runtime-1',
|
||||
instance: {
|
||||
pid,
|
||||
startedAtMs,
|
||||
...(requestCarriesIncarnation && recorded ? recorded : {}),
|
||||
nonce,
|
||||
lockPath: lock.path
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Alive for `aliveFor` observations, then gone. */
|
||||
function linuxHost(
|
||||
actual: OrcadProcessIncarnation | null,
|
||||
aliveFor = Number.POSITIVE_INFINITY
|
||||
): OrcadManagedStopCompletionOptions {
|
||||
let observations = 0
|
||||
return {
|
||||
platform: 'linux',
|
||||
probeProcess: () => (observations++ < aliveFor ? 'alive' : 'missing'),
|
||||
startedAtMs: () => DRIFTED_STARTED_AT_MS,
|
||||
incarnation: () => actual,
|
||||
sleep: async () => {},
|
||||
attempts: 3,
|
||||
now: () => new Date('2026-10-01T00:00:00.000Z')
|
||||
}
|
||||
}
|
||||
|
||||
const requestWritten = (request: OrcadManagedStopRequest): boolean =>
|
||||
existsSync(orcadManagedStopRequestPath(request.instance))
|
||||
|
||||
describe('managed stop instance identity', () => {
|
||||
it.each([
|
||||
['the lock', false],
|
||||
['the request', true]
|
||||
])(
|
||||
'keeps a drifted-btime instance live by its ticks from %s, writes the request, then proves exit',
|
||||
async (_source, requestCarriesIncarnation) => {
|
||||
const request = running(RECORDED, { requestCarriesIncarnation })
|
||||
expect(await completeOrcadManagedStop(request, linuxHost(RECORDED, 2))).toBe('exited')
|
||||
expect(requestWritten(request)).toBe(true)
|
||||
expect(existsSync(orcadStopReceiptPath(request, 'completed'))).toBe(true)
|
||||
}
|
||||
)
|
||||
|
||||
it.each([
|
||||
['different start ticks', { ...RECORDED, linuxStartTicks: '9999' }],
|
||||
['a different boot', { ...RECORDED, bootId: 'boot-b' }]
|
||||
])('proves exit for a PID now naming a process with %s', async (_name, actual) => {
|
||||
const request = running(RECORDED)
|
||||
expect(await completeOrcadManagedStop(request, linuxHost(actual))).toBe('exited')
|
||||
expect(requestWritten(request)).toBe(false)
|
||||
})
|
||||
|
||||
it('stays live, never exited, while the incarnation cannot be read', async () => {
|
||||
const request = running(RECORDED)
|
||||
expect(await completeOrcadManagedStop(request, linuxHost(null))).toBe('live')
|
||||
expect(requestWritten(request)).toBe(true)
|
||||
})
|
||||
|
||||
it('answers unverifiable, not exited, for an older lock whose wall-clock time drifted', async () => {
|
||||
const request = running(null)
|
||||
expect(await completeOrcadManagedStop(request, linuxHost(RECORDED))).toBe('unverifiable')
|
||||
expect(existsSync(orcadStopReceiptPath(request, 'completed'))).toBe(false)
|
||||
})
|
||||
|
||||
it('stays readable by peers on either side of the change', async () => {
|
||||
const request = running(RECORDED, { requestCarriesIncarnation: true })
|
||||
// The instance shape slots before this change parse: the incarnation is dropped, not refused.
|
||||
const olderInstance = z.object({
|
||||
pid: z.number().int().positive(),
|
||||
startedAtMs: z.number().finite().nonnegative().nullable(),
|
||||
nonce: z.string().min(1),
|
||||
lockPath: z.string().min(1)
|
||||
})
|
||||
const { linuxStartTicks: _ticks, bootId: _boot, ...withoutIncarnation } = request.instance
|
||||
expect(olderInstance.parse(request.instance)).toEqual(withoutIncarnation)
|
||||
// And a newer slot reads what an older client wrote, taking the incarnation from the lock.
|
||||
const fromOlderClient = OrcadManagedStopRequestSchema.parse({
|
||||
...request,
|
||||
instance: withoutIncarnation
|
||||
})
|
||||
expect(fromOlderClient.instance).not.toHaveProperty('linuxStartTicks')
|
||||
expect(await completeOrcadManagedStop(fromOlderClient, linuxHost(RECORDED, 2))).toBe('exited')
|
||||
expect(JSON.parse(readFileSync(orcadManagedStopRequestPath(request.instance), 'utf8'))).toEqual(
|
||||
fromOlderClient
|
||||
)
|
||||
})
|
||||
|
||||
it('matches its own receipt when an older client retries without the incarnation', async () => {
|
||||
const request = running(RECORDED, { requestCarriesIncarnation: true })
|
||||
expect(await completeOrcadManagedStop(request, linuxHost(RECORDED, 2))).toBe('exited')
|
||||
const { linuxStartTicks: _ticks, bootId: _boot, ...withoutIncarnation } = request.instance
|
||||
const retried = { ...request, instance: withoutIncarnation }
|
||||
expect(await completeOrcadManagedStop(retried, linuxHost(RECORDED, 0))).toBe('exited')
|
||||
})
|
||||
})
|
||||
@@ -11,7 +11,7 @@ import {
|
||||
type OrcadManagedStopInstance,
|
||||
type OrcadManagedStopRequest
|
||||
} from '../../shared/orcad-stop-request'
|
||||
import { readOrcadInstanceLockRecord } from './orcad-instance-lock'
|
||||
import { readOrcadInstanceLockRecord, type OrcadLockRecord } from './orcad-instance-lock'
|
||||
|
||||
/** Keyed by the lock nonce, so a request for a previous instance is never this one's. */
|
||||
export function orcadManagedStopRequestPath(instance: OrcadManagedStopInstance): string {
|
||||
@@ -38,11 +38,18 @@ export function sameOrcadManagedStopInstance(
|
||||
|
||||
/** Whether the instance lock still names exactly this instance. */
|
||||
export function orcadInstanceLockNames(instance: OrcadManagedStopInstance): boolean {
|
||||
return readOrcadInstanceLockNaming(instance) !== null
|
||||
}
|
||||
|
||||
/** The lock record, only while it still names exactly this instance. */
|
||||
export function readOrcadInstanceLockNaming(
|
||||
instance: OrcadManagedStopInstance
|
||||
): OrcadLockRecord | null {
|
||||
if (!lstatSync(instance.lockPath).isFile()) {
|
||||
return false
|
||||
return null
|
||||
}
|
||||
const record = readOrcadInstanceLockRecord(instance.lockPath)
|
||||
return record !== null && sameOrcadManagedStopInstance(instance, record)
|
||||
return record !== null && sameOrcadManagedStopInstance(instance, record) ? record : null
|
||||
}
|
||||
|
||||
/** Throws unless the request names this running instance and its lock is still ours. */
|
||||
|
||||
@@ -73,7 +73,8 @@ function running(): { lock: OrcadInstanceLock; request: OrcadManagedStopRequest
|
||||
const lock = acquireOrcadInstanceLock(root, {
|
||||
identity: () => 'uid-1000',
|
||||
version: () => '1.0.0',
|
||||
startedAtMs: () => 5_000
|
||||
startedAtMs: () => 5_000,
|
||||
incarnation: () => null
|
||||
})
|
||||
const { pid, startedAtMs, nonce } = lock.record
|
||||
return {
|
||||
@@ -189,11 +190,14 @@ describe('completing a managed stop', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('treats a reused PID with a different start time as proof of exit', async () => {
|
||||
it('treats a reused PID with a different stored creation time as proof of exit', async () => {
|
||||
const { request } = running()
|
||||
expect(await completeOrcadManagedStop(request, options({ startedAtMs: () => 99_000 }))).toBe(
|
||||
'exited'
|
||||
)
|
||||
expect(
|
||||
await completeOrcadManagedStop(
|
||||
request,
|
||||
options({ startedAtMs: () => 99_000, platform: 'win32' })
|
||||
)
|
||||
).toBe('exited')
|
||||
})
|
||||
|
||||
it.each([
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
import { spawn } from 'node:child_process'
|
||||
import { mkdtempSync, readFileSync, rmSync, symlinkSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type * as DaemonProcessStartTime from '../daemon/daemon-process-start-time'
|
||||
|
||||
@@ -5,14 +9,17 @@ const { readWindowsProcessCreationTime, getProcessStartedAtMs } = vi.hoisted(()
|
||||
readWindowsProcessCreationTime: vi.fn<(pid: number) => number | null>(),
|
||||
getProcessStartedAtMs: vi.fn<(pid: number) => number | null>()
|
||||
}))
|
||||
vi.mock('../windows/windows-process-table', () => ({ readWindowsProcessCreationTime }))
|
||||
vi.mock('../windows/windows-process-table', () => ({
|
||||
readWindowsProcessCreationTime
|
||||
}))
|
||||
vi.mock('../daemon/daemon-process-start-time', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof DaemonProcessStartTime>()),
|
||||
getProcessStartedAtMs
|
||||
}))
|
||||
|
||||
import {
|
||||
orcadProcessStartTimeMatches,
|
||||
compareOrcadProcessIdentity,
|
||||
readOrcadProcessIncarnation,
|
||||
readOrcadProcessStartedAtMs
|
||||
} from './orcad-process-start-time'
|
||||
|
||||
@@ -45,9 +52,86 @@ describe('orcad process start time', () => {
|
||||
it('tells a reused Windows PID apart, and fails open when the addon cannot answer', () => {
|
||||
onPlatform('win32')
|
||||
readWindowsProcessCreationTime.mockReturnValue(9_000_000)
|
||||
expect(orcadProcessStartTimeMatches(42, 1_000)).toBe(false)
|
||||
expect(orcadProcessStartTimeMatches(42, 9_000_100)).toBe(true)
|
||||
expect(compareOrcadProcessIdentity(42, { startedAtMs: 1_000 })).toBe('differs')
|
||||
expect(compareOrcadProcessIdentity(42, { startedAtMs: 9_000_100 })).toBe('matches')
|
||||
readWindowsProcessCreationTime.mockReturnValue(null)
|
||||
expect(orcadProcessStartTimeMatches(42, 1_000)).toBe(true)
|
||||
expect(compareOrcadProcessIdentity(42, { startedAtMs: 1_000 })).toBe('unknown')
|
||||
})
|
||||
})
|
||||
|
||||
describe('orcad process identity', () => {
|
||||
const incarnation = { linuxStartTicks: '1234', bootId: 'boot-a' }
|
||||
const recorded = { startedAtMs: 1_000, ...incarnation }
|
||||
|
||||
it('matches on ticks and boot id even when the btime-derived start time drifted', () => {
|
||||
expect(
|
||||
compareOrcadProcessIdentity(42, recorded, {
|
||||
startedAtMs: () => 158_000,
|
||||
incarnation: () => incarnation,
|
||||
platform: 'linux'
|
||||
})
|
||||
).toBe('matches')
|
||||
})
|
||||
|
||||
it.each([
|
||||
['different start ticks', { ...incarnation, linuxStartTicks: '9999' }],
|
||||
['a different boot', { ...incarnation, bootId: 'boot-b' }]
|
||||
])('proves reuse on %s', (_name, actual) => {
|
||||
expect(compareOrcadProcessIdentity(42, recorded, { incarnation: () => actual })).toBe('differs')
|
||||
})
|
||||
|
||||
it('is unknown, never different, when the incarnation cannot be read', () => {
|
||||
expect(compareOrcadProcessIdentity(42, recorded, { incarnation: () => null })).toBe('unknown')
|
||||
})
|
||||
|
||||
it('reads a Linux wall-clock mismatch on a record without ticks as unproven', () => {
|
||||
expect(
|
||||
compareOrcadProcessIdentity(
|
||||
42,
|
||||
{ startedAtMs: 1_000 },
|
||||
{
|
||||
startedAtMs: () => 158_000,
|
||||
incarnation: () => incarnation,
|
||||
platform: 'linux'
|
||||
}
|
||||
)
|
||||
).toBe('unproven')
|
||||
})
|
||||
})
|
||||
|
||||
describe.runIf(platform === 'linux')('orcad process incarnation on a real /proc', () => {
|
||||
it('reads ticks and the boot id, even for a command name with spaces and parens', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'orcad-incarnation-'))
|
||||
const odd = join(dir, 'x) y (z')
|
||||
symlinkSync('/bin/sleep', odd)
|
||||
const child = spawn(odd, ['30'], { stdio: 'ignore' })
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
child.once('spawn', resolve)
|
||||
child.once('error', reject)
|
||||
})
|
||||
const pid = child.pid ?? 0
|
||||
const bootId = readFileSync('/proc/sys/kernel/random/boot_id', 'utf8').trim()
|
||||
const own = readOrcadProcessIncarnation(process.pid)
|
||||
const other = readOrcadProcessIncarnation(pid)
|
||||
expect(own).toEqual({
|
||||
linuxStartTicks: expect.stringMatching(/^\d+$/),
|
||||
bootId
|
||||
})
|
||||
expect(other).toEqual({
|
||||
linuxStartTicks: expect.stringMatching(/^\d+$/),
|
||||
bootId
|
||||
})
|
||||
expect(compareOrcadProcessIdentity(pid, { startedAtMs: null, ...other })).toBe('matches')
|
||||
// This test process started well before the child, so its ticks name another process.
|
||||
expect(compareOrcadProcessIdentity(pid, { startedAtMs: null, ...own })).toBe('differs')
|
||||
} finally {
|
||||
child.kill('SIGKILL')
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('answers null for a PID that does not exist', () => {
|
||||
expect(readOrcadProcessIncarnation(2 ** 30)).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
/**
|
||||
* A process's start time as orcad's identity records carry it (instance lock, managed stop).
|
||||
* A process's identity as orcad's records carry it (instance lock, managed stop).
|
||||
*
|
||||
* Windows reuses PIDs aggressively and `getProcessStartedAtMs` answers null there, so a PID
|
||||
* alone would be the whole identity. The process-tree addon orcad's Windows slot stages reads
|
||||
* the kernel creation time for one PID without a table snapshot or a PowerShell spawn.
|
||||
* Linux derives wall-clock start time from `/proc/stat` btime, which drifts (WSL2, NTP slews),
|
||||
* so there the kernel start ticks plus boot id are the identity and wall time is only a hint.
|
||||
* Null still means "cannot say": callers must never read it as proof of exit.
|
||||
*/
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { parseLinuxStartTicks } from '../agent-hooks/managed-hook-owner-identity'
|
||||
import {
|
||||
getProcessStartedAtMs,
|
||||
START_TIME_TOLERANCE_MS,
|
||||
@@ -13,17 +17,80 @@ import {
|
||||
} from '../daemon/daemon-process-start-time'
|
||||
import { readWindowsProcessCreationTime } from '../windows/windows-process-table'
|
||||
|
||||
export type OrcadProcessIncarnation = { linuxStartTicks: string; bootId: string }
|
||||
|
||||
/** What a lock or stop request recorded; the incarnation is absent from older writers. */
|
||||
export type OrcadRecordedIdentity = {
|
||||
startedAtMs: number | null
|
||||
linuxStartTicks?: string
|
||||
bootId?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* `differs` proves the PID now names another process. `unproven` is a wall-clock disagreement
|
||||
* that proves nothing, because the clock it was derived from can drift.
|
||||
*/
|
||||
export type OrcadProcessIdentityVerdict = 'matches' | 'differs' | 'unknown' | 'unproven'
|
||||
|
||||
export type OrcadProcessIdentityReaders = {
|
||||
startedAtMs?: (pid: number) => number | null
|
||||
incarnation?: (pid: number) => OrcadProcessIncarnation | null
|
||||
platform?: NodeJS.Platform
|
||||
}
|
||||
|
||||
export function readOrcadProcessStartedAtMs(pid: number): number | null {
|
||||
return process.platform === 'win32'
|
||||
? readWindowsProcessCreationTime(pid)
|
||||
: getProcessStartedAtMs(pid)
|
||||
}
|
||||
|
||||
/** Fails open on an unknown time on either side: an unprovable mismatch keeps the holder. */
|
||||
export function orcadProcessStartTimeMatches(pid: number, expected: number | null): boolean {
|
||||
return startTimesWithinTolerance(
|
||||
readOrcadProcessStartedAtMs(pid),
|
||||
expected,
|
||||
START_TIME_TOLERANCE_MS
|
||||
)
|
||||
/** Linux only: `/proc/<pid>/stat` field 22 and the kernel boot id, neither tied to a clock. */
|
||||
export function readOrcadProcessIncarnation(pid: number): OrcadProcessIncarnation | null {
|
||||
if (process.platform !== 'linux') {
|
||||
return null
|
||||
}
|
||||
try {
|
||||
const linuxStartTicks = parseLinuxStartTicks(readFileSync(`/proc/${pid}/stat`, 'utf8'))
|
||||
const bootId = readFileSync('/proc/sys/kernel/random/boot_id', 'utf8').trim()
|
||||
return linuxStartTicks && /^\d+$/.test(linuxStartTicks) && bootId
|
||||
? { linuxStartTicks, bootId }
|
||||
: null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
export function recordedOrcadProcessIncarnation(
|
||||
recorded: OrcadRecordedIdentity
|
||||
): OrcadProcessIncarnation | null {
|
||||
return recorded.linuxStartTicks && recorded.bootId
|
||||
? { linuxStartTicks: recorded.linuxStartTicks, bootId: recorded.bootId }
|
||||
: null
|
||||
}
|
||||
|
||||
/** Prefers the recorded incarnation; wall time decides only for records that lack one. */
|
||||
export function compareOrcadProcessIdentity(
|
||||
pid: number,
|
||||
recorded: OrcadRecordedIdentity,
|
||||
readers: OrcadProcessIdentityReaders = {}
|
||||
): OrcadProcessIdentityVerdict {
|
||||
const expected = recordedOrcadProcessIncarnation(recorded)
|
||||
if (expected) {
|
||||
const actual = (readers.incarnation ?? readOrcadProcessIncarnation)(pid)
|
||||
if (!actual) {
|
||||
return 'unknown'
|
||||
}
|
||||
return actual.bootId === expected.bootId && actual.linuxStartTicks === expected.linuxStartTicks
|
||||
? 'matches'
|
||||
: 'differs'
|
||||
}
|
||||
const actual = (readers.startedAtMs ?? readOrcadProcessStartedAtMs)(pid)
|
||||
if (actual === null || recorded.startedAtMs === null) {
|
||||
return 'unknown'
|
||||
}
|
||||
if (startTimesWithinTolerance(actual, recorded.startedAtMs, START_TIME_TOLERANCE_MS)) {
|
||||
return 'matches'
|
||||
}
|
||||
// Windows and macOS report a stored creation time; Linux recomputes it from a drifting btime.
|
||||
return (readers.platform ?? process.platform) === 'linux' ? 'unproven' : 'differs'
|
||||
}
|
||||
|
||||
@@ -84,7 +84,16 @@ export async function readRemoteOrcadManagedStopTarget(
|
||||
context: {
|
||||
version,
|
||||
runtimeId,
|
||||
instance: { pid: lock.pid, startedAtMs: lock.startedAtMs, nonce: lock.nonce, lockPath }
|
||||
instance: {
|
||||
pid: lock.pid,
|
||||
startedAtMs: lock.startedAtMs,
|
||||
// Older slots drop these; newer ones prove identity without the drifting wall clock.
|
||||
...(lock.linuxStartTicks && lock.bootId
|
||||
? { linuxStartTicks: lock.linuxStartTicks, bootId: lock.bootId }
|
||||
: {}),
|
||||
nonce: lock.nonce,
|
||||
lockPath
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,9 @@ export const ORCAD_MANAGED_STOP_REQUEST_FILE_FLAG = '--request-file'
|
||||
export const OrcadManagedStopInstanceSchema = z.object({
|
||||
pid: z.number().int().positive().max(Number.MAX_SAFE_INTEGER),
|
||||
startedAtMs: z.number().finite().nonnegative().nullable(),
|
||||
/** Linux only, copied from the lock; older clients omit it and older slots drop it. */
|
||||
linuxStartTicks: z.string().regex(/^\d+$/).max(32).optional().catch(undefined),
|
||||
bootId: z.string().min(1).max(255).optional().catch(undefined),
|
||||
nonce: z.string().min(1).max(255),
|
||||
lockPath: z.string().min(1).max(4096)
|
||||
})
|
||||
@@ -40,6 +43,13 @@ export const OrcadManagedStopRequestSchema = z.object({
|
||||
|
||||
export type OrcadManagedStopInstance = z.infer<typeof OrcadManagedStopInstanceSchema>
|
||||
export type OrcadManagedStopRequest = z.infer<typeof OrcadManagedStopRequestSchema>
|
||||
|
||||
/** Equality key for a request; omits the incarnation, which an older client recovering it drops. */
|
||||
export function orcadManagedStopRequestKey(request: OrcadManagedStopRequest): string {
|
||||
const { instance, ...rest } = OrcadManagedStopRequestSchema.parse(request)
|
||||
const { linuxStartTicks: _ticks, bootId: _bootId, ...bound } = instance
|
||||
return JSON.stringify({ ...rest, instance: bound })
|
||||
}
|
||||
/** What a stopping orcad must match before a managed request may stop it. */
|
||||
export type OrcadManagedStopContext = Omit<
|
||||
OrcadManagedStopRequest,
|
||||
|
||||
Reference in New Issue
Block a user