mirror of
https://github.com/stablyai/orca.git
synced 2026-11-05 03:27:44 +00:00
fix(orcad): keep an artifact slot whose serve or daemon record names another machine or boot (#29952)
On a shared home, a peer lock's PID is evidence only on the machine that issued it. Cache retention judged the instance lock and daemon PID records by PID alone, so a second machine evicted the slot another machine's live serve or daemon runs from.
This commit is contained in:
@@ -8,6 +8,7 @@ import {
|
||||
pruneDesktopOrcadArtifactCache,
|
||||
pruneOrcadArtifactCache
|
||||
} from './orcad-artifact-cache-retention'
|
||||
import { liveDaemonOrcadSlots, type OrcadSlotPeerJudge } from './orcad-daemon-slot-pins'
|
||||
|
||||
const roots: string[] = []
|
||||
afterEach(() => {
|
||||
@@ -145,4 +146,78 @@ describe('orcad artifact cache retention', () => {
|
||||
writeFileSync(join(userData, 'daemon', 'daemon-v30.pid'), '')
|
||||
expect(await pruneDesktopOrcadArtifactCache(userData)).toEqual([])
|
||||
})
|
||||
|
||||
describe('a peer record from another machine or boot (a shared home)', () => {
|
||||
const here: OrcadSlotPeerJudge = {
|
||||
self: { machineIdentity: 'machine-a', bootId: 'boot-a' },
|
||||
dataRootIsOnLocalDisk: () => true
|
||||
}
|
||||
const dead = () => false
|
||||
const lock = (userData: string, fields: Record<string, unknown>) =>
|
||||
writeFileSync(
|
||||
join(userData, ORCAD_LOCK_FILE_NAME),
|
||||
JSON.stringify({
|
||||
pid: 4242,
|
||||
startedAtMs: null,
|
||||
nonce: 'nonce',
|
||||
identity: 'uid',
|
||||
version: '0.1.0+peer',
|
||||
acquiredAt: new Date().toISOString(),
|
||||
role: 'orcad',
|
||||
...fields
|
||||
})
|
||||
)
|
||||
|
||||
it('keeps the slot of a serve whose lock names another machine, though its PID is dead here', () => {
|
||||
const userData = cacheRoot()
|
||||
lock(userData, { machineIdentity: 'machine-b', bootId: 'boot-b' })
|
||||
expect(liveLocalOrcadServeVersion(userData, dead, here)).toBe('0.1.0+peer')
|
||||
})
|
||||
|
||||
it('keeps the slot of a serve on the same machine id but another boot', () => {
|
||||
const userData = cacheRoot()
|
||||
lock(userData, { machineIdentity: 'machine-a', bootId: 'boot-b' })
|
||||
expect(liveLocalOrcadServeVersion(userData, dead, here)).toBe('0.1.0+peer')
|
||||
})
|
||||
|
||||
it('keeps the slot of a lock naming no machine on a data root other machines can reach', () => {
|
||||
const userData = cacheRoot()
|
||||
lock(userData, {})
|
||||
expect(
|
||||
liveLocalOrcadServeVersion(userData, dead, { ...here, dataRootIsOnLocalDisk: () => false })
|
||||
).toBe('0.1.0+peer')
|
||||
expect(liveLocalOrcadServeVersion(userData, dead, here)).toBeNull()
|
||||
})
|
||||
|
||||
it('still frees the slot of a dead serve on this machine and boot', () => {
|
||||
const userData = cacheRoot()
|
||||
lock(userData, { machineIdentity: 'machine-a', bootId: 'boot-a' })
|
||||
expect(liveLocalOrcadServeVersion(userData, dead, here)).toBeNull()
|
||||
expect(liveLocalOrcadServeVersion(userData, () => true, here)).toBe('0.1.0+peer')
|
||||
})
|
||||
|
||||
it("keeps a daemon's slot when its record names another boot or a network home", () => {
|
||||
const userData = cacheRoot()
|
||||
const root = join(userData, 'orcad-artifacts')
|
||||
const daemonSlot = slot(root, 'linux-x64-glibc', '0.1.0+old', 90)
|
||||
mkdirSync(join(userData, 'daemon'))
|
||||
const record = (bootId?: string) =>
|
||||
writeFileSync(
|
||||
join(userData, 'daemon', 'daemon-v30.pid'),
|
||||
JSON.stringify({
|
||||
pid: 4242,
|
||||
entryPath: join(daemonSlot, 'out', 'daemon-entry.js'),
|
||||
...(bootId ? { bootId } : {})
|
||||
})
|
||||
)
|
||||
record('boot-b')
|
||||
expect(liveDaemonOrcadSlots(userData, root, dead, here)).toEqual(['0.1.0+old'])
|
||||
record()
|
||||
expect(
|
||||
liveDaemonOrcadSlots(userData, root, dead, { ...here, dataRootIsOnLocalDisk: () => false })
|
||||
).toEqual(['0.1.0+old'])
|
||||
record('boot-a')
|
||||
expect(liveDaemonOrcadSlots(userData, root, dead, here)).toEqual([])
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -4,15 +4,24 @@
|
||||
*
|
||||
* Never evicted: a slot this process materialized (an SSH deploy may be reading it), the slot
|
||||
* a live local orcad serve runs from (named by the profile's instance lock), and the slot a
|
||||
* surviving terminal daemon runs from (named by its PID record).
|
||||
* surviving terminal daemon runs from (named by its PID record). A record from another machine or
|
||||
* boot is unverifiable, so its slot is kept too.
|
||||
*/
|
||||
import { readdir, rm, stat } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { nodeRuntimeAsset } from '../../shared/node-runtime-pin'
|
||||
import { ORCAD_LOCK_FILE_NAME, readOrcadInstanceLockRecord } from './orcad-instance-lock'
|
||||
import {
|
||||
ORCAD_LOCK_FILE_NAME,
|
||||
orcadPeerPidIsJudgeableHere,
|
||||
readOrcadInstanceLockRecord
|
||||
} from './orcad-instance-lock'
|
||||
import { materializedOrcadArtifactVersions } from './orcad-artifact-materializer'
|
||||
import { isProcessAlive } from '../daemon/daemon-process-inspection'
|
||||
import { liveDaemonOrcadSlots } from './orcad-daemon-slot-pins'
|
||||
import {
|
||||
defaultOrcadSlotPeerJudge,
|
||||
liveDaemonOrcadSlots,
|
||||
type OrcadSlotPeerJudge
|
||||
} from './orcad-daemon-slot-pins'
|
||||
|
||||
/** The in-use slot plus the two most recent others (the newest three when none is in use). */
|
||||
export const ORCAD_ARTIFACT_CACHE_KEEP = 3
|
||||
@@ -76,12 +85,13 @@ export function pruneDesktopOrcadArtifactCache(
|
||||
alsoInUse: readonly string[] = []
|
||||
): Promise<string[]> {
|
||||
const cacheRoot = join(userDataPath, 'orcad-artifacts')
|
||||
const daemonSlots = liveDaemonOrcadSlots(userDataPath, cacheRoot)
|
||||
const judge = defaultOrcadSlotPeerJudge()
|
||||
const daemonSlots = liveDaemonOrcadSlots(userDataPath, cacheRoot, isProcessAlive, judge)
|
||||
if (!daemonSlots) {
|
||||
// An unreadable daemon record could name any slot; evicting nothing is the safe answer.
|
||||
return Promise.resolve([])
|
||||
}
|
||||
const live = liveLocalOrcadServeVersion(userDataPath)
|
||||
const live = liveLocalOrcadServeVersion(userDataPath, isProcessAlive, judge)
|
||||
return pruneOrcadArtifactCache(cacheRoot, {
|
||||
inUseVersions: new Set([
|
||||
...materializedOrcadArtifactVersions(),
|
||||
@@ -92,13 +102,23 @@ export function pruneDesktopOrcadArtifactCache(
|
||||
})
|
||||
}
|
||||
|
||||
/** The slot version a live local orcad serve runs from, or null when none holds the profile. */
|
||||
/**
|
||||
* The slot version a live local orcad serve runs from, or null when none holds the profile. A
|
||||
* serve on another machine or boot (a shared home) is unverifiable from here, so its slot stays.
|
||||
*/
|
||||
export function liveLocalOrcadServeVersion(
|
||||
userDataPath: string,
|
||||
isAlive: (pid: number) => boolean = isProcessAlive
|
||||
isAlive: (pid: number) => boolean = isProcessAlive,
|
||||
judge: OrcadSlotPeerJudge = defaultOrcadSlotPeerJudge()
|
||||
): string | null {
|
||||
const record = readOrcadInstanceLockRecord(join(userDataPath, ORCAD_LOCK_FILE_NAME))
|
||||
return record && record.role !== 'desktop' && isAlive(record.pid) ? record.version : null
|
||||
if (!record || record.role === 'desktop') {
|
||||
return null
|
||||
}
|
||||
const judgeable = orcadPeerPidIsJudgeableHere(record, judge.self, () =>
|
||||
judge.dataRootIsOnLocalDisk(userDataPath)
|
||||
)
|
||||
return !judgeable || isAlive(record.pid) ? record.version : null
|
||||
}
|
||||
|
||||
async function listNames(directory: string): Promise<string[]> {
|
||||
|
||||
@@ -6,6 +6,22 @@ import { readdirSync, readFileSync } from 'node:fs'
|
||||
import { isAbsolute, join, relative, resolve, sep } from 'node:path'
|
||||
import { parseDaemonPidFile } from '../daemon/daemon-pid-file-parse'
|
||||
import { hasErrorCode, isProcessAlive } from '../daemon/daemon-process-inspection'
|
||||
import {
|
||||
currentOrcadPeerMachine,
|
||||
orcadPeerPidIsJudgeableHere,
|
||||
type OrcadPeerMachine
|
||||
} from './orcad-instance-lock'
|
||||
import { orcadDataRootIsOnLocalDisk } from './orcad-data-root-locality'
|
||||
|
||||
/** This machine, for telling a peer record's PID apart from one another machine issued. */
|
||||
export type OrcadSlotPeerJudge = {
|
||||
self: OrcadPeerMachine
|
||||
dataRootIsOnLocalDisk: (dataRoot: string) => boolean
|
||||
}
|
||||
|
||||
export function defaultOrcadSlotPeerJudge(): OrcadSlotPeerJudge {
|
||||
return { self: currentOrcadPeerMachine(), dataRootIsOnLocalDisk: orcadDataRootIsOnLocalDisk }
|
||||
}
|
||||
|
||||
const DAEMON_PID_FILE = /^daemon-v\d+\.pid$/u
|
||||
|
||||
@@ -13,7 +29,8 @@ const DAEMON_PID_FILE = /^daemon-v\d+\.pid$/u
|
||||
export function liveDaemonOrcadSlots(
|
||||
userDataPath: string,
|
||||
cacheRoot: string,
|
||||
isAlive: (pid: number) => boolean = isProcessAlive
|
||||
isAlive: (pid: number) => boolean = isProcessAlive,
|
||||
judge: OrcadSlotPeerJudge = defaultOrcadSlotPeerJudge()
|
||||
): string[] | null {
|
||||
const runtimeDir = join(userDataPath, 'daemon')
|
||||
let names: string[]
|
||||
@@ -35,7 +52,13 @@ export function liveDaemonOrcadSlots(
|
||||
// A record being written or torn names no process we could rule out.
|
||||
return null
|
||||
}
|
||||
if (!isAlive(record.pid)) {
|
||||
// A daemon on another machine or boot (a shared home) is unverifiable here; keep its slot.
|
||||
const judgeable = orcadPeerPidIsJudgeableHere(
|
||||
{ bootId: record.bootId ?? undefined },
|
||||
judge.self,
|
||||
() => judge.dataRootIsOnLocalDisk(userDataPath)
|
||||
)
|
||||
if (judgeable && !isAlive(record.pid)) {
|
||||
continue
|
||||
}
|
||||
for (const path of [record.entryPath, record.spawnerExecPath]) {
|
||||
|
||||
@@ -210,7 +210,7 @@ export function acquireOrcadInstanceLock(
|
||||
return Boolean(
|
||||
owner &&
|
||||
owner.identity === identity &&
|
||||
holderLocality(owner, record, dataRootIsLocal) === 'local' &&
|
||||
orcadLockHolderLocality(owner, record, dataRootIsLocal) === 'local' &&
|
||||
(!isAlive(owner.pid) || holderIdentity(owner) === 'differs')
|
||||
)
|
||||
}
|
||||
@@ -247,7 +247,7 @@ export function acquireOrcadInstanceLock(
|
||||
'root corrupts it. Give each its own ORCA_USER_DATA.'
|
||||
)
|
||||
}
|
||||
const locality = holderLocality(existing, record, dataRootIsLocal)
|
||||
const locality = orcadLockHolderLocality(existing, record, dataRootIsLocal)
|
||||
if (locality !== 'local') {
|
||||
// Its PID means nothing here, so neither a missing nor a reused one proves it exited.
|
||||
throw new OrcadInstanceLockError(
|
||||
@@ -282,9 +282,9 @@ export function acquireOrcadInstanceLock(
|
||||
* Same machine: the same machine identity, or (records without one) the same kernel boot. A record
|
||||
* that names neither is local only when no other machine can reach the data root.
|
||||
*/
|
||||
function holderLocality(
|
||||
holder: OrcadLockRecord,
|
||||
self: OrcadLockRecord,
|
||||
function orcadLockHolderLocality(
|
||||
holder: OrcadPeerMachine,
|
||||
self: OrcadPeerMachine,
|
||||
dataRootIsOnLocalDisk: () => boolean
|
||||
): 'local' | 'foreign' | 'unknown' {
|
||||
if (holder.machineIdentity && self.machineIdentity) {
|
||||
@@ -296,6 +296,29 @@ function holderLocality(
|
||||
return dataRootIsOnLocalDisk() ? 'local' : 'unknown'
|
||||
}
|
||||
|
||||
export type OrcadPeerMachine = Pick<OrcadLockRecord, 'machineIdentity' | 'bootId'>
|
||||
|
||||
export function currentOrcadPeerMachine(): OrcadPeerMachine {
|
||||
const machineIdentity = profileStateAccessMachineIdentity()
|
||||
const bootId = readOrcadProcessIncarnation(process.pid)?.bootId
|
||||
return { ...(machineIdentity ? { machineIdentity } : {}), ...(bootId ? { bootId } : {}) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a peer's recorded PID means anything here: same machine and kernel boot. A different or
|
||||
* unprovable one leaves the peer unverifiable, never exited.
|
||||
*/
|
||||
export function orcadPeerPidIsJudgeableHere(
|
||||
peer: OrcadPeerMachine,
|
||||
self: OrcadPeerMachine,
|
||||
dataRootIsOnLocalDisk: () => boolean
|
||||
): boolean {
|
||||
if (peer.bootId && self.bootId && peer.bootId !== self.bootId) {
|
||||
return false
|
||||
}
|
||||
return orcadLockHolderLocality(peer, self, dataRootIsOnLocalDisk) === 'local'
|
||||
}
|
||||
|
||||
/** A record without a boot id cannot tell its holder from a reused PID; say so. */
|
||||
function possiblyReusedPidError(
|
||||
lockPath: string,
|
||||
|
||||
Reference in New Issue
Block a user