mirror of
https://github.com/stablyai/orca.git
synced 2026-11-06 12:07:58 +00:00
fix(orcad): never claim or reclaim another machine's server, and recover a stale lock without displacing a live one (#29902)
* fix(orcad): never claim or reclaim another machine's orcad, and serialize stale-lock recovery * fix(orcad): judge data-root locality without a synchronous spawn * fix(orcad): break a recovery lock only once its owner provably exited * fix(orcad): publish the recovery lock with its owner in one rename * fix(orcad): recover a stale lock in generations so no live reclaimer is displaced * fix(orcad): keep a released recovery generation until the stale lock is replaced * fix(orcad): fail closed on an unreadable recovery owner, and never let cleanup fail a won lock * fix(orcad): key recovery locks by the stale record alone
This commit is contained in:
@@ -0,0 +1,27 @@
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { orcadDataRootIsOnLocalDisk } from './orcad-data-root-locality'
|
||||
|
||||
describe('orcadDataRootIsOnLocalDisk', () => {
|
||||
it.runIf(process.platform === 'linux' || process.platform === 'darwin')(
|
||||
'recognises a local temp directory, and refuses a path it cannot inspect',
|
||||
() => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'orcad-locality-'))
|
||||
try {
|
||||
expect(orcadDataRootIsOnLocalDisk(root)).toBe(true)
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true })
|
||||
}
|
||||
expect(orcadDataRootIsOnLocalDisk(join(root, 'missing'))).toBe(false)
|
||||
}
|
||||
)
|
||||
|
||||
it('treats a Windows UNC data root as reachable from other machines', () => {
|
||||
expect(orcadDataRootIsOnLocalDisk('\\\\filer\\home\\orca', 'win32')).toBe(false)
|
||||
expect(orcadDataRootIsOnLocalDisk('\\\\?\\UNC\\filer\\home\\orca', 'win32')).toBe(false)
|
||||
expect(orcadDataRootIsOnLocalDisk('C:\\Users\\u\\AppData\\Roaming\\orca', 'win32')).toBe(true)
|
||||
expect(orcadDataRootIsOnLocalDisk('\\\\?\\C:\\Users\\u\\orca', 'win32')).toBe(true)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,49 @@
|
||||
/**
|
||||
* Whether a data root sits on a disk only this machine mounts. An instance-lock record that names
|
||||
* no machine is judged by its PID only when this holds: on a network home another machine may be
|
||||
* running it, and a PID is evidence only on the machine that issued it.
|
||||
*/
|
||||
import { statfsSync } from 'node:fs'
|
||||
import process from 'node:process'
|
||||
|
||||
// Linux `statfs` f_type magics (linux/magic.h) of local disk and memory filesystems.
|
||||
const LINUX_LOCAL_FILESYSTEM_MAGICS = new Set([
|
||||
0xef53, // ext2/3/4
|
||||
0x58465342, // xfs
|
||||
0x9123683e, // btrfs
|
||||
0x2fc12fc1, // zfs
|
||||
0xca451a4e, // bcachefs
|
||||
0xf2f52010, // f2fs
|
||||
0x3153464a, // jfs
|
||||
0x52654973, // reiserfs
|
||||
0x3434, // nilfs
|
||||
0x01021994, // tmpfs
|
||||
0x858458f6, // ramfs
|
||||
0x794c7630, // overlayfs
|
||||
0xf15f, // ecryptfs
|
||||
0x4d44, // vfat
|
||||
0x2011bab0, // exfat
|
||||
0x482b // hfsplus
|
||||
])
|
||||
|
||||
export function orcadDataRootIsOnLocalDisk(
|
||||
dataRoot: string,
|
||||
platform: NodeJS.Platform = process.platform
|
||||
): boolean {
|
||||
if (platform === 'win32') {
|
||||
// `\\?\C:\` and `\\.\C:\` name local drives; any other `\\` path is a share. Mapped network
|
||||
// drives look local here, but every Windows writer records its MachineGuid anyway.
|
||||
return !/^[\\/]{2}/u.test(dataRoot) || /^[\\/]{2}[?.][\\/][a-z]:/iu.test(dataRoot)
|
||||
}
|
||||
try {
|
||||
const type = statfsSync(dataRoot).type
|
||||
if (platform === 'linux') {
|
||||
return LINUX_LOCAL_FILESYSTEM_MAGICS.has(type)
|
||||
}
|
||||
// macOS numbers filesystem types as their drivers load, so compare against the boot volume's,
|
||||
// which is always a local disk; network filesystems never share its type.
|
||||
return platform === 'darwin' && type === statfsSync('/').type
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
/**
|
||||
* Replacing a stale orcad instance lock without ever displacing a live one (see
|
||||
* `orcad-instance-lock.ts`).
|
||||
*/
|
||||
import { createHash, randomUUID } from 'node:crypto'
|
||||
import {
|
||||
existsSync,
|
||||
linkSync,
|
||||
lstatSync,
|
||||
mkdirSync,
|
||||
readdirSync,
|
||||
renameSync,
|
||||
rmSync,
|
||||
unlinkSync,
|
||||
writeFileSync
|
||||
} from 'node:fs'
|
||||
import { basename, dirname, join } from 'node:path'
|
||||
import process from 'node:process'
|
||||
import { OrcadInstanceLockError } from './orcad-data-root-privacy'
|
||||
import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader'
|
||||
import { hasErrorCode } from '../daemon/daemon-process-inspection'
|
||||
|
||||
export const MAX_ORCAD_LOCK_BYTES = 64 * 1024
|
||||
// Serializes stale-lock recovery, so no reclaimer ever renames away a lock another just published.
|
||||
const RECOVERY_DIR_SUFFIX = '.recovery'
|
||||
const RECOVERY_OWNER_FILE = 'owner'
|
||||
const RECOVERY_RELEASED_FILE = 'released'
|
||||
|
||||
/** Who is recovering, as a lock record, and whether a recovery owner's holder provably exited. */
|
||||
export type OrcadLockRecoveryOwner = {
|
||||
record: string
|
||||
exited: (ownerRecord: string) => boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Why rename-and-then-publish rather than unlink-and-write: rename claims one exact directory
|
||||
* entry, so a replacement written between our read and our write stays at the canonical path
|
||||
* and wins — we never delete a record we did not inspect.
|
||||
*/
|
||||
export function reclaimAndPublish(
|
||||
lockPath: string,
|
||||
dataRoot: string,
|
||||
inspectedContents: string,
|
||||
publish: () => boolean,
|
||||
owner: OrcadLockRecoveryOwner
|
||||
): void {
|
||||
return withRecoveryLock(lockPath, inspectedContents, owner, () => {
|
||||
// Another reclaimer finished first: whatever it published is authoritative, not stale.
|
||||
if (readBoundedLockFile(lockPath) !== inspectedContents) {
|
||||
throw new OrcadInstanceLockError(
|
||||
'orcad_instance_lock_held',
|
||||
`Another process took the data root ${dataRoot} while this one was reclaiming a stale lock.`
|
||||
)
|
||||
}
|
||||
renameAndPublish(lockPath, dataRoot, inspectedContents, publish)
|
||||
})
|
||||
}
|
||||
|
||||
function renameAndPublish(
|
||||
lockPath: string,
|
||||
dataRoot: string,
|
||||
inspectedContents: string,
|
||||
publish: () => boolean
|
||||
): void {
|
||||
const claimPath = `${lockPath}.stale-${process.pid}-${randomUUID()}`
|
||||
try {
|
||||
renameSync(lockPath, claimPath)
|
||||
} catch {
|
||||
throw new OrcadInstanceLockError(
|
||||
'orcad_instance_lock_held',
|
||||
`Could not reclaim the stale orcad instance lock at ${lockPath}; another process is ` +
|
||||
'holding it. Retry, or stop the other orcad.'
|
||||
)
|
||||
}
|
||||
// A contender may have replaced the entry after the liveness check; never displace its record.
|
||||
const claimedContents = readBoundedLockFile(claimPath)
|
||||
if (claimedContents !== inspectedContents) {
|
||||
restoreDisplacedLock(claimPath, lockPath, claimedContents)
|
||||
throw new OrcadInstanceLockError(
|
||||
'orcad_instance_lock_held',
|
||||
`The orcad instance lock at ${lockPath} changed while reclaiming a stale record.`
|
||||
)
|
||||
}
|
||||
const published = publish()
|
||||
// A uniquely named claim is inert either way.
|
||||
unlinkQuietly(claimPath)
|
||||
if (!published) {
|
||||
// Someone else claimed it first. Their record is authoritative; ours is not.
|
||||
throw new OrcadInstanceLockError(
|
||||
'orcad_instance_lock_held',
|
||||
`Another orcad took the data root ${dataRoot} while this one was reclaiming a stale lock.`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One recovery lock per stale record (named by its contents), in generations: a dead
|
||||
* owner's generation is never removed or renamed, the next one is created beside it. Taking one is
|
||||
* exclusive, and a later generation exists only once every earlier owner provably exited, so the
|
||||
* newest generation's owner is the only live reclaimer and no live owner is ever displaced.
|
||||
*/
|
||||
function withRecoveryLock<T>(
|
||||
lockPath: string,
|
||||
inspectedContents: string,
|
||||
owner: OrcadLockRecoveryOwner,
|
||||
recover: () => T
|
||||
): T {
|
||||
const prefix = orcadLockRecoveryPrefix(lockPath, inspectedContents)
|
||||
const generations = recoveryGenerations(prefix)
|
||||
const newest = generations.at(-1)
|
||||
const newestDir = `${prefix}${newest}`
|
||||
const newestOwner = newest === undefined ? null : readRecoveryOwner(newestDir)
|
||||
const free =
|
||||
newestOwner === null ||
|
||||
// A generation is never published without its owner; one without is a cut-short cleanup's.
|
||||
newestOwner.state === 'absent' ||
|
||||
(newestOwner.state === 'present' &&
|
||||
(existsSync(join(newestDir, RECOVERY_RELEASED_FILE)) || owner.exited(newestOwner.record)))
|
||||
const generation = newest === undefined ? 0 : newest + 1
|
||||
if (!free || !takeRecoveryLock(`${prefix}${generation}`, owner.record)) {
|
||||
throw new OrcadInstanceLockError(
|
||||
'orcad_instance_lock_held',
|
||||
`Another process is reclaiming the stale orcad instance lock at ${lockPath}. Retry.`
|
||||
)
|
||||
}
|
||||
try {
|
||||
return recover()
|
||||
} finally {
|
||||
const current = readBoundedLockFile(lockPath)
|
||||
if (current === null || current === inspectedContents) {
|
||||
// Kept, marked released: a reused generation number could be judged by a contender that
|
||||
// read this generation's owner before we let go.
|
||||
writeQuietly(join(`${prefix}${generation}`, RECOVERY_RELEASED_FILE))
|
||||
} else {
|
||||
// Once the stale record is gone no reclaimer can use its recovery locks again.
|
||||
for (const stale of recoveryGenerations(prefix)) {
|
||||
try {
|
||||
rmSync(`${prefix}${stale}`, { recursive: true, force: true })
|
||||
} catch {
|
||||
// A late contender writing into it; it is inert once the stale record is gone.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Keyed by contents alone: everyone who inspected the same stale record must share one recovery
|
||||
* lock, whatever the path held when they looked again (another inode, or nothing mid-reclaim).
|
||||
*/
|
||||
export function orcadLockRecoveryPrefix(lockPath: string, inspectedContents: string): string {
|
||||
const key = createHash('sha256').update(inspectedContents).digest('hex')
|
||||
return `${lockPath}${RECOVERY_DIR_SUFFIX}-${key.slice(0, 16)}-`
|
||||
}
|
||||
|
||||
function recoveryGenerations(prefix: string): number[] {
|
||||
const name = basename(prefix)
|
||||
try {
|
||||
return readdirSync(dirname(prefix))
|
||||
.filter((entry) => entry.startsWith(name) && /^\d{1,9}$/u.test(entry.slice(name.length)))
|
||||
.map((entry) => Number(entry.slice(name.length)))
|
||||
.sort((a, b) => a - b)
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Staged with its owner, then renamed into place: a recovery lock never exists without a complete
|
||||
* owner, and a rename onto one that does (a non-empty directory) fails instead of replacing it.
|
||||
*/
|
||||
function takeRecoveryLock(dir: string, ownerRecord: string): boolean {
|
||||
const staged = `${dir}.staged-${process.pid}-${randomUUID()}`
|
||||
try {
|
||||
mkdirSync(staged, { mode: 0o700 })
|
||||
writeFileSync(join(staged, RECOVERY_OWNER_FILE), ownerRecord, { flag: 'wx', mode: 0o600 })
|
||||
} catch (error) {
|
||||
rmSync(staged, { recursive: true, force: true })
|
||||
throw new OrcadInstanceLockError(
|
||||
'orcad_data_root_unusable',
|
||||
`Cannot create ${staged}: ${error instanceof Error ? error.message : String(error)}`
|
||||
)
|
||||
}
|
||||
try {
|
||||
renameSync(staged, dir)
|
||||
return true
|
||||
} catch {
|
||||
// Whatever holds the name, it was not created by this attempt; only our staging is removed.
|
||||
rmSync(staged, { recursive: true, force: true })
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/** No-clobber restore: a third contender's newer record at the canonical path stays authoritative. */
|
||||
function restoreDisplacedLock(
|
||||
claimPath: string,
|
||||
lockPath: string,
|
||||
claimedContents: string | null
|
||||
): void {
|
||||
try {
|
||||
linkSync(claimPath, lockPath)
|
||||
unlinkSync(claimPath)
|
||||
} catch {
|
||||
if (claimedContents === null) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
writeFileSync(lockPath, claimedContents, { flag: 'wx', mode: 0o600 })
|
||||
unlinkSync(claimPath)
|
||||
} catch {
|
||||
// A newer contender won, or restoration is unavailable; fail closed.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** An unreadable owner fails closed: it proves nothing about whether its reclaimer exited. */
|
||||
function readRecoveryOwner(
|
||||
dir: string
|
||||
): { state: 'absent' } | { state: 'present'; record: string } | { state: 'unreadable' } {
|
||||
const path = join(dir, RECOVERY_OWNER_FILE)
|
||||
try {
|
||||
lstatSync(path)
|
||||
} catch (error) {
|
||||
return hasErrorCode(error, 'ENOENT') ? { state: 'absent' } : { state: 'unreadable' }
|
||||
}
|
||||
const record = readBoundedLockFile(path)
|
||||
return record === null ? { state: 'unreadable' } : { state: 'present', record }
|
||||
}
|
||||
|
||||
function writeQuietly(path: string): void {
|
||||
try {
|
||||
writeFileSync(path, '', { flag: 'wx', mode: 0o600 })
|
||||
} catch {
|
||||
// Unwritable: the owner record still releases it once this process exits.
|
||||
}
|
||||
}
|
||||
|
||||
export function unlinkQuietly(path: string): void {
|
||||
try {
|
||||
unlinkSync(path)
|
||||
} catch {
|
||||
// Already gone, or inert under its unique name.
|
||||
}
|
||||
}
|
||||
|
||||
export function readBoundedLockFile(path: string): string | null {
|
||||
try {
|
||||
const { buffer, stats } = readNodeFileSyncWithinLimit(path, MAX_ORCAD_LOCK_BYTES)
|
||||
return stats.isFile() ? buffer.toString('utf8') : null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,8 @@ import {
|
||||
} from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type * as NodeFs from 'node:fs'
|
||||
import { z } from 'zod'
|
||||
import {
|
||||
acquireOrcadInstanceLock,
|
||||
@@ -22,6 +23,31 @@ import {
|
||||
type OrcadLockRecord
|
||||
} from './orcad-instance-lock'
|
||||
import { OrcadInstanceLockError } from './orcad-data-root-privacy'
|
||||
import { orcadLockRecoveryPrefix } from './orcad-instance-lock-reclaim'
|
||||
|
||||
// Runs once, right after the next rename out of the canonical lock path: a contender's turn.
|
||||
const afterLockRename = vi.hoisted((): { next: (() => void) | null; failNext: boolean } => ({
|
||||
next: null,
|
||||
failNext: false
|
||||
}))
|
||||
vi.mock('node:fs', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof NodeFs>()
|
||||
return {
|
||||
...actual,
|
||||
renameSync: (from: NodeFs.PathLike, to: NodeFs.PathLike) => {
|
||||
if (afterLockRename.failNext && String(from).endsWith('orcad.lock')) {
|
||||
afterLockRename.failNext = false
|
||||
throw Object.assign(new Error('EBUSY: resource busy'), { code: 'EBUSY' })
|
||||
}
|
||||
actual.renameSync(from, to)
|
||||
const contender = afterLockRename.next
|
||||
if (contender && String(from).endsWith('orcad.lock')) {
|
||||
afterLockRename.next = null
|
||||
contender()
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
const roots: string[] = []
|
||||
|
||||
@@ -40,6 +66,8 @@ function hooks(overrides: OrcadInstanceLockHooks = {}): OrcadInstanceLockHooks {
|
||||
incarnation: () => null,
|
||||
holderIdentity: () => 'matches',
|
||||
processIsAlive: () => false,
|
||||
machineIdentity: () => 'machine-a',
|
||||
dataRootIsOnLocalDisk: () => true,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
@@ -57,6 +85,8 @@ function persistedRecord(overrides: Partial<OrcadLockRecord> = {}): OrcadLockRec
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
afterLockRename.next = null
|
||||
afterLockRename.failNext = false
|
||||
for (const root of roots.splice(0)) {
|
||||
rmSync(root, { recursive: true, force: true })
|
||||
}
|
||||
@@ -456,6 +486,205 @@ describe('acquireOrcadInstanceLock', () => {
|
||||
lock.release()
|
||||
})
|
||||
|
||||
it('never judges a holder on another machine by a PID this machine issued', () => {
|
||||
const root = makeRoot()
|
||||
const lockPath = join(root, ORCAD_LOCK_FILE_NAME)
|
||||
// Machine B's orcad holds a data root on a shared home; its PID does not exist here.
|
||||
const remote = persistedRecord({ nonce: 'machine-b', machineIdentity: 'machine-b' })
|
||||
writeFileSync(lockPath, JSON.stringify(remote))
|
||||
for (const role of ['orcad', 'desktop'] as const) {
|
||||
expect(() => acquireOrcadInstanceLock(root, hooks({ role }))).toThrow(
|
||||
expect.objectContaining({
|
||||
code: 'orcad_instance_lock_held',
|
||||
message: expect.stringContaining('from another machine')
|
||||
})
|
||||
)
|
||||
}
|
||||
expect(readOrcadInstanceLockRecord(lockPath)?.nonce).toBe('machine-b')
|
||||
expect(readdirSync(root)).toEqual([ORCAD_LOCK_FILE_NAME])
|
||||
|
||||
// The same machine's crashed holder is still reclaimed.
|
||||
writeFileSync(lockPath, JSON.stringify({ ...remote, machineIdentity: 'machine-a' }))
|
||||
const lock = acquireOrcadInstanceLock(root, hooks())
|
||||
expect(readOrcadInstanceLockRecord(lockPath)).toMatchObject({
|
||||
nonce: lock.record.nonce,
|
||||
machineIdentity: 'machine-a'
|
||||
})
|
||||
})
|
||||
|
||||
it('reclaims a record naming no machine only when it is provably from this one', () => {
|
||||
const root = makeRoot()
|
||||
const lockPath = join(root, ORCAD_LOCK_FILE_NAME)
|
||||
// An older writer's record, on a data root other machines may share.
|
||||
writeFileSync(lockPath, JSON.stringify(persistedRecord({ bootId: 'boot-1' })))
|
||||
const shared = { dataRootIsOnLocalDisk: () => false }
|
||||
expect(() => acquireOrcadInstanceLock(root, hooks(shared))).toThrow(
|
||||
expect.objectContaining({ code: 'orcad_instance_lock_held' })
|
||||
)
|
||||
expect(() =>
|
||||
acquireOrcadInstanceLock(root, hooks({ ...shared, machineIdentity: () => null }))
|
||||
).toThrow(expect.objectContaining({ code: 'orcad_instance_lock_held' }))
|
||||
expect(readOrcadInstanceLockRecord(lockPath)?.nonce).toBe('stale')
|
||||
|
||||
// The same kernel boot is this machine.
|
||||
const sameBoot = hooks({
|
||||
...shared,
|
||||
incarnation: () => ({ linuxStartTicks: '1', bootId: 'boot-1' })
|
||||
})
|
||||
acquireOrcadInstanceLock(root, sameBoot).release()
|
||||
// So is a data root only this machine mounts.
|
||||
writeFileSync(lockPath, JSON.stringify(persistedRecord()))
|
||||
expect(acquireOrcadInstanceLock(root, hooks()).record.machineIdentity).toBe('machine-a')
|
||||
})
|
||||
|
||||
it('admits one owner when reclaimers of the same stale lock interleave', () => {
|
||||
const root = makeRoot()
|
||||
const lockPath = join(root, ORCAD_LOCK_FILE_NAME)
|
||||
writeFileSync(lockPath, JSON.stringify(persistedRecord()))
|
||||
const owners: { record: OrcadLockRecord }[] = []
|
||||
const failures: unknown[] = []
|
||||
const attempt = (lockHooks: OrcadInstanceLockHooks): void => {
|
||||
try {
|
||||
owners.push(acquireOrcadInstanceLock(root, lockHooks))
|
||||
} catch (error) {
|
||||
failures.push(error)
|
||||
}
|
||||
}
|
||||
let raced = false
|
||||
// A has read the stale record when B reclaims it and publishes its own; then, the moment
|
||||
// anything renames B's live lock away, C tries the momentarily vacant path.
|
||||
attempt(
|
||||
hooks({
|
||||
processIsAlive: () => {
|
||||
if (!raced) {
|
||||
raced = true
|
||||
attempt(hooks())
|
||||
afterLockRename.next = () => attempt(hooks())
|
||||
}
|
||||
return false
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
expect(owners).toHaveLength(1)
|
||||
expect(readOrcadInstanceLockRecord(lockPath)?.nonce).toBe(owners[0]?.record.nonce)
|
||||
expect(failures).toEqual([expect.objectContaining({ code: 'orcad_instance_lock_held' })])
|
||||
expect(readdirSync(root)).toEqual([ORCAD_LOCK_FILE_NAME])
|
||||
})
|
||||
|
||||
it('takes a new recovery generation only once every earlier owner provably exited', () => {
|
||||
const root = makeRoot()
|
||||
const lockPath = join(root, ORCAD_LOCK_FILE_NAME)
|
||||
const stale = JSON.stringify(persistedRecord({ machineIdentity: 'machine-a' }))
|
||||
const generation = (n: number): string => `${orcadLockRecoveryPrefix(lockPath, stale)}${n}`
|
||||
const leaveRecovery = (n: number, owner: string | null): void => {
|
||||
mkdirSync(generation(n))
|
||||
if (owner !== null) {
|
||||
writeFileSync(join(generation(n), 'owner'), owner)
|
||||
}
|
||||
}
|
||||
const reclaimer = (overrides: Partial<OrcadLockRecord>): string =>
|
||||
JSON.stringify(persistedRecord({ pid: 5150, nonce: 'reclaimer', ...overrides }))
|
||||
const refused = expect.objectContaining({ message: expect.stringContaining('is reclaiming') })
|
||||
// Only pid 5150, a reclaimer paused however long ago (stalled NFS, a suspended laptop), runs.
|
||||
const paused = hooks({ processIsAlive: (pid) => pid === 5150 })
|
||||
const reset = (): void => {
|
||||
for (const entry of readdirSync(root)) {
|
||||
rmSync(join(root, entry), { recursive: true, force: true })
|
||||
}
|
||||
writeFileSync(lockPath, stale)
|
||||
}
|
||||
|
||||
for (const owner of [
|
||||
reclaimer({ machineIdentity: 'machine-a' }),
|
||||
// Dead here, but on another machine.
|
||||
reclaimer({ pid: 6160, machineIdentity: 'machine-b' }),
|
||||
'not a record'
|
||||
]) {
|
||||
reset()
|
||||
leaveRecovery(0, owner)
|
||||
expect(() => acquireOrcadInstanceLock(root, paused)).toThrow(refused)
|
||||
expect(readOrcadInstanceLockRecord(lockPath)?.nonce).toBe('stale')
|
||||
expect(readFileSync(join(generation(0), 'owner'), 'utf8')).toBe(owner)
|
||||
}
|
||||
// An owner that cannot be read proves nothing either.
|
||||
reset()
|
||||
mkdirSync(join(generation(0), 'owner'), { recursive: true })
|
||||
expect(() => acquireOrcadInstanceLock(root, paused)).toThrow(refused)
|
||||
// Dead here, on a machine this cannot tell apart from this one.
|
||||
reset()
|
||||
leaveRecovery(0, reclaimer({ pid: 6160 }))
|
||||
expect(() =>
|
||||
acquireOrcadInstanceLock(root, hooks({ ...paused, dataRootIsOnLocalDisk: () => false }))
|
||||
).toThrow(refused)
|
||||
|
||||
// An owner that exited on this machine is a crash's; so is a generation a cleanup cut short.
|
||||
for (const owner of [reclaimer({ pid: 6160, machineIdentity: 'machine-a' }), null]) {
|
||||
reset()
|
||||
leaveRecovery(0, owner)
|
||||
const lock = acquireOrcadInstanceLock(root, paused)
|
||||
expect(readOrcadInstanceLockRecord(lockPath)?.nonce).toBe(lock.record.nonce)
|
||||
expect(readdirSync(root)).toEqual([ORCAD_LOCK_FILE_NAME])
|
||||
lock.release()
|
||||
}
|
||||
})
|
||||
|
||||
it('never reuses a generation a failed recovery let go of', () => {
|
||||
const root = makeRoot()
|
||||
const lockPath = join(root, ORCAD_LOCK_FILE_NAME)
|
||||
const stale = JSON.stringify(persistedRecord({ machineIdentity: 'machine-a' }))
|
||||
const generation = (n: number): string => `${orcadLockRecoveryPrefix(lockPath, stale)}${n}`
|
||||
writeFileSync(lockPath, stale)
|
||||
// Its rename of the stale lock fails, so the stale record stays in place.
|
||||
afterLockRename.failNext = true
|
||||
expect(() => acquireOrcadInstanceLock(root, hooks())).toThrow(
|
||||
expect.objectContaining({ code: 'orcad_instance_lock_held' })
|
||||
)
|
||||
expect(readdirSync(generation(0)).sort()).toEqual(['owner', 'released'])
|
||||
|
||||
// The next reclaimer, though this process is still alive, moves to a new generation.
|
||||
const lock = acquireOrcadInstanceLock(
|
||||
root,
|
||||
hooks({ processIsAlive: (pid) => pid === process.pid })
|
||||
)
|
||||
expect(readOrcadInstanceLockRecord(lockPath)?.nonce).toBe(lock.record.nonce)
|
||||
expect(readdirSync(root)).toEqual([ORCAD_LOCK_FILE_NAME])
|
||||
})
|
||||
|
||||
it('never displaces a reclaimer that broke the same dead recovery owner first', () => {
|
||||
const root = makeRoot()
|
||||
const lockPath = join(root, ORCAD_LOCK_FILE_NAME)
|
||||
const stale = JSON.stringify(persistedRecord({ machineIdentity: 'machine-a' }))
|
||||
const generation = (n: number): string => `${orcadLockRecoveryPrefix(lockPath, stale)}${n}`
|
||||
writeFileSync(lockPath, stale)
|
||||
mkdirSync(generation(0))
|
||||
const dead = JSON.stringify(persistedRecord({ pid: 6160, machineIdentity: 'machine-a' }))
|
||||
writeFileSync(join(generation(0), 'owner'), dead)
|
||||
const live = JSON.stringify(persistedRecord({ pid: 5150, machineIdentity: 'machine-a' }))
|
||||
let raced = false
|
||||
// B judges generation 0's owner dead at the same moment and takes generation 1 first.
|
||||
const racing = hooks({
|
||||
processIsAlive: (pid) => {
|
||||
if (!raced && pid === 6160) {
|
||||
raced = true
|
||||
mkdirSync(generation(1))
|
||||
writeFileSync(join(generation(1), 'owner'), live)
|
||||
}
|
||||
return pid === 5150
|
||||
}
|
||||
})
|
||||
const refused = expect.objectContaining({ message: expect.stringContaining('is reclaiming') })
|
||||
expect(() => acquireOrcadInstanceLock(root, racing)).toThrow(refused)
|
||||
// And a third starter defers to B, the newest generation's live owner.
|
||||
expect(() =>
|
||||
acquireOrcadInstanceLock(root, hooks({ processIsAlive: (pid) => pid === 5150 }))
|
||||
).toThrow(refused)
|
||||
expect(readFileSync(join(generation(1), 'owner'), 'utf8')).toBe(live)
|
||||
expect(readFileSync(join(generation(0), 'owner'), 'utf8')).toBe(dead)
|
||||
expect(readOrcadInstanceLockRecord(lockPath)?.nonce).toBe('stale')
|
||||
expect(readdirSync(root)).toHaveLength(3)
|
||||
})
|
||||
|
||||
it.runIf(process.platform !== 'win32')(
|
||||
"leaves the desktop profile's permissions as they were",
|
||||
() => {
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
* it says nothing about the daemon, which is what makes a non-destructive restart possible.
|
||||
*/
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { linkSync, mkdirSync, renameSync, statSync, unlinkSync, writeFileSync } from 'node:fs'
|
||||
import { linkSync, mkdirSync, statSync, unlinkSync, writeFileSync } from 'node:fs'
|
||||
import { userInfo } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import process from 'node:process'
|
||||
@@ -31,11 +31,18 @@ import {
|
||||
OrcadInstanceLockError,
|
||||
type OrcadDataRootPrivacyHooks
|
||||
} from './orcad-data-root-privacy'
|
||||
import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader'
|
||||
import { hasErrorCode, isProcessAlive } from '../daemon/daemon-process-inspection'
|
||||
import { profileStateAccessMachineIdentity } from '../persistence/profile-state/profile-state-access-identity'
|
||||
import { orcadDataRootIsOnLocalDisk } from './orcad-data-root-locality'
|
||||
import {
|
||||
MAX_ORCAD_LOCK_BYTES,
|
||||
readBoundedLockFile,
|
||||
reclaimAndPublish,
|
||||
unlinkQuietly,
|
||||
type OrcadLockRecoveryOwner
|
||||
} from './orcad-instance-lock-reclaim'
|
||||
|
||||
export const ORCAD_LOCK_FILE_NAME = 'orcad.lock'
|
||||
const MAX_ORCAD_LOCK_BYTES = 64 * 1024
|
||||
// A writer finishes in milliseconds; a garbled record older than this has no live writer.
|
||||
const GARBLED_LOCK_GRACE_MS = 10_000
|
||||
|
||||
@@ -56,7 +63,12 @@ const OrcadLockRecordSchema = z.object({
|
||||
/** Distinguishes our record from a replacement written after we lost the race. */
|
||||
nonce: z.string().min(1).max(255),
|
||||
/** Absent in records orcad wrote before the desktop app shared this lock. */
|
||||
role: z.enum(['orcad', 'desktop']).optional()
|
||||
role: z.enum(['orcad', 'desktop']).optional(),
|
||||
/**
|
||||
* The holder's machine (as profile-state owners record it). Absent from older writers. A PID is
|
||||
* evidence only on the machine that issued it, and a shared home can put two on one data root.
|
||||
*/
|
||||
machineIdentity: z.string().min(1).max(255).optional().catch(undefined)
|
||||
})
|
||||
|
||||
export type OrcadLockRecord = z.infer<typeof OrcadLockRecordSchema>
|
||||
@@ -89,6 +101,9 @@ export type OrcadInstanceLockHooks = OrcadDataRootPrivacyHooks & {
|
||||
* hold Electron's single-instance lock first, which is what lets it reclaim a desktop record.
|
||||
*/
|
||||
role?: 'orcad' | 'desktop'
|
||||
machineIdentity?: () => string | null
|
||||
/** Whether no other machine can reach the data root, for records that name no machine. */
|
||||
dataRootIsOnLocalDisk?: (dataRoot: string) => boolean
|
||||
}
|
||||
|
||||
function defaultIdentity(): string {
|
||||
@@ -147,6 +162,7 @@ export function acquireOrcadInstanceLock(
|
||||
}
|
||||
|
||||
const lockPath = join(dataRoot, ORCAD_LOCK_FILE_NAME)
|
||||
const machineIdentity = (hooks.machineIdentity ?? profileStateAccessMachineIdentity)()
|
||||
const record: OrcadLockRecord = {
|
||||
pid: process.pid,
|
||||
startedAtMs: readStartedAt(process.pid),
|
||||
@@ -155,7 +171,8 @@ export function acquireOrcadInstanceLock(
|
||||
version: (hooks.version ?? (() => process.env.ORCA_VERSION ?? 'unknown'))(),
|
||||
acquiredAt: (hooks.now ?? (() => new Date()))().toISOString(),
|
||||
nonce: randomUUID(),
|
||||
role: hooks.role ?? 'orcad'
|
||||
role: hooks.role ?? 'orcad',
|
||||
...(machineIdentity ? { machineIdentity } : {})
|
||||
}
|
||||
const serialized = JSON.stringify(record)
|
||||
|
||||
@@ -183,6 +200,22 @@ export function acquireOrcadInstanceLock(
|
||||
return makeLock(lockPath, record)
|
||||
}
|
||||
|
||||
const dataRootIsLocal = (): boolean =>
|
||||
(hooks.dataRootIsOnLocalDisk ?? orcadDataRootIsOnLocalDisk)(dataRoot)
|
||||
// A recovery lock is ours by the same record, and broken only on the same proof of exit.
|
||||
const recoveryOwner: OrcadLockRecoveryOwner = {
|
||||
record: serialized,
|
||||
exited: (ownerRecord) => {
|
||||
const owner = parseOrcadInstanceLockRecord(ownerRecord)
|
||||
return Boolean(
|
||||
owner &&
|
||||
owner.identity === identity &&
|
||||
holderLocality(owner, record, dataRootIsLocal) === 'local' &&
|
||||
(!isAlive(owner.pid) || holderIdentity(owner) === 'differs')
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const existingContents = readBoundedLockFile(lockPath)
|
||||
if (existingContents === null) {
|
||||
// Why fail closed: a record we cannot read proves nothing about its holder having exited.
|
||||
@@ -198,7 +231,8 @@ export function acquireOrcadInstanceLock(
|
||||
// Only a torn write (an older build, or a crash mid-write) leaves this; once it has aged
|
||||
// past any live writer, no owner can be behind it.
|
||||
if (garbledLockIsAbandoned(lockPath)) {
|
||||
return reclaimAndPublish(lockPath, dataRoot, existingContents, publish, record)
|
||||
reclaimAndPublish(lockPath, dataRoot, existingContents, publish, recoveryOwner)
|
||||
return makeLock(lockPath, record)
|
||||
}
|
||||
throw new OrcadInstanceLockError(
|
||||
'orcad_instance_lock_held',
|
||||
@@ -213,6 +247,18 @@ export function acquireOrcadInstanceLock(
|
||||
'root corrupts it. Give each its own ORCA_USER_DATA.'
|
||||
)
|
||||
}
|
||||
const locality = holderLocality(existing, record, dataRootIsLocal)
|
||||
if (locality !== 'local') {
|
||||
// Its PID means nothing here, so neither a missing nor a reused one proves it exited.
|
||||
throw new OrcadInstanceLockError(
|
||||
'orcad_instance_lock_held',
|
||||
`${describeLockHolder(existing)} (pid ${existing.pid}, started ${existing.acquiredAt}) ` +
|
||||
`holds the data root ${dataRoot}${locality === 'foreign' ? ' from another machine' : ''}. ` +
|
||||
'Orca cannot check from here whether it is still running, because this data root may be ' +
|
||||
`shared with other machines. Stop it there, or if it is not running anywhere, delete ` +
|
||||
`${lockPath} and try again.`
|
||||
)
|
||||
}
|
||||
// The desktop takes this lock only after Electron's single-instance lock, which already proves
|
||||
// no other desktop runs; a desktop record is then stale even when its PID was reused.
|
||||
const staleDesktopRecord = hooks.role === 'desktop' && existing.role === 'desktop'
|
||||
@@ -228,7 +274,26 @@ export function acquireOrcadInstanceLock(
|
||||
'before starting another, or use a different ORCA_USER_DATA.'
|
||||
)
|
||||
}
|
||||
return reclaimAndPublish(lockPath, dataRoot, existingContents, publish, record)
|
||||
reclaimAndPublish(lockPath, dataRoot, existingContents, publish, recoveryOwner)
|
||||
return makeLock(lockPath, record)
|
||||
}
|
||||
|
||||
/**
|
||||
* Same machine: the same machine identity, or (records without one) the same kernel boot. A record
|
||||
* that names neither is local only when no other machine can reach the data root.
|
||||
*/
|
||||
function holderLocality(
|
||||
holder: OrcadLockRecord,
|
||||
self: OrcadLockRecord,
|
||||
dataRootIsOnLocalDisk: () => boolean
|
||||
): 'local' | 'foreign' | 'unknown' {
|
||||
if (holder.machineIdentity && self.machineIdentity) {
|
||||
return holder.machineIdentity === self.machineIdentity ? 'local' : 'foreign'
|
||||
}
|
||||
if (holder.bootId && holder.bootId === self.bootId) {
|
||||
return 'local'
|
||||
}
|
||||
return dataRootIsOnLocalDisk() ? 'local' : 'unknown'
|
||||
}
|
||||
|
||||
/** A record without a boot id cannot tell its holder from a reused PID; say so. */
|
||||
@@ -250,50 +315,6 @@ function possiblyReusedPidError(
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Why rename-and-then-publish rather than unlink-and-write: rename claims one exact directory
|
||||
* entry, so a replacement written between our read and our write stays at the canonical path
|
||||
* and wins — we never delete a record we did not inspect.
|
||||
*/
|
||||
function reclaimAndPublish(
|
||||
lockPath: string,
|
||||
dataRoot: string,
|
||||
inspectedContents: string,
|
||||
publish: () => boolean,
|
||||
record: OrcadLockRecord
|
||||
): OrcadInstanceLock {
|
||||
const claimPath = `${lockPath}.stale-${process.pid}-${randomUUID()}`
|
||||
try {
|
||||
renameSync(lockPath, claimPath)
|
||||
} catch {
|
||||
throw new OrcadInstanceLockError(
|
||||
'orcad_instance_lock_held',
|
||||
`Could not reclaim the stale orcad instance lock at ${lockPath}; another process is ` +
|
||||
'holding it. Retry, or stop the other orcad.'
|
||||
)
|
||||
}
|
||||
// A contender may have replaced the entry after the liveness check; never displace its record.
|
||||
const claimedContents = readBoundedLockFile(claimPath)
|
||||
if (claimedContents !== inspectedContents) {
|
||||
restoreDisplacedLock(claimPath, lockPath, claimedContents)
|
||||
throw new OrcadInstanceLockError(
|
||||
'orcad_instance_lock_held',
|
||||
`The orcad instance lock at ${lockPath} changed while reclaiming a stale record.`
|
||||
)
|
||||
}
|
||||
const published = publish()
|
||||
// A uniquely named claim is inert either way.
|
||||
unlinkQuietly(claimPath)
|
||||
if (!published) {
|
||||
// Someone else claimed it first. Their record is authoritative; ours is not.
|
||||
throw new OrcadInstanceLockError(
|
||||
'orcad_instance_lock_held',
|
||||
`Another orcad took the data root ${dataRoot} while this one was reclaiming a stale lock.`
|
||||
)
|
||||
}
|
||||
return makeLock(lockPath, record)
|
||||
}
|
||||
|
||||
function garbledLockIsAbandoned(lockPath: string): boolean {
|
||||
try {
|
||||
return Date.now() - statSync(lockPath).mtimeMs > GARBLED_LOCK_GRACE_MS
|
||||
@@ -302,49 +323,10 @@ function garbledLockIsAbandoned(lockPath: string): boolean {
|
||||
}
|
||||
}
|
||||
|
||||
function unlinkQuietly(path: string): void {
|
||||
try {
|
||||
unlinkSync(path)
|
||||
} catch {
|
||||
// Already gone, or inert under its unique name.
|
||||
}
|
||||
}
|
||||
|
||||
function describeLockHolder(record: OrcadLockRecord): string {
|
||||
return record.role === 'desktop' ? 'The Orca desktop app' : 'Another orcad'
|
||||
}
|
||||
|
||||
/** No-clobber restore: a third contender's newer record at the canonical path stays authoritative. */
|
||||
function restoreDisplacedLock(
|
||||
claimPath: string,
|
||||
lockPath: string,
|
||||
claimedContents: string | null
|
||||
): void {
|
||||
try {
|
||||
linkSync(claimPath, lockPath)
|
||||
unlinkSync(claimPath)
|
||||
} catch {
|
||||
if (claimedContents === null) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
writeFileSync(lockPath, claimedContents, { flag: 'wx', mode: 0o600 })
|
||||
unlinkSync(claimPath)
|
||||
} catch {
|
||||
// A newer contender won, or restoration is unavailable; fail closed.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function readBoundedLockFile(path: string): string | null {
|
||||
try {
|
||||
const { buffer, stats } = readNodeFileSyncWithinLimit(path, MAX_ORCAD_LOCK_BYTES)
|
||||
return stats.isFile() ? buffer.toString('utf8') : null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function makeLock(lockPath: string, record: OrcadLockRecord): OrcadInstanceLock {
|
||||
let released = false
|
||||
return {
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { shellEscape } from './ssh-connection-utils'
|
||||
import { RELAY_REMOTE_DIR } from './relay-protocol'
|
||||
import {
|
||||
ORCAD_MACHINE_CLAIM_FILENAME,
|
||||
ORCAD_MACHINE_CLAIM_SCHEMA_VERSION
|
||||
} from './orcad-machine-home'
|
||||
import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform'
|
||||
|
||||
/** Under `~/.orca-remote`: decides the claim when hard links cannot (see the claim command). */
|
||||
export const ORCAD_MACHINE_CLAIM_ARBITER_DIRNAME = 'orcad-machine.arbiter'
|
||||
|
||||
/**
|
||||
* Publishes the claim exclusively and prints whichever claim won. Why a hard link: `ln` is atomic
|
||||
* and exclusive on NFS too, and a reader never sees a half-written claim. Where `ln` cannot decide
|
||||
* (no hard links, or an empty claim a full disk left), a directory rename picks the winner: it is
|
||||
* atomic everywhere, and the arbiter arrives already holding the winner's claim, so a crash leaves
|
||||
* no half-taken lock. Every machine then republishes that same claim, so concurrent writes agree.
|
||||
*/
|
||||
export function orcadMachineClaimCommand(
|
||||
host: RemoteHostPlatform,
|
||||
loginHome: string,
|
||||
machineKey: string
|
||||
): string {
|
||||
const remoteDir = (...parts: string[]): string =>
|
||||
shellEscape(joinRemotePath(host, loginHome, RELAY_REMOTE_DIR, ...parts))
|
||||
const attempt = randomUUID()
|
||||
const base = remoteDir()
|
||||
const claim = remoteDir(ORCAD_MACHINE_CLAIM_FILENAME)
|
||||
const partial = remoteDir(`${ORCAD_MACHINE_CLAIM_FILENAME}.partial.${attempt}`)
|
||||
const stageName = `${ORCAD_MACHINE_CLAIM_ARBITER_DIRNAME}.partial.${attempt}`
|
||||
const stage = remoteDir(stageName)
|
||||
const stageClaim = remoteDir(stageName, ORCAD_MACHINE_CLAIM_FILENAME)
|
||||
const arbiter = remoteDir(ORCAD_MACHINE_CLAIM_ARBITER_DIRNAME)
|
||||
const arbiterClaim = remoteDir(ORCAD_MACHINE_CLAIM_ARBITER_DIRNAME, ORCAD_MACHINE_CLAIM_FILENAME)
|
||||
// `mv` into an existing arbiter nests the stage inside it instead of replacing it.
|
||||
const nestedStage = remoteDir(ORCAD_MACHINE_CLAIM_ARBITER_DIRNAME, stageName)
|
||||
const body = shellEscape(
|
||||
JSON.stringify({
|
||||
schemaVersion: ORCAD_MACHINE_CLAIM_SCHEMA_VERSION,
|
||||
machineKey
|
||||
})
|
||||
)
|
||||
return [
|
||||
`umask 077; mkdir -p ${base} || exit 65;`,
|
||||
`printf %s ${body} > ${partial} || exit 65;`,
|
||||
// Once an arbiter exists it decides any claim that later goes missing or empty.
|
||||
`if { [ -d ${arbiter} ] || ! ln ${partial} ${claim} 2>/dev/null; } && [ ! -s ${claim} ]; then`,
|
||||
`{ mkdir ${stage} && printf %s ${body} > ${stageClaim}; } || { rm -rf ${stage} ${partial}; exit 65; };`,
|
||||
`mv ${stage} ${arbiter} 2>/dev/null; rm -rf ${stage} ${nestedStage};`,
|
||||
`[ -s ${arbiterClaim} ] && cat ${arbiterClaim} > ${partial} || { rm -f ${partial}; exit 65; };`,
|
||||
`[ -s ${claim} ] || mv -f ${partial} ${claim};`,
|
||||
`fi; rm -f ${partial}; cat ${claim}`
|
||||
].join(' ')
|
||||
}
|
||||
@@ -46,10 +46,10 @@ const {
|
||||
orcadMachineIdentity,
|
||||
orcadMachineOwnershipUnresolvedMessage,
|
||||
parseOrcadMachineHomeProbe,
|
||||
ORCAD_MACHINE_CLAIM_ARBITER_DIRNAME,
|
||||
ORCAD_MACHINE_CLAIM_FILENAME,
|
||||
ORCAD_MACHINE_HOMES_DIRNAME
|
||||
} = await import('./orcad-machine-home')
|
||||
const { ORCAD_MACHINE_CLAIM_ARBITER_DIRNAME } = await import('./orcad-machine-claim-command')
|
||||
const { computeRemoteInstallDir } = await import('./remote-versioned-install')
|
||||
const { ORCAD_INSTALL_MODEL } = await import('./remote-install-model')
|
||||
const { orcadLivenessProbeCommand, ORCAD_PID_FILENAME } = await import('./orcad-remote-launch')
|
||||
@@ -263,6 +263,28 @@ describe.runIf(POSIX)('resolveOrcadMachineHome on a home several machines share'
|
||||
expect(await resolveOrcadMachineHome(a, true)).toEqual(loginHomeResult())
|
||||
})
|
||||
|
||||
it('never lets another Mac claim a live install whose owner recorded no machine identity', async () => {
|
||||
const nfs = { homeSource: 'filer:/export/home' }
|
||||
const b = machine('node-b', 'node-b', nfs)
|
||||
mkdirSync(join(home, '.orca-remote', 'orcad-0.1.0+abc'), { recursive: true })
|
||||
writeFileSync(join(home, '.orca-remote', 'orcad-active.json'), '{"schemaVersion":1}')
|
||||
// node-a's older orcad is running: its owner names only node-a's hostname.
|
||||
legacyOwner('node-a', null)
|
||||
|
||||
await expect(resolveOrcadMachineHome(b, true)).rejects.toThrow(
|
||||
/recorded only a hostname[\s\S]*If node-b set it up/
|
||||
)
|
||||
expect(existsSync(claimPath())).toBe(false)
|
||||
// The same Mac under a drifted hostname is just as unprovable, so it is not claimed either.
|
||||
await expect(
|
||||
resolveOrcadMachineHome(machine('node-a', 'cafe-wifi', nfs), true)
|
||||
).rejects.toThrow(/recorded only a hostname/)
|
||||
// node-a, still under the name its owner recorded, keeps its own install.
|
||||
expect(await resolveOrcadMachineHome(machine('node-a', 'node-a', nfs), true)).toEqual(
|
||||
loginHomeResult()
|
||||
)
|
||||
})
|
||||
|
||||
it('lets exactly one machine claim a home on a filesystem without hard links', async () => {
|
||||
const a = machine('node-a', 'node-a', { noHardLinks: true, claimWriteDelay: 0 })
|
||||
const b = machine('node-b', 'node-b', { noHardLinks: true, claimWriteDelay: 0.5 })
|
||||
@@ -443,6 +465,24 @@ describe('decideOrcadMachineHome', () => {
|
||||
expect(decideOrcadMachineHome(unrecorded, linux).kind).toBe('machine-home')
|
||||
})
|
||||
|
||||
it('claims no install whose owner names neither its machine nor a matching hostname', () => {
|
||||
const uuid = '12345678-1234-1234-1234-123456789abc'
|
||||
const mac = orcadMachineIdentity(darwin, `"IOPlatformUUID" = "${uuid}"`, 'mac-b')
|
||||
const otherMac = { host: 'mac-a', machineIdentity: null }
|
||||
expect(
|
||||
decideOrcadMachineHome(probe(darwin, { identity: mac, dataRootOwners: [otherMac] }), darwin)
|
||||
).toEqual({ kind: 'unresolved', machineKey: mac?.key, cause: 'unattributed-owner' })
|
||||
// A local disk cannot be another machine's.
|
||||
expect(
|
||||
decideOrcadMachineHome(
|
||||
probe(darwin, { identity: mac, dataRootOwners: [otherMac], homeOnLocalDisk: true }),
|
||||
darwin
|
||||
).kind
|
||||
).toBe('claim')
|
||||
const nameless = probe(linux, { dataRootOwners: [{ host: null, machineIdentity: null }] })
|
||||
expect(decideOrcadMachineHome(nameless, linux).kind).toBe('unresolved')
|
||||
})
|
||||
|
||||
it('reads a Linux home’s filesystem type before its df source', () => {
|
||||
const output = (installed: string, type: string, source: string): string =>
|
||||
[
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
* immutable install stays per machine: the runtime store's GC judges "in use" from processes it
|
||||
* can see, so a store shared across machines could be collected under another machine's server.
|
||||
*/
|
||||
import { createHash, randomUUID } from 'node:crypto'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { shellEscape } from './ssh-connection-utils'
|
||||
import { RELAY_REMOTE_DIR } from './relay-protocol'
|
||||
import { ORCAD_ACTIVATION_FILENAME } from './orcad-activation-record'
|
||||
@@ -27,8 +27,6 @@ export const ORCAD_MACHINE_CLAIM_FILENAME = 'orcad-machine.json'
|
||||
/** Under `~/.orca-remote`: one orcad home per other machine sharing this `$HOME`. */
|
||||
export const ORCAD_MACHINE_HOMES_DIRNAME = 'hosts'
|
||||
export const ORCAD_MACHINE_CLAIM_SCHEMA_VERSION = 1
|
||||
/** Under `~/.orca-remote`: decides the claim when hard links cannot (see the claim command). */
|
||||
export const ORCAD_MACHINE_CLAIM_ARBITER_DIRNAME = 'orcad-machine.arbiter'
|
||||
|
||||
const SECTION = '__ORCA_MACHINE_HOME_SECTION__'
|
||||
const CLAIM_ABSENT = 'ABSENT'
|
||||
@@ -92,8 +90,11 @@ export type OrcadMachineHomeDecision =
|
||||
| { kind: 'login-home' }
|
||||
| { kind: 'claim'; machineKey: string }
|
||||
| { kind: 'machine-home'; machineKey: string }
|
||||
/** A stopped pre-claim install on a shared home: no record says which machine ran it. */
|
||||
| { kind: 'unresolved'; machineKey: string }
|
||||
/** A pre-claim install on a shared home that no record attributes to a machine. */
|
||||
| { kind: 'unresolved'; machineKey: string; cause: OrcadMachineOwnershipUnresolvedCause }
|
||||
|
||||
/** A stopped install left no owner; an older orcad's owner recorded no machine identity. */
|
||||
export type OrcadMachineOwnershipUnresolvedCause = 'stopped-install' | 'unattributed-owner'
|
||||
|
||||
/**
|
||||
* Reads the login home, this machine's identity, the claim, and who holds the shared data root,
|
||||
@@ -242,20 +243,24 @@ function parseOwnerLine(line: string): OrcadMachineHomeProbe['dataRootOwners'][n
|
||||
}
|
||||
}
|
||||
|
||||
/** True when a data-root owner names a machine other than this one. */
|
||||
function isForeignOwner(
|
||||
/** Whether a data-root owner is this machine, another one, or cannot be told apart. */
|
||||
function ownerMachine(
|
||||
owner: OrcadMachineHomeProbe['dataRootOwners'][number],
|
||||
identity: OrcadMachineIdentity,
|
||||
host: RemoteHostPlatform
|
||||
): boolean {
|
||||
): 'this' | 'foreign' | 'unknown' {
|
||||
if (owner.machineIdentity) {
|
||||
return owner.machineIdentity.toLowerCase() !== identity.machineIdentity
|
||||
return owner.machineIdentity.toLowerCase() === identity.machineIdentity ? 'this' : 'foreign'
|
||||
}
|
||||
// An owner too old to record its machine is judged by hostname, except on drifting macOS names.
|
||||
return (
|
||||
host.os !== 'darwin' &&
|
||||
Boolean(owner.host && identity.hostname && owner.host !== identity.hostname)
|
||||
)
|
||||
// An owner too old to record its machine is judged by hostname.
|
||||
if (!owner.host || !identity.hostname) {
|
||||
return 'unknown'
|
||||
}
|
||||
if (owner.host === identity.hostname) {
|
||||
return 'this'
|
||||
}
|
||||
// A macOS hostname follows the network, so a different one may still be this Mac.
|
||||
return host.os === 'darwin' ? 'unknown' : 'foreign'
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -263,8 +268,9 @@ function isForeignOwner(
|
||||
* data root: that one is the install's owner, and claiming it here would judge its PIDs on the
|
||||
* wrong machine. A machine that already has its own home keeps it, so a later claim cannot move
|
||||
* it off a server still running there. An unreadable claim throws: guessing can strand a server.
|
||||
* A cleanly stopped install leaves no owner, so on a home that may be shared its machine is
|
||||
* unknown: claiming it could hand one machine's pairing and projects to another.
|
||||
* A cleanly stopped install leaves no owner, and an older orcad's owner may name no machine
|
||||
* (macOS hostnames drift), so on a home that may be shared its machine is unknown: claiming it
|
||||
* could hand one machine's pairing, projects and instance lock to another.
|
||||
*/
|
||||
export function decideOrcadMachineHome(
|
||||
probe: OrcadMachineHomeProbe,
|
||||
@@ -284,27 +290,37 @@ export function decideOrcadMachineHome(
|
||||
? { kind: 'login-home' }
|
||||
: { kind: 'machine-home', machineKey: identity.key }
|
||||
}
|
||||
if (
|
||||
probe.machineHomeKeys.includes(identity.key) ||
|
||||
probe.dataRootOwners.some((owner) => isForeignOwner(owner, identity, host))
|
||||
) {
|
||||
const owners = probe.dataRootOwners.map((owner) => ownerMachine(owner, identity, host))
|
||||
if (probe.machineHomeKeys.includes(identity.key) || owners.includes('foreign')) {
|
||||
return { kind: 'machine-home', machineKey: identity.key }
|
||||
}
|
||||
return probe.loginHomeInstalled && probe.dataRootOwners.length === 0 && !probe.homeOnLocalDisk
|
||||
? { kind: 'unresolved', machineKey: identity.key }
|
||||
if (probe.homeOnLocalDisk) {
|
||||
return { kind: 'claim', machineKey: identity.key }
|
||||
}
|
||||
// Claiming another machine's install would judge its PIDs here and admit a second writer.
|
||||
if (owners.includes('unknown')) {
|
||||
return { kind: 'unresolved', machineKey: identity.key, cause: 'unattributed-owner' }
|
||||
}
|
||||
return probe.loginHomeInstalled && owners.length === 0
|
||||
? { kind: 'unresolved', machineKey: identity.key, cause: 'stopped-install' }
|
||||
: { kind: 'claim', machineKey: identity.key }
|
||||
}
|
||||
|
||||
/** Names the one-line claim the user can run on the machine that set the server up. */
|
||||
export function orcadMachineOwnershipUnresolvedMessage(
|
||||
machineKey: string,
|
||||
hostname: string | null
|
||||
hostname: string | null,
|
||||
cause: OrcadMachineOwnershipUnresolvedCause = 'stopped-install'
|
||||
): string {
|
||||
const claim = JSON.stringify({ schemaVersion: ORCAD_MACHINE_CLAIM_SCHEMA_VERSION, machineKey })
|
||||
const path = `~/${RELAY_REMOTE_DIR}/${ORCAD_MACHINE_CLAIM_FILENAME}`
|
||||
const why =
|
||||
cause === 'stopped-install'
|
||||
? 'that stopped server left no record of its own'
|
||||
: 'the older Orca server there recorded only a hostname, which does not identify its machine'
|
||||
return (
|
||||
`Orca can't tell which machine set up the stopped Orca server in ~/${RELAY_REMOTE_DIR}: ` +
|
||||
'this home may be shared by several machines, and that server left no record of its own. ' +
|
||||
`Orca can't tell which machine set up the Orca server in ~/${RELAY_REMOTE_DIR}: ` +
|
||||
`this home may be shared by several machines, and ${why}. ` +
|
||||
'If another machine set it up, connect to that one and run the command shown there. ' +
|
||||
`If ${hostname ?? 'this machine'} set it up, run this on it, then reconnect: ` +
|
||||
`mkdir -p ~/${RELAY_REMOTE_DIR} && printf '%s' '${claim}' > ${path}`
|
||||
@@ -330,47 +346,3 @@ export function orcadMachineDataRoot(
|
||||
): string {
|
||||
return joinRemotePath(host, loginHome, `.orca-${machineKey.slice(0, 8)}`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Publishes the claim exclusively and prints whichever claim won. Why a hard link: `ln` is atomic
|
||||
* and exclusive on NFS too, and a reader never sees a half-written claim. Where `ln` cannot decide
|
||||
* (no hard links, or an empty claim a full disk left), a directory rename picks the winner: it is
|
||||
* atomic everywhere, and the arbiter arrives already holding the winner's claim, so a crash leaves
|
||||
* no half-taken lock. Every machine then republishes that same claim, so concurrent writes agree.
|
||||
*/
|
||||
export function orcadMachineClaimCommand(
|
||||
host: RemoteHostPlatform,
|
||||
loginHome: string,
|
||||
machineKey: string
|
||||
): string {
|
||||
const remoteDir = (...parts: string[]): string =>
|
||||
shellEscape(joinRemotePath(host, loginHome, RELAY_REMOTE_DIR, ...parts))
|
||||
const attempt = randomUUID()
|
||||
const base = remoteDir()
|
||||
const claim = remoteDir(ORCAD_MACHINE_CLAIM_FILENAME)
|
||||
const partial = remoteDir(`${ORCAD_MACHINE_CLAIM_FILENAME}.partial.${attempt}`)
|
||||
const stageName = `${ORCAD_MACHINE_CLAIM_ARBITER_DIRNAME}.partial.${attempt}`
|
||||
const stage = remoteDir(stageName)
|
||||
const stageClaim = remoteDir(stageName, ORCAD_MACHINE_CLAIM_FILENAME)
|
||||
const arbiter = remoteDir(ORCAD_MACHINE_CLAIM_ARBITER_DIRNAME)
|
||||
const arbiterClaim = remoteDir(ORCAD_MACHINE_CLAIM_ARBITER_DIRNAME, ORCAD_MACHINE_CLAIM_FILENAME)
|
||||
// `mv` into an existing arbiter nests the stage inside it instead of replacing it.
|
||||
const nestedStage = remoteDir(ORCAD_MACHINE_CLAIM_ARBITER_DIRNAME, stageName)
|
||||
const body = shellEscape(
|
||||
JSON.stringify({
|
||||
schemaVersion: ORCAD_MACHINE_CLAIM_SCHEMA_VERSION,
|
||||
machineKey
|
||||
})
|
||||
)
|
||||
return [
|
||||
`umask 077; mkdir -p ${base} || exit 65;`,
|
||||
`printf %s ${body} > ${partial} || exit 65;`,
|
||||
// Once an arbiter exists it decides any claim that later goes missing or empty.
|
||||
`if { [ -d ${arbiter} ] || ! ln ${partial} ${claim} 2>/dev/null; } && [ ! -s ${claim} ]; then`,
|
||||
`{ mkdir ${stage} && printf %s ${body} > ${stageClaim}; } || { rm -rf ${stage} ${partial}; exit 65; };`,
|
||||
`mv ${stage} ${arbiter} 2>/dev/null; rm -rf ${stage} ${nestedStage};`,
|
||||
`[ -s ${arbiterClaim} ] && cat ${arbiterClaim} > ${partial} || { rm -f ${partial}; exit 65; };`,
|
||||
`[ -s ${claim} ] || mv -f ${partial} ${claim};`,
|
||||
`fi; rm -f ${partial}; cat ${claim}`
|
||||
].join(' ')
|
||||
}
|
||||
|
||||
@@ -12,7 +12,6 @@ import {
|
||||
} from './orcad-host-transport'
|
||||
import {
|
||||
decideOrcadMachineHome,
|
||||
orcadMachineClaimCommand,
|
||||
orcadMachineDataRoot,
|
||||
orcadMachineHomeDir,
|
||||
orcadMachineHomeProbeCommand,
|
||||
@@ -20,6 +19,7 @@ import {
|
||||
parseOrcadMachineClaim,
|
||||
parseOrcadMachineHomeProbe
|
||||
} from './orcad-machine-home'
|
||||
import { orcadMachineClaimCommand } from './orcad-machine-claim-command'
|
||||
import { execOrcadRemote, type OrcadRemoteExecTarget } from './orcad-remote-runtime-control'
|
||||
import { readRemoteHomeCommand } from './ssh-remote-commands'
|
||||
import { shellEscape } from './ssh-connection-utils'
|
||||
@@ -130,7 +130,11 @@ export async function resolveOrcadMachineHome(
|
||||
let decision = decideOrcadMachineHome(probe, host)
|
||||
if (decision.kind === 'unresolved') {
|
||||
throw new Error(
|
||||
orcadMachineOwnershipUnresolvedMessage(decision.machineKey, probe.identity?.hostname ?? null)
|
||||
orcadMachineOwnershipUnresolvedMessage(
|
||||
decision.machineKey,
|
||||
probe.identity?.hostname ?? null,
|
||||
decision.cause
|
||||
)
|
||||
)
|
||||
}
|
||||
if (decision.kind === 'claim') {
|
||||
|
||||
Reference in New Issue
Block a user