feat(ssh): remote orcad primitives on the pinned Node runtime (#16741 T6-1) (#24419)

- orcad-remote-runtime-control: the one launch-and-poll-readiness loop that
  deployOrcad and rollbackOrcad now share, plus exec and recovery helpers.
- orcad-remote-record-file: bounded, marker-delimited reads and atomic writes
  for host records. A read that returns no verifiable answer rejects instead
  of reading as "no record".
- The activation record store reads through it (a lost read no longer becomes
  an empty record) and writeOrcadActivationRecord refuses to replace a record
  this client cannot read, such as a newer schema; deploy and rollback use it.
- orcad-active-readiness: prove a recorded-active or relaunched slot against
  the activation gate, reporting exited / unverifiable / rejected.
- orcad-remote-build-hash and orcad-remote-context (host, home, server target
  via orcad-deployment-target, activation record; POSIX-only).
- Readiness reads are capped at 256 KiB, and a finished invalid JSON line is
  malformed rather than pending forever.

Inert: nothing in the app calls managed orcad deploy yet. Ported from #16741
(a68b6f3531) onto main's pinned-Node deploy core; the branch's Bun runtime,
slot eligibility, target detection and bundle installation are superseded by
main's orcad-remote-runtime, orcad-deployment-target and orcad-remote-install.

Co-authored-by: m4air <m4air@m4airs-Air.localdomain>
This commit is contained in:
OrcaWin
2026-10-01 11:09:04 -07:00
committed by GitHub
co-authored by m4air
parent 92cb71765e
commit dd87ae578d
12 changed files with 618 additions and 117 deletions
+45 -15
View File
@@ -6,32 +6,50 @@
* activated" would deploy over a live install and lose its rollback target.
*/
import type { SshConnection } from './ssh-connection'
import { execCommand } from './ssh-relay-deploy-helpers'
import { RELAY_REMOTE_DIR } from './relay-protocol'
import {
ORCAD_ACTIVATION_FILENAME,
emptyOrcadActivationRecord,
parseOrcadActivationRecord,
serializeOrcadActivationRecord,
type OrcadActivationReadResult,
type OrcadActivationRecord
} from './orcad-activation-record'
import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform'
import {
readBoundedOrcadRemoteRecord,
writeAtomicOrcadRemoteRecord
} from './orcad-remote-record-file'
const ORCAD_ACTIVATION_RECORD_MAX_BYTES = 64 * 1024
type ActivationRecordTarget = {
conn: SshConnection
host: RemoteHostPlatform
remoteHome: string
signal?: AbortSignal
}
export function orcadActivationPath(host: RemoteHostPlatform, remoteHome: string): string {
return joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR, ORCAD_ACTIVATION_FILENAME)
}
export async function readOrcadActivationRecord(options: {
conn: SshConnection
host: RemoteHostPlatform
remoteHome: string
signal?: AbortSignal
}): Promise<OrcadActivationRecord> {
const path = orcadActivationPath(options.host, options.remoteHome)
const raw = await execCommand(options.conn, `cat ${shellQuote(path)} 2>/dev/null || true`, {
wrapCommand: options.host.commandDialect !== 'powershell',
signal: options.signal
}).catch(() => '')
const parsed = parseOrcadActivationRecord(raw)
/** A failed read rejects; it never reads as absent, because absence would admit a fresh deploy. */
async function readActivationRecordState(
options: ActivationRecordTarget
): Promise<OrcadActivationReadResult> {
const read = await readBoundedOrcadRemoteRecord(
options,
orcadActivationPath(options.host, options.remoteHome),
ORCAD_ACTIVATION_RECORD_MAX_BYTES
)
return read.state === 'absent' ? read : parseOrcadActivationRecord(read.raw)
}
export async function readOrcadActivationRecord(
options: ActivationRecordTarget
): Promise<OrcadActivationRecord> {
const parsed = await readActivationRecordState(options)
if (parsed.state === 'ok') {
return parsed.record
}
@@ -43,6 +61,18 @@ export async function readOrcadActivationRecord(options: {
return emptyOrcadActivationRecord()
}
function shellQuote(value: string): string {
return `'${value.replaceAll("'", `'\\''`)}'`
/** Refuses to replace a record this client cannot read, e.g. one a newer client wrote. */
export async function writeOrcadActivationRecord(
options: ActivationRecordTarget,
record: OrcadActivationRecord
): Promise<void> {
const existing = await readActivationRecordState(options)
if (existing.state === 'unreadable') {
throw new Error(`Refusing to overwrite this host's orcad activation record: ${existing.reason}`)
}
await writeAtomicOrcadRemoteRecord(
options,
orcadActivationPath(options.host, options.remoteHome),
serializeOrcadActivationRecord(record)
)
}
+90
View File
@@ -0,0 +1,90 @@
/** Proving that the orcad an activation record names is the one actually serving. */
import { evaluateOrcadActivation, type OrcadActivationExpectation } from './orcad-activation-gate'
import {
orcadLivenessProbeCommand,
parseOrcadLiveness,
parseOrcadReadinessOutput,
readOrcadReadinessCommand,
type OrcadLaunchSpec,
type OrcadReadinessParse
} from './orcad-remote-launch'
import {
execOrcadRemote,
launchOrcadAndAwaitReadiness,
type OrcadRemoteExecTarget
} from './orcad-remote-runtime-control'
import type { ServeReadiness } from '../server/serve-readiness'
/** `exited` is proven absence; `unverifiable` means the host could not say, which is not death. */
export type OrcadReadinessFailureVerdict = 'exited' | 'unverifiable' | 'rejected'
export class OrcadActiveReadinessError extends Error {
constructor(
readonly verdict: OrcadReadinessFailureVerdict,
message: string
) {
super(message)
this.name = 'OrcadActiveReadinessError'
}
}
function gatedReadiness(
parsed: OrcadReadinessParse,
expectation: OrcadActivationExpectation,
label: string
): ServeReadiness {
const readiness = parsed.state === 'ready' ? parsed.readiness : null
const verdict = evaluateOrcadActivation(readiness, expectation)
if (verdict.decision === 'reject') {
throw new OrcadActiveReadinessError(
'rejected',
`${label} failed its readiness check: ${verdict.reason}`
)
}
if (!readiness) {
throw new OrcadActiveReadinessError(
'rejected',
`${label} passed activation without a readiness payload.`
)
}
return readiness
}
/** Checks a recorded-active slot without starting anything. */
export async function probeActiveOrcadReadiness(
target: OrcadRemoteExecTarget & { remoteInstallDir: string },
expectation: OrcadActivationExpectation
): Promise<ServeReadiness> {
const liveness = parseOrcadLiveness(
await execOrcadRemote(target, orcadLivenessProbeCommand(target.host, target.remoteInstallDir))
)
if (liveness === 'DEAD') {
throw new OrcadActiveReadinessError(
'exited',
`orcad ${expectation.fullVersion} is recorded active but its process has exited.`
)
}
if (liveness !== 'LIVE') {
throw new OrcadActiveReadinessError(
'unverifiable',
`orcad ${expectation.fullVersion} process state is unverifiable.`
)
}
const parsed = parseOrcadReadinessOutput(
await execOrcadRemote(target, readOrcadReadinessCommand(target.host, target.remoteInstallDir))
)
return gatedReadiness(parsed, expectation, 'The active orcad')
}
/** Starts a slot (recovery or rollback) and accepts it only if it proves the expected build. */
export async function launchOrcadSlotAndAwaitReadiness(
target: OrcadRemoteExecTarget & {
readinessTimeoutMs?: number
sleep?: (ms: number) => Promise<void>
},
spec: OrcadLaunchSpec,
expectation: OrcadActivationExpectation
): Promise<ServeReadiness> {
const parsed = await launchOrcadAndAwaitReadiness(target, spec)
return gatedReadiness(parsed, expectation, `orcad ${spec.fullVersion}`)
}
+36
View File
@@ -0,0 +1,36 @@
/** The installed slot's `orcad.js` identity, the same 16-hex prefix orcad reports in its health. */
import { shellEscape } from './ssh-connection-utils'
import { assertPosixOrcadHost } from './orcad-remote-host-support'
import { execOrcadRemote, type OrcadRemoteExecTarget } from './orcad-remote-runtime-control'
import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform'
const BUILD_HASH_MARKER = '__ORCAD_BUILD_HASH__'
export async function readRemoteOrcadBuildHash(
target: OrcadRemoteExecTarget,
remoteInstallDir: string
): Promise<string> {
const output = await execOrcadRemote(
target,
remoteOrcadBuildHashCommand(target.host, remoteInstallDir)
)
const match = output.match(/__ORCAD_BUILD_HASH__\s+([a-fA-F0-9]{16})/u)
if (!match?.[1]) {
throw new Error('Could not verify the installed orcad build hash.')
}
return match[1].toLowerCase()
}
export function remoteOrcadBuildHashCommand(
host: RemoteHostPlatform,
remoteInstallDir: string
): string {
assertPosixOrcadHost(host)
const path = shellEscape(joinRemotePath(host, remoteInstallDir, 'orcad.js'))
// Why both tools: GNU/busybox ship sha256sum, macOS ships shasum; either prints the digest first.
return [
`orca_hash=$(if command -v sha256sum >/dev/null 2>&1; then sha256sum ${path} | awk '{print $1}';`,
`elif command -v shasum >/dev/null 2>&1; then shasum -a 256 ${path} | awk '{print $1}'; fi);`,
`case "$orca_hash" in [0-9a-fA-F][0-9a-fA-F]*) printf '%s %.16s\\n' ${shellEscape(BUILD_HASH_MARKER)} "$orca_hash";; esac`
].join(' ')
}
+59
View File
@@ -0,0 +1,59 @@
/** Everything a managed-orcad operation needs to know about one SSH host before it acts. */
import type { ServerTarget } from '../../shared/node-runtime-pin'
import type { SshTarget } from '../../shared/ssh-types'
import type { OrcadActivationRecord } from './orcad-activation-record'
import { readOrcadActivationRecord } from './orcad-activation-record-store'
import { resolveOrcadDeploymentTarget } from './orcad-deployment-target'
import { assertPosixOrcadHost } from './orcad-remote-host-support'
import { execOrcadRemote } from './orcad-remote-runtime-control'
import type { SshConnection } from './ssh-connection'
import { readRemoteHomeCommand } from './ssh-remote-commands'
import {
joinRemotePath,
normalizeRemoteHome,
validateRemoteHome,
type RemoteHostPlatform
} from './ssh-remote-platform'
import { detectRemoteHostPlatform } from './ssh-remote-platform-detection'
export type OrcadRemoteContext = {
activationRecord: OrcadActivationRecord
serverTarget: ServerTarget
connection: SshConnection
host: RemoteHostPlatform
remoteHome: string
target: SshTarget
userDataDir: string
}
export async function resolveOrcadRemoteContext(
target: SshTarget,
connection: SshConnection,
signal?: AbortSignal
): Promise<OrcadRemoteContext> {
const host = await detectRemoteHostPlatform(connection, { signal })
if (!host) {
throw new Error('This SSH host platform is not supported by managed orcad.')
}
// Why first: every lifecycle step after this is POSIX-only, so refuse before probing further.
assertPosixOrcadHost(host)
const remote = { conn: connection, host, signal }
const remoteHome = normalizeRemoteHome(
await execOrcadRemote(remote, readRemoteHomeCommand(host)),
host
)
if (!validateRemoteHome(remoteHome, host)) {
throw new Error(`Remote home is not a valid path: ${remoteHome.slice(0, 100)}`)
}
const serverTarget = await resolveOrcadDeploymentTarget({ conn: connection, host, signal })
const activationRecord = await readOrcadActivationRecord({ ...remote, remoteHome })
return {
activationRecord,
serverTarget,
connection,
host,
remoteHome,
target,
userDataDir: joinRemotePath(host, remoteHome, '.orca')
}
}
+19 -11
View File
@@ -1,4 +1,5 @@
import { chmodSync, mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from 'node:fs'
import type * as RecordFile from './orcad-remote-record-file'
import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path'
import { beforeEach, describe, expect, it, vi } from 'vitest'
@@ -17,6 +18,10 @@ vi.mock('./ssh-relay-install-transfers', () => ({
uploadRelayDirectory: vi.fn().mockResolvedValue(undefined),
writeRelayFile: vi.fn().mockResolvedValue(undefined)
}))
vi.mock('./orcad-remote-record-file', async (importOriginal) => ({
...(await importOriginal<typeof RecordFile>()),
writeAtomicOrcadRemoteRecord: vi.fn().mockResolvedValue(undefined)
}))
vi.mock('./orcad-remote-node-runtime', () => ({
ensureRemoteOrcadNodeRuntime: vi.fn().mockResolvedValue(undefined)
}))
@@ -26,7 +31,8 @@ vi.mock('./orcad-local-build-hash', () => ({
import { execCommand } from './ssh-relay-deploy-helpers'
import { acquireInstallLock } from './ssh-relay-install-lock'
import { uploadRelayDirectory, writeRelayFile } from './ssh-relay-install-transfers'
import { uploadRelayDirectory } from './ssh-relay-install-transfers'
import { writeAtomicOrcadRemoteRecord } from './orcad-remote-record-file'
import { deployOrcad, type OrcadDeployOptions } from './orcad-remote-deploy'
import { installOrcadBundle } from './orcad-remote-install'
import { ensureRemoteOrcadNodeRuntime } from './orcad-remote-node-runtime'
@@ -106,10 +112,12 @@ type HostScript = {
function scriptHost(script: HostScript): void {
mockExec.mockImplementation(async (_conn, command: string) => {
const text = String(command)
if (text.startsWith('cat ') && text.includes('orcad-active.json')) {
if (text.includes('__ORCAD_RECORD_PRESENT__') && text.includes('orcad-active.json')) {
return script.activationRecord
? `__ORCAD_RECORD_PRESENT__\n${script.activationRecord}`
: '__ORCAD_RECORD_ABSENT__\n'
}
if (text.includes('.orcad-readiness') && text.startsWith('cat ')) {
if (text.includes('.orcad-readiness') && text.startsWith('head -c ')) {
if (script.readinessAtMs !== undefined && Date.now() < script.readinessAtMs) {
return ''
}
@@ -272,8 +280,8 @@ describe('deployOrcad', () => {
expect(script.log).toEqual(['preflight'])
expect(
vi
.mocked(writeRelayFile)
.mock.calls.some(([, , path]) => path.includes('orcad-active.json'))
.mocked(writeAtomicOrcadRemoteRecord)
.mock.calls.some(([, path]) => path.includes('orcad-active.json'))
).toBe(false)
}
)
@@ -356,7 +364,7 @@ describe('deployOrcad', () => {
if (String(command).startsWith('chmod 755 ')) {
throw new Error('chmod failed')
}
return ''
return String(command).includes('__ORCAD_RECORD_ABSENT__') ? '__ORCAD_RECORD_ABSENT__\n' : ''
})
await expect(deployOrcad(options())).rejects.toThrow('chmod failed')
expect(vi.mocked(finalizeInstall)).not.toHaveBeenCalled()
@@ -441,9 +449,9 @@ describe('deployOrcad', () => {
const result = await deployOrcad(options())
expect(result).toMatchObject({ outcome: 'installed-and-activated', fullVersion: NEW_VERSION })
const written = vi
.mocked(writeRelayFile)
.mock.calls.find((call) => String(call[2]).endsWith('orcad-active.json'))
expect(JSON.parse(String(written?.[3]))).toMatchObject({
.mocked(writeAtomicOrcadRemoteRecord)
.mock.calls.find((call) => String(call[1]).endsWith('orcad-active.json'))
expect(JSON.parse(String(written?.[2]))).toMatchObject({
active: NEW_VERSION,
previous: OLD_VERSION
})
@@ -492,8 +500,8 @@ describe('deployOrcad', () => {
expect(result).toMatchObject({ code: 'orcad_activation_daemon_degraded' })
expect(
vi
.mocked(writeRelayFile)
.mock.calls.some((call) => String(call[2]).endsWith('orcad-active.json'))
.mocked(writeAtomicOrcadRemoteRecord)
.mock.calls.some((call) => String(call[1]).endsWith('orcad-active.json'))
).toBe(false)
})
+12 -37
View File
@@ -4,28 +4,24 @@
* preserve current state and the prelaunch snapshot for explicit recovery.
*/
import type { SshConnection } from './ssh-connection'
import { ORCAD_STARTUP_READINESS_TIMEOUT_MS } from '../../shared/orcad-profile-preflight'
import { execCommand } from './ssh-relay-deploy-helpers'
import { ORCAD_INSTALL_MODEL } from './remote-install-model'
import { writeRelayFile } from './ssh-relay-install-transfers'
import { computeRemoteInstallDir, readLocalFullVersion } from './ssh-relay-versioned-install'
import { RELAY_REMOTE_DIR } from './relay-protocol'
import {
ORCAD_STATE_SNAPSHOT_DIR,
serializeOrcadActivationRecord,
withActivatedVersion,
type OrcadActivationRecord,
type OrcadStateSnapshot
} from './orcad-activation-record'
import { orcadActivationPath, readOrcadActivationRecord } from './orcad-activation-record-store'
import {
readOrcadActivationRecord,
writeOrcadActivationRecord
} from './orcad-activation-record-store'
import { evaluateOrcadActivation, type OrcadActivationVerdict } from './orcad-activation-gate'
import { planOrcadUpdate, type OrcadTerminalCensus } from './orcad-update-plan'
import {
ORCAD_LOG_FILENAME,
orcadLaunchCommand,
parseOrcadReadinessOutput,
readOrcadReadinessCommand
} from './orcad-remote-launch'
import { ORCAD_LOG_FILENAME } from './orcad-remote-launch'
import { launchOrcadAndAwaitReadiness } from './orcad-remote-runtime-control'
import { rejectedOrcadStateRecoveryRefusal, stopOutgoingOrcad } from './orcad-remote-deploy-stop'
import {
captureOrcadStateSnapshotCommand,
@@ -81,7 +77,6 @@ export type OrcadDeployResult =
| { outcome: 'already-active'; fullVersion: string }
| { outcome: 'installed-not-activated'; fullVersion: string; code: string; reason: string }
const READINESS_POLL_MS = 500
const STOP_WAIT_SECONDS = 20
function exec(options: OrcadDeployOptions, command: string): Promise<string> {
@@ -135,29 +130,12 @@ async function captureSnapshot(
}
}
async function launchAndAwaitReadiness(
function launchAndAwaitReadiness(
options: OrcadDeployOptions,
remoteInstallDir: string,
fullVersion: string
): Promise<ReturnType<typeof parseOrcadReadinessOutput>> {
await exec(
options,
orcadLaunchCommand(options.host, { ...options, remoteInstallDir, fullVersion })
)
const deadline = Date.now() + (options.readinessTimeoutMs ?? ORCAD_STARTUP_READINESS_TIMEOUT_MS)
const sleep = options.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms)))
let last = parseOrcadReadinessOutput('')
while (Date.now() < deadline) {
options.signal?.throwIfAborted()
last = parseOrcadReadinessOutput(
await exec(options, readOrcadReadinessCommand(options.host, remoteInstallDir))
)
if (last.state !== 'pending') {
return last
}
await sleep(READINESS_POLL_MS)
}
return last
): ReturnType<typeof launchOrcadAndAwaitReadiness> {
return launchOrcadAndAwaitReadiness(options, { ...options, remoteInstallDir, fullVersion })
}
/** Restart the incumbent only when the candidate left shared state unchanged. */
@@ -320,12 +298,9 @@ export async function deployOrcad(input: OrcadDeployOptions): Promise<OrcadDeplo
}
}
await writeRelayFile(
options.conn,
options.host,
orcadActivationPath(options.host, options.remoteHome),
serializeOrcadActivationRecord(withActivatedVersion(record, fullVersion, snapshot, now())),
{ signal: options.signal }
await writeOrcadActivationRecord(
options,
withActivatedVersion(record, fullVersion, snapshot, now())
)
return { outcome: 'installed-and-activated', fullVersion, verdict }
}
+13 -6
View File
@@ -26,6 +26,8 @@ import { selectOrcadSlotRuntimeCommand } from './orcad-remote-runtime'
export const ORCAD_READINESS_FILENAME = '.orcad-readiness'
/** Stderr, including the bind-exposure line and every supervision message. */
export const ORCAD_LOG_FILENAME = 'orcad.log'
// Why a cap: the readiness file is candidate-written stdout, and a runaway writer must not be read whole.
const ORCAD_READINESS_MAX_BYTES = 256 * 1024
export { ORCAD_PID_FILENAME, OrcadRemoteLaunchUnsupportedError } from './orcad-remote-host-support'
export type OrcadLaunchSpec = {
@@ -78,7 +80,8 @@ export function readOrcadReadinessCommand(
): string {
assertPosixHost(host)
const readiness = shellEscape(joinRemotePath(host, remoteInstallDir, ORCAD_READINESS_FILENAME))
return `cat ${readiness} 2>/dev/null || true`
// One byte over the cap is enough to tell an oversized payload from a full one.
return `head -c ${ORCAD_READINESS_MAX_BYTES + 1} ${readiness} 2>/dev/null || true`
}
/**
@@ -134,19 +137,23 @@ export type OrcadReadinessParse =
* not `malformed` — reporting a parse failure for a race would fail deploys that were fine.
*/
export function parseOrcadReadinessOutput(raw: string): OrcadReadinessParse {
if (Buffer.byteLength(raw, 'utf8') > ORCAD_READINESS_MAX_BYTES) {
return { state: 'malformed', reason: 'readiness payload exceeds the 256 KiB limit' }
}
const lines = raw.split('\n')
let sawCandidate = false
for (const line of lines) {
for (const [index, line] of lines.entries()) {
const trimmed = line.trim()
if (!trimmed.startsWith('{')) {
continue
}
sawCandidate = true
let parsed: unknown
try {
parsed = JSON.parse(trimmed)
} catch {
continue
// Only the unterminated last line can still be mid-write; a finished bad line will stay bad.
return index === lines.length - 1
? { state: 'pending' }
: { state: 'malformed', reason: 'readiness line is not valid JSON' }
}
if (typeof parsed !== 'object' || parsed === null) {
continue
@@ -160,7 +167,7 @@ export function parseOrcadReadinessOutput(raw: string): OrcadReadinessParse {
}
return { state: 'ready', readiness: toServeReadiness(payload as Record<string, unknown>) }
}
return sawCandidate ? { state: 'pending' } : { state: 'pending' }
return { state: 'pending' }
}
function toServeReadiness(payload: Record<string, unknown>): ServeReadiness {
@@ -0,0 +1,186 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
vi.mock('./ssh-relay-deploy-helpers', () => ({
execCommand: vi.fn(),
isUnconfirmedSshCommandTermination: (error: unknown) =>
error instanceof Error && error.message === 'channel lost'
}))
vi.mock('./ssh-remote-platform-detection', () => ({ detectRemoteHostPlatform: vi.fn() }))
import { execCommand } from './ssh-relay-deploy-helpers'
import { detectRemoteHostPlatform } from './ssh-remote-platform-detection'
import { resolveOrcadRemoteContext } from './orcad-remote-context'
import { getRemoteHostPlatform } from './ssh-remote-platform'
import type { SshConnection } from './ssh-connection'
import {
readBoundedOrcadRemoteRecord,
writeAtomicOrcadRemoteRecord
} from './orcad-remote-record-file'
import {
readOrcadActivationRecord,
writeOrcadActivationRecord
} from './orcad-activation-record-store'
import { emptyOrcadActivationRecord } from './orcad-activation-record'
import { readRemoteOrcadBuildHash } from './orcad-remote-build-hash'
import { OrcadActiveReadinessError, probeActiveOrcadReadiness } from './orcad-active-readiness'
import { parseOrcadReadinessOutput } from './orcad-remote-launch'
const mockExec = vi.mocked(execCommand)
const linux = getRemoteHostPlatform('linux-x64')
const windows = getRemoteHostPlatform('win32-x64')
const conn: SshConnection = Object.create(null)
const target = { conn, host: linux }
const BUILD_HASH = 'abc123def4567890'
function readyLine(buildHash = BUILD_HASH): string {
return JSON.stringify({
type: 'orca_server_ready',
runtimeId: 'r1',
boundEndpoint: 'ws://127.0.0.1:7777',
advertisedEndpoint: null,
managedWslCliReconciliation: 'settled',
pairing: { available: false, reason: 'disabled_by_operator', guidance: 'n/a' },
health: {
buildHash,
buildVersion: '0.2.0+bb01',
nodeVersion: '24.21.0',
nodeAbi: '137',
platform: 'linux',
arch: 'x64',
pid: 1,
terminalDaemon: {
state: 'live',
ownsFreshSessions: true,
pid: 2,
buildVersion: '0.2.0+bb01',
entryPath: '/x/daemon-entry.js',
protocolVersion: 38,
selfTest: { ok: true, coverage: 'pty-spawn', verdict: 'healthy', durationMs: 5 }
}
}
})
}
beforeEach(() => {
mockExec.mockReset()
})
describe('orcad host record files', () => {
it('tells an absent record from one whose read gave no answer', async () => {
mockExec.mockResolvedValueOnce('__ORCAD_RECORD_ABSENT__\n')
await expect(readBoundedOrcadRemoteRecord(target, '/r.json', 64)).resolves.toEqual({
state: 'absent'
})
mockExec.mockResolvedValueOnce('__ORCAD_RECORD_PRESENT__\n{"a":1}')
await expect(readBoundedOrcadRemoteRecord(target, '/r.json', 64)).resolves.toEqual({
state: 'present',
raw: '{"a":1}'
})
mockExec.mockResolvedValueOnce('')
await expect(readBoundedOrcadRemoteRecord(target, '/r.json', 64)).rejects.toThrow(
'no verifiable answer'
)
})
it('keeps the partial file when the write may still be running on the host', async () => {
mockExec.mockRejectedValueOnce(new Error('channel lost'))
await expect(writeAtomicOrcadRemoteRecord(target, '/r.json', '{}')).rejects.toThrow()
expect(mockExec).toHaveBeenCalledOnce()
mockExec.mockRejectedValueOnce(new Error('exit 1')).mockResolvedValueOnce('')
await expect(writeAtomicOrcadRemoteRecord(target, '/r.json', '{}')).rejects.toThrow()
expect(String(mockExec.mock.calls.at(-1)?.[1])).toContain('rm -f')
})
it('refuses Windows hosts, which the orcad lifecycle does not support', async () => {
await expect(
readBoundedOrcadRemoteRecord({ conn, host: windows }, 'C:/r.json', 64)
).rejects.toThrow()
expect(mockExec).not.toHaveBeenCalled()
})
})
describe('activation record store', () => {
const options = { conn, host: linux, remoteHome: '/home/u' }
const newer = JSON.stringify({ ...emptyOrcadActivationRecord(), schemaVersion: 2 })
it('reads a lost read as an error, never as an empty record', async () => {
mockExec.mockRejectedValueOnce(new Error('channel lost'))
await expect(readOrcadActivationRecord(options)).rejects.toThrow('channel lost')
})
it('reads a newer schema as unreadable and never overwrites it', async () => {
mockExec.mockResolvedValue(`__ORCAD_RECORD_PRESENT__\n${newer}`)
await expect(readOrcadActivationRecord(options)).rejects.toThrow('schemaVersion 2')
await expect(writeOrcadActivationRecord(options, emptyOrcadActivationRecord())).rejects.toThrow(
'Refusing to overwrite'
)
expect(mockExec.mock.calls.some(([, command]) => String(command).includes('mv -f'))).toBe(false)
})
it('writes atomically when the host has no record yet', async () => {
mockExec.mockResolvedValueOnce('__ORCAD_RECORD_ABSENT__\n').mockResolvedValueOnce('')
await writeOrcadActivationRecord(options, emptyOrcadActivationRecord())
expect(String(mockExec.mock.calls[1]?.[1])).toMatch(/printf %s .* && mv -f /s)
})
})
describe('installed build identity and readiness', () => {
const slot = { ...target, remoteInstallDir: '/home/u/.orca-remote/orcad-0.2.0+bb01' }
const expectation = { buildHash: BUILD_HASH, fullVersion: '0.2.0+bb01' }
it('reads the 16-hex build hash the slot reports', async () => {
mockExec.mockResolvedValueOnce(`noise\n__ORCAD_BUILD_HASH__ ${BUILD_HASH.toUpperCase()}\n`)
await expect(readRemoteOrcadBuildHash(target, '/slot')).resolves.toBe(BUILD_HASH)
mockExec.mockResolvedValueOnce('')
await expect(readRemoteOrcadBuildHash(target, '/slot')).rejects.toThrow()
})
it.each([
['DEAD', 'exited'],
['UNKNOWN', 'unverifiable'],
['', 'unverifiable']
])('reports a %j liveness probe as %s', async (liveness, verdict) => {
mockExec.mockResolvedValueOnce(liveness)
await expect(probeActiveOrcadReadiness(slot, expectation)).rejects.toMatchObject({
verdict
})
})
it('accepts only the expected build once the process is live', async () => {
mockExec.mockResolvedValueOnce('LIVE').mockResolvedValueOnce(`${readyLine()}\n`)
await expect(probeActiveOrcadReadiness(slot, expectation)).resolves.toMatchObject({
runtimeId: 'r1'
})
mockExec
.mockResolvedValueOnce('LIVE')
.mockResolvedValueOnce(`${readyLine('ffffffffffffffff')}\n`)
const rejected = probeActiveOrcadReadiness(slot, expectation)
await expect(rejected).rejects.toBeInstanceOf(OrcadActiveReadinessError)
await expect(rejected).rejects.toMatchObject({ verdict: 'rejected' })
})
})
describe('readiness parsing bounds', () => {
it('waits on a half-written last line but rejects a finished invalid one', () => {
expect(parseOrcadReadinessOutput('{"type":"orca_ser')).toEqual({ state: 'pending' })
expect(parseOrcadReadinessOutput('{"type":"orca_ser\n')).toMatchObject({
state: 'malformed'
})
})
it('rejects a payload over the size cap', () => {
expect(parseOrcadReadinessOutput('x'.repeat(256 * 1024 + 1))).toMatchObject({
state: 'malformed'
})
})
})
describe('orcad remote context', () => {
it('refuses a Windows host before probing anything else', async () => {
vi.mocked(detectRemoteHostPlatform).mockResolvedValueOnce(windows)
const sshTarget = { id: 't', label: 't', host: 'h', port: 22, username: 'u' }
await expect(resolveOrcadRemoteContext(sshTarget, conn)).rejects.toThrow()
expect(mockExec).not.toHaveBeenCalled()
})
})
+68
View File
@@ -0,0 +1,68 @@
/**
* Small JSON records Orca keeps on an orcad host (activation, transactions, stop receipts).
*
* Reads are bounded and never swallow a failure: a record that could not be read is not an
* absent one, and treating it as absent is how a client deploys over a live install.
*/
import { randomUUID } from 'node:crypto'
import { shellEscape } from './ssh-connection-utils'
import { assertPosixOrcadHost } from './orcad-remote-host-support'
import { removeRemoteFileCommand } from './ssh-remote-commands'
import { execOrcadRemote, type OrcadRemoteExecTarget } from './orcad-remote-runtime-control'
import { isUnconfirmedSshCommandTermination } from './ssh-relay-deploy-helpers'
const ABSENT_MARKER = '__ORCAD_RECORD_ABSENT__'
const PRESENT_MARKER = '__ORCAD_RECORD_PRESENT__'
/** `present` carries raw bytes; schema checks belong to the caller that owns the format. */
export type OrcadRemoteRecordRead = { state: 'absent' } | { state: 'present'; raw: string }
export async function readBoundedOrcadRemoteRecord(
target: OrcadRemoteExecTarget,
path: string,
maxBytes: number
): Promise<OrcadRemoteRecordRead> {
assertPosixOrcadHost(target.host)
const file = shellEscape(path)
// Why markers: an empty stdout must never be mistaken for "no record" when the read failed.
const output = await execOrcadRemote(
target,
`if [ ! -e ${file} ] && [ ! -L ${file} ]; then printf '%s\\n' ${ABSENT_MARKER}; exit 0; fi; ` +
`[ -f ${file} ] || exit 65; size=$(wc -c < ${file}) || exit 65; ` +
`[ "$size" -le ${maxBytes} ] || exit 65; ` +
`printf '%s\\n' ${PRESENT_MARKER}; cat ${file}`
)
const newline = output.indexOf('\n')
const marker = (newline === -1 ? output : output.slice(0, newline)).trim()
if (marker === ABSENT_MARKER) {
return { state: 'absent' }
}
if (marker !== PRESENT_MARKER) {
throw new Error('orcad host record read returned no verifiable answer')
}
return { state: 'present', raw: newline === -1 ? '' : output.slice(newline + 1) }
}
export async function writeAtomicOrcadRemoteRecord(
target: OrcadRemoteExecTarget,
path: string,
contents: string
): Promise<void> {
assertPosixOrcadHost(target.host)
const partialPath = `${path}.partial.${process.pid}.${randomUUID()}`
try {
await execOrcadRemote(
target,
`umask 077; printf %s ${shellEscape(contents)} > ${shellEscape(partialPath)} && ` +
`mv -f ${shellEscape(partialPath)} ${shellEscape(path)}`
)
} catch (error) {
// Why keep the partial on an unconfirmed termination: the write may still be running there.
if (!isUnconfirmedSshCommandTermination(error)) {
await execOrcadRemote(target, removeRemoteFileCommand(target.host, partialPath)).catch(
() => {}
)
}
throw error
}
}
+15 -7
View File
@@ -1,4 +1,5 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import type * as RecordFile from './orcad-remote-record-file'
vi.mock('./ssh-relay-deploy-helpers', () => ({
execCommand: vi.fn(),
@@ -9,9 +10,13 @@ vi.mock('./ssh-relay-install-transfers', () => ({
writeRelayFile: vi.fn().mockResolvedValue(undefined),
uploadRelayDirectory: vi.fn().mockResolvedValue(undefined)
}))
vi.mock('./orcad-remote-record-file', async (importOriginal) => ({
...(await importOriginal<typeof RecordFile>()),
writeAtomicOrcadRemoteRecord: vi.fn().mockResolvedValue(undefined)
}))
import { execCommand } from './ssh-relay-deploy-helpers'
import { writeRelayFile } from './ssh-relay-install-transfers'
import { writeAtomicOrcadRemoteRecord } from './orcad-remote-record-file'
import { rollbackOrcad, type OrcadRollbackOptions } from './orcad-remote-rollback'
import { emptyOrcadActivationRecord, type OrcadActivationRecord } from './orcad-activation-record'
import { getRemoteHostPlatform } from './ssh-remote-platform'
@@ -74,6 +79,9 @@ function scriptHost(
): void {
mockExec.mockImplementation(async (_conn, command: string) => {
const text = String(command)
if (text.includes('__ORCAD_RECORD_PRESENT__')) {
return `__ORCAD_RECORD_PRESENT__\n${JSON.stringify(record())}`
}
if (text.includes('state.tar') && text.includes('test -f') && !text.includes('tar -C')) {
return 'PRESENT'
}
@@ -92,7 +100,7 @@ function scriptHost(
log.push(`launch:${text.includes(ACTIVE) ? ACTIVE : TARGET}`)
return '9999'
}
if (text.startsWith('cat ') && text.includes('.orcad-readiness')) {
if (text.startsWith('head -c ') && text.includes('.orcad-readiness')) {
if (overrides.readinessAtMs !== undefined && Date.now() < overrides.readinessAtMs) {
return ''
}
@@ -169,7 +177,7 @@ describe('rollbackOrcad', () => {
code: 'orcad_rollback_orphans_live_terminals'
})
expect(log).toEqual([])
expect(vi.mocked(writeRelayFile)).not.toHaveBeenCalled()
expect(vi.mocked(writeAtomicOrcadRemoteRecord)).not.toHaveBeenCalled()
})
it('refuses when the snapshot is gone from the host', async () => {
@@ -212,7 +220,7 @@ describe('rollbackOrcad', () => {
expect(result).toMatchObject({ outcome: 'failed', code: 'orcad_activation_no_readiness' })
// Until the target is proven serving, `active` must still name the version an operator
// would have to bring back.
expect(vi.mocked(writeRelayFile)).not.toHaveBeenCalled()
expect(vi.mocked(writeAtomicOrcadRemoteRecord)).not.toHaveBeenCalled()
})
it('records the rollback only after the target answers healthy', async () => {
@@ -220,9 +228,9 @@ describe('rollbackOrcad', () => {
scriptHost(log)
await rollbackOrcad(options())
const written = vi
.mocked(writeRelayFile)
.mock.calls.find((call) => String(call[2]).endsWith('orcad-active.json'))
expect(JSON.parse(String(written?.[3]))).toMatchObject({
.mocked(writeAtomicOrcadRemoteRecord)
.mock.calls.find((call) => String(call[1]).endsWith('orcad-active.json'))
expect(JSON.parse(String(written?.[2]))).toMatchObject({
active: TARGET,
previous: null,
snapshot: null
+12 -41
View File
@@ -14,26 +14,19 @@
* failure this is meant to avoid, arrived at from the other side.
*/
import type { SshConnection } from './ssh-connection'
import { ORCAD_STARTUP_READINESS_TIMEOUT_MS } from '../../shared/orcad-profile-preflight'
import { execCommand } from './ssh-relay-deploy-helpers'
import { ORCAD_INSTALL_MODEL } from './remote-install-model'
import { computeRemoteInstallDir } from './ssh-relay-versioned-install'
import { writeRelayFile } from './ssh-relay-install-transfers'
import { RELAY_REMOTE_DIR } from './relay-protocol'
import {
ORCAD_STATE_SNAPSHOT_DIR,
serializeOrcadActivationRecord,
withRolledBackVersion,
type OrcadActivationRecord
} from './orcad-activation-record'
import { assessOrcadRollback, type OrcadTerminalCensus } from './orcad-update-plan'
import { evaluateOrcadActivation, type OrcadActivationVerdict } from './orcad-activation-gate'
import {
ORCAD_LOG_FILENAME,
orcadLaunchCommand,
parseOrcadReadinessOutput,
readOrcadReadinessCommand
} from './orcad-remote-launch'
import { ORCAD_LOG_FILENAME } from './orcad-remote-launch'
import { launchOrcadAndAwaitReadiness } from './orcad-remote-runtime-control'
import {
newestStateMtimeCommand,
parseNewestStateMtimeSeconds,
@@ -46,7 +39,7 @@ import {
parseOrcadStopOutcome,
stopOrcadCommand
} from './orcad-remote-process-control'
import { orcadActivationPath } from './orcad-activation-record-store'
import { writeOrcadActivationRecord } from './orcad-activation-record-store'
import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform'
export type OrcadRollbackOptions = {
@@ -72,7 +65,6 @@ export type OrcadRollbackResult =
| { outcome: 'refused'; code: string; reason: string }
| { outcome: 'failed'; code: string; reason: string }
const READINESS_POLL_MS = 500
const STOP_WAIT_SECONDS = 20
function exec(options: OrcadRollbackOptions, command: string): Promise<string> {
@@ -185,29 +177,14 @@ export async function rollbackOrcad(options: OrcadRollbackOptions): Promise<Orca
}
const targetDir = computeRemoteInstallDir(ORCAD_INSTALL_MODEL, options.remoteHome, safety.target)
await exec(
options,
orcadLaunchCommand(options.host, {
remoteInstallDir: targetDir,
nodePath: options.nodePath,
fullVersion: safety.target,
userDataDir: options.userDataDir,
bindHost: options.bindHost,
port: options.port
})
)
const deadline = Date.now() + (options.readinessTimeoutMs ?? ORCAD_STARTUP_READINESS_TIMEOUT_MS)
const sleep = options.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms)))
let parsed = parseOrcadReadinessOutput('')
while (Date.now() < deadline && parsed.state === 'pending') {
options.signal?.throwIfAborted()
parsed = parseOrcadReadinessOutput(
await exec(options, readOrcadReadinessCommand(options.host, targetDir))
)
if (parsed.state === 'pending') {
await sleep(READINESS_POLL_MS)
}
}
const parsed = await launchOrcadAndAwaitReadiness(options, {
remoteInstallDir: targetDir,
nodePath: options.nodePath,
fullVersion: safety.target,
userDataDir: options.userDataDir,
bindHost: options.bindHost,
port: options.port
})
const verdict = evaluateOrcadActivation(parsed.state === 'ready' ? parsed.readiness : null, {
buildHash: options.targetBuildHash,
fullVersion: safety.target
@@ -225,13 +202,7 @@ export async function rollbackOrcad(options: OrcadRollbackOptions): Promise<Orca
// Why the record is written last: until the target is proven serving, `active` still names
// the version an operator would need to bring back, and `previous` still names this target.
await writeRelayFile(
options.conn,
options.host,
orcadActivationPath(options.host, options.remoteHome),
serializeOrcadActivationRecord(withRolledBackVersion(options.record, now())),
{ signal: options.signal }
)
await writeOrcadActivationRecord(options, withRolledBackVersion(options.record, now()))
return {
outcome: 'rolled-back',
target: safety.target,
@@ -0,0 +1,63 @@
/** Launch a slot and poll its readiness file: the one loop deploy, rollback and recovery share. */
import { ORCAD_STARTUP_READINESS_TIMEOUT_MS } from '../../shared/orcad-profile-preflight'
import {
orcadLaunchCommand,
parseOrcadReadinessOutput,
readOrcadReadinessCommand,
type OrcadLaunchSpec,
type OrcadReadinessParse
} from './orcad-remote-launch'
import type { SshConnection } from './ssh-connection'
import { execCommand } from './ssh-relay-deploy-helpers'
import type { RemoteHostPlatform } from './ssh-remote-platform'
const READINESS_POLL_MS = 500
export type OrcadRemoteExecTarget = {
conn: SshConnection
host: RemoteHostPlatform
signal?: AbortSignal
}
export function execOrcadRemote(
target: OrcadRemoteExecTarget,
command: string,
signal = target.signal
): Promise<string> {
return execCommand(target.conn, command, {
wrapCommand: target.host.commandDialect !== 'powershell',
signal
})
}
/** Recovery paths must finish even when the request that started them was cancelled. */
export function withoutAbortSignal<T extends { signal?: AbortSignal }>(
options: T
): Omit<T, 'signal'> {
const { signal: _signal, ...rest } = options
return rest
}
export async function launchOrcadAndAwaitReadiness(
target: OrcadRemoteExecTarget & {
readinessTimeoutMs?: number
sleep?: (ms: number) => Promise<void>
},
spec: OrcadLaunchSpec
): Promise<OrcadReadinessParse> {
await execOrcadRemote(target, orcadLaunchCommand(target.host, spec))
const deadline = Date.now() + (target.readinessTimeoutMs ?? ORCAD_STARTUP_READINESS_TIMEOUT_MS)
const sleep = target.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms)))
let last = parseOrcadReadinessOutput('')
while (Date.now() < deadline) {
target.signal?.throwIfAborted()
last = parseOrcadReadinessOutput(
await execOrcadRemote(target, readOrcadReadinessCommand(target.host, spec.remoteInstallDir))
)
if (last.state !== 'pending') {
return last
}
await sleep(READINESS_POLL_MS)
}
return last
}