diff --git a/src/main/ssh/orcad-activation-record-store.ts b/src/main/ssh/orcad-activation-record-store.ts index d60634ec525..a36dcc728e0 100644 --- a/src/main/ssh/orcad-activation-record-store.ts +++ b/src/main/ssh/orcad-activation-record-store.ts @@ -6,32 +6,50 @@ * activated" would deploy over a live install and lose its rollback target. */ import type { SshConnection } from './ssh-connection' -import { execCommand } from './ssh-relay-deploy-helpers' import { RELAY_REMOTE_DIR } from './relay-protocol' import { ORCAD_ACTIVATION_FILENAME, emptyOrcadActivationRecord, parseOrcadActivationRecord, + serializeOrcadActivationRecord, + type OrcadActivationReadResult, type OrcadActivationRecord } from './orcad-activation-record' import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { + readBoundedOrcadRemoteRecord, + writeAtomicOrcadRemoteRecord +} from './orcad-remote-record-file' + +const ORCAD_ACTIVATION_RECORD_MAX_BYTES = 64 * 1024 + +type ActivationRecordTarget = { + conn: SshConnection + host: RemoteHostPlatform + remoteHome: string + signal?: AbortSignal +} export function orcadActivationPath(host: RemoteHostPlatform, remoteHome: string): string { return joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR, ORCAD_ACTIVATION_FILENAME) } -export async function readOrcadActivationRecord(options: { - conn: SshConnection - host: RemoteHostPlatform - remoteHome: string - signal?: AbortSignal -}): Promise { - const path = orcadActivationPath(options.host, options.remoteHome) - const raw = await execCommand(options.conn, `cat ${shellQuote(path)} 2>/dev/null || true`, { - wrapCommand: options.host.commandDialect !== 'powershell', - signal: options.signal - }).catch(() => '') - const parsed = parseOrcadActivationRecord(raw) +/** A failed read rejects; it never reads as absent, because absence would admit a fresh deploy. */ +async function readActivationRecordState( + options: ActivationRecordTarget +): Promise { + const read = await readBoundedOrcadRemoteRecord( + options, + orcadActivationPath(options.host, options.remoteHome), + ORCAD_ACTIVATION_RECORD_MAX_BYTES + ) + return read.state === 'absent' ? read : parseOrcadActivationRecord(read.raw) +} + +export async function readOrcadActivationRecord( + options: ActivationRecordTarget +): Promise { + const parsed = await readActivationRecordState(options) if (parsed.state === 'ok') { return parsed.record } @@ -43,6 +61,18 @@ export async function readOrcadActivationRecord(options: { return emptyOrcadActivationRecord() } -function shellQuote(value: string): string { - return `'${value.replaceAll("'", `'\\''`)}'` +/** Refuses to replace a record this client cannot read, e.g. one a newer client wrote. */ +export async function writeOrcadActivationRecord( + options: ActivationRecordTarget, + record: OrcadActivationRecord +): Promise { + const existing = await readActivationRecordState(options) + if (existing.state === 'unreadable') { + throw new Error(`Refusing to overwrite this host's orcad activation record: ${existing.reason}`) + } + await writeAtomicOrcadRemoteRecord( + options, + orcadActivationPath(options.host, options.remoteHome), + serializeOrcadActivationRecord(record) + ) } diff --git a/src/main/ssh/orcad-active-readiness.ts b/src/main/ssh/orcad-active-readiness.ts new file mode 100644 index 00000000000..40beecb236c --- /dev/null +++ b/src/main/ssh/orcad-active-readiness.ts @@ -0,0 +1,90 @@ +/** Proving that the orcad an activation record names is the one actually serving. */ +import { evaluateOrcadActivation, type OrcadActivationExpectation } from './orcad-activation-gate' +import { + orcadLivenessProbeCommand, + parseOrcadLiveness, + parseOrcadReadinessOutput, + readOrcadReadinessCommand, + type OrcadLaunchSpec, + type OrcadReadinessParse +} from './orcad-remote-launch' +import { + execOrcadRemote, + launchOrcadAndAwaitReadiness, + type OrcadRemoteExecTarget +} from './orcad-remote-runtime-control' +import type { ServeReadiness } from '../server/serve-readiness' + +/** `exited` is proven absence; `unverifiable` means the host could not say, which is not death. */ +export type OrcadReadinessFailureVerdict = 'exited' | 'unverifiable' | 'rejected' + +export class OrcadActiveReadinessError extends Error { + constructor( + readonly verdict: OrcadReadinessFailureVerdict, + message: string + ) { + super(message) + this.name = 'OrcadActiveReadinessError' + } +} + +function gatedReadiness( + parsed: OrcadReadinessParse, + expectation: OrcadActivationExpectation, + label: string +): ServeReadiness { + const readiness = parsed.state === 'ready' ? parsed.readiness : null + const verdict = evaluateOrcadActivation(readiness, expectation) + if (verdict.decision === 'reject') { + throw new OrcadActiveReadinessError( + 'rejected', + `${label} failed its readiness check: ${verdict.reason}` + ) + } + if (!readiness) { + throw new OrcadActiveReadinessError( + 'rejected', + `${label} passed activation without a readiness payload.` + ) + } + return readiness +} + +/** Checks a recorded-active slot without starting anything. */ +export async function probeActiveOrcadReadiness( + target: OrcadRemoteExecTarget & { remoteInstallDir: string }, + expectation: OrcadActivationExpectation +): Promise { + const liveness = parseOrcadLiveness( + await execOrcadRemote(target, orcadLivenessProbeCommand(target.host, target.remoteInstallDir)) + ) + if (liveness === 'DEAD') { + throw new OrcadActiveReadinessError( + 'exited', + `orcad ${expectation.fullVersion} is recorded active but its process has exited.` + ) + } + if (liveness !== 'LIVE') { + throw new OrcadActiveReadinessError( + 'unverifiable', + `orcad ${expectation.fullVersion} process state is unverifiable.` + ) + } + const parsed = parseOrcadReadinessOutput( + await execOrcadRemote(target, readOrcadReadinessCommand(target.host, target.remoteInstallDir)) + ) + return gatedReadiness(parsed, expectation, 'The active orcad') +} + +/** Starts a slot (recovery or rollback) and accepts it only if it proves the expected build. */ +export async function launchOrcadSlotAndAwaitReadiness( + target: OrcadRemoteExecTarget & { + readinessTimeoutMs?: number + sleep?: (ms: number) => Promise + }, + spec: OrcadLaunchSpec, + expectation: OrcadActivationExpectation +): Promise { + const parsed = await launchOrcadAndAwaitReadiness(target, spec) + return gatedReadiness(parsed, expectation, `orcad ${spec.fullVersion}`) +} diff --git a/src/main/ssh/orcad-remote-build-hash.ts b/src/main/ssh/orcad-remote-build-hash.ts new file mode 100644 index 00000000000..7ed6384dfe7 --- /dev/null +++ b/src/main/ssh/orcad-remote-build-hash.ts @@ -0,0 +1,36 @@ +/** The installed slot's `orcad.js` identity, the same 16-hex prefix orcad reports in its health. */ +import { shellEscape } from './ssh-connection-utils' +import { assertPosixOrcadHost } from './orcad-remote-host-support' +import { execOrcadRemote, type OrcadRemoteExecTarget } from './orcad-remote-runtime-control' +import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' + +const BUILD_HASH_MARKER = '__ORCAD_BUILD_HASH__' + +export async function readRemoteOrcadBuildHash( + target: OrcadRemoteExecTarget, + remoteInstallDir: string +): Promise { + const output = await execOrcadRemote( + target, + remoteOrcadBuildHashCommand(target.host, remoteInstallDir) + ) + const match = output.match(/__ORCAD_BUILD_HASH__\s+([a-fA-F0-9]{16})/u) + if (!match?.[1]) { + throw new Error('Could not verify the installed orcad build hash.') + } + return match[1].toLowerCase() +} + +export function remoteOrcadBuildHashCommand( + host: RemoteHostPlatform, + remoteInstallDir: string +): string { + assertPosixOrcadHost(host) + const path = shellEscape(joinRemotePath(host, remoteInstallDir, 'orcad.js')) + // Why both tools: GNU/busybox ship sha256sum, macOS ships shasum; either prints the digest first. + return [ + `orca_hash=$(if command -v sha256sum >/dev/null 2>&1; then sha256sum ${path} | awk '{print $1}';`, + `elif command -v shasum >/dev/null 2>&1; then shasum -a 256 ${path} | awk '{print $1}'; fi);`, + `case "$orca_hash" in [0-9a-fA-F][0-9a-fA-F]*) printf '%s %.16s\\n' ${shellEscape(BUILD_HASH_MARKER)} "$orca_hash";; esac` + ].join(' ') +} diff --git a/src/main/ssh/orcad-remote-context.ts b/src/main/ssh/orcad-remote-context.ts new file mode 100644 index 00000000000..6b893ba2950 --- /dev/null +++ b/src/main/ssh/orcad-remote-context.ts @@ -0,0 +1,59 @@ +/** Everything a managed-orcad operation needs to know about one SSH host before it acts. */ +import type { ServerTarget } from '../../shared/node-runtime-pin' +import type { SshTarget } from '../../shared/ssh-types' +import type { OrcadActivationRecord } from './orcad-activation-record' +import { readOrcadActivationRecord } from './orcad-activation-record-store' +import { resolveOrcadDeploymentTarget } from './orcad-deployment-target' +import { assertPosixOrcadHost } from './orcad-remote-host-support' +import { execOrcadRemote } from './orcad-remote-runtime-control' +import type { SshConnection } from './ssh-connection' +import { readRemoteHomeCommand } from './ssh-remote-commands' +import { + joinRemotePath, + normalizeRemoteHome, + validateRemoteHome, + type RemoteHostPlatform +} from './ssh-remote-platform' +import { detectRemoteHostPlatform } from './ssh-remote-platform-detection' + +export type OrcadRemoteContext = { + activationRecord: OrcadActivationRecord + serverTarget: ServerTarget + connection: SshConnection + host: RemoteHostPlatform + remoteHome: string + target: SshTarget + userDataDir: string +} + +export async function resolveOrcadRemoteContext( + target: SshTarget, + connection: SshConnection, + signal?: AbortSignal +): Promise { + const host = await detectRemoteHostPlatform(connection, { signal }) + if (!host) { + throw new Error('This SSH host platform is not supported by managed orcad.') + } + // Why first: every lifecycle step after this is POSIX-only, so refuse before probing further. + assertPosixOrcadHost(host) + const remote = { conn: connection, host, signal } + const remoteHome = normalizeRemoteHome( + await execOrcadRemote(remote, readRemoteHomeCommand(host)), + host + ) + if (!validateRemoteHome(remoteHome, host)) { + throw new Error(`Remote home is not a valid path: ${remoteHome.slice(0, 100)}`) + } + const serverTarget = await resolveOrcadDeploymentTarget({ conn: connection, host, signal }) + const activationRecord = await readOrcadActivationRecord({ ...remote, remoteHome }) + return { + activationRecord, + serverTarget, + connection, + host, + remoteHome, + target, + userDataDir: joinRemotePath(host, remoteHome, '.orca') + } +} diff --git a/src/main/ssh/orcad-remote-deploy.test.ts b/src/main/ssh/orcad-remote-deploy.test.ts index 8308ff52db5..be2dd5223d2 100644 --- a/src/main/ssh/orcad-remote-deploy.test.ts +++ b/src/main/ssh/orcad-remote-deploy.test.ts @@ -1,4 +1,5 @@ import { chmodSync, mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from 'node:fs' +import type * as RecordFile from './orcad-remote-record-file' import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' import { beforeEach, describe, expect, it, vi } from 'vitest' @@ -17,6 +18,10 @@ vi.mock('./ssh-relay-install-transfers', () => ({ uploadRelayDirectory: vi.fn().mockResolvedValue(undefined), writeRelayFile: vi.fn().mockResolvedValue(undefined) })) +vi.mock('./orcad-remote-record-file', async (importOriginal) => ({ + ...(await importOriginal()), + writeAtomicOrcadRemoteRecord: vi.fn().mockResolvedValue(undefined) +})) vi.mock('./orcad-remote-node-runtime', () => ({ ensureRemoteOrcadNodeRuntime: vi.fn().mockResolvedValue(undefined) })) @@ -26,7 +31,8 @@ vi.mock('./orcad-local-build-hash', () => ({ import { execCommand } from './ssh-relay-deploy-helpers' import { acquireInstallLock } from './ssh-relay-install-lock' -import { uploadRelayDirectory, writeRelayFile } from './ssh-relay-install-transfers' +import { uploadRelayDirectory } from './ssh-relay-install-transfers' +import { writeAtomicOrcadRemoteRecord } from './orcad-remote-record-file' import { deployOrcad, type OrcadDeployOptions } from './orcad-remote-deploy' import { installOrcadBundle } from './orcad-remote-install' import { ensureRemoteOrcadNodeRuntime } from './orcad-remote-node-runtime' @@ -106,10 +112,12 @@ type HostScript = { function scriptHost(script: HostScript): void { mockExec.mockImplementation(async (_conn, command: string) => { const text = String(command) - if (text.startsWith('cat ') && text.includes('orcad-active.json')) { + if (text.includes('__ORCAD_RECORD_PRESENT__') && text.includes('orcad-active.json')) { return script.activationRecord + ? `__ORCAD_RECORD_PRESENT__\n${script.activationRecord}` + : '__ORCAD_RECORD_ABSENT__\n' } - if (text.includes('.orcad-readiness') && text.startsWith('cat ')) { + if (text.includes('.orcad-readiness') && text.startsWith('head -c ')) { if (script.readinessAtMs !== undefined && Date.now() < script.readinessAtMs) { return '' } @@ -272,8 +280,8 @@ describe('deployOrcad', () => { expect(script.log).toEqual(['preflight']) expect( vi - .mocked(writeRelayFile) - .mock.calls.some(([, , path]) => path.includes('orcad-active.json')) + .mocked(writeAtomicOrcadRemoteRecord) + .mock.calls.some(([, path]) => path.includes('orcad-active.json')) ).toBe(false) } ) @@ -356,7 +364,7 @@ describe('deployOrcad', () => { if (String(command).startsWith('chmod 755 ')) { throw new Error('chmod failed') } - return '' + return String(command).includes('__ORCAD_RECORD_ABSENT__') ? '__ORCAD_RECORD_ABSENT__\n' : '' }) await expect(deployOrcad(options())).rejects.toThrow('chmod failed') expect(vi.mocked(finalizeInstall)).not.toHaveBeenCalled() @@ -441,9 +449,9 @@ describe('deployOrcad', () => { const result = await deployOrcad(options()) expect(result).toMatchObject({ outcome: 'installed-and-activated', fullVersion: NEW_VERSION }) const written = vi - .mocked(writeRelayFile) - .mock.calls.find((call) => String(call[2]).endsWith('orcad-active.json')) - expect(JSON.parse(String(written?.[3]))).toMatchObject({ + .mocked(writeAtomicOrcadRemoteRecord) + .mock.calls.find((call) => String(call[1]).endsWith('orcad-active.json')) + expect(JSON.parse(String(written?.[2]))).toMatchObject({ active: NEW_VERSION, previous: OLD_VERSION }) @@ -492,8 +500,8 @@ describe('deployOrcad', () => { expect(result).toMatchObject({ code: 'orcad_activation_daemon_degraded' }) expect( vi - .mocked(writeRelayFile) - .mock.calls.some((call) => String(call[2]).endsWith('orcad-active.json')) + .mocked(writeAtomicOrcadRemoteRecord) + .mock.calls.some((call) => String(call[1]).endsWith('orcad-active.json')) ).toBe(false) }) diff --git a/src/main/ssh/orcad-remote-deploy.ts b/src/main/ssh/orcad-remote-deploy.ts index f296e08bd36..95c184e47b4 100644 --- a/src/main/ssh/orcad-remote-deploy.ts +++ b/src/main/ssh/orcad-remote-deploy.ts @@ -4,28 +4,24 @@ * preserve current state and the prelaunch snapshot for explicit recovery. */ import type { SshConnection } from './ssh-connection' -import { ORCAD_STARTUP_READINESS_TIMEOUT_MS } from '../../shared/orcad-profile-preflight' import { execCommand } from './ssh-relay-deploy-helpers' import { ORCAD_INSTALL_MODEL } from './remote-install-model' -import { writeRelayFile } from './ssh-relay-install-transfers' import { computeRemoteInstallDir, readLocalFullVersion } from './ssh-relay-versioned-install' import { RELAY_REMOTE_DIR } from './relay-protocol' import { ORCAD_STATE_SNAPSHOT_DIR, - serializeOrcadActivationRecord, withActivatedVersion, type OrcadActivationRecord, type OrcadStateSnapshot } from './orcad-activation-record' -import { orcadActivationPath, readOrcadActivationRecord } from './orcad-activation-record-store' +import { + readOrcadActivationRecord, + writeOrcadActivationRecord +} from './orcad-activation-record-store' import { evaluateOrcadActivation, type OrcadActivationVerdict } from './orcad-activation-gate' import { planOrcadUpdate, type OrcadTerminalCensus } from './orcad-update-plan' -import { - ORCAD_LOG_FILENAME, - orcadLaunchCommand, - parseOrcadReadinessOutput, - readOrcadReadinessCommand -} from './orcad-remote-launch' +import { ORCAD_LOG_FILENAME } from './orcad-remote-launch' +import { launchOrcadAndAwaitReadiness } from './orcad-remote-runtime-control' import { rejectedOrcadStateRecoveryRefusal, stopOutgoingOrcad } from './orcad-remote-deploy-stop' import { captureOrcadStateSnapshotCommand, @@ -81,7 +77,6 @@ export type OrcadDeployResult = | { outcome: 'already-active'; fullVersion: string } | { outcome: 'installed-not-activated'; fullVersion: string; code: string; reason: string } -const READINESS_POLL_MS = 500 const STOP_WAIT_SECONDS = 20 function exec(options: OrcadDeployOptions, command: string): Promise { @@ -135,29 +130,12 @@ async function captureSnapshot( } } -async function launchAndAwaitReadiness( +function launchAndAwaitReadiness( options: OrcadDeployOptions, remoteInstallDir: string, fullVersion: string -): Promise> { - await exec( - options, - orcadLaunchCommand(options.host, { ...options, remoteInstallDir, fullVersion }) - ) - const deadline = Date.now() + (options.readinessTimeoutMs ?? ORCAD_STARTUP_READINESS_TIMEOUT_MS) - const sleep = options.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))) - let last = parseOrcadReadinessOutput('') - while (Date.now() < deadline) { - options.signal?.throwIfAborted() - last = parseOrcadReadinessOutput( - await exec(options, readOrcadReadinessCommand(options.host, remoteInstallDir)) - ) - if (last.state !== 'pending') { - return last - } - await sleep(READINESS_POLL_MS) - } - return last +): ReturnType { + return launchOrcadAndAwaitReadiness(options, { ...options, remoteInstallDir, fullVersion }) } /** Restart the incumbent only when the candidate left shared state unchanged. */ @@ -320,12 +298,9 @@ export async function deployOrcad(input: OrcadDeployOptions): Promise/dev/null || true` + // One byte over the cap is enough to tell an oversized payload from a full one. + return `head -c ${ORCAD_READINESS_MAX_BYTES + 1} ${readiness} 2>/dev/null || true` } /** @@ -134,19 +137,23 @@ export type OrcadReadinessParse = * not `malformed` — reporting a parse failure for a race would fail deploys that were fine. */ export function parseOrcadReadinessOutput(raw: string): OrcadReadinessParse { + if (Buffer.byteLength(raw, 'utf8') > ORCAD_READINESS_MAX_BYTES) { + return { state: 'malformed', reason: 'readiness payload exceeds the 256 KiB limit' } + } const lines = raw.split('\n') - let sawCandidate = false - for (const line of lines) { + for (const [index, line] of lines.entries()) { const trimmed = line.trim() if (!trimmed.startsWith('{')) { continue } - sawCandidate = true let parsed: unknown try { parsed = JSON.parse(trimmed) } catch { - continue + // Only the unterminated last line can still be mid-write; a finished bad line will stay bad. + return index === lines.length - 1 + ? { state: 'pending' } + : { state: 'malformed', reason: 'readiness line is not valid JSON' } } if (typeof parsed !== 'object' || parsed === null) { continue @@ -160,7 +167,7 @@ export function parseOrcadReadinessOutput(raw: string): OrcadReadinessParse { } return { state: 'ready', readiness: toServeReadiness(payload as Record) } } - return sawCandidate ? { state: 'pending' } : { state: 'pending' } + return { state: 'pending' } } function toServeReadiness(payload: Record): ServeReadiness { diff --git a/src/main/ssh/orcad-remote-primitives.test.ts b/src/main/ssh/orcad-remote-primitives.test.ts new file mode 100644 index 00000000000..ab7e6813f98 --- /dev/null +++ b/src/main/ssh/orcad-remote-primitives.test.ts @@ -0,0 +1,186 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('./ssh-relay-deploy-helpers', () => ({ + execCommand: vi.fn(), + isUnconfirmedSshCommandTermination: (error: unknown) => + error instanceof Error && error.message === 'channel lost' +})) + +vi.mock('./ssh-remote-platform-detection', () => ({ detectRemoteHostPlatform: vi.fn() })) + +import { execCommand } from './ssh-relay-deploy-helpers' +import { detectRemoteHostPlatform } from './ssh-remote-platform-detection' +import { resolveOrcadRemoteContext } from './orcad-remote-context' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import type { SshConnection } from './ssh-connection' +import { + readBoundedOrcadRemoteRecord, + writeAtomicOrcadRemoteRecord +} from './orcad-remote-record-file' +import { + readOrcadActivationRecord, + writeOrcadActivationRecord +} from './orcad-activation-record-store' +import { emptyOrcadActivationRecord } from './orcad-activation-record' +import { readRemoteOrcadBuildHash } from './orcad-remote-build-hash' +import { OrcadActiveReadinessError, probeActiveOrcadReadiness } from './orcad-active-readiness' +import { parseOrcadReadinessOutput } from './orcad-remote-launch' + +const mockExec = vi.mocked(execCommand) +const linux = getRemoteHostPlatform('linux-x64') +const windows = getRemoteHostPlatform('win32-x64') +const conn: SshConnection = Object.create(null) +const target = { conn, host: linux } +const BUILD_HASH = 'abc123def4567890' + +function readyLine(buildHash = BUILD_HASH): string { + return JSON.stringify({ + type: 'orca_server_ready', + runtimeId: 'r1', + boundEndpoint: 'ws://127.0.0.1:7777', + advertisedEndpoint: null, + managedWslCliReconciliation: 'settled', + pairing: { available: false, reason: 'disabled_by_operator', guidance: 'n/a' }, + health: { + buildHash, + buildVersion: '0.2.0+bb01', + nodeVersion: '24.21.0', + nodeAbi: '137', + platform: 'linux', + arch: 'x64', + pid: 1, + terminalDaemon: { + state: 'live', + ownsFreshSessions: true, + pid: 2, + buildVersion: '0.2.0+bb01', + entryPath: '/x/daemon-entry.js', + protocolVersion: 38, + selfTest: { ok: true, coverage: 'pty-spawn', verdict: 'healthy', durationMs: 5 } + } + } + }) +} + +beforeEach(() => { + mockExec.mockReset() +}) + +describe('orcad host record files', () => { + it('tells an absent record from one whose read gave no answer', async () => { + mockExec.mockResolvedValueOnce('__ORCAD_RECORD_ABSENT__\n') + await expect(readBoundedOrcadRemoteRecord(target, '/r.json', 64)).resolves.toEqual({ + state: 'absent' + }) + mockExec.mockResolvedValueOnce('__ORCAD_RECORD_PRESENT__\n{"a":1}') + await expect(readBoundedOrcadRemoteRecord(target, '/r.json', 64)).resolves.toEqual({ + state: 'present', + raw: '{"a":1}' + }) + mockExec.mockResolvedValueOnce('') + await expect(readBoundedOrcadRemoteRecord(target, '/r.json', 64)).rejects.toThrow( + 'no verifiable answer' + ) + }) + + it('keeps the partial file when the write may still be running on the host', async () => { + mockExec.mockRejectedValueOnce(new Error('channel lost')) + await expect(writeAtomicOrcadRemoteRecord(target, '/r.json', '{}')).rejects.toThrow() + expect(mockExec).toHaveBeenCalledOnce() + mockExec.mockRejectedValueOnce(new Error('exit 1')).mockResolvedValueOnce('') + await expect(writeAtomicOrcadRemoteRecord(target, '/r.json', '{}')).rejects.toThrow() + expect(String(mockExec.mock.calls.at(-1)?.[1])).toContain('rm -f') + }) + + it('refuses Windows hosts, which the orcad lifecycle does not support', async () => { + await expect( + readBoundedOrcadRemoteRecord({ conn, host: windows }, 'C:/r.json', 64) + ).rejects.toThrow() + expect(mockExec).not.toHaveBeenCalled() + }) +}) + +describe('activation record store', () => { + const options = { conn, host: linux, remoteHome: '/home/u' } + const newer = JSON.stringify({ ...emptyOrcadActivationRecord(), schemaVersion: 2 }) + + it('reads a lost read as an error, never as an empty record', async () => { + mockExec.mockRejectedValueOnce(new Error('channel lost')) + await expect(readOrcadActivationRecord(options)).rejects.toThrow('channel lost') + }) + + it('reads a newer schema as unreadable and never overwrites it', async () => { + mockExec.mockResolvedValue(`__ORCAD_RECORD_PRESENT__\n${newer}`) + await expect(readOrcadActivationRecord(options)).rejects.toThrow('schemaVersion 2') + await expect(writeOrcadActivationRecord(options, emptyOrcadActivationRecord())).rejects.toThrow( + 'Refusing to overwrite' + ) + expect(mockExec.mock.calls.some(([, command]) => String(command).includes('mv -f'))).toBe(false) + }) + + it('writes atomically when the host has no record yet', async () => { + mockExec.mockResolvedValueOnce('__ORCAD_RECORD_ABSENT__\n').mockResolvedValueOnce('') + await writeOrcadActivationRecord(options, emptyOrcadActivationRecord()) + expect(String(mockExec.mock.calls[1]?.[1])).toMatch(/printf %s .* && mv -f /s) + }) +}) + +describe('installed build identity and readiness', () => { + const slot = { ...target, remoteInstallDir: '/home/u/.orca-remote/orcad-0.2.0+bb01' } + const expectation = { buildHash: BUILD_HASH, fullVersion: '0.2.0+bb01' } + + it('reads the 16-hex build hash the slot reports', async () => { + mockExec.mockResolvedValueOnce(`noise\n__ORCAD_BUILD_HASH__ ${BUILD_HASH.toUpperCase()}\n`) + await expect(readRemoteOrcadBuildHash(target, '/slot')).resolves.toBe(BUILD_HASH) + mockExec.mockResolvedValueOnce('') + await expect(readRemoteOrcadBuildHash(target, '/slot')).rejects.toThrow() + }) + + it.each([ + ['DEAD', 'exited'], + ['UNKNOWN', 'unverifiable'], + ['', 'unverifiable'] + ])('reports a %j liveness probe as %s', async (liveness, verdict) => { + mockExec.mockResolvedValueOnce(liveness) + await expect(probeActiveOrcadReadiness(slot, expectation)).rejects.toMatchObject({ + verdict + }) + }) + + it('accepts only the expected build once the process is live', async () => { + mockExec.mockResolvedValueOnce('LIVE').mockResolvedValueOnce(`${readyLine()}\n`) + await expect(probeActiveOrcadReadiness(slot, expectation)).resolves.toMatchObject({ + runtimeId: 'r1' + }) + mockExec + .mockResolvedValueOnce('LIVE') + .mockResolvedValueOnce(`${readyLine('ffffffffffffffff')}\n`) + const rejected = probeActiveOrcadReadiness(slot, expectation) + await expect(rejected).rejects.toBeInstanceOf(OrcadActiveReadinessError) + await expect(rejected).rejects.toMatchObject({ verdict: 'rejected' }) + }) +}) + +describe('readiness parsing bounds', () => { + it('waits on a half-written last line but rejects a finished invalid one', () => { + expect(parseOrcadReadinessOutput('{"type":"orca_ser')).toEqual({ state: 'pending' }) + expect(parseOrcadReadinessOutput('{"type":"orca_ser\n')).toMatchObject({ + state: 'malformed' + }) + }) + + it('rejects a payload over the size cap', () => { + expect(parseOrcadReadinessOutput('x'.repeat(256 * 1024 + 1))).toMatchObject({ + state: 'malformed' + }) + }) +}) + +describe('orcad remote context', () => { + it('refuses a Windows host before probing anything else', async () => { + vi.mocked(detectRemoteHostPlatform).mockResolvedValueOnce(windows) + const sshTarget = { id: 't', label: 't', host: 'h', port: 22, username: 'u' } + await expect(resolveOrcadRemoteContext(sshTarget, conn)).rejects.toThrow() + expect(mockExec).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/ssh/orcad-remote-record-file.ts b/src/main/ssh/orcad-remote-record-file.ts new file mode 100644 index 00000000000..5c29c3b3ef5 --- /dev/null +++ b/src/main/ssh/orcad-remote-record-file.ts @@ -0,0 +1,68 @@ +/** + * Small JSON records Orca keeps on an orcad host (activation, transactions, stop receipts). + * + * Reads are bounded and never swallow a failure: a record that could not be read is not an + * absent one, and treating it as absent is how a client deploys over a live install. + */ +import { randomUUID } from 'node:crypto' +import { shellEscape } from './ssh-connection-utils' +import { assertPosixOrcadHost } from './orcad-remote-host-support' +import { removeRemoteFileCommand } from './ssh-remote-commands' +import { execOrcadRemote, type OrcadRemoteExecTarget } from './orcad-remote-runtime-control' +import { isUnconfirmedSshCommandTermination } from './ssh-relay-deploy-helpers' + +const ABSENT_MARKER = '__ORCAD_RECORD_ABSENT__' +const PRESENT_MARKER = '__ORCAD_RECORD_PRESENT__' + +/** `present` carries raw bytes; schema checks belong to the caller that owns the format. */ +export type OrcadRemoteRecordRead = { state: 'absent' } | { state: 'present'; raw: string } + +export async function readBoundedOrcadRemoteRecord( + target: OrcadRemoteExecTarget, + path: string, + maxBytes: number +): Promise { + assertPosixOrcadHost(target.host) + const file = shellEscape(path) + // Why markers: an empty stdout must never be mistaken for "no record" when the read failed. + const output = await execOrcadRemote( + target, + `if [ ! -e ${file} ] && [ ! -L ${file} ]; then printf '%s\\n' ${ABSENT_MARKER}; exit 0; fi; ` + + `[ -f ${file} ] || exit 65; size=$(wc -c < ${file}) || exit 65; ` + + `[ "$size" -le ${maxBytes} ] || exit 65; ` + + `printf '%s\\n' ${PRESENT_MARKER}; cat ${file}` + ) + const newline = output.indexOf('\n') + const marker = (newline === -1 ? output : output.slice(0, newline)).trim() + if (marker === ABSENT_MARKER) { + return { state: 'absent' } + } + if (marker !== PRESENT_MARKER) { + throw new Error('orcad host record read returned no verifiable answer') + } + return { state: 'present', raw: newline === -1 ? '' : output.slice(newline + 1) } +} + +export async function writeAtomicOrcadRemoteRecord( + target: OrcadRemoteExecTarget, + path: string, + contents: string +): Promise { + assertPosixOrcadHost(target.host) + const partialPath = `${path}.partial.${process.pid}.${randomUUID()}` + try { + await execOrcadRemote( + target, + `umask 077; printf %s ${shellEscape(contents)} > ${shellEscape(partialPath)} && ` + + `mv -f ${shellEscape(partialPath)} ${shellEscape(path)}` + ) + } catch (error) { + // Why keep the partial on an unconfirmed termination: the write may still be running there. + if (!isUnconfirmedSshCommandTermination(error)) { + await execOrcadRemote(target, removeRemoteFileCommand(target.host, partialPath)).catch( + () => {} + ) + } + throw error + } +} diff --git a/src/main/ssh/orcad-remote-rollback.test.ts b/src/main/ssh/orcad-remote-rollback.test.ts index 4d2f9150a75..5d2dd2e558e 100644 --- a/src/main/ssh/orcad-remote-rollback.test.ts +++ b/src/main/ssh/orcad-remote-rollback.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' +import type * as RecordFile from './orcad-remote-record-file' vi.mock('./ssh-relay-deploy-helpers', () => ({ execCommand: vi.fn(), @@ -9,9 +10,13 @@ vi.mock('./ssh-relay-install-transfers', () => ({ writeRelayFile: vi.fn().mockResolvedValue(undefined), uploadRelayDirectory: vi.fn().mockResolvedValue(undefined) })) +vi.mock('./orcad-remote-record-file', async (importOriginal) => ({ + ...(await importOriginal()), + writeAtomicOrcadRemoteRecord: vi.fn().mockResolvedValue(undefined) +})) import { execCommand } from './ssh-relay-deploy-helpers' -import { writeRelayFile } from './ssh-relay-install-transfers' +import { writeAtomicOrcadRemoteRecord } from './orcad-remote-record-file' import { rollbackOrcad, type OrcadRollbackOptions } from './orcad-remote-rollback' import { emptyOrcadActivationRecord, type OrcadActivationRecord } from './orcad-activation-record' import { getRemoteHostPlatform } from './ssh-remote-platform' @@ -74,6 +79,9 @@ function scriptHost( ): void { mockExec.mockImplementation(async (_conn, command: string) => { const text = String(command) + if (text.includes('__ORCAD_RECORD_PRESENT__')) { + return `__ORCAD_RECORD_PRESENT__\n${JSON.stringify(record())}` + } if (text.includes('state.tar') && text.includes('test -f') && !text.includes('tar -C')) { return 'PRESENT' } @@ -92,7 +100,7 @@ function scriptHost( log.push(`launch:${text.includes(ACTIVE) ? ACTIVE : TARGET}`) return '9999' } - if (text.startsWith('cat ') && text.includes('.orcad-readiness')) { + if (text.startsWith('head -c ') && text.includes('.orcad-readiness')) { if (overrides.readinessAtMs !== undefined && Date.now() < overrides.readinessAtMs) { return '' } @@ -169,7 +177,7 @@ describe('rollbackOrcad', () => { code: 'orcad_rollback_orphans_live_terminals' }) expect(log).toEqual([]) - expect(vi.mocked(writeRelayFile)).not.toHaveBeenCalled() + expect(vi.mocked(writeAtomicOrcadRemoteRecord)).not.toHaveBeenCalled() }) it('refuses when the snapshot is gone from the host', async () => { @@ -212,7 +220,7 @@ describe('rollbackOrcad', () => { expect(result).toMatchObject({ outcome: 'failed', code: 'orcad_activation_no_readiness' }) // Until the target is proven serving, `active` must still name the version an operator // would have to bring back. - expect(vi.mocked(writeRelayFile)).not.toHaveBeenCalled() + expect(vi.mocked(writeAtomicOrcadRemoteRecord)).not.toHaveBeenCalled() }) it('records the rollback only after the target answers healthy', async () => { @@ -220,9 +228,9 @@ describe('rollbackOrcad', () => { scriptHost(log) await rollbackOrcad(options()) const written = vi - .mocked(writeRelayFile) - .mock.calls.find((call) => String(call[2]).endsWith('orcad-active.json')) - expect(JSON.parse(String(written?.[3]))).toMatchObject({ + .mocked(writeAtomicOrcadRemoteRecord) + .mock.calls.find((call) => String(call[1]).endsWith('orcad-active.json')) + expect(JSON.parse(String(written?.[2]))).toMatchObject({ active: TARGET, previous: null, snapshot: null diff --git a/src/main/ssh/orcad-remote-rollback.ts b/src/main/ssh/orcad-remote-rollback.ts index 412021c2f22..09dc885d5ae 100644 --- a/src/main/ssh/orcad-remote-rollback.ts +++ b/src/main/ssh/orcad-remote-rollback.ts @@ -14,26 +14,19 @@ * failure this is meant to avoid, arrived at from the other side. */ import type { SshConnection } from './ssh-connection' -import { ORCAD_STARTUP_READINESS_TIMEOUT_MS } from '../../shared/orcad-profile-preflight' import { execCommand } from './ssh-relay-deploy-helpers' import { ORCAD_INSTALL_MODEL } from './remote-install-model' import { computeRemoteInstallDir } from './ssh-relay-versioned-install' -import { writeRelayFile } from './ssh-relay-install-transfers' import { RELAY_REMOTE_DIR } from './relay-protocol' import { ORCAD_STATE_SNAPSHOT_DIR, - serializeOrcadActivationRecord, withRolledBackVersion, type OrcadActivationRecord } from './orcad-activation-record' import { assessOrcadRollback, type OrcadTerminalCensus } from './orcad-update-plan' import { evaluateOrcadActivation, type OrcadActivationVerdict } from './orcad-activation-gate' -import { - ORCAD_LOG_FILENAME, - orcadLaunchCommand, - parseOrcadReadinessOutput, - readOrcadReadinessCommand -} from './orcad-remote-launch' +import { ORCAD_LOG_FILENAME } from './orcad-remote-launch' +import { launchOrcadAndAwaitReadiness } from './orcad-remote-runtime-control' import { newestStateMtimeCommand, parseNewestStateMtimeSeconds, @@ -46,7 +39,7 @@ import { parseOrcadStopOutcome, stopOrcadCommand } from './orcad-remote-process-control' -import { orcadActivationPath } from './orcad-activation-record-store' +import { writeOrcadActivationRecord } from './orcad-activation-record-store' import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' export type OrcadRollbackOptions = { @@ -72,7 +65,6 @@ export type OrcadRollbackResult = | { outcome: 'refused'; code: string; reason: string } | { outcome: 'failed'; code: string; reason: string } -const READINESS_POLL_MS = 500 const STOP_WAIT_SECONDS = 20 function exec(options: OrcadRollbackOptions, command: string): Promise { @@ -185,29 +177,14 @@ export async function rollbackOrcad(options: OrcadRollbackOptions): Promise new Promise((r) => setTimeout(r, ms))) - let parsed = parseOrcadReadinessOutput('') - while (Date.now() < deadline && parsed.state === 'pending') { - options.signal?.throwIfAborted() - parsed = parseOrcadReadinessOutput( - await exec(options, readOrcadReadinessCommand(options.host, targetDir)) - ) - if (parsed.state === 'pending') { - await sleep(READINESS_POLL_MS) - } - } + const parsed = await launchOrcadAndAwaitReadiness(options, { + remoteInstallDir: targetDir, + nodePath: options.nodePath, + fullVersion: safety.target, + userDataDir: options.userDataDir, + bindHost: options.bindHost, + port: options.port + }) const verdict = evaluateOrcadActivation(parsed.state === 'ready' ? parsed.readiness : null, { buildHash: options.targetBuildHash, fullVersion: safety.target @@ -225,13 +202,7 @@ export async function rollbackOrcad(options: OrcadRollbackOptions): Promise { + return execCommand(target.conn, command, { + wrapCommand: target.host.commandDialect !== 'powershell', + signal + }) +} + +/** Recovery paths must finish even when the request that started them was cancelled. */ +export function withoutAbortSignal( + options: T +): Omit { + const { signal: _signal, ...rest } = options + return rest +} + +export async function launchOrcadAndAwaitReadiness( + target: OrcadRemoteExecTarget & { + readinessTimeoutMs?: number + sleep?: (ms: number) => Promise + }, + spec: OrcadLaunchSpec +): Promise { + await execOrcadRemote(target, orcadLaunchCommand(target.host, spec)) + const deadline = Date.now() + (target.readinessTimeoutMs ?? ORCAD_STARTUP_READINESS_TIMEOUT_MS) + const sleep = target.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))) + let last = parseOrcadReadinessOutput('') + while (Date.now() < deadline) { + target.signal?.throwIfAborted() + last = parseOrcadReadinessOutput( + await execOrcadRemote(target, readOrcadReadinessCommand(target.host, spec.remoteInstallDir)) + ) + if (last.state !== 'pending') { + return last + } + await sleep(READINESS_POLL_MS) + } + return last +}