mirror of
https://github.com/stablyai/orca.git
synced 2026-10-06 08:02:28 +00:00
- orcad-remote-runtime-control: the one launch-and-poll-readiness loop that
deployOrcad and rollbackOrcad now share, plus exec and recovery helpers.
- orcad-remote-record-file: bounded, marker-delimited reads and atomic writes
for host records. A read that returns no verifiable answer rejects instead
of reading as "no record".
- The activation record store reads through it (a lost read no longer becomes
an empty record) and writeOrcadActivationRecord refuses to replace a record
this client cannot read, such as a newer schema; deploy and rollback use it.
- orcad-active-readiness: prove a recorded-active or relaunched slot against
the activation gate, reporting exited / unverifiable / rejected.
- orcad-remote-build-hash and orcad-remote-context (host, home, server target
via orcad-deployment-target, activation record; POSIX-only).
- Readiness reads are capped at 256 KiB, and a finished invalid JSON line is
malformed rather than pending forever.
Inert: nothing in the app calls managed orcad deploy yet. Ported from #16741
(a68b6f3531) onto main's pinned-Node deploy core; the branch's Bun runtime,
slot eligibility, target detection and bundle installation are superseded by
main's orcad-remote-runtime, orcad-deployment-target and orcad-remote-install.
Co-authored-by: m4air <m4air@m4airs-Air.localdomain>
This commit is contained in:
@@ -6,32 +6,50 @@
|
||||
* activated" would deploy over a live install and lose its rollback target.
|
||||
*/
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import { execCommand } from './ssh-relay-deploy-helpers'
|
||||
import { RELAY_REMOTE_DIR } from './relay-protocol'
|
||||
import {
|
||||
ORCAD_ACTIVATION_FILENAME,
|
||||
emptyOrcadActivationRecord,
|
||||
parseOrcadActivationRecord,
|
||||
serializeOrcadActivationRecord,
|
||||
type OrcadActivationReadResult,
|
||||
type OrcadActivationRecord
|
||||
} from './orcad-activation-record'
|
||||
import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform'
|
||||
import {
|
||||
readBoundedOrcadRemoteRecord,
|
||||
writeAtomicOrcadRemoteRecord
|
||||
} from './orcad-remote-record-file'
|
||||
|
||||
const ORCAD_ACTIVATION_RECORD_MAX_BYTES = 64 * 1024
|
||||
|
||||
type ActivationRecordTarget = {
|
||||
conn: SshConnection
|
||||
host: RemoteHostPlatform
|
||||
remoteHome: string
|
||||
signal?: AbortSignal
|
||||
}
|
||||
|
||||
export function orcadActivationPath(host: RemoteHostPlatform, remoteHome: string): string {
|
||||
return joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR, ORCAD_ACTIVATION_FILENAME)
|
||||
}
|
||||
|
||||
export async function readOrcadActivationRecord(options: {
|
||||
conn: SshConnection
|
||||
host: RemoteHostPlatform
|
||||
remoteHome: string
|
||||
signal?: AbortSignal
|
||||
}): Promise<OrcadActivationRecord> {
|
||||
const path = orcadActivationPath(options.host, options.remoteHome)
|
||||
const raw = await execCommand(options.conn, `cat ${shellQuote(path)} 2>/dev/null || true`, {
|
||||
wrapCommand: options.host.commandDialect !== 'powershell',
|
||||
signal: options.signal
|
||||
}).catch(() => '')
|
||||
const parsed = parseOrcadActivationRecord(raw)
|
||||
/** A failed read rejects; it never reads as absent, because absence would admit a fresh deploy. */
|
||||
async function readActivationRecordState(
|
||||
options: ActivationRecordTarget
|
||||
): Promise<OrcadActivationReadResult> {
|
||||
const read = await readBoundedOrcadRemoteRecord(
|
||||
options,
|
||||
orcadActivationPath(options.host, options.remoteHome),
|
||||
ORCAD_ACTIVATION_RECORD_MAX_BYTES
|
||||
)
|
||||
return read.state === 'absent' ? read : parseOrcadActivationRecord(read.raw)
|
||||
}
|
||||
|
||||
export async function readOrcadActivationRecord(
|
||||
options: ActivationRecordTarget
|
||||
): Promise<OrcadActivationRecord> {
|
||||
const parsed = await readActivationRecordState(options)
|
||||
if (parsed.state === 'ok') {
|
||||
return parsed.record
|
||||
}
|
||||
@@ -43,6 +61,18 @@ export async function readOrcadActivationRecord(options: {
|
||||
return emptyOrcadActivationRecord()
|
||||
}
|
||||
|
||||
function shellQuote(value: string): string {
|
||||
return `'${value.replaceAll("'", `'\\''`)}'`
|
||||
/** Refuses to replace a record this client cannot read, e.g. one a newer client wrote. */
|
||||
export async function writeOrcadActivationRecord(
|
||||
options: ActivationRecordTarget,
|
||||
record: OrcadActivationRecord
|
||||
): Promise<void> {
|
||||
const existing = await readActivationRecordState(options)
|
||||
if (existing.state === 'unreadable') {
|
||||
throw new Error(`Refusing to overwrite this host's orcad activation record: ${existing.reason}`)
|
||||
}
|
||||
await writeAtomicOrcadRemoteRecord(
|
||||
options,
|
||||
orcadActivationPath(options.host, options.remoteHome),
|
||||
serializeOrcadActivationRecord(record)
|
||||
)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
/** Proving that the orcad an activation record names is the one actually serving. */
|
||||
import { evaluateOrcadActivation, type OrcadActivationExpectation } from './orcad-activation-gate'
|
||||
import {
|
||||
orcadLivenessProbeCommand,
|
||||
parseOrcadLiveness,
|
||||
parseOrcadReadinessOutput,
|
||||
readOrcadReadinessCommand,
|
||||
type OrcadLaunchSpec,
|
||||
type OrcadReadinessParse
|
||||
} from './orcad-remote-launch'
|
||||
import {
|
||||
execOrcadRemote,
|
||||
launchOrcadAndAwaitReadiness,
|
||||
type OrcadRemoteExecTarget
|
||||
} from './orcad-remote-runtime-control'
|
||||
import type { ServeReadiness } from '../server/serve-readiness'
|
||||
|
||||
/** `exited` is proven absence; `unverifiable` means the host could not say, which is not death. */
|
||||
export type OrcadReadinessFailureVerdict = 'exited' | 'unverifiable' | 'rejected'
|
||||
|
||||
export class OrcadActiveReadinessError extends Error {
|
||||
constructor(
|
||||
readonly verdict: OrcadReadinessFailureVerdict,
|
||||
message: string
|
||||
) {
|
||||
super(message)
|
||||
this.name = 'OrcadActiveReadinessError'
|
||||
}
|
||||
}
|
||||
|
||||
function gatedReadiness(
|
||||
parsed: OrcadReadinessParse,
|
||||
expectation: OrcadActivationExpectation,
|
||||
label: string
|
||||
): ServeReadiness {
|
||||
const readiness = parsed.state === 'ready' ? parsed.readiness : null
|
||||
const verdict = evaluateOrcadActivation(readiness, expectation)
|
||||
if (verdict.decision === 'reject') {
|
||||
throw new OrcadActiveReadinessError(
|
||||
'rejected',
|
||||
`${label} failed its readiness check: ${verdict.reason}`
|
||||
)
|
||||
}
|
||||
if (!readiness) {
|
||||
throw new OrcadActiveReadinessError(
|
||||
'rejected',
|
||||
`${label} passed activation without a readiness payload.`
|
||||
)
|
||||
}
|
||||
return readiness
|
||||
}
|
||||
|
||||
/** Checks a recorded-active slot without starting anything. */
|
||||
export async function probeActiveOrcadReadiness(
|
||||
target: OrcadRemoteExecTarget & { remoteInstallDir: string },
|
||||
expectation: OrcadActivationExpectation
|
||||
): Promise<ServeReadiness> {
|
||||
const liveness = parseOrcadLiveness(
|
||||
await execOrcadRemote(target, orcadLivenessProbeCommand(target.host, target.remoteInstallDir))
|
||||
)
|
||||
if (liveness === 'DEAD') {
|
||||
throw new OrcadActiveReadinessError(
|
||||
'exited',
|
||||
`orcad ${expectation.fullVersion} is recorded active but its process has exited.`
|
||||
)
|
||||
}
|
||||
if (liveness !== 'LIVE') {
|
||||
throw new OrcadActiveReadinessError(
|
||||
'unverifiable',
|
||||
`orcad ${expectation.fullVersion} process state is unverifiable.`
|
||||
)
|
||||
}
|
||||
const parsed = parseOrcadReadinessOutput(
|
||||
await execOrcadRemote(target, readOrcadReadinessCommand(target.host, target.remoteInstallDir))
|
||||
)
|
||||
return gatedReadiness(parsed, expectation, 'The active orcad')
|
||||
}
|
||||
|
||||
/** Starts a slot (recovery or rollback) and accepts it only if it proves the expected build. */
|
||||
export async function launchOrcadSlotAndAwaitReadiness(
|
||||
target: OrcadRemoteExecTarget & {
|
||||
readinessTimeoutMs?: number
|
||||
sleep?: (ms: number) => Promise<void>
|
||||
},
|
||||
spec: OrcadLaunchSpec,
|
||||
expectation: OrcadActivationExpectation
|
||||
): Promise<ServeReadiness> {
|
||||
const parsed = await launchOrcadAndAwaitReadiness(target, spec)
|
||||
return gatedReadiness(parsed, expectation, `orcad ${spec.fullVersion}`)
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
/** The installed slot's `orcad.js` identity, the same 16-hex prefix orcad reports in its health. */
|
||||
import { shellEscape } from './ssh-connection-utils'
|
||||
import { assertPosixOrcadHost } from './orcad-remote-host-support'
|
||||
import { execOrcadRemote, type OrcadRemoteExecTarget } from './orcad-remote-runtime-control'
|
||||
import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform'
|
||||
|
||||
const BUILD_HASH_MARKER = '__ORCAD_BUILD_HASH__'
|
||||
|
||||
export async function readRemoteOrcadBuildHash(
|
||||
target: OrcadRemoteExecTarget,
|
||||
remoteInstallDir: string
|
||||
): Promise<string> {
|
||||
const output = await execOrcadRemote(
|
||||
target,
|
||||
remoteOrcadBuildHashCommand(target.host, remoteInstallDir)
|
||||
)
|
||||
const match = output.match(/__ORCAD_BUILD_HASH__\s+([a-fA-F0-9]{16})/u)
|
||||
if (!match?.[1]) {
|
||||
throw new Error('Could not verify the installed orcad build hash.')
|
||||
}
|
||||
return match[1].toLowerCase()
|
||||
}
|
||||
|
||||
export function remoteOrcadBuildHashCommand(
|
||||
host: RemoteHostPlatform,
|
||||
remoteInstallDir: string
|
||||
): string {
|
||||
assertPosixOrcadHost(host)
|
||||
const path = shellEscape(joinRemotePath(host, remoteInstallDir, 'orcad.js'))
|
||||
// Why both tools: GNU/busybox ship sha256sum, macOS ships shasum; either prints the digest first.
|
||||
return [
|
||||
`orca_hash=$(if command -v sha256sum >/dev/null 2>&1; then sha256sum ${path} | awk '{print $1}';`,
|
||||
`elif command -v shasum >/dev/null 2>&1; then shasum -a 256 ${path} | awk '{print $1}'; fi);`,
|
||||
`case "$orca_hash" in [0-9a-fA-F][0-9a-fA-F]*) printf '%s %.16s\\n' ${shellEscape(BUILD_HASH_MARKER)} "$orca_hash";; esac`
|
||||
].join(' ')
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
/** Everything a managed-orcad operation needs to know about one SSH host before it acts. */
|
||||
import type { ServerTarget } from '../../shared/node-runtime-pin'
|
||||
import type { SshTarget } from '../../shared/ssh-types'
|
||||
import type { OrcadActivationRecord } from './orcad-activation-record'
|
||||
import { readOrcadActivationRecord } from './orcad-activation-record-store'
|
||||
import { resolveOrcadDeploymentTarget } from './orcad-deployment-target'
|
||||
import { assertPosixOrcadHost } from './orcad-remote-host-support'
|
||||
import { execOrcadRemote } from './orcad-remote-runtime-control'
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import { readRemoteHomeCommand } from './ssh-remote-commands'
|
||||
import {
|
||||
joinRemotePath,
|
||||
normalizeRemoteHome,
|
||||
validateRemoteHome,
|
||||
type RemoteHostPlatform
|
||||
} from './ssh-remote-platform'
|
||||
import { detectRemoteHostPlatform } from './ssh-remote-platform-detection'
|
||||
|
||||
export type OrcadRemoteContext = {
|
||||
activationRecord: OrcadActivationRecord
|
||||
serverTarget: ServerTarget
|
||||
connection: SshConnection
|
||||
host: RemoteHostPlatform
|
||||
remoteHome: string
|
||||
target: SshTarget
|
||||
userDataDir: string
|
||||
}
|
||||
|
||||
export async function resolveOrcadRemoteContext(
|
||||
target: SshTarget,
|
||||
connection: SshConnection,
|
||||
signal?: AbortSignal
|
||||
): Promise<OrcadRemoteContext> {
|
||||
const host = await detectRemoteHostPlatform(connection, { signal })
|
||||
if (!host) {
|
||||
throw new Error('This SSH host platform is not supported by managed orcad.')
|
||||
}
|
||||
// Why first: every lifecycle step after this is POSIX-only, so refuse before probing further.
|
||||
assertPosixOrcadHost(host)
|
||||
const remote = { conn: connection, host, signal }
|
||||
const remoteHome = normalizeRemoteHome(
|
||||
await execOrcadRemote(remote, readRemoteHomeCommand(host)),
|
||||
host
|
||||
)
|
||||
if (!validateRemoteHome(remoteHome, host)) {
|
||||
throw new Error(`Remote home is not a valid path: ${remoteHome.slice(0, 100)}`)
|
||||
}
|
||||
const serverTarget = await resolveOrcadDeploymentTarget({ conn: connection, host, signal })
|
||||
const activationRecord = await readOrcadActivationRecord({ ...remote, remoteHome })
|
||||
return {
|
||||
activationRecord,
|
||||
serverTarget,
|
||||
connection,
|
||||
host,
|
||||
remoteHome,
|
||||
target,
|
||||
userDataDir: joinRemotePath(host, remoteHome, '.orca')
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { chmodSync, mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
||||
import type * as RecordFile from './orcad-remote-record-file'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
@@ -17,6 +18,10 @@ vi.mock('./ssh-relay-install-transfers', () => ({
|
||||
uploadRelayDirectory: vi.fn().mockResolvedValue(undefined),
|
||||
writeRelayFile: vi.fn().mockResolvedValue(undefined)
|
||||
}))
|
||||
vi.mock('./orcad-remote-record-file', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof RecordFile>()),
|
||||
writeAtomicOrcadRemoteRecord: vi.fn().mockResolvedValue(undefined)
|
||||
}))
|
||||
vi.mock('./orcad-remote-node-runtime', () => ({
|
||||
ensureRemoteOrcadNodeRuntime: vi.fn().mockResolvedValue(undefined)
|
||||
}))
|
||||
@@ -26,7 +31,8 @@ vi.mock('./orcad-local-build-hash', () => ({
|
||||
|
||||
import { execCommand } from './ssh-relay-deploy-helpers'
|
||||
import { acquireInstallLock } from './ssh-relay-install-lock'
|
||||
import { uploadRelayDirectory, writeRelayFile } from './ssh-relay-install-transfers'
|
||||
import { uploadRelayDirectory } from './ssh-relay-install-transfers'
|
||||
import { writeAtomicOrcadRemoteRecord } from './orcad-remote-record-file'
|
||||
import { deployOrcad, type OrcadDeployOptions } from './orcad-remote-deploy'
|
||||
import { installOrcadBundle } from './orcad-remote-install'
|
||||
import { ensureRemoteOrcadNodeRuntime } from './orcad-remote-node-runtime'
|
||||
@@ -106,10 +112,12 @@ type HostScript = {
|
||||
function scriptHost(script: HostScript): void {
|
||||
mockExec.mockImplementation(async (_conn, command: string) => {
|
||||
const text = String(command)
|
||||
if (text.startsWith('cat ') && text.includes('orcad-active.json')) {
|
||||
if (text.includes('__ORCAD_RECORD_PRESENT__') && text.includes('orcad-active.json')) {
|
||||
return script.activationRecord
|
||||
? `__ORCAD_RECORD_PRESENT__\n${script.activationRecord}`
|
||||
: '__ORCAD_RECORD_ABSENT__\n'
|
||||
}
|
||||
if (text.includes('.orcad-readiness') && text.startsWith('cat ')) {
|
||||
if (text.includes('.orcad-readiness') && text.startsWith('head -c ')) {
|
||||
if (script.readinessAtMs !== undefined && Date.now() < script.readinessAtMs) {
|
||||
return ''
|
||||
}
|
||||
@@ -272,8 +280,8 @@ describe('deployOrcad', () => {
|
||||
expect(script.log).toEqual(['preflight'])
|
||||
expect(
|
||||
vi
|
||||
.mocked(writeRelayFile)
|
||||
.mock.calls.some(([, , path]) => path.includes('orcad-active.json'))
|
||||
.mocked(writeAtomicOrcadRemoteRecord)
|
||||
.mock.calls.some(([, path]) => path.includes('orcad-active.json'))
|
||||
).toBe(false)
|
||||
}
|
||||
)
|
||||
@@ -356,7 +364,7 @@ describe('deployOrcad', () => {
|
||||
if (String(command).startsWith('chmod 755 ')) {
|
||||
throw new Error('chmod failed')
|
||||
}
|
||||
return ''
|
||||
return String(command).includes('__ORCAD_RECORD_ABSENT__') ? '__ORCAD_RECORD_ABSENT__\n' : ''
|
||||
})
|
||||
await expect(deployOrcad(options())).rejects.toThrow('chmod failed')
|
||||
expect(vi.mocked(finalizeInstall)).not.toHaveBeenCalled()
|
||||
@@ -441,9 +449,9 @@ describe('deployOrcad', () => {
|
||||
const result = await deployOrcad(options())
|
||||
expect(result).toMatchObject({ outcome: 'installed-and-activated', fullVersion: NEW_VERSION })
|
||||
const written = vi
|
||||
.mocked(writeRelayFile)
|
||||
.mock.calls.find((call) => String(call[2]).endsWith('orcad-active.json'))
|
||||
expect(JSON.parse(String(written?.[3]))).toMatchObject({
|
||||
.mocked(writeAtomicOrcadRemoteRecord)
|
||||
.mock.calls.find((call) => String(call[1]).endsWith('orcad-active.json'))
|
||||
expect(JSON.parse(String(written?.[2]))).toMatchObject({
|
||||
active: NEW_VERSION,
|
||||
previous: OLD_VERSION
|
||||
})
|
||||
@@ -492,8 +500,8 @@ describe('deployOrcad', () => {
|
||||
expect(result).toMatchObject({ code: 'orcad_activation_daemon_degraded' })
|
||||
expect(
|
||||
vi
|
||||
.mocked(writeRelayFile)
|
||||
.mock.calls.some((call) => String(call[2]).endsWith('orcad-active.json'))
|
||||
.mocked(writeAtomicOrcadRemoteRecord)
|
||||
.mock.calls.some((call) => String(call[1]).endsWith('orcad-active.json'))
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
|
||||
@@ -4,28 +4,24 @@
|
||||
* preserve current state and the prelaunch snapshot for explicit recovery.
|
||||
*/
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import { ORCAD_STARTUP_READINESS_TIMEOUT_MS } from '../../shared/orcad-profile-preflight'
|
||||
import { execCommand } from './ssh-relay-deploy-helpers'
|
||||
import { ORCAD_INSTALL_MODEL } from './remote-install-model'
|
||||
import { writeRelayFile } from './ssh-relay-install-transfers'
|
||||
import { computeRemoteInstallDir, readLocalFullVersion } from './ssh-relay-versioned-install'
|
||||
import { RELAY_REMOTE_DIR } from './relay-protocol'
|
||||
import {
|
||||
ORCAD_STATE_SNAPSHOT_DIR,
|
||||
serializeOrcadActivationRecord,
|
||||
withActivatedVersion,
|
||||
type OrcadActivationRecord,
|
||||
type OrcadStateSnapshot
|
||||
} from './orcad-activation-record'
|
||||
import { orcadActivationPath, readOrcadActivationRecord } from './orcad-activation-record-store'
|
||||
import {
|
||||
readOrcadActivationRecord,
|
||||
writeOrcadActivationRecord
|
||||
} from './orcad-activation-record-store'
|
||||
import { evaluateOrcadActivation, type OrcadActivationVerdict } from './orcad-activation-gate'
|
||||
import { planOrcadUpdate, type OrcadTerminalCensus } from './orcad-update-plan'
|
||||
import {
|
||||
ORCAD_LOG_FILENAME,
|
||||
orcadLaunchCommand,
|
||||
parseOrcadReadinessOutput,
|
||||
readOrcadReadinessCommand
|
||||
} from './orcad-remote-launch'
|
||||
import { ORCAD_LOG_FILENAME } from './orcad-remote-launch'
|
||||
import { launchOrcadAndAwaitReadiness } from './orcad-remote-runtime-control'
|
||||
import { rejectedOrcadStateRecoveryRefusal, stopOutgoingOrcad } from './orcad-remote-deploy-stop'
|
||||
import {
|
||||
captureOrcadStateSnapshotCommand,
|
||||
@@ -81,7 +77,6 @@ export type OrcadDeployResult =
|
||||
| { outcome: 'already-active'; fullVersion: string }
|
||||
| { outcome: 'installed-not-activated'; fullVersion: string; code: string; reason: string }
|
||||
|
||||
const READINESS_POLL_MS = 500
|
||||
const STOP_WAIT_SECONDS = 20
|
||||
|
||||
function exec(options: OrcadDeployOptions, command: string): Promise<string> {
|
||||
@@ -135,29 +130,12 @@ async function captureSnapshot(
|
||||
}
|
||||
}
|
||||
|
||||
async function launchAndAwaitReadiness(
|
||||
function launchAndAwaitReadiness(
|
||||
options: OrcadDeployOptions,
|
||||
remoteInstallDir: string,
|
||||
fullVersion: string
|
||||
): Promise<ReturnType<typeof parseOrcadReadinessOutput>> {
|
||||
await exec(
|
||||
options,
|
||||
orcadLaunchCommand(options.host, { ...options, remoteInstallDir, fullVersion })
|
||||
)
|
||||
const deadline = Date.now() + (options.readinessTimeoutMs ?? ORCAD_STARTUP_READINESS_TIMEOUT_MS)
|
||||
const sleep = options.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms)))
|
||||
let last = parseOrcadReadinessOutput('')
|
||||
while (Date.now() < deadline) {
|
||||
options.signal?.throwIfAborted()
|
||||
last = parseOrcadReadinessOutput(
|
||||
await exec(options, readOrcadReadinessCommand(options.host, remoteInstallDir))
|
||||
)
|
||||
if (last.state !== 'pending') {
|
||||
return last
|
||||
}
|
||||
await sleep(READINESS_POLL_MS)
|
||||
}
|
||||
return last
|
||||
): ReturnType<typeof launchOrcadAndAwaitReadiness> {
|
||||
return launchOrcadAndAwaitReadiness(options, { ...options, remoteInstallDir, fullVersion })
|
||||
}
|
||||
|
||||
/** Restart the incumbent only when the candidate left shared state unchanged. */
|
||||
@@ -320,12 +298,9 @@ export async function deployOrcad(input: OrcadDeployOptions): Promise<OrcadDeplo
|
||||
}
|
||||
}
|
||||
|
||||
await writeRelayFile(
|
||||
options.conn,
|
||||
options.host,
|
||||
orcadActivationPath(options.host, options.remoteHome),
|
||||
serializeOrcadActivationRecord(withActivatedVersion(record, fullVersion, snapshot, now())),
|
||||
{ signal: options.signal }
|
||||
await writeOrcadActivationRecord(
|
||||
options,
|
||||
withActivatedVersion(record, fullVersion, snapshot, now())
|
||||
)
|
||||
return { outcome: 'installed-and-activated', fullVersion, verdict }
|
||||
}
|
||||
|
||||
@@ -26,6 +26,8 @@ import { selectOrcadSlotRuntimeCommand } from './orcad-remote-runtime'
|
||||
export const ORCAD_READINESS_FILENAME = '.orcad-readiness'
|
||||
/** Stderr, including the bind-exposure line and every supervision message. */
|
||||
export const ORCAD_LOG_FILENAME = 'orcad.log'
|
||||
// Why a cap: the readiness file is candidate-written stdout, and a runaway writer must not be read whole.
|
||||
const ORCAD_READINESS_MAX_BYTES = 256 * 1024
|
||||
export { ORCAD_PID_FILENAME, OrcadRemoteLaunchUnsupportedError } from './orcad-remote-host-support'
|
||||
|
||||
export type OrcadLaunchSpec = {
|
||||
@@ -78,7 +80,8 @@ export function readOrcadReadinessCommand(
|
||||
): string {
|
||||
assertPosixHost(host)
|
||||
const readiness = shellEscape(joinRemotePath(host, remoteInstallDir, ORCAD_READINESS_FILENAME))
|
||||
return `cat ${readiness} 2>/dev/null || true`
|
||||
// One byte over the cap is enough to tell an oversized payload from a full one.
|
||||
return `head -c ${ORCAD_READINESS_MAX_BYTES + 1} ${readiness} 2>/dev/null || true`
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -134,19 +137,23 @@ export type OrcadReadinessParse =
|
||||
* not `malformed` — reporting a parse failure for a race would fail deploys that were fine.
|
||||
*/
|
||||
export function parseOrcadReadinessOutput(raw: string): OrcadReadinessParse {
|
||||
if (Buffer.byteLength(raw, 'utf8') > ORCAD_READINESS_MAX_BYTES) {
|
||||
return { state: 'malformed', reason: 'readiness payload exceeds the 256 KiB limit' }
|
||||
}
|
||||
const lines = raw.split('\n')
|
||||
let sawCandidate = false
|
||||
for (const line of lines) {
|
||||
for (const [index, line] of lines.entries()) {
|
||||
const trimmed = line.trim()
|
||||
if (!trimmed.startsWith('{')) {
|
||||
continue
|
||||
}
|
||||
sawCandidate = true
|
||||
let parsed: unknown
|
||||
try {
|
||||
parsed = JSON.parse(trimmed)
|
||||
} catch {
|
||||
continue
|
||||
// Only the unterminated last line can still be mid-write; a finished bad line will stay bad.
|
||||
return index === lines.length - 1
|
||||
? { state: 'pending' }
|
||||
: { state: 'malformed', reason: 'readiness line is not valid JSON' }
|
||||
}
|
||||
if (typeof parsed !== 'object' || parsed === null) {
|
||||
continue
|
||||
@@ -160,7 +167,7 @@ export function parseOrcadReadinessOutput(raw: string): OrcadReadinessParse {
|
||||
}
|
||||
return { state: 'ready', readiness: toServeReadiness(payload as Record<string, unknown>) }
|
||||
}
|
||||
return sawCandidate ? { state: 'pending' } : { state: 'pending' }
|
||||
return { state: 'pending' }
|
||||
}
|
||||
|
||||
function toServeReadiness(payload: Record<string, unknown>): ServeReadiness {
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
vi.mock('./ssh-relay-deploy-helpers', () => ({
|
||||
execCommand: vi.fn(),
|
||||
isUnconfirmedSshCommandTermination: (error: unknown) =>
|
||||
error instanceof Error && error.message === 'channel lost'
|
||||
}))
|
||||
|
||||
vi.mock('./ssh-remote-platform-detection', () => ({ detectRemoteHostPlatform: vi.fn() }))
|
||||
|
||||
import { execCommand } from './ssh-relay-deploy-helpers'
|
||||
import { detectRemoteHostPlatform } from './ssh-remote-platform-detection'
|
||||
import { resolveOrcadRemoteContext } from './orcad-remote-context'
|
||||
import { getRemoteHostPlatform } from './ssh-remote-platform'
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import {
|
||||
readBoundedOrcadRemoteRecord,
|
||||
writeAtomicOrcadRemoteRecord
|
||||
} from './orcad-remote-record-file'
|
||||
import {
|
||||
readOrcadActivationRecord,
|
||||
writeOrcadActivationRecord
|
||||
} from './orcad-activation-record-store'
|
||||
import { emptyOrcadActivationRecord } from './orcad-activation-record'
|
||||
import { readRemoteOrcadBuildHash } from './orcad-remote-build-hash'
|
||||
import { OrcadActiveReadinessError, probeActiveOrcadReadiness } from './orcad-active-readiness'
|
||||
import { parseOrcadReadinessOutput } from './orcad-remote-launch'
|
||||
|
||||
const mockExec = vi.mocked(execCommand)
|
||||
const linux = getRemoteHostPlatform('linux-x64')
|
||||
const windows = getRemoteHostPlatform('win32-x64')
|
||||
const conn: SshConnection = Object.create(null)
|
||||
const target = { conn, host: linux }
|
||||
const BUILD_HASH = 'abc123def4567890'
|
||||
|
||||
function readyLine(buildHash = BUILD_HASH): string {
|
||||
return JSON.stringify({
|
||||
type: 'orca_server_ready',
|
||||
runtimeId: 'r1',
|
||||
boundEndpoint: 'ws://127.0.0.1:7777',
|
||||
advertisedEndpoint: null,
|
||||
managedWslCliReconciliation: 'settled',
|
||||
pairing: { available: false, reason: 'disabled_by_operator', guidance: 'n/a' },
|
||||
health: {
|
||||
buildHash,
|
||||
buildVersion: '0.2.0+bb01',
|
||||
nodeVersion: '24.21.0',
|
||||
nodeAbi: '137',
|
||||
platform: 'linux',
|
||||
arch: 'x64',
|
||||
pid: 1,
|
||||
terminalDaemon: {
|
||||
state: 'live',
|
||||
ownsFreshSessions: true,
|
||||
pid: 2,
|
||||
buildVersion: '0.2.0+bb01',
|
||||
entryPath: '/x/daemon-entry.js',
|
||||
protocolVersion: 38,
|
||||
selfTest: { ok: true, coverage: 'pty-spawn', verdict: 'healthy', durationMs: 5 }
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
mockExec.mockReset()
|
||||
})
|
||||
|
||||
describe('orcad host record files', () => {
|
||||
it('tells an absent record from one whose read gave no answer', async () => {
|
||||
mockExec.mockResolvedValueOnce('__ORCAD_RECORD_ABSENT__\n')
|
||||
await expect(readBoundedOrcadRemoteRecord(target, '/r.json', 64)).resolves.toEqual({
|
||||
state: 'absent'
|
||||
})
|
||||
mockExec.mockResolvedValueOnce('__ORCAD_RECORD_PRESENT__\n{"a":1}')
|
||||
await expect(readBoundedOrcadRemoteRecord(target, '/r.json', 64)).resolves.toEqual({
|
||||
state: 'present',
|
||||
raw: '{"a":1}'
|
||||
})
|
||||
mockExec.mockResolvedValueOnce('')
|
||||
await expect(readBoundedOrcadRemoteRecord(target, '/r.json', 64)).rejects.toThrow(
|
||||
'no verifiable answer'
|
||||
)
|
||||
})
|
||||
|
||||
it('keeps the partial file when the write may still be running on the host', async () => {
|
||||
mockExec.mockRejectedValueOnce(new Error('channel lost'))
|
||||
await expect(writeAtomicOrcadRemoteRecord(target, '/r.json', '{}')).rejects.toThrow()
|
||||
expect(mockExec).toHaveBeenCalledOnce()
|
||||
mockExec.mockRejectedValueOnce(new Error('exit 1')).mockResolvedValueOnce('')
|
||||
await expect(writeAtomicOrcadRemoteRecord(target, '/r.json', '{}')).rejects.toThrow()
|
||||
expect(String(mockExec.mock.calls.at(-1)?.[1])).toContain('rm -f')
|
||||
})
|
||||
|
||||
it('refuses Windows hosts, which the orcad lifecycle does not support', async () => {
|
||||
await expect(
|
||||
readBoundedOrcadRemoteRecord({ conn, host: windows }, 'C:/r.json', 64)
|
||||
).rejects.toThrow()
|
||||
expect(mockExec).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe('activation record store', () => {
|
||||
const options = { conn, host: linux, remoteHome: '/home/u' }
|
||||
const newer = JSON.stringify({ ...emptyOrcadActivationRecord(), schemaVersion: 2 })
|
||||
|
||||
it('reads a lost read as an error, never as an empty record', async () => {
|
||||
mockExec.mockRejectedValueOnce(new Error('channel lost'))
|
||||
await expect(readOrcadActivationRecord(options)).rejects.toThrow('channel lost')
|
||||
})
|
||||
|
||||
it('reads a newer schema as unreadable and never overwrites it', async () => {
|
||||
mockExec.mockResolvedValue(`__ORCAD_RECORD_PRESENT__\n${newer}`)
|
||||
await expect(readOrcadActivationRecord(options)).rejects.toThrow('schemaVersion 2')
|
||||
await expect(writeOrcadActivationRecord(options, emptyOrcadActivationRecord())).rejects.toThrow(
|
||||
'Refusing to overwrite'
|
||||
)
|
||||
expect(mockExec.mock.calls.some(([, command]) => String(command).includes('mv -f'))).toBe(false)
|
||||
})
|
||||
|
||||
it('writes atomically when the host has no record yet', async () => {
|
||||
mockExec.mockResolvedValueOnce('__ORCAD_RECORD_ABSENT__\n').mockResolvedValueOnce('')
|
||||
await writeOrcadActivationRecord(options, emptyOrcadActivationRecord())
|
||||
expect(String(mockExec.mock.calls[1]?.[1])).toMatch(/printf %s .* && mv -f /s)
|
||||
})
|
||||
})
|
||||
|
||||
describe('installed build identity and readiness', () => {
|
||||
const slot = { ...target, remoteInstallDir: '/home/u/.orca-remote/orcad-0.2.0+bb01' }
|
||||
const expectation = { buildHash: BUILD_HASH, fullVersion: '0.2.0+bb01' }
|
||||
|
||||
it('reads the 16-hex build hash the slot reports', async () => {
|
||||
mockExec.mockResolvedValueOnce(`noise\n__ORCAD_BUILD_HASH__ ${BUILD_HASH.toUpperCase()}\n`)
|
||||
await expect(readRemoteOrcadBuildHash(target, '/slot')).resolves.toBe(BUILD_HASH)
|
||||
mockExec.mockResolvedValueOnce('')
|
||||
await expect(readRemoteOrcadBuildHash(target, '/slot')).rejects.toThrow()
|
||||
})
|
||||
|
||||
it.each([
|
||||
['DEAD', 'exited'],
|
||||
['UNKNOWN', 'unverifiable'],
|
||||
['', 'unverifiable']
|
||||
])('reports a %j liveness probe as %s', async (liveness, verdict) => {
|
||||
mockExec.mockResolvedValueOnce(liveness)
|
||||
await expect(probeActiveOrcadReadiness(slot, expectation)).rejects.toMatchObject({
|
||||
verdict
|
||||
})
|
||||
})
|
||||
|
||||
it('accepts only the expected build once the process is live', async () => {
|
||||
mockExec.mockResolvedValueOnce('LIVE').mockResolvedValueOnce(`${readyLine()}\n`)
|
||||
await expect(probeActiveOrcadReadiness(slot, expectation)).resolves.toMatchObject({
|
||||
runtimeId: 'r1'
|
||||
})
|
||||
mockExec
|
||||
.mockResolvedValueOnce('LIVE')
|
||||
.mockResolvedValueOnce(`${readyLine('ffffffffffffffff')}\n`)
|
||||
const rejected = probeActiveOrcadReadiness(slot, expectation)
|
||||
await expect(rejected).rejects.toBeInstanceOf(OrcadActiveReadinessError)
|
||||
await expect(rejected).rejects.toMatchObject({ verdict: 'rejected' })
|
||||
})
|
||||
})
|
||||
|
||||
describe('readiness parsing bounds', () => {
|
||||
it('waits on a half-written last line but rejects a finished invalid one', () => {
|
||||
expect(parseOrcadReadinessOutput('{"type":"orca_ser')).toEqual({ state: 'pending' })
|
||||
expect(parseOrcadReadinessOutput('{"type":"orca_ser\n')).toMatchObject({
|
||||
state: 'malformed'
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects a payload over the size cap', () => {
|
||||
expect(parseOrcadReadinessOutput('x'.repeat(256 * 1024 + 1))).toMatchObject({
|
||||
state: 'malformed'
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('orcad remote context', () => {
|
||||
it('refuses a Windows host before probing anything else', async () => {
|
||||
vi.mocked(detectRemoteHostPlatform).mockResolvedValueOnce(windows)
|
||||
const sshTarget = { id: 't', label: 't', host: 'h', port: 22, username: 'u' }
|
||||
await expect(resolveOrcadRemoteContext(sshTarget, conn)).rejects.toThrow()
|
||||
expect(mockExec).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* Small JSON records Orca keeps on an orcad host (activation, transactions, stop receipts).
|
||||
*
|
||||
* Reads are bounded and never swallow a failure: a record that could not be read is not an
|
||||
* absent one, and treating it as absent is how a client deploys over a live install.
|
||||
*/
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { shellEscape } from './ssh-connection-utils'
|
||||
import { assertPosixOrcadHost } from './orcad-remote-host-support'
|
||||
import { removeRemoteFileCommand } from './ssh-remote-commands'
|
||||
import { execOrcadRemote, type OrcadRemoteExecTarget } from './orcad-remote-runtime-control'
|
||||
import { isUnconfirmedSshCommandTermination } from './ssh-relay-deploy-helpers'
|
||||
|
||||
const ABSENT_MARKER = '__ORCAD_RECORD_ABSENT__'
|
||||
const PRESENT_MARKER = '__ORCAD_RECORD_PRESENT__'
|
||||
|
||||
/** `present` carries raw bytes; schema checks belong to the caller that owns the format. */
|
||||
export type OrcadRemoteRecordRead = { state: 'absent' } | { state: 'present'; raw: string }
|
||||
|
||||
export async function readBoundedOrcadRemoteRecord(
|
||||
target: OrcadRemoteExecTarget,
|
||||
path: string,
|
||||
maxBytes: number
|
||||
): Promise<OrcadRemoteRecordRead> {
|
||||
assertPosixOrcadHost(target.host)
|
||||
const file = shellEscape(path)
|
||||
// Why markers: an empty stdout must never be mistaken for "no record" when the read failed.
|
||||
const output = await execOrcadRemote(
|
||||
target,
|
||||
`if [ ! -e ${file} ] && [ ! -L ${file} ]; then printf '%s\\n' ${ABSENT_MARKER}; exit 0; fi; ` +
|
||||
`[ -f ${file} ] || exit 65; size=$(wc -c < ${file}) || exit 65; ` +
|
||||
`[ "$size" -le ${maxBytes} ] || exit 65; ` +
|
||||
`printf '%s\\n' ${PRESENT_MARKER}; cat ${file}`
|
||||
)
|
||||
const newline = output.indexOf('\n')
|
||||
const marker = (newline === -1 ? output : output.slice(0, newline)).trim()
|
||||
if (marker === ABSENT_MARKER) {
|
||||
return { state: 'absent' }
|
||||
}
|
||||
if (marker !== PRESENT_MARKER) {
|
||||
throw new Error('orcad host record read returned no verifiable answer')
|
||||
}
|
||||
return { state: 'present', raw: newline === -1 ? '' : output.slice(newline + 1) }
|
||||
}
|
||||
|
||||
export async function writeAtomicOrcadRemoteRecord(
|
||||
target: OrcadRemoteExecTarget,
|
||||
path: string,
|
||||
contents: string
|
||||
): Promise<void> {
|
||||
assertPosixOrcadHost(target.host)
|
||||
const partialPath = `${path}.partial.${process.pid}.${randomUUID()}`
|
||||
try {
|
||||
await execOrcadRemote(
|
||||
target,
|
||||
`umask 077; printf %s ${shellEscape(contents)} > ${shellEscape(partialPath)} && ` +
|
||||
`mv -f ${shellEscape(partialPath)} ${shellEscape(path)}`
|
||||
)
|
||||
} catch (error) {
|
||||
// Why keep the partial on an unconfirmed termination: the write may still be running there.
|
||||
if (!isUnconfirmedSshCommandTermination(error)) {
|
||||
await execOrcadRemote(target, removeRemoteFileCommand(target.host, partialPath)).catch(
|
||||
() => {}
|
||||
)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type * as RecordFile from './orcad-remote-record-file'
|
||||
|
||||
vi.mock('./ssh-relay-deploy-helpers', () => ({
|
||||
execCommand: vi.fn(),
|
||||
@@ -9,9 +10,13 @@ vi.mock('./ssh-relay-install-transfers', () => ({
|
||||
writeRelayFile: vi.fn().mockResolvedValue(undefined),
|
||||
uploadRelayDirectory: vi.fn().mockResolvedValue(undefined)
|
||||
}))
|
||||
vi.mock('./orcad-remote-record-file', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof RecordFile>()),
|
||||
writeAtomicOrcadRemoteRecord: vi.fn().mockResolvedValue(undefined)
|
||||
}))
|
||||
|
||||
import { execCommand } from './ssh-relay-deploy-helpers'
|
||||
import { writeRelayFile } from './ssh-relay-install-transfers'
|
||||
import { writeAtomicOrcadRemoteRecord } from './orcad-remote-record-file'
|
||||
import { rollbackOrcad, type OrcadRollbackOptions } from './orcad-remote-rollback'
|
||||
import { emptyOrcadActivationRecord, type OrcadActivationRecord } from './orcad-activation-record'
|
||||
import { getRemoteHostPlatform } from './ssh-remote-platform'
|
||||
@@ -74,6 +79,9 @@ function scriptHost(
|
||||
): void {
|
||||
mockExec.mockImplementation(async (_conn, command: string) => {
|
||||
const text = String(command)
|
||||
if (text.includes('__ORCAD_RECORD_PRESENT__')) {
|
||||
return `__ORCAD_RECORD_PRESENT__\n${JSON.stringify(record())}`
|
||||
}
|
||||
if (text.includes('state.tar') && text.includes('test -f') && !text.includes('tar -C')) {
|
||||
return 'PRESENT'
|
||||
}
|
||||
@@ -92,7 +100,7 @@ function scriptHost(
|
||||
log.push(`launch:${text.includes(ACTIVE) ? ACTIVE : TARGET}`)
|
||||
return '9999'
|
||||
}
|
||||
if (text.startsWith('cat ') && text.includes('.orcad-readiness')) {
|
||||
if (text.startsWith('head -c ') && text.includes('.orcad-readiness')) {
|
||||
if (overrides.readinessAtMs !== undefined && Date.now() < overrides.readinessAtMs) {
|
||||
return ''
|
||||
}
|
||||
@@ -169,7 +177,7 @@ describe('rollbackOrcad', () => {
|
||||
code: 'orcad_rollback_orphans_live_terminals'
|
||||
})
|
||||
expect(log).toEqual([])
|
||||
expect(vi.mocked(writeRelayFile)).not.toHaveBeenCalled()
|
||||
expect(vi.mocked(writeAtomicOrcadRemoteRecord)).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('refuses when the snapshot is gone from the host', async () => {
|
||||
@@ -212,7 +220,7 @@ describe('rollbackOrcad', () => {
|
||||
expect(result).toMatchObject({ outcome: 'failed', code: 'orcad_activation_no_readiness' })
|
||||
// Until the target is proven serving, `active` must still name the version an operator
|
||||
// would have to bring back.
|
||||
expect(vi.mocked(writeRelayFile)).not.toHaveBeenCalled()
|
||||
expect(vi.mocked(writeAtomicOrcadRemoteRecord)).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('records the rollback only after the target answers healthy', async () => {
|
||||
@@ -220,9 +228,9 @@ describe('rollbackOrcad', () => {
|
||||
scriptHost(log)
|
||||
await rollbackOrcad(options())
|
||||
const written = vi
|
||||
.mocked(writeRelayFile)
|
||||
.mock.calls.find((call) => String(call[2]).endsWith('orcad-active.json'))
|
||||
expect(JSON.parse(String(written?.[3]))).toMatchObject({
|
||||
.mocked(writeAtomicOrcadRemoteRecord)
|
||||
.mock.calls.find((call) => String(call[1]).endsWith('orcad-active.json'))
|
||||
expect(JSON.parse(String(written?.[2]))).toMatchObject({
|
||||
active: TARGET,
|
||||
previous: null,
|
||||
snapshot: null
|
||||
|
||||
@@ -14,26 +14,19 @@
|
||||
* failure this is meant to avoid, arrived at from the other side.
|
||||
*/
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import { ORCAD_STARTUP_READINESS_TIMEOUT_MS } from '../../shared/orcad-profile-preflight'
|
||||
import { execCommand } from './ssh-relay-deploy-helpers'
|
||||
import { ORCAD_INSTALL_MODEL } from './remote-install-model'
|
||||
import { computeRemoteInstallDir } from './ssh-relay-versioned-install'
|
||||
import { writeRelayFile } from './ssh-relay-install-transfers'
|
||||
import { RELAY_REMOTE_DIR } from './relay-protocol'
|
||||
import {
|
||||
ORCAD_STATE_SNAPSHOT_DIR,
|
||||
serializeOrcadActivationRecord,
|
||||
withRolledBackVersion,
|
||||
type OrcadActivationRecord
|
||||
} from './orcad-activation-record'
|
||||
import { assessOrcadRollback, type OrcadTerminalCensus } from './orcad-update-plan'
|
||||
import { evaluateOrcadActivation, type OrcadActivationVerdict } from './orcad-activation-gate'
|
||||
import {
|
||||
ORCAD_LOG_FILENAME,
|
||||
orcadLaunchCommand,
|
||||
parseOrcadReadinessOutput,
|
||||
readOrcadReadinessCommand
|
||||
} from './orcad-remote-launch'
|
||||
import { ORCAD_LOG_FILENAME } from './orcad-remote-launch'
|
||||
import { launchOrcadAndAwaitReadiness } from './orcad-remote-runtime-control'
|
||||
import {
|
||||
newestStateMtimeCommand,
|
||||
parseNewestStateMtimeSeconds,
|
||||
@@ -46,7 +39,7 @@ import {
|
||||
parseOrcadStopOutcome,
|
||||
stopOrcadCommand
|
||||
} from './orcad-remote-process-control'
|
||||
import { orcadActivationPath } from './orcad-activation-record-store'
|
||||
import { writeOrcadActivationRecord } from './orcad-activation-record-store'
|
||||
import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform'
|
||||
|
||||
export type OrcadRollbackOptions = {
|
||||
@@ -72,7 +65,6 @@ export type OrcadRollbackResult =
|
||||
| { outcome: 'refused'; code: string; reason: string }
|
||||
| { outcome: 'failed'; code: string; reason: string }
|
||||
|
||||
const READINESS_POLL_MS = 500
|
||||
const STOP_WAIT_SECONDS = 20
|
||||
|
||||
function exec(options: OrcadRollbackOptions, command: string): Promise<string> {
|
||||
@@ -185,29 +177,14 @@ export async function rollbackOrcad(options: OrcadRollbackOptions): Promise<Orca
|
||||
}
|
||||
|
||||
const targetDir = computeRemoteInstallDir(ORCAD_INSTALL_MODEL, options.remoteHome, safety.target)
|
||||
await exec(
|
||||
options,
|
||||
orcadLaunchCommand(options.host, {
|
||||
remoteInstallDir: targetDir,
|
||||
nodePath: options.nodePath,
|
||||
fullVersion: safety.target,
|
||||
userDataDir: options.userDataDir,
|
||||
bindHost: options.bindHost,
|
||||
port: options.port
|
||||
})
|
||||
)
|
||||
const deadline = Date.now() + (options.readinessTimeoutMs ?? ORCAD_STARTUP_READINESS_TIMEOUT_MS)
|
||||
const sleep = options.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms)))
|
||||
let parsed = parseOrcadReadinessOutput('')
|
||||
while (Date.now() < deadline && parsed.state === 'pending') {
|
||||
options.signal?.throwIfAborted()
|
||||
parsed = parseOrcadReadinessOutput(
|
||||
await exec(options, readOrcadReadinessCommand(options.host, targetDir))
|
||||
)
|
||||
if (parsed.state === 'pending') {
|
||||
await sleep(READINESS_POLL_MS)
|
||||
}
|
||||
}
|
||||
const parsed = await launchOrcadAndAwaitReadiness(options, {
|
||||
remoteInstallDir: targetDir,
|
||||
nodePath: options.nodePath,
|
||||
fullVersion: safety.target,
|
||||
userDataDir: options.userDataDir,
|
||||
bindHost: options.bindHost,
|
||||
port: options.port
|
||||
})
|
||||
const verdict = evaluateOrcadActivation(parsed.state === 'ready' ? parsed.readiness : null, {
|
||||
buildHash: options.targetBuildHash,
|
||||
fullVersion: safety.target
|
||||
@@ -225,13 +202,7 @@ export async function rollbackOrcad(options: OrcadRollbackOptions): Promise<Orca
|
||||
|
||||
// Why the record is written last: until the target is proven serving, `active` still names
|
||||
// the version an operator would need to bring back, and `previous` still names this target.
|
||||
await writeRelayFile(
|
||||
options.conn,
|
||||
options.host,
|
||||
orcadActivationPath(options.host, options.remoteHome),
|
||||
serializeOrcadActivationRecord(withRolledBackVersion(options.record, now())),
|
||||
{ signal: options.signal }
|
||||
)
|
||||
await writeOrcadActivationRecord(options, withRolledBackVersion(options.record, now()))
|
||||
return {
|
||||
outcome: 'rolled-back',
|
||||
target: safety.target,
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
/** Launch a slot and poll its readiness file: the one loop deploy, rollback and recovery share. */
|
||||
import { ORCAD_STARTUP_READINESS_TIMEOUT_MS } from '../../shared/orcad-profile-preflight'
|
||||
import {
|
||||
orcadLaunchCommand,
|
||||
parseOrcadReadinessOutput,
|
||||
readOrcadReadinessCommand,
|
||||
type OrcadLaunchSpec,
|
||||
type OrcadReadinessParse
|
||||
} from './orcad-remote-launch'
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import { execCommand } from './ssh-relay-deploy-helpers'
|
||||
import type { RemoteHostPlatform } from './ssh-remote-platform'
|
||||
|
||||
const READINESS_POLL_MS = 500
|
||||
|
||||
export type OrcadRemoteExecTarget = {
|
||||
conn: SshConnection
|
||||
host: RemoteHostPlatform
|
||||
signal?: AbortSignal
|
||||
}
|
||||
|
||||
export function execOrcadRemote(
|
||||
target: OrcadRemoteExecTarget,
|
||||
command: string,
|
||||
signal = target.signal
|
||||
): Promise<string> {
|
||||
return execCommand(target.conn, command, {
|
||||
wrapCommand: target.host.commandDialect !== 'powershell',
|
||||
signal
|
||||
})
|
||||
}
|
||||
|
||||
/** Recovery paths must finish even when the request that started them was cancelled. */
|
||||
export function withoutAbortSignal<T extends { signal?: AbortSignal }>(
|
||||
options: T
|
||||
): Omit<T, 'signal'> {
|
||||
const { signal: _signal, ...rest } = options
|
||||
return rest
|
||||
}
|
||||
|
||||
export async function launchOrcadAndAwaitReadiness(
|
||||
target: OrcadRemoteExecTarget & {
|
||||
readinessTimeoutMs?: number
|
||||
sleep?: (ms: number) => Promise<void>
|
||||
},
|
||||
spec: OrcadLaunchSpec
|
||||
): Promise<OrcadReadinessParse> {
|
||||
await execOrcadRemote(target, orcadLaunchCommand(target.host, spec))
|
||||
const deadline = Date.now() + (target.readinessTimeoutMs ?? ORCAD_STARTUP_READINESS_TIMEOUT_MS)
|
||||
const sleep = target.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms)))
|
||||
let last = parseOrcadReadinessOutput('')
|
||||
while (Date.now() < deadline) {
|
||||
target.signal?.throwIfAborted()
|
||||
last = parseOrcadReadinessOutput(
|
||||
await execOrcadRemote(target, readOrcadReadinessCommand(target.host, spec.remoteInstallDir))
|
||||
)
|
||||
if (last.state !== 'pending') {
|
||||
return last
|
||||
}
|
||||
await sleep(READINESS_POLL_MS)
|
||||
}
|
||||
return last
|
||||
}
|
||||
Reference in New Issue
Block a user