diff --git a/src/main/managed-server/install/orcad-remote-install-cached-probe.test.ts b/src/main/managed-server/install/orcad-remote-install-cached-probe.test.ts new file mode 100644 index 00000000000..4aa25a822e1 --- /dev/null +++ b/src/main/managed-server/install/orcad-remote-install-cached-probe.test.ts @@ -0,0 +1,95 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type * as ExecCommandModule from '../../ssh/remote-exec/remote-exec-command' + +vi.mock('../../ssh/remote-exec/remote-exec-command', async (importOriginal) => ({ + ...(await importOriginal()), + execCommand: vi.fn() +})) +vi.mock('./orcad-remote-node-runtime', async (importOriginal) => ({ + ...(await importOriginal>()), + ensureRemoteOrcadNodeRuntime: vi.fn(async () => ({ executable: '/n', transfer: 'cached' })) +})) +vi.mock('./remote-versioned-install', async (importOriginal) => ({ + ...(await importOriginal>()), + isRemoteInstallComplete: vi.fn(async () => true) +})) + +import { execCommand } from '../../ssh/remote-exec/remote-exec-command' +import { getRemoteHostPlatform } from '../../ssh/ssh-remote-platform' +import type { OrcadHostTransport } from '../../ssh/orcad-host-transport' +import { ensureRemoteOrcadNodeRuntime } from './orcad-remote-node-runtime' +import { isRemoteInstallComplete } from './remote-versioned-install' +import { installOrcadBundle } from './orcad-remote-install' +import { cachedOrcadSlotInstalled } from './orcad-remote-install-cached-probe' + +const mockExec = vi.mocked(execCommand) +const END = '\n__ORCA_CACHED_SLOT_PROBE_END__\n' +const SLOT = '/home/u/.orca-remote/orcad-0.1.0+aa' +// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: execCommand is mocked, so the transport is never used. +const conn = {} as OrcadHostTransport +const linux = getRemoteHostPlatform('linux-x64') +const options = { conn, host: linux, target: 'linux-x64-glibc' as const } + +function bundle(recordedNodeRuntimeProbe?: boolean) { + return installOrcadBundle( + { + ...options, + localOrcadDir: '/local/orcad', + nodeRuntimeArchive: async () => '/cache/node.tar.gz', + ...(recordedNodeRuntimeProbe ? { recordedNodeRuntimeProbe } : {}) + }, + '0.1.0+aa', + SLOT + ) +} + +beforeEach(() => { + vi.clearAllMocks() +}) + +describe('cachedOrcadSlotInstalled', () => { + it('skips the install only when the runtime is cached and the slot complete', async () => { + mockExec.mockResolvedValueOnce(`ORCA_NODE_RUNTIME_READY${END}OK\n`) + await expect(cachedOrcadSlotInstalled(options, SLOT)).resolves.toBe(true) + expect(mockExec).toHaveBeenCalledOnce() + for (const answer of [ + `ORCA_NODE_RUNTIME_MISSING${END}OK\n`, + `ORCA_NODE_RUNTIME_READY${END}MISSING\n`, + 'ORCA_NODE_RUNTIME_READY\n' + ]) { + mockExec.mockResolvedValueOnce(answer) + await expect(cachedOrcadSlotInstalled(options, SLOT)).resolves.toBe(false) + } + mockExec.mockRejectedValueOnce(new Error('exit 1')) + await expect(cachedOrcadSlotInstalled(options, SLOT)).resolves.toBe(false) + }) + + it('leaves Windows hosts to the separate probes', async () => { + const windows = { ...options, host: getRemoteHostPlatform('win32-x64') } + await expect(cachedOrcadSlotInstalled(windows, SLOT)).resolves.toBe(false) + expect(mockExec).not.toHaveBeenCalled() + }) +}) + +describe('installOrcadBundle with the WSL cached-slot probe', () => { + it('returns after one exec when both answers are cached', async () => { + mockExec.mockResolvedValueOnce(`ORCA_NODE_RUNTIME_READY${END}OK\n`) + await bundle(true) + expect(ensureRemoteOrcadNodeRuntime).not.toHaveBeenCalled() + expect(isRemoteInstallComplete).not.toHaveBeenCalled() + }) + + it('runs the separate probes when the combined one does not say both', async () => { + mockExec.mockResolvedValueOnce(`ORCA_NODE_RUNTIME_MISSING${END}OK\n`) + await bundle(true) + expect(ensureRemoteOrcadNodeRuntime).toHaveBeenCalledOnce() + expect(isRemoteInstallComplete).toHaveBeenCalledOnce() + }) + + it('never sends the combined probe without the option (SSH)', async () => { + await bundle() + expect(mockExec).not.toHaveBeenCalled() + expect(ensureRemoteOrcadNodeRuntime).toHaveBeenCalledOnce() + expect(isRemoteInstallComplete).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/managed-server/install/orcad-remote-install-cached-probe.ts b/src/main/managed-server/install/orcad-remote-install-cached-probe.ts new file mode 100644 index 00000000000..e66a7d4d1af --- /dev/null +++ b/src/main/managed-server/install/orcad-remote-install-cached-probe.ts @@ -0,0 +1,66 @@ +// WSL opt-in: one exec asks both "is the pinned runtime cached?" and "is this slot complete?". +import type { NodeRuntimeTarget } from '../../../shared/node-runtime-pin' +import type { OrcadHostTransport } from '../../ssh/orcad-host-transport' +import { REMOTE_NODE_RUNTIME_READY } from '../../ssh/orcad-remote-node-runtime-report' +import { probeRemoteInstallCompleteCommand } from '../../ssh/ssh-remote-commands' +import { + execCommand, + isUnconfirmedSshCommandTermination +} from '../../ssh/remote-exec/remote-exec-command' +import { isWindowsRemoteHost, type RemoteHostPlatform } from '../../ssh/ssh-remote-platform' +import { NODE_RUNTIME_PROMOTE_TIMEOUT_MS, remoteNodeRuntimeDir } from './orcad-remote-node-runtime' +import { recordedRemoteNodeRuntimeProbeCommand } from './orcad-remote-node-runtime-recorded-probe' +import { ORCAD_INSTALL_MODEL } from './remote-install-model' + +const SECTION_END = '__ORCA_CACHED_SLOT_PROBE_END__' + +export function cachedOrcadSlotProbeCommand( + host: RemoteHostPlatform, + remoteDir: string, + target: NodeRuntimeTarget +): string { + const runtimeDir = remoteNodeRuntimeDir(host, remoteDir, target) + return [ + recordedRemoteNodeRuntimeProbeCommand(host, runtimeDir, target), + `printf '\\n%s\\n' ${SECTION_END}`, + probeRemoteInstallCompleteCommand(host, remoteDir, [ + ...ORCAD_INSTALL_MODEL.requiredArtifacts(host), + ORCAD_INSTALL_MODEL.installCompleteFilename + ]) + ].join('; ') +} + +/** + * True only when both answers say the install can be skipped, which is where the separate + * probes return too; anything else (including a failed exec) leaves them to run as usual. + */ +export async function cachedOrcadSlotInstalled( + options: { + conn: OrcadHostTransport + host: RemoteHostPlatform + target: NodeRuntimeTarget + signal?: AbortSignal + }, + remoteDir: string +): Promise { + if (isWindowsRemoteHost(options.host)) { + return false + } + try { + const output = await execCommand( + options.conn, + cachedOrcadSlotProbeCommand(options.host, remoteDir, options.target), + { signal: options.signal, timeoutMs: NODE_RUNTIME_PROMOTE_TIMEOUT_MS, wrapCommand: true } + ) + const [runtime, install, ...rest] = output.split(`\n${SECTION_END}\n`) + return ( + rest.length === 0 && runtime?.trim() === REMOTE_NODE_RUNTIME_READY && install?.trim() === 'OK' + ) + } catch (error) { + if (isUnconfirmedSshCommandTermination(error)) { + throw error + } + options.signal?.throwIfAborted() + return false + } +} diff --git a/src/main/managed-server/install/orcad-remote-install.ts b/src/main/managed-server/install/orcad-remote-install.ts index eaaf4042267..a1907b7ea60 100644 --- a/src/main/managed-server/install/orcad-remote-install.ts +++ b/src/main/managed-server/install/orcad-remote-install.ts @@ -4,6 +4,7 @@ import { orcadAgentBrowserNativeName } from '../../../shared/orcad-agent-browser import type { NodeRuntimeTarget } from '../../../shared/node-runtime-pin' import { ensureRemoteOrcadNodeRuntime } from './orcad-remote-node-runtime' import { recordedRemoteNodeRuntimeProbeCommand } from './orcad-remote-node-runtime-recorded-probe' +import { cachedOrcadSlotInstalled } from './orcad-remote-install-cached-probe' import { execCommand, isUnconfirmedSshCommandTermination @@ -41,6 +42,9 @@ export async function installOrcadBundle( fullVersion: string, remoteDir: string ): Promise { + if (options.recordedNodeRuntimeProbe && (await cachedOrcadSlotInstalled(options, remoteDir))) { + return + } // Existing slot bytes are unusable without their referenced runtime. await ensureRemoteOrcadNodeRuntime({ conn: options.conn, diff --git a/src/main/managed-server/posix-host-op-byte-budget.test.ts b/src/main/managed-server/posix-host-op-byte-budget.test.ts index 20dc33056c7..116b0187fc7 100644 --- a/src/main/managed-server/posix-host-op-byte-budget.test.ts +++ b/src/main/managed-server/posix-host-op-byte-budget.test.ts @@ -73,6 +73,7 @@ import { recordedRemoteNodeRuntimeProbeCommand } from './install/orcad-remote-node-runtime-recorded-probe' import { runtimeStoreInventoryCommand } from './install/remote-node-runtime-store-inventory' +import { cachedOrcadSlotProbeCommand } from './install/orcad-remote-install-cached-probe' import { posixReleaseOwnedLockCommand } from './install/remote-node-runtime-store-lock' import { ORCAD_INSTALL_MODEL } from './install/remote-install-model' import { stopOrcadCommand } from './install/orcad-remote-process-control' @@ -101,7 +102,9 @@ const OVER_BUDGET_CEILINGS: Readonly> = { tryStealInstallLockCommand: 9_022, orcadLaunchCommand: 6_896, // WSL-only: wsl.exe argv, never SSH-framed; carries the Node bridge as its fallback. - orcadBashStdioBridgeCommand: 6_300 + orcadBashStdioBridgeCommand: 6_300, + // WSL-only (sent only under recordedNodeRuntimeProbe): wsl.exe argv, never SSH-framed. + cachedOrcadSlotProbeCommand: 7_069 } type HostCase = { host: RemoteHostPlatform; target: NodeRuntimeTarget; nodePath: string } @@ -257,6 +260,8 @@ const BUILDERS: Readonly string>> = { ...ORCAD_INSTALL_MODEL.requiredArtifacts(host), ORCAD_INSTALL_MODEL.installCompleteFilename ]), + cachedOrcadSlotProbeCommand: ({ host, target }) => + cachedOrcadSlotProbeCommand(host, slotDir, target), listRemoteInstallBaseDirsCommand: ({ host }) => listRemoteInstallBaseDirsCommand(host, baseDir, ORCAD_INSTALL_MODEL), probeFileExistsCommand: ({ host }) => probeFileExistsCommand(host, `${slotDir}/.complete`), diff --git a/src/main/ssh/orcad-active-readiness.ts b/src/main/ssh/orcad-active-readiness.ts index 43aa571747a..3f3c1955116 100644 --- a/src/main/ssh/orcad-active-readiness.ts +++ b/src/main/ssh/orcad-active-readiness.ts @@ -19,6 +19,7 @@ import { } from '../managed-server/install/orcad-remote-readiness-wait' import type { ServeReadiness } from '../server/serve-readiness' import { withOrcadLogTail } from '../managed-server/install/orcad-remote-log-tail' +import type { RemoteHostPlatform } from './ssh-remote-platform' type OrcadReadinessFailureVerdict = 'exited' | 'unverifiable' | 'rejected' @@ -78,9 +79,26 @@ export async function probeActiveOrcadReadiness( target: OrcadRemoteExecTarget & { remoteInstallDir: string }, expectation: OrcadActivationExpectation ): Promise { - const liveness = parseOrcadLiveness( - await execOrcadRemote(target, orcadLivenessProbeCommand(target.host, target.remoteInstallDir)) + assertActiveOrcadLive( + await execOrcadRemote(target, orcadLivenessProbeCommand(target.host, target.remoteInstallDir)), + expectation ) + return activeOrcadReadinessFromOutput( + target.host, + await execOrcadRemote( + target, + readOrcadReadinessNowCommand(target.host, target.remoteInstallDir) + ), + expectation + ) +} + +/** Throws unless the liveness probe's answer proves the active slot's process is running. */ +export function assertActiveOrcadLive( + livenessOutput: string, + expectation: OrcadActivationExpectation +): void { + const liveness = parseOrcadLiveness(livenessOutput) if (liveness === 'DEAD') { throw new OrcadActiveReadinessError( 'exited', @@ -99,14 +117,18 @@ export async function probeActiveOrcadReadiness( `orcad ${expectation.fullVersion} process state is unverifiable.` ) } - const parsed = parseOrcadReadinessWaitOutput( - target.host, - await execOrcadRemote( - target, - readOrcadReadinessNowCommand(target.host, target.remoteInstallDir) - ) +} + +export function activeOrcadReadinessFromOutput( + host: RemoteHostPlatform, + readinessOutput: string, + expectation: OrcadActivationExpectation +): ServeReadiness { + return gatedReadiness( + parseOrcadReadinessWaitOutput(host, readinessOutput), + expectation, + 'The active orcad' ) - return gatedReadiness(parsed, expectation, 'The active orcad') } /** Accept recovery/rollback launches only when they prove the expected build. */ diff --git a/src/main/wsl/wsl-guest-process-identity.ts b/src/main/wsl/wsl-guest-process-identity.ts index b17b80ffc1c..0364d928645 100644 --- a/src/main/wsl/wsl-guest-process-identity.ts +++ b/src/main/wsl/wsl-guest-process-identity.ts @@ -15,10 +15,14 @@ const IDENTITY_TIMEOUT_MS = 10_000 /** One exec, so a classification never mixes two distro incarnations. */ export function wslGuestIdentityCommand(orcadPid: number | null): string { - const orcad = - orcadPid !== null && Number.isSafeInteger(orcadPid) && orcadPid > 0 - ? `cat /proc/${orcadPid}/stat 2>/dev/null; ` - : '' + return wslGuestIdentityShellCommand( + orcadPid !== null && Number.isSafeInteger(orcadPid) && orcadPid > 0 ? String(orcadPid) : null + ) +} + +/** `pidWord` is a digits-only word, literal or a shell expansion the caller has validated. */ +export function wslGuestIdentityShellCommand(pidWord: string | null): string { + const orcad = pidWord !== null ? `cat /proc/${pidWord}/stat 2>/dev/null; ` : '' return [ `cat /proc/sys/kernel/random/boot_id; echo ${SECTION}; cat /proc/1/stat; echo ${SECTION}; `, `${orcad}echo ${SECTION}` diff --git a/src/main/wsl/wsl-orcad-active-probe.test.ts b/src/main/wsl/wsl-orcad-active-probe.test.ts new file mode 100644 index 00000000000..58ba0faef5f --- /dev/null +++ b/src/main/wsl/wsl-orcad-active-probe.test.ts @@ -0,0 +1,108 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type * as ExecCommandModule from '../ssh/remote-exec/remote-exec-command' + +vi.mock('../ssh/remote-exec/remote-exec-command', async (importOriginal) => ({ + ...(await importOriginal()), + execCommand: vi.fn() +})) +vi.mock('../ssh/orcad-local-build-hash', () => ({ + computeLocalOrcadBuildHash: () => 'abc123def4567890' +})) + +import { execCommand } from '../ssh/remote-exec/remote-exec-command' +import { getRemoteHostPlatform } from '../ssh/ssh-remote-platform' +import type { OrcadHostTransport } from '../ssh/orcad-host-transport' +import { emptyOrcadActivationRecord } from '../managed-server/activation/orcad-activation-record' +import { OrcadActiveReadinessError, probeActiveOrcadReadiness } from '../ssh/orcad-active-readiness' +import { orcadLivenessProbeCommand } from '../managed-server/install/orcad-remote-launch' +import { readOrcadReadinessNowCommand } from '../managed-server/install/orcad-remote-readiness-wait' +import { probeWslActiveOrcad, wslOrcadActiveProbeCommand } from './wsl-orcad-active-probe' + +const mockExec = vi.mocked(execCommand) +const VERSION = '0.1.0+aa' +const END = '\n__ORCA_WSL_ACTIVE_PROBE_END__\n' +const host = getRemoteHostPlatform('linux-x64') +const context = { + activationRecord: emptyOrcadActivationRecord(), + serverTarget: 'linux-x64-glibc' as const, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: execCommand is mocked, so the transport is never used. + connection: {} as OrcadHostTransport, + host, + remoteHome: '/home/u', + target: { kind: 'wsl' as const, id: 'wsl:u', distro: 'U', label: 'U' }, + userDataDir: '/home/u/.orca' +} +const READY = JSON.stringify({ + type: 'orca_server_ready', + schemaVersion: 1, + runtimeId: 'r1', + boundEndpoint: 'ws://127.0.0.1:6790', + advertisedEndpoint: null, + managedWslCliReconciliation: 'settled', + pairing: { available: false, reason: 'disabled_by_operator', guidance: 'n/a' }, + health: { + buildHash: 'abc123def4567890', + buildVersion: VERSION, + nodeVersion: '20.11.0', + nodeAbi: '115', + platform: 'linux', + arch: 'x64', + pid: 1, + terminalDaemon: { + state: 'live', + ownsFreshSessions: true, + pid: 2, + buildVersion: VERSION, + entryPath: '/x/daemon-entry.js', + protocolVersion: 3, + selfTest: { ok: true, coverage: 'pty-spawn', verdict: 'healthy', durationMs: 5 } + } + } +}) + +const probe = () => probeWslActiveOrcad(context, '/local/orcad', VERSION) + +beforeEach(() => { + mockExec.mockReset() +}) + +describe('probeWslActiveOrcad', () => { + it('reads liveness, readiness and the pid in one exec, in that order', async () => { + mockExec.mockResolvedValueOnce(`LIVE\n${END}${READY}\n${END}4242\n`) + await expect(probe()).resolves.toMatchObject({ readiness: { runtimeId: 'r1' }, pid: 4242 }) + expect(mockExec).toHaveBeenCalledOnce() + const command = wslOrcadActiveProbeCommand(context, VERSION) + const at = (needle: string) => command.lastIndexOf(needle) + expect(at('orcad_alive')).toBeLessThan(at('head -c')) + expect(at('head -c')).toBeLessThan(at("cat '/home/u/.orca-remote/orcad-0.1.0+aa/.orcad-pid'")) + }) + + it('judges the answers as the separate probes do', async () => { + mockExec.mockResolvedValueOnce(`DEAD\n${END}${END}\n`) + await expect(probe()).rejects.toMatchObject({ verdict: 'exited' }) + mockExec.mockResolvedValueOnce(`UNKNOWN\n${END}${READY}\n${END}4242\n`) + await expect(probe()).rejects.toMatchObject({ verdict: 'unverifiable' }) + mockExec.mockResolvedValueOnce(`LIVE\n${END}${READY.replace('abc123', 'fff123')}\n${END}1\n`) + await expect(probe()).rejects.toMatchObject({ verdict: 'rejected' }) + }) + + it('reports an answer of another shape as unverifiable, never exited', async () => { + mockExec.mockResolvedValueOnce('LIVE\n') + const error = await probe().catch((caught: unknown) => caught) + expect(error).toBeInstanceOf(OrcadActiveReadinessError) + expect(error).toMatchObject({ verdict: 'unverifiable' }) + }) + + it('leaves the SSH probe its two separate reads, byte for byte', async () => { + const dir = '/home/u/.orca-remote/orcad-0.1.0+aa' + mockExec.mockResolvedValueOnce('LIVE\n').mockResolvedValueOnce(`${READY}\n`) + await probeActiveOrcadReadiness( + { conn: context.connection, host, remoteInstallDir: dir }, + { buildHash: 'abc123def4567890', fullVersion: VERSION } + ) + expect(mockExec.mock.calls.map(([, command]) => command)).toEqual([ + orcadLivenessProbeCommand(host, dir), + readOrcadReadinessNowCommand(host, dir) + ]) + }) +}) diff --git a/src/main/wsl/wsl-orcad-active-probe.ts b/src/main/wsl/wsl-orcad-active-probe.ts new file mode 100644 index 00000000000..cafc0daaa11 --- /dev/null +++ b/src/main/wsl/wsl-orcad-active-probe.ts @@ -0,0 +1,88 @@ +/** + * WSL reads an active slot's liveness, readiness and pid in one guest command; SSH keeps + * `probeActiveOrcadReadiness`'s separate reads. The answers are judged exactly as those are. + */ +import type { OrcadHostTarget } from '../ssh/orcad-host-transport' +import type { OrcadRemoteContext } from '../managed-server/install/orcad-remote-context' +import { orcadLivenessProbeCommand } from '../managed-server/install/orcad-remote-launch' +import { readOrcadReadinessNowCommand } from '../managed-server/install/orcad-remote-readiness-wait' +import { execOrcadRemote } from '../managed-server/install/orcad-remote-runtime-control' +import { ORCAD_PID_FILENAME } from '../managed-server/install/orcad-remote-host-support' +import { managedOrcadInstallDir } from '../managed-server/lifecycle/orcad-managed-runtime-context' +import { + activeOrcadReadinessFromOutput, + assertActiveOrcadLive, + OrcadActiveReadinessError +} from '../ssh/orcad-active-readiness' +import { computeLocalOrcadBuildHash } from '../ssh/orcad-local-build-hash' +import type { ServeReadiness } from '../server/serve-readiness' +import { quotePosixShell } from '../../shared/shell-argument-quoting' +import { execCommand } from '../ssh/remote-exec/remote-exec-command' +import { joinRemotePath } from '../ssh/ssh-remote-platform' + +const SECTION_END = '__ORCA_WSL_ACTIVE_PROBE_END__' + +type WslContext = OrcadRemoteContext + +function pidCommand(context: WslContext, fullVersion: string): string { + const pidFile = joinRemotePath( + context.host, + managedOrcadInstallDir(context, fullVersion), + ORCAD_PID_FILENAME + ) + return `cat ${quotePosixShell(pidFile)} 2>/dev/null || true` +} + +function parsePid(output: string): number | null { + const pid = Number.parseInt(output, 10) + return Number.isSafeInteger(pid) && pid > 0 ? pid : null +} + +export async function readWslOrcadPid( + context: WslContext, + fullVersion: string, + signal?: AbortSignal +): Promise { + return parsePid( + await execCommand(context.connection, pidCommand(context, fullVersion), { signal }) + ) +} + +/** Each section ends with a marker on its own line; the newline before it is the probe's own. */ +export function wslOrcadActiveProbeCommand(context: WslContext, fullVersion: string): string { + const installDir = managedOrcadInstallDir(context, fullVersion) + const end = `printf '\\n%s\\n' ${SECTION_END}` + return [ + orcadLivenessProbeCommand(context.host, installDir), + end, + readOrcadReadinessNowCommand(context.host, installDir), + end, + pidCommand(context, fullVersion) + ].join('\n') +} + +export async function probeWslActiveOrcad( + context: WslContext, + localOrcadDir: string, + fullVersion: string, + signal?: AbortSignal +): Promise<{ readiness: ServeReadiness; pid: number | null }> { + const output = await execOrcadRemote( + { conn: context.connection, host: context.host, signal }, + wslOrcadActiveProbeCommand(context, fullVersion) + ) + const sections = output.split(`\n${SECTION_END}\n`) + if (sections.length !== 3) { + throw new OrcadActiveReadinessError( + 'unverifiable', + `orcad ${fullVersion} process state is unverifiable.` + ) + } + const [liveness = '', readiness = '', pid = ''] = sections + const expectation = { buildHash: computeLocalOrcadBuildHash(localOrcadDir), fullVersion } + assertActiveOrcadLive(liveness, expectation) + return { + readiness: activeOrcadReadinessFromOutput(context.host, readiness, expectation), + pid: parsePid(pid) + } +} diff --git a/src/main/wsl/wsl-orcad-connect-state-snapshot.test.ts b/src/main/wsl/wsl-orcad-connect-state-snapshot.test.ts index 292a9e870d8..5457ac15029 100644 --- a/src/main/wsl/wsl-orcad-connect-state-snapshot.test.ts +++ b/src/main/wsl/wsl-orcad-connect-state-snapshot.test.ts @@ -135,7 +135,7 @@ describe.skipIf(process.platform === 'win32')('WSL connect state snapshot fallba mockExec.mockImplementationOnce( async () => 'UNREADABLE\n__ORCA_WSL_CONNECT_STATE_END__\nABSENT\n__ORCA_WSL_CONNECT_STATE_END__\n' + - 'ABSENT\n__ORCA_WSL_CONNECT_STATE_END__\nOPEN\n' + 'ABSENT\n__ORCA_WSL_CONNECT_STATE_END__\nNONE\n__ORCA_WSL_CONNECT_STATE_END__\nOPEN\n' ) const reader = createWslOrcadConnectStateReader() @@ -144,7 +144,42 @@ describe.skipIf(process.platform === 'win32')('WSL connect state snapshot fallba }) }) +describe.runIf(process.platform === 'linux')('WSL connect state snapshot on a Linux guest', () => { + it("reads the lock holder's identity in the same command", async () => { + write('.orca/orcad.lock', `${JSON.stringify({ pid: process.pid, version: '0.1.0' })}\n`) + const reader = createWslOrcadConnectStateReader() + + await reader.readActivationRecord(target()) + + expect(mockExec).toHaveBeenCalledOnce() + const lock = reader.snapshot()?.instanceLock + expect(lock?.state === 'present' && lock.holder?.pid).toBe(process.pid) + expect(lock?.state === 'present' && lock.holder?.guest.orcad).toMatch(`${process.pid}:`) + }) +}) + describe('parseWslOrcadConnectState', () => { + const end = '\n__ORCA_WSL_CONNECT_STATE_END__\n' + const stat = (pid: number) => + `${pid} (node) S ${Array.from({ length: 18 }, () => '0').join(' ')} 77 0` + const identity = (pid: number) => + `PID ${pid}\n0f6e1d2c-3b4a-4958-8776-655443322110\n--orca-identity--\n${stat(1)}\n` + + `--orca-identity--\n${stat(pid)}\n--orca-identity--\n` + const answer = (holder: string) => + `ABSENT${end}ABSENT${end}PRESENT\n{"pid":9}${end}${holder}${end}OPEN\n` + + it('attaches the holder identity to a present lock, and nothing it cannot parse', () => { + const lock = parseWslOrcadConnectState(answer(identity(9)))?.instanceLock + expect(lock).toMatchObject({ state: 'present', holder: { pid: 9, guest: { orcad: '9:77' } } }) + expect(parseWslOrcadConnectState(answer('NONE'))?.instanceLock).toEqual({ + state: 'present', + raw: '{"pid":9}' + }) + expect(parseWslOrcadConnectState(answer('PID 9\nnoise'))?.instanceLock).not.toHaveProperty( + 'holder' + ) + }) + it('rejects a truncated answer', () => { expect(parseWslOrcadConnectState('ABSENT\n__ORCA_WSL_CONNECT_STATE_END__\n')).toBeNull() }) diff --git a/src/main/wsl/wsl-orcad-connect-state-snapshot.ts b/src/main/wsl/wsl-orcad-connect-state-snapshot.ts index ebbb3a4d234..7b68ec1681c 100644 --- a/src/main/wsl/wsl-orcad-connect-state-snapshot.ts +++ b/src/main/wsl/wsl-orcad-connect-state-snapshot.ts @@ -22,6 +22,11 @@ import { import { execOrcadRemote } from '../managed-server/install/orcad-remote-runtime-control' import { probeInstallLockExistsCommand } from '../managed-server/install/remote-install-lock-commands' import { ORCAD_LOCK_FILE_NAME } from '../orcad/orcad-instance-lock' +import { + parseWslGuestIdentity, + wslGuestIdentityShellCommand, + type WslGuestIdentity +} from './wsl-guest-process-identity' import { quotePosixShell } from '../../shared/shell-argument-quoting' import type { OrcadHostTransport } from '../ssh/orcad-host-transport' import { @@ -34,7 +39,12 @@ import { const RECORD_MAX_BYTES = 64 * 1024 const SECTION_END = '__ORCA_WSL_CONNECT_STATE_END__' -export type WslOrcadInstanceLockRead = { state: 'absent' } | { state: 'present'; raw: string } +/** The guest's identity read for the pid the lock named, so liveness needs no second exec. */ +export type WslOrcadLockHolderIdentity = { pid: number; guest: WslGuestIdentity } + +export type WslOrcadInstanceLockRead = + | { state: 'absent' } + | { state: 'present'; raw: string; holder?: WslOrcadLockHolderIdentity } export type WslOrcadConnectStateSnapshot = { activation: OrcadActivationReadResult @@ -75,10 +85,18 @@ export function wslOrcadConnectStateCommand( `printf '\\n%s\\n' ${SECTION_END}` ) } + const lock = quotePosixShell(paths.instanceLock) + // The pid as digits only; the parser drops the identity unless it matches the parsed lock. + const holderIdentity = + `orca_lock_pid=$(sed -n 's/.*"pid"[[:space:]]*:[[:space:]]*\\([0-9][0-9]*\\).*/\\1/p' ${lock} 2>/dev/null | head -n 1); ` + + `if [ -n "$orca_lock_pid" ]; then printf 'PID %s\\n' "$orca_lock_pid"; ` + + `${wslGuestIdentityShellCommand('"$orca_lock_pid"')}; else printf '%s' NONE; fi; ` + + `printf '\\n%s\\n' ${SECTION_END}` return [ read(paths.activation), read(paths.transaction), read(paths.instanceLock), + holderIdentity, probeInstallLockExistsCommand(host, paths.fence) ].join('; ') } @@ -95,14 +113,27 @@ function parseSection(text: string): Section | null { return text.startsWith('PRESENT\n') ? { state: 'present', raw: text.slice(8) } : null } +function withHolderIdentity( + lock: { state: 'present'; raw: string }, + section: string +): WslOrcadInstanceLockRead { + const match = /^PID (\d+)\n([\s\S]*)$/.exec(section) + if (!match) { + return lock + } + const pid = Number(match[1]) + const guest = parseWslGuestIdentity(match[2] ?? '') + return guest && Number.isSafeInteger(pid) ? { ...lock, holder: { pid, guest } } : lock +} + /** Null unless the output has exactly the shape the command prints. */ export function parseWslOrcadConnectState(output: string): WslOrcadConnectStateSnapshot | null { const parts = output.split(`\n${SECTION_END}\n`) - if (parts.length !== 4) { + if (parts.length !== 5) { return null } const [activation, transaction, instanceLock] = parts.slice(0, 3).map(parseSection) - const fence = parts[3]?.trim() + const fence = parts[4]?.trim() if (!activation || !transaction || !instanceLock || (fence !== 'LOCKED' && fence !== 'OPEN')) { return null } @@ -115,7 +146,12 @@ export function parseWslOrcadConnectState(output: string): WslOrcadConnectStateS transaction.state !== 'unreadable' && parseOrcadActivationTransaction(transaction.state === 'present' ? transaction.raw : null) .state === 'absent', - instanceLock: instanceLock.state === 'unreadable' ? null : instanceLock, + instanceLock: + instanceLock.state === 'unreadable' + ? null + : instanceLock.state === 'present' + ? withHolderIdentity(instanceLock, parts[3] ?? '') + : instanceLock, fenceOpen: fence === 'OPEN' } } diff --git a/src/main/wsl/wsl-orcad-deploy-app-update.test.ts b/src/main/wsl/wsl-orcad-deploy-app-update.test.ts new file mode 100644 index 00000000000..172223f30c9 --- /dev/null +++ b/src/main/wsl/wsl-orcad-deploy-app-update.test.ts @@ -0,0 +1,619 @@ +// Split from wsl-orcad-deploy.test.ts: the connect across an app update (E1), with the same host mocks. +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { OrcadHostTransport } from '../ssh/orcad-host-transport' +import type * as ActiveProbe from './wsl-orcad-active-probe' +import type * as ManagedRuntimeContext from '../managed-server/lifecycle/orcad-managed-runtime-context' +import type * as RemoteVersionedInstall from '../managed-server/install/remote-versioned-install' +import { + emptyOrcadActivationRecord, + type OrcadActivationRecord +} from '../managed-server/activation/orcad-activation-record' +import { getRemoteHostPlatform } from '../ssh/ssh-remote-platform' +import type { WslDistroHolder } from './wsl-distro-holder' +import type { WslOrcadHostTarget } from './wsl-orcad-host-target' + +const mocks = vi.hoisted(() => ({ + steps: new Array(), + exec: vi.fn(), + resolveContext: vi.fn(), + recover: vi.fn(), + deploy: vi.fn(), + probe: vi.fn(), + prune: vi.fn(), + wake: vi.fn(), + adopt: vi.fn(), + incumbent: vi.fn(), + adoptRunning: vi.fn(), + localVersion: vi.fn(), + readRecord: vi.fn(), + transaction: vi.fn(async (): Promise => null) +})) +vi.mock('../managed-server/activation/orcad-activation-transaction-store', () => ({ + readOrcadActivationTransaction: mocks.transaction +})) +vi.mock('../managed-server/activation/orcad-activation-record-store', () => ({ + readOrcadActivationRecord: mocks.readRecord, + orcadActivationPath: (_host: unknown, home: string) => `${home}/.orca-remote/orcad-active.json` +})) +vi.mock('../managed-server/install/remote-versioned-install', async (importOriginal) => ({ + ...(await importOriginal()), + readLocalFullVersion: mocks.localVersion +})) +vi.mock('./wsl-orcad-instance-adoption', () => ({ adoptUnrecordedWslOrcad: mocks.adopt })) +vi.mock('../ssh/orcad-running-version-adoption', () => ({ + adoptRunningOrcadVersion: mocks.adoptRunning +})) + +vi.mock('../../shared/app-environment', () => ({ + getAppEnvironment: () => ({ getVersion: () => '1.5.0', getPath: () => '/user-data' }) +})) +vi.mock('../ssh/remote-exec/remote-exec-command', () => ({ execCommand: mocks.exec })) +vi.mock('../managed-server/install/orcad-remote-context', () => ({ + resolveOrcadRemoteContext: mocks.resolveContext +})) +vi.mock('../managed-server/activation/orcad-activation-recovery', () => ({ + recoverInterruptedOrcadActivation: mocks.recover +})) +vi.mock('../orcad/orcad-artifact-materializer', () => ({ + materializeOrcadArtifact: async () => '/local/orcad' +})) +vi.mock('../ssh/orcad-remote-deploy', () => ({ deployOrcad: mocks.deploy })) +vi.mock('./wsl-orcad-active-probe', async (importOriginal) => ({ + ...(await importOriginal()), + probeWslActiveOrcad: async (...args: unknown[]) => ({ + readiness: await mocks.probe(...args), + pid: 4242 + }) +})) +vi.mock('../managed-server/lifecycle/orcad-managed-version-gc', () => ({ + provenLiveDaemonVersion: () => null, + pruneManagedOrcadVersions: mocks.prune +})) +vi.mock('../managed-server/lifecycle/orcad-managed-wake', () => ({ + wakeStoppedManagedOrcad: mocks.wake +})) +vi.mock('../managed-server/lifecycle/orcad-managed-runtime-context', async (importOriginal) => ({ + ...(await importOriginal()), + probeManagedOrcadReadiness: mocks.probe +})) + +import { deployAndLaunchWslOrcad } from './wsl-orcad-deploy' +import { resetOrcadUpdateFailuresForTests } from '../managed-server/lifecycle/orcad-managed-update-failure-memory' + +beforeEach(() => resetOrcadUpdateFailuresForTests()) + +const host = getRemoteHostPlatform('linux-x64') +const READINESS = { + runtimeId: 'r1', + boundEndpoint: 'ws://127.0.0.1:6790', + advertisedEndpoint: null, + managedWslCliReconciliation: 'settled' as const, + pairing: { available: false as const, reason: 'disabled_by_operator', guidance: '' } +} + +function holder(): WslDistroHolder { + return { + ensureHeld: vi.fn(async (distro: string) => { + mocks.steps.push(`hold:${distro}`) + return { + outcome: 'spawned' as const, + record: { version: 1 as const, distro, pid: 9, creationTimeMs: 1, startedAtMs: 1 } + } + }), + adoptHolders: vi.fn(), + release: vi.fn(), + releaseAll: vi.fn(), + holderState: vi.fn() + } +} + +function transport(target: WslOrcadHostTarget): OrcadHostTransport { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: every host call is mocked at its module. + return { getTarget: () => target } as unknown as OrcadHostTransport +} + +function launch(kernel = '5.15.167.4-microsoft-standard-WSL2', options = {}) { + mocks.exec.mockImplementation(async (_conn: unknown, command: string) => { + if (command.includes('uname -r')) { + mocks.steps.push('uname') + return `__ORCA_WSL_KERNEL__ ${kernel}\n__ORCA_WSL_HOME__ /home/u\n\n__ORCA_REMOTE_PLATFORM__ Linux x86_64\n\n__ORCA_WSL_LIBC__\nldd (GNU libc) 2.39\n` + } + const end = '\n__ORCA_WSL_CONNECT_STATE_END__\n' + return command.includes(end.trim()) ? `ABSENT${end}ABSENT${end}ABSENT${end}OPEN\n` : '4242\n' + }) + return deployAndLaunchWslOrcad('Ubuntu-24.04', { + holder: holder(), + createTransport: transport, + readRegistryId: async () => '0283592d-be56-40d4-b935-3dc18c3aa007', + readIncumbent: mocks.incumbent, + ...options + }) +} + +describe('deployAndLaunchWslOrcad across an app update (E1)', () => { + const INCUMBENT_READINESS = { + ...READINESS, + runtimeId: 'r-old', + boundEndpoint: 'ws://127.0.0.1:6791' + } + const LIVE = { + liveSessions: 2, + startedSinceActivation: 0, + daemonProtocolVersion: 3 + } + const activeOld = { + ...emptyOrcadActivationRecord(), + active: '1.4.0+aa', + activeAppVersion: '1.4.0', + activatedAt: '2026-10-12T01:00:00.000Z' + } + + beforeEach(() => { + vi.clearAllMocks() + mocks.steps.length = 0 + mocks.resolveContext.mockImplementation(async (target: WslOrcadHostTarget, connection) => ({ + activationRecord: activeOld, + serverTarget: 'linux-x64-glibc', + connection, + host, + remoteHome: '/home/u', + target, + userDataDir: '/home/u/.orca' + })) + mocks.recover.mockResolvedValue({ outcome: 'none' }) + mocks.adopt.mockResolvedValue({ outcome: 'none' }) + mocks.probe.mockResolvedValue(READINESS) + mocks.localVersion.mockReturnValue('1.5.0+ab') + mocks.incumbent.mockResolvedValue({ census: LIVE, readiness: INCUMBENT_READINESS }) + mocks.readRecord.mockResolvedValue(activeOld) + }) + + it('plans the update on the census the running orcad reports, not an unknown one', async () => { + mocks.deploy.mockResolvedValueOnce({ + outcome: 'installed-and-activated', + fullVersion: '1.5.0+ab', + verdict: {} + }) + await expect(launch()).resolves.toMatchObject({ + outcome: 'ready', + fullVersion: '1.5.0+ab' + }) + expect(mocks.incumbent).toHaveBeenCalledTimes(1) + expect(mocks.deploy.mock.calls[0][0]).toMatchObject({ + census: LIVE, + censusRecord: activeOld + }) + }) + + it('keeps the incumbent serving, with a forceable deferral, while terminals run', async () => { + mocks.deploy.mockResolvedValueOnce({ + outcome: 'installed-not-activated', + fullVersion: '1.5.0+ab', + code: 'orcad_update_terminals_running', + reason: '2 terminals are running' + }) + await expect(launch()).resolves.toEqual({ + outcome: 'ready', + distro: 'Ubuntu-24.04', + distroId: '0283592d-be56-40d4-b935-3dc18c3aa007', + port: 6791, + fullVersion: '1.4.0+aa', + pid: 4242, + readiness: INCUMBENT_READINESS, + holder: 'spawned', + deferredUpdate: { + candidateVersion: '1.5.0+ab', + code: 'orcad_update_terminals_running', + reason: '2 terminals are running', + forceable: true + } + }) + expect(mocks.exec).toHaveBeenLastCalledWith( + expect.anything(), + "cat '/home/u/.orca-remote/orcad-1.4.0+aa/.orcad-pid' 2>/dev/null || true", + expect.anything() + ) + // Keep the installed candidate for later explicit Force activation. + expect(mocks.prune).not.toHaveBeenCalled() + expect(mocks.probe).not.toHaveBeenCalled() + }) + + it('does not serve an incumbent another client replaced during the deploy', async () => { + mocks.readRecord.mockResolvedValueOnce({ ...activeOld, active: '1.4.5+bb' }) + mocks.deploy.mockResolvedValueOnce({ + outcome: 'installed-not-activated', + fullVersion: '1.5.0+ab', + code: 'orcad_update_terminals_running', + reason: 'terminals are running' + }) + await expect(launch()).resolves.toMatchObject({ outcome: 'deferred', forceable: true }) + }) + + it('forwards force to the update planner', async () => { + mocks.deploy.mockResolvedValueOnce({ + outcome: 'installed-and-activated', + fullVersion: '1.5.0+ab', + verdict: {} + }) + await launch(undefined, { intent: 'force' }) + expect(mocks.deploy.mock.calls[0][0]).toMatchObject({ + force: true, + census: LIVE + }) + }) + + it('reports a deferral when the incumbent could not be proven serving', async () => { + mocks.incumbent.mockResolvedValueOnce({ + census: { + liveSessions: null, + startedSinceActivation: null, + daemonProtocolVersion: null + }, + readiness: null + }) + mocks.deploy.mockResolvedValueOnce({ + outcome: 'installed-not-activated', + fullVersion: '1.5.0+ab', + code: 'orcad_update_terminal_census_unavailable', + reason: 'no census' + }) + await expect(launch()).resolves.toMatchObject({ + outcome: 'deferred', + forceable: false + }) + }) + + it('does not serve the incumbent past a refusal that is not a wait', async () => { + mocks.deploy.mockResolvedValueOnce({ + outcome: 'installed-not-activated', + fullVersion: '1.5.0+ab', + code: 'orcad_activation_policy_refused', + reason: 'newer Orca ran this host' + }) + await expect(launch()).resolves.toMatchObject({ + outcome: 'deferred', + forceable: true + }) + }) + + describe('the SSH auto-update rules', () => { + const newer = { + ...activeOld, + active: '2.0.0+cd', + activeAppVersion: '2.0.0', + previous: '1.5.0+ab' + } + const NEWER_READINESS = { ...INCUMBENT_READINESS, runtimeId: 'r-new' } + const policyRefused = { + outcome: 'installed-not-activated', + fullVersion: '1.5.0+ab', + code: 'orcad_activation_policy_refused', + reason: 'newer Orca ran this host' + } + + function withRecord(record: OrcadActivationRecord): void { + mocks.resolveContext.mockImplementation(async (target: WslOrcadHostTarget, connection) => ({ + activationRecord: record, + serverTarget: 'linux-x64-glibc', + connection, + host, + remoteHome: '/home/u', + target, + userDataDir: '/home/u/.orca' + })) + mocks.readRecord.mockResolvedValue(record) + } + + const adoptedAs = (version: string, readiness: typeof INCUMBENT_READINESS) => ({ + version, + readiness, + reprobe: async () => readiness + }) + + it('serves a newer build another Orca activated instead of deploying over it', async () => { + withRecord(newer) + mocks.deploy.mockResolvedValue(policyRefused) + mocks.adoptRunning.mockResolvedValueOnce(adoptedAs('2.0.0+cd', NEWER_READINESS)) + await expect(launch()).resolves.toEqual({ + outcome: 'ready', + distro: 'Ubuntu-24.04', + distroId: '0283592d-be56-40d4-b935-3dc18c3aa007', + port: 6791, + fullVersion: '2.0.0+cd', + pid: 4242, + readiness: NEWER_READINESS, + holder: 'spawned' + }) + expect(mocks.adoptRunning).toHaveBeenCalledWith(expect.anything(), newer, '1.5.0+ab') + expect(mocks.deploy).not.toHaveBeenCalled() + expect(mocks.incumbent).not.toHaveBeenCalled() + expect(mocks.prune).not.toHaveBeenCalled() + expect(mocks.exec).toHaveBeenLastCalledWith( + expect.anything(), + "cat '/home/u/.orca-remote/orcad-2.0.0+cd/.orcad-pid' 2>/dev/null || true", + expect.anything() + ) + }) + + it('does not re-activate the build this host rolled back from', async () => { + withRecord({ ...activeOld, rolledBackFrom: '1.5.0+ab' }) + mocks.deploy.mockResolvedValue({ + outcome: 'installed-and-activated', + fullVersion: '1.5.0+ab', + verdict: {} + }) + mocks.adoptRunning.mockResolvedValueOnce(adoptedAs('1.4.0+aa', INCUMBENT_READINESS)) + await expect(launch()).resolves.toMatchObject({ + outcome: 'ready', + fullVersion: '1.4.0+aa', + readiness: INCUMBENT_READINESS, + deferredUpdate: { + candidateVersion: '1.5.0+ab', + code: 'orcad_activation_policy_refused', + reason: expect.stringMatching(/rolled back from 1\.5\.0\+ab/), + forceable: true + } + }) + expect(mocks.deploy).not.toHaveBeenCalled() + }) + + it('labels the launch with the build the adoption found serving, not the stale record', async () => { + withRecord(newer) + mocks.adoptRunning.mockResolvedValueOnce(adoptedAs('2.1.0+ef', NEWER_READINESS)) + await expect(launch()).resolves.toMatchObject({ outcome: 'ready', fullVersion: '2.1.0+ef' }) + }) + + it('defers while another run holds the host fence', async () => { + withRecord(newer) + mocks.adoptRunning.mockResolvedValueOnce({ + outcome: 'deferred', + candidateVersion: '1.5.0+ab', + code: 'orcad_activation_fence_busy', + reason: 'busy' + }) + await expect(launch()).resolves.toEqual({ + outcome: 'deferred', + distro: 'Ubuntu-24.04', + candidateVersion: '1.5.0+ab', + code: 'orcad_activation_fence_busy', + reason: 'busy', + forceable: false + }) + expect(mocks.deploy).not.toHaveBeenCalled() + }) + + it('lets an explicit force deploy past both skips', async () => { + withRecord({ ...newer, rolledBackFrom: '1.5.0+ab' }) + mocks.deploy.mockResolvedValueOnce({ + outcome: 'installed-and-activated', + fullVersion: '1.5.0+ab', + verdict: {} + }) + await expect(launch(undefined, { intent: 'force' })).resolves.toMatchObject({ + outcome: 'ready', + fullVersion: '1.5.0+ab' + }) + expect(mocks.adoptRunning).not.toHaveBeenCalled() + }) + + it('refuses the rolled-back build under the fence when nothing could be adopted', async () => { + const rolledBack = { ...activeOld, rolledBackFrom: '1.5.0+ab' } + withRecord(rolledBack) + mocks.deploy.mockResolvedValueOnce(policyRefused) + mocks.adoptRunning.mockResolvedValueOnce(null) + await expect(launch()).resolves.toMatchObject({ outcome: 'deferred', forceable: true }) + const { admitRecord } = mocks.deploy.mock.calls[0][0] + expect(admitRecord(rolledBack, '1.5.0+ab')).toMatch(/rolled back from 1\.5\.0\+ab/) + expect(admitRecord(activeOld, '1.5.0+ab')).toBeNull() + expect(admitRecord(newer, '1.5.0+ab')).toMatch(/Orca 2\.0\.0 last ran/) + }) + + const rejected = { + outcome: 'installed-not-activated', + fullVersion: '1.5.0+ab', + code: 'orcad_candidate_readiness_rejected', + reason: 'the candidate never became ready; the incumbent was restored' + } + + it('does not retry a candidate rejected earlier this launch, and serves the incumbent', async () => { + mocks.deploy.mockResolvedValue(rejected) + await expect(launch()).resolves.toMatchObject({ outcome: 'deferred' }) + mocks.adoptRunning.mockResolvedValueOnce(adoptedAs('1.4.0+aa', INCUMBENT_READINESS)) + await expect(launch()).resolves.toMatchObject({ + outcome: 'ready', + fullVersion: '1.4.0+aa', + deferredUpdate: { + candidateVersion: '1.5.0+ab', + code: 'orcad_candidate_readiness_rejected', + reason: rejected.reason, + forceable: true + } + }) + expect(mocks.deploy).toHaveBeenCalledTimes(1) + await launch(undefined, { intent: 'force' }) + expect(mocks.deploy).toHaveBeenCalledTimes(2) + }) + + it('serves the restored incumbent on the very connect that rejected the candidate', async () => { + mocks.deploy.mockResolvedValueOnce(rejected) + mocks.adoptRunning.mockResolvedValueOnce(adoptedAs('1.4.0+aa', INCUMBENT_READINESS)) + await expect(launch()).resolves.toMatchObject({ + outcome: 'ready', + fullVersion: '1.4.0+aa', + deferredUpdate: { code: 'orcad_candidate_readiness_rejected', forceable: true } + }) + }) + + it('retries a failed candidate on Update, and still defers over live terminals', async () => { + mocks.deploy.mockResolvedValueOnce(rejected) + await launch() + mocks.adoptRunning.mockClear() + mocks.deploy.mockResolvedValueOnce({ + outcome: 'installed-not-activated', + fullVersion: '1.5.0+ab', + code: 'orcad_update_terminals_running', + reason: '2 terminals are running' + }) + await expect(launch(undefined, { intent: 'update' })).resolves.toMatchObject({ + outcome: 'ready', + fullVersion: '1.4.0+aa', + deferredUpdate: { code: 'orcad_update_terminals_running', forceable: true } + }) + expect(mocks.adoptRunning).not.toHaveBeenCalled() + // Update still gates on incumbent terminals; only Force bypasses them. + expect(mocks.deploy.mock.calls[1][0]).toMatchObject({ force: false, census: LIVE }) + const { admitRecord } = mocks.deploy.mock.calls[1][0] + expect(admitRecord({ ...activeOld, rolledBackFrom: '1.5.0+ab' }, '1.5.0+ab')).toBeNull() + }) + + it('reports the rejection as is when the incumbent was not restored', async () => { + mocks.deploy.mockResolvedValueOnce(rejected) + mocks.transaction.mockResolvedValueOnce({ transactionId: 't1' }) + await expect(launch()).resolves.toMatchObject({ + outcome: 'deferred', + code: 'orcad_candidate_readiness_rejected', + reason: rejected.reason + }) + // Unreadable host state proves no rollback restoration. + mocks.deploy.mockResolvedValueOnce(rejected) + mocks.transaction.mockRejectedValueOnce(new Error('wsl.exe did not answer')) + await expect(launch(undefined, { intent: 'update' })).resolves.toMatchObject({ + outcome: 'deferred', + code: 'orcad_candidate_readiness_rejected' + }) + expect(mocks.adoptRunning).not.toHaveBeenCalled() + }) + + it('serves a newer host on Update rather than downgrading it, and only Force deploys', async () => { + withRecord(newer) + mocks.adoptRunning.mockResolvedValueOnce(adoptedAs('2.0.0+cd', NEWER_READINESS)) + await expect(launch(undefined, { intent: 'update' })).resolves.toMatchObject({ + outcome: 'ready', + fullVersion: '2.0.0+cd' + }) + expect(mocks.deploy).not.toHaveBeenCalled() + mocks.adoptRunning.mockResolvedValueOnce(null) + mocks.deploy.mockResolvedValue(policyRefused) + await launch(undefined, { intent: 'update' }) + expect(mocks.deploy.mock.calls[0][0].admitRecord(newer, '1.5.0+ab')).toMatch(/last ran/) + await launch(undefined, { intent: 'force' }) + expect(mocks.deploy.mock.calls[1][0]).toMatchObject({ force: true }) + expect(mocks.deploy.mock.calls[1][0].admitRecord(newer, '1.5.0+ab')).toBeNull() + }) + + it('still skips a failed build on the next connect when an explicit Update only waited', async () => { + mocks.deploy.mockResolvedValueOnce(rejected) + await launch() + mocks.deploy.mockResolvedValueOnce({ + outcome: 'installed-not-activated', + fullVersion: '1.5.0+ab', + code: 'orcad_update_terminals_running', + reason: 'busy' + }) + await launch(undefined, { intent: 'update' }) + mocks.adoptRunning.mockResolvedValueOnce(adoptedAs('1.4.0+aa', INCUMBENT_READINESS)) + await expect(launch()).resolves.toMatchObject({ outcome: 'ready', fullVersion: '1.4.0+aa' }) + expect(mocks.deploy).toHaveBeenCalledTimes(2) + }) + + it('names a newer host refusal apart from a rolled-back one, so only Force is offered', async () => { + withRecord(newer) + mocks.adoptRunning.mockResolvedValueOnce(null) + mocks.deploy.mockResolvedValueOnce(policyRefused) + await expect(launch(undefined, { intent: 'update' })).resolves.toMatchObject({ + outcome: 'deferred', + code: 'orcad_host_newer', + forceable: true + }) + withRecord({ ...activeOld, rolledBackFrom: '1.5.0+ab' }) + mocks.adoptRunning.mockResolvedValueOnce(null) + mocks.deploy.mockResolvedValueOnce(policyRefused) + await expect(launch()).resolves.toMatchObject({ + outcome: 'deferred', + code: 'orcad_activation_policy_refused' + }) + }) + + it('keeps retrying an update that only waited, or that a newer host refused', async () => { + mocks.deploy.mockResolvedValue({ + outcome: 'installed-not-activated', + fullVersion: '1.5.0+ab', + code: 'orcad_update_terminals_running', + reason: 'busy' + }) + await launch() + await launch() + mocks.deploy.mockResolvedValue(policyRefused) + await launch() + await launch() + expect(mocks.deploy).toHaveBeenCalledTimes(4) + expect(mocks.adoptRunning).not.toHaveBeenCalled() + }) + + it('lets the next connect try again once an explicit Force went through', async () => { + mocks.deploy.mockResolvedValueOnce(rejected) + await launch() + mocks.deploy.mockResolvedValueOnce({ + outcome: 'installed-and-activated', + fullVersion: '1.5.0+ab', + verdict: {} + }) + await launch(undefined, { intent: 'force' }) + mocks.deploy.mockResolvedValueOnce(rejected) + await launch() + expect(mocks.deploy).toHaveBeenCalledTimes(3) + }) + + it('deploys on the record the wake read when the skipped build stopped meanwhile', async () => { + withRecord(newer) + const stopped = { ...newer, active: null, activatedAt: null } + mocks.adoptRunning.mockResolvedValueOnce(null) + mocks.readRecord.mockResolvedValueOnce(stopped) + mocks.deploy.mockResolvedValueOnce(policyRefused) + await launch() + expect(mocks.incumbent).not.toHaveBeenCalled() + expect(mocks.deploy.mock.calls[0][0]).toMatchObject({ + censusRecord: stopped, + census: { liveSessions: 0 } + }) + }) + + it('leaves a host stopped after a rollback for Force, not the build it rolled back from', async () => { + // Desktop lacks the previous bundle and cannot restart it. + const stopped = { + ...emptyOrcadActivationRecord(), + previous: '1.4.0+aa', + rolledBackFrom: '1.5.0+ab' + } + withRecord(stopped) + mocks.deploy.mockResolvedValueOnce(policyRefused) + await expect(launch()).resolves.toMatchObject({ outcome: 'deferred', forceable: true }) + expect(mocks.adoptRunning).not.toHaveBeenCalled() + expect(mocks.deploy.mock.calls[0][0].admitRecord(stopped, '1.5.0+ab')).toMatch( + /rolled back from 1\.5\.0\+ab/ + ) + }) + + it('updates an older host as before, without asking to adopt it', async () => { + mocks.deploy.mockResolvedValueOnce({ + outcome: 'installed-and-activated', + fullVersion: '1.5.0+ab', + verdict: {} + }) + await expect(launch()).resolves.toMatchObject({ outcome: 'ready', fullVersion: '1.5.0+ab' }) + expect(mocks.adoptRunning).not.toHaveBeenCalled() + }) + }) + + it('never asks the incumbent when it already runs this build', async () => { + mocks.localVersion.mockReturnValue('1.4.0+aa') + mocks.deploy.mockResolvedValueOnce({ + outcome: 'already-active', + fullVersion: '1.4.0+aa' + }) + mocks.wake.mockResolvedValueOnce({ outcome: 'serving' }) + await launch() + expect(mocks.incumbent).not.toHaveBeenCalled() + expect(mocks.deploy.mock.calls[0][0].census.liveSessions).toBeNull() + }) +}) diff --git a/src/main/wsl/wsl-orcad-deploy.test.ts b/src/main/wsl/wsl-orcad-deploy.test.ts index 2286da3acc9..a814d0f7e7c 100644 --- a/src/main/wsl/wsl-orcad-deploy.test.ts +++ b/src/main/wsl/wsl-orcad-deploy.test.ts @@ -1,11 +1,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import type { OrcadHostTransport } from '../ssh/orcad-host-transport' +import type * as ActiveProbe from './wsl-orcad-active-probe' import type * as ManagedRuntimeContext from '../managed-server/lifecycle/orcad-managed-runtime-context' import type * as RemoteVersionedInstall from '../managed-server/install/remote-versioned-install' -import { - emptyOrcadActivationRecord, - type OrcadActivationRecord -} from '../managed-server/activation/orcad-activation-record' +import { emptyOrcadActivationRecord } from '../managed-server/activation/orcad-activation-record' import { getRemoteHostPlatform } from '../ssh/ssh-remote-platform' import { OrcadActiveReadinessError } from '../ssh/orcad-active-readiness' import { WslDistroNotRunningError } from './wsl-orcad-connection-errors' @@ -58,6 +56,13 @@ vi.mock('../orcad/orcad-artifact-materializer', () => ({ materializeOrcadArtifact: async () => '/local/orcad' })) vi.mock('../ssh/orcad-remote-deploy', () => ({ deployOrcad: mocks.deploy })) +vi.mock('./wsl-orcad-active-probe', async (importOriginal) => ({ + ...(await importOriginal()), + probeWslActiveOrcad: async (...args: unknown[]) => ({ + readiness: await mocks.probe(...args), + pid: 4242 + }) +})) vi.mock('../managed-server/lifecycle/orcad-managed-version-gc', () => ({ provenLiveDaemonVersion: () => null, pruneManagedOrcadVersions: mocks.prune @@ -189,11 +194,10 @@ describe('deployAndLaunchWslOrcad', () => { distro: 'Ubuntu-24.04', label: 'Ubuntu-24.04' }) - expect(mocks.exec).toHaveBeenLastCalledWith( - expect.anything(), - "cat '/home/u/.orca-remote/orcad-1.5.0+ab/.orcad-pid' 2>/dev/null || true", - expect.anything() - ) + // The serving probe read the pid; nothing reads it again. + expect( + mocks.exec.mock.calls.some(([, command]) => String(command).includes('.orcad-pid')) + ).toBe(false) }) it.each([ @@ -201,7 +205,7 @@ describe('deployAndLaunchWslOrcad', () => { ['a held fence', 'ABSENT', 'LOCKED', { transactionAbsent: true, fenceOpen: false }], ['a pending journal', 'PRESENT\n{}', 'OPEN', { transactionAbsent: false, fenceOpen: true }] ])('passes its connect-state read (%s) on as hints', async (_, journal, fence, preRead) => { - snapshotAnswer = `${section('ABSENT')}${section(journal)}${section('ABSENT')}${fence}\n` + snapshotAnswer = `${section('ABSENT')}${section(journal)}${section('ABSENT')}${section('NONE')}${fence}\n` const resolve = mocks.resolveContext.getMockImplementation() mocks.resolveContext.mockImplementationOnce(async (target, connection, signal, h, known) => { const context = await resolve?.(target, connection, signal, h, known) @@ -373,491 +377,3 @@ describe('deployAndLaunchWslOrcad', () => { expect(mocks.probe).not.toHaveBeenCalled() }) }) - -describe('deployAndLaunchWslOrcad across an app update (E1)', () => { - const INCUMBENT_READINESS = { - ...READINESS, - runtimeId: 'r-old', - boundEndpoint: 'ws://127.0.0.1:6791' - } - const LIVE = { - liveSessions: 2, - startedSinceActivation: 0, - daemonProtocolVersion: 3 - } - const activeOld = { - ...emptyOrcadActivationRecord(), - active: '1.4.0+aa', - activeAppVersion: '1.4.0', - activatedAt: '2026-10-12T01:00:00.000Z' - } - - beforeEach(() => { - vi.clearAllMocks() - mocks.steps.length = 0 - mocks.resolveContext.mockImplementation(async (target: WslOrcadHostTarget, connection) => ({ - activationRecord: activeOld, - serverTarget: 'linux-x64-glibc', - connection, - host, - remoteHome: '/home/u', - target, - userDataDir: '/home/u/.orca' - })) - mocks.recover.mockResolvedValue({ outcome: 'none' }) - mocks.adopt.mockResolvedValue({ outcome: 'none' }) - mocks.probe.mockResolvedValue(READINESS) - mocks.localVersion.mockReturnValue('1.5.0+ab') - mocks.incumbent.mockResolvedValue({ census: LIVE, readiness: INCUMBENT_READINESS }) - mocks.readRecord.mockResolvedValue(activeOld) - }) - - it('plans the update on the census the running orcad reports, not an unknown one', async () => { - mocks.deploy.mockResolvedValueOnce({ - outcome: 'installed-and-activated', - fullVersion: '1.5.0+ab', - verdict: {} - }) - await expect(launch()).resolves.toMatchObject({ - outcome: 'ready', - fullVersion: '1.5.0+ab' - }) - expect(mocks.incumbent).toHaveBeenCalledTimes(1) - expect(mocks.deploy.mock.calls[0][0]).toMatchObject({ - census: LIVE, - censusRecord: activeOld - }) - }) - - it('keeps the incumbent serving, with a forceable deferral, while terminals run', async () => { - mocks.deploy.mockResolvedValueOnce({ - outcome: 'installed-not-activated', - fullVersion: '1.5.0+ab', - code: 'orcad_update_terminals_running', - reason: '2 terminals are running' - }) - await expect(launch()).resolves.toEqual({ - outcome: 'ready', - distro: 'Ubuntu-24.04', - distroId: '0283592d-be56-40d4-b935-3dc18c3aa007', - port: 6791, - fullVersion: '1.4.0+aa', - pid: 4242, - readiness: INCUMBENT_READINESS, - holder: 'spawned', - deferredUpdate: { - candidateVersion: '1.5.0+ab', - code: 'orcad_update_terminals_running', - reason: '2 terminals are running', - forceable: true - } - }) - expect(mocks.exec).toHaveBeenLastCalledWith( - expect.anything(), - "cat '/home/u/.orca-remote/orcad-1.4.0+aa/.orcad-pid' 2>/dev/null || true", - expect.anything() - ) - // Keep the installed candidate for later explicit Force activation. - expect(mocks.prune).not.toHaveBeenCalled() - expect(mocks.probe).not.toHaveBeenCalled() - }) - - it('does not serve an incumbent another client replaced during the deploy', async () => { - mocks.readRecord.mockResolvedValueOnce({ ...activeOld, active: '1.4.5+bb' }) - mocks.deploy.mockResolvedValueOnce({ - outcome: 'installed-not-activated', - fullVersion: '1.5.0+ab', - code: 'orcad_update_terminals_running', - reason: 'terminals are running' - }) - await expect(launch()).resolves.toMatchObject({ outcome: 'deferred', forceable: true }) - }) - - it('forwards force to the update planner', async () => { - mocks.deploy.mockResolvedValueOnce({ - outcome: 'installed-and-activated', - fullVersion: '1.5.0+ab', - verdict: {} - }) - await launch(undefined, { intent: 'force' }) - expect(mocks.deploy.mock.calls[0][0]).toMatchObject({ - force: true, - census: LIVE - }) - }) - - it('reports a deferral when the incumbent could not be proven serving', async () => { - mocks.incumbent.mockResolvedValueOnce({ - census: { - liveSessions: null, - startedSinceActivation: null, - daemonProtocolVersion: null - }, - readiness: null - }) - mocks.deploy.mockResolvedValueOnce({ - outcome: 'installed-not-activated', - fullVersion: '1.5.0+ab', - code: 'orcad_update_terminal_census_unavailable', - reason: 'no census' - }) - await expect(launch()).resolves.toMatchObject({ - outcome: 'deferred', - forceable: false - }) - }) - - it('does not serve the incumbent past a refusal that is not a wait', async () => { - mocks.deploy.mockResolvedValueOnce({ - outcome: 'installed-not-activated', - fullVersion: '1.5.0+ab', - code: 'orcad_activation_policy_refused', - reason: 'newer Orca ran this host' - }) - await expect(launch()).resolves.toMatchObject({ - outcome: 'deferred', - forceable: true - }) - }) - - describe('the SSH auto-update rules', () => { - const newer = { - ...activeOld, - active: '2.0.0+cd', - activeAppVersion: '2.0.0', - previous: '1.5.0+ab' - } - const NEWER_READINESS = { ...INCUMBENT_READINESS, runtimeId: 'r-new' } - const policyRefused = { - outcome: 'installed-not-activated', - fullVersion: '1.5.0+ab', - code: 'orcad_activation_policy_refused', - reason: 'newer Orca ran this host' - } - - function withRecord(record: OrcadActivationRecord): void { - mocks.resolveContext.mockImplementation(async (target: WslOrcadHostTarget, connection) => ({ - activationRecord: record, - serverTarget: 'linux-x64-glibc', - connection, - host, - remoteHome: '/home/u', - target, - userDataDir: '/home/u/.orca' - })) - mocks.readRecord.mockResolvedValue(record) - } - - const adoptedAs = (version: string, readiness: typeof INCUMBENT_READINESS) => ({ - version, - readiness, - reprobe: async () => readiness - }) - - it('serves a newer build another Orca activated instead of deploying over it', async () => { - withRecord(newer) - mocks.deploy.mockResolvedValue(policyRefused) - mocks.adoptRunning.mockResolvedValueOnce(adoptedAs('2.0.0+cd', NEWER_READINESS)) - await expect(launch()).resolves.toEqual({ - outcome: 'ready', - distro: 'Ubuntu-24.04', - distroId: '0283592d-be56-40d4-b935-3dc18c3aa007', - port: 6791, - fullVersion: '2.0.0+cd', - pid: 4242, - readiness: NEWER_READINESS, - holder: 'spawned' - }) - expect(mocks.adoptRunning).toHaveBeenCalledWith(expect.anything(), newer, '1.5.0+ab') - expect(mocks.deploy).not.toHaveBeenCalled() - expect(mocks.incumbent).not.toHaveBeenCalled() - expect(mocks.prune).not.toHaveBeenCalled() - expect(mocks.exec).toHaveBeenLastCalledWith( - expect.anything(), - "cat '/home/u/.orca-remote/orcad-2.0.0+cd/.orcad-pid' 2>/dev/null || true", - expect.anything() - ) - }) - - it('does not re-activate the build this host rolled back from', async () => { - withRecord({ ...activeOld, rolledBackFrom: '1.5.0+ab' }) - mocks.deploy.mockResolvedValue({ - outcome: 'installed-and-activated', - fullVersion: '1.5.0+ab', - verdict: {} - }) - mocks.adoptRunning.mockResolvedValueOnce(adoptedAs('1.4.0+aa', INCUMBENT_READINESS)) - await expect(launch()).resolves.toMatchObject({ - outcome: 'ready', - fullVersion: '1.4.0+aa', - readiness: INCUMBENT_READINESS, - deferredUpdate: { - candidateVersion: '1.5.0+ab', - code: 'orcad_activation_policy_refused', - reason: expect.stringMatching(/rolled back from 1\.5\.0\+ab/), - forceable: true - } - }) - expect(mocks.deploy).not.toHaveBeenCalled() - }) - - it('labels the launch with the build the adoption found serving, not the stale record', async () => { - withRecord(newer) - mocks.adoptRunning.mockResolvedValueOnce(adoptedAs('2.1.0+ef', NEWER_READINESS)) - await expect(launch()).resolves.toMatchObject({ outcome: 'ready', fullVersion: '2.1.0+ef' }) - }) - - it('defers while another run holds the host fence', async () => { - withRecord(newer) - mocks.adoptRunning.mockResolvedValueOnce({ - outcome: 'deferred', - candidateVersion: '1.5.0+ab', - code: 'orcad_activation_fence_busy', - reason: 'busy' - }) - await expect(launch()).resolves.toEqual({ - outcome: 'deferred', - distro: 'Ubuntu-24.04', - candidateVersion: '1.5.0+ab', - code: 'orcad_activation_fence_busy', - reason: 'busy', - forceable: false - }) - expect(mocks.deploy).not.toHaveBeenCalled() - }) - - it('lets an explicit force deploy past both skips', async () => { - withRecord({ ...newer, rolledBackFrom: '1.5.0+ab' }) - mocks.deploy.mockResolvedValueOnce({ - outcome: 'installed-and-activated', - fullVersion: '1.5.0+ab', - verdict: {} - }) - await expect(launch(undefined, { intent: 'force' })).resolves.toMatchObject({ - outcome: 'ready', - fullVersion: '1.5.0+ab' - }) - expect(mocks.adoptRunning).not.toHaveBeenCalled() - }) - - it('refuses the rolled-back build under the fence when nothing could be adopted', async () => { - const rolledBack = { ...activeOld, rolledBackFrom: '1.5.0+ab' } - withRecord(rolledBack) - mocks.deploy.mockResolvedValueOnce(policyRefused) - mocks.adoptRunning.mockResolvedValueOnce(null) - await expect(launch()).resolves.toMatchObject({ outcome: 'deferred', forceable: true }) - const { admitRecord } = mocks.deploy.mock.calls[0][0] - expect(admitRecord(rolledBack, '1.5.0+ab')).toMatch(/rolled back from 1\.5\.0\+ab/) - expect(admitRecord(activeOld, '1.5.0+ab')).toBeNull() - expect(admitRecord(newer, '1.5.0+ab')).toMatch(/Orca 2\.0\.0 last ran/) - }) - - const rejected = { - outcome: 'installed-not-activated', - fullVersion: '1.5.0+ab', - code: 'orcad_candidate_readiness_rejected', - reason: 'the candidate never became ready; the incumbent was restored' - } - - it('does not retry a candidate rejected earlier this launch, and serves the incumbent', async () => { - mocks.deploy.mockResolvedValue(rejected) - await expect(launch()).resolves.toMatchObject({ outcome: 'deferred' }) - mocks.adoptRunning.mockResolvedValueOnce(adoptedAs('1.4.0+aa', INCUMBENT_READINESS)) - await expect(launch()).resolves.toMatchObject({ - outcome: 'ready', - fullVersion: '1.4.0+aa', - deferredUpdate: { - candidateVersion: '1.5.0+ab', - code: 'orcad_candidate_readiness_rejected', - reason: rejected.reason, - forceable: true - } - }) - expect(mocks.deploy).toHaveBeenCalledTimes(1) - await launch(undefined, { intent: 'force' }) - expect(mocks.deploy).toHaveBeenCalledTimes(2) - }) - - it('serves the restored incumbent on the very connect that rejected the candidate', async () => { - mocks.deploy.mockResolvedValueOnce(rejected) - mocks.adoptRunning.mockResolvedValueOnce(adoptedAs('1.4.0+aa', INCUMBENT_READINESS)) - await expect(launch()).resolves.toMatchObject({ - outcome: 'ready', - fullVersion: '1.4.0+aa', - deferredUpdate: { code: 'orcad_candidate_readiness_rejected', forceable: true } - }) - }) - - it('retries a failed candidate on Update, and still defers over live terminals', async () => { - mocks.deploy.mockResolvedValueOnce(rejected) - await launch() - mocks.adoptRunning.mockClear() - mocks.deploy.mockResolvedValueOnce({ - outcome: 'installed-not-activated', - fullVersion: '1.5.0+ab', - code: 'orcad_update_terminals_running', - reason: '2 terminals are running' - }) - await expect(launch(undefined, { intent: 'update' })).resolves.toMatchObject({ - outcome: 'ready', - fullVersion: '1.4.0+aa', - deferredUpdate: { code: 'orcad_update_terminals_running', forceable: true } - }) - expect(mocks.adoptRunning).not.toHaveBeenCalled() - // Update still gates on incumbent terminals; only Force bypasses them. - expect(mocks.deploy.mock.calls[1][0]).toMatchObject({ force: false, census: LIVE }) - const { admitRecord } = mocks.deploy.mock.calls[1][0] - expect(admitRecord({ ...activeOld, rolledBackFrom: '1.5.0+ab' }, '1.5.0+ab')).toBeNull() - }) - - it('reports the rejection as is when the incumbent was not restored', async () => { - mocks.deploy.mockResolvedValueOnce(rejected) - mocks.transaction.mockResolvedValueOnce({ transactionId: 't1' }) - await expect(launch()).resolves.toMatchObject({ - outcome: 'deferred', - code: 'orcad_candidate_readiness_rejected', - reason: rejected.reason - }) - // Unreadable host state proves no rollback restoration. - mocks.deploy.mockResolvedValueOnce(rejected) - mocks.transaction.mockRejectedValueOnce(new Error('wsl.exe did not answer')) - await expect(launch(undefined, { intent: 'update' })).resolves.toMatchObject({ - outcome: 'deferred', - code: 'orcad_candidate_readiness_rejected' - }) - expect(mocks.adoptRunning).not.toHaveBeenCalled() - }) - - it('serves a newer host on Update rather than downgrading it, and only Force deploys', async () => { - withRecord(newer) - mocks.adoptRunning.mockResolvedValueOnce(adoptedAs('2.0.0+cd', NEWER_READINESS)) - await expect(launch(undefined, { intent: 'update' })).resolves.toMatchObject({ - outcome: 'ready', - fullVersion: '2.0.0+cd' - }) - expect(mocks.deploy).not.toHaveBeenCalled() - mocks.adoptRunning.mockResolvedValueOnce(null) - mocks.deploy.mockResolvedValue(policyRefused) - await launch(undefined, { intent: 'update' }) - expect(mocks.deploy.mock.calls[0][0].admitRecord(newer, '1.5.0+ab')).toMatch(/last ran/) - await launch(undefined, { intent: 'force' }) - expect(mocks.deploy.mock.calls[1][0]).toMatchObject({ force: true }) - expect(mocks.deploy.mock.calls[1][0].admitRecord(newer, '1.5.0+ab')).toBeNull() - }) - - it('still skips a failed build on the next connect when an explicit Update only waited', async () => { - mocks.deploy.mockResolvedValueOnce(rejected) - await launch() - mocks.deploy.mockResolvedValueOnce({ - outcome: 'installed-not-activated', - fullVersion: '1.5.0+ab', - code: 'orcad_update_terminals_running', - reason: 'busy' - }) - await launch(undefined, { intent: 'update' }) - mocks.adoptRunning.mockResolvedValueOnce(adoptedAs('1.4.0+aa', INCUMBENT_READINESS)) - await expect(launch()).resolves.toMatchObject({ outcome: 'ready', fullVersion: '1.4.0+aa' }) - expect(mocks.deploy).toHaveBeenCalledTimes(2) - }) - - it('names a newer host refusal apart from a rolled-back one, so only Force is offered', async () => { - withRecord(newer) - mocks.adoptRunning.mockResolvedValueOnce(null) - mocks.deploy.mockResolvedValueOnce(policyRefused) - await expect(launch(undefined, { intent: 'update' })).resolves.toMatchObject({ - outcome: 'deferred', - code: 'orcad_host_newer', - forceable: true - }) - withRecord({ ...activeOld, rolledBackFrom: '1.5.0+ab' }) - mocks.adoptRunning.mockResolvedValueOnce(null) - mocks.deploy.mockResolvedValueOnce(policyRefused) - await expect(launch()).resolves.toMatchObject({ - outcome: 'deferred', - code: 'orcad_activation_policy_refused' - }) - }) - - it('keeps retrying an update that only waited, or that a newer host refused', async () => { - mocks.deploy.mockResolvedValue({ - outcome: 'installed-not-activated', - fullVersion: '1.5.0+ab', - code: 'orcad_update_terminals_running', - reason: 'busy' - }) - await launch() - await launch() - mocks.deploy.mockResolvedValue(policyRefused) - await launch() - await launch() - expect(mocks.deploy).toHaveBeenCalledTimes(4) - expect(mocks.adoptRunning).not.toHaveBeenCalled() - }) - - it('lets the next connect try again once an explicit Force went through', async () => { - mocks.deploy.mockResolvedValueOnce(rejected) - await launch() - mocks.deploy.mockResolvedValueOnce({ - outcome: 'installed-and-activated', - fullVersion: '1.5.0+ab', - verdict: {} - }) - await launch(undefined, { intent: 'force' }) - mocks.deploy.mockResolvedValueOnce(rejected) - await launch() - expect(mocks.deploy).toHaveBeenCalledTimes(3) - }) - - it('deploys on the record the wake read when the skipped build stopped meanwhile', async () => { - withRecord(newer) - const stopped = { ...newer, active: null, activatedAt: null } - mocks.adoptRunning.mockResolvedValueOnce(null) - mocks.readRecord.mockResolvedValueOnce(stopped) - mocks.deploy.mockResolvedValueOnce(policyRefused) - await launch() - expect(mocks.incumbent).not.toHaveBeenCalled() - expect(mocks.deploy.mock.calls[0][0]).toMatchObject({ - censusRecord: stopped, - census: { liveSessions: 0 } - }) - }) - - it('leaves a host stopped after a rollback for Force, not the build it rolled back from', async () => { - // Desktop lacks the previous bundle and cannot restart it. - const stopped = { - ...emptyOrcadActivationRecord(), - previous: '1.4.0+aa', - rolledBackFrom: '1.5.0+ab' - } - withRecord(stopped) - mocks.deploy.mockResolvedValueOnce(policyRefused) - await expect(launch()).resolves.toMatchObject({ outcome: 'deferred', forceable: true }) - expect(mocks.adoptRunning).not.toHaveBeenCalled() - expect(mocks.deploy.mock.calls[0][0].admitRecord(stopped, '1.5.0+ab')).toMatch( - /rolled back from 1\.5\.0\+ab/ - ) - }) - - it('updates an older host as before, without asking to adopt it', async () => { - mocks.deploy.mockResolvedValueOnce({ - outcome: 'installed-and-activated', - fullVersion: '1.5.0+ab', - verdict: {} - }) - await expect(launch()).resolves.toMatchObject({ outcome: 'ready', fullVersion: '1.5.0+ab' }) - expect(mocks.adoptRunning).not.toHaveBeenCalled() - }) - }) - - it('never asks the incumbent when it already runs this build', async () => { - mocks.localVersion.mockReturnValue('1.4.0+aa') - mocks.deploy.mockResolvedValueOnce({ - outcome: 'already-active', - fullVersion: '1.4.0+aa' - }) - mocks.wake.mockResolvedValueOnce({ outcome: 'serving' }) - await launch() - expect(mocks.incumbent).not.toHaveBeenCalled() - expect(mocks.deploy.mock.calls[0][0].census.liveSessions).toBeNull() - }) -}) diff --git a/src/main/wsl/wsl-orcad-deploy.ts b/src/main/wsl/wsl-orcad-deploy.ts index e9c77165d94..42a93355208 100644 --- a/src/main/wsl/wsl-orcad-deploy.ts +++ b/src/main/wsl/wsl-orcad-deploy.ts @@ -13,9 +13,7 @@ import { orcadBoundPort } from '../managed-server/lifecycle/orcad-managed-bound- import { isForceableOrcadDeferral, managedOrcadDeployPolicy, - managedOrcadInstallDir, - managedOrcadSlot, - probeManagedOrcadReadiness + managedOrcadSlot } from '../managed-server/lifecycle/orcad-managed-runtime-context' import { isWaitingOrcadUpdateDeferral } from '../managed-server/lifecycle/orcad-managed-auto-update' import { clearOrcadUpdateFailedThisLaunch } from '../managed-server/lifecycle/orcad-managed-update-failure-memory' @@ -28,10 +26,6 @@ import { OrcadActiveReadinessError } from '../ssh/orcad-active-readiness' import { WslDistroNotRunningError } from './wsl-orcad-connection-errors' import { deployOrcad, type OrcadDeployOptions } from '../ssh/orcad-remote-deploy' import { isWslMoveUnderwayOnHost } from '../persistence/moving-wsl-projects/wsl-move-running-moves' -import { ORCAD_PID_FILENAME } from '../managed-server/install/orcad-remote-host-support' -import { quotePosixShell } from '../../shared/shell-argument-quoting' -import { execCommand } from '../ssh/remote-exec/remote-exec-command' -import { joinRemotePath } from '../ssh/ssh-remote-platform' import { readLocalFullVersion } from '../managed-server/install/remote-versioned-install' import { createWslDistroHolder, type WslDistroHolder } from './wsl-distro-holder' import { wslDistroHolderKey } from './wsl-distro-holder-record' @@ -40,6 +34,7 @@ import type { WslOrcadHostTarget } from './wsl-orcad-host-target' import { createWslOrcadHostTransport, type WslOrcadHostTransport } from './wsl-orcad-host-transport' import { holdWslDistroWithContext } from './wsl-orcad-held-context' import { deployWslOrcadWithOneFence } from './wsl-orcad-one-fence-deploy' +import { probeWslActiveOrcad, readWslOrcadPid } from './wsl-orcad-active-probe' import { wslOrcadConnectStateHints } from './wsl-orcad-connect-state-snapshot' import { readWslOrcadIncumbent, type WslOrcadIncumbent } from './wsl-orcad-incumbent' import { adoptUnrecordedWslOrcad } from './wsl-orcad-instance-adoption' @@ -129,6 +124,8 @@ async function launchOverTransport( ): Promise { const { signal } = options let ownBuildVersion: string | null = null + // Set by the serving probe; readyLaunch reads the pid itself for any other version. + let probed: { fullVersion: string; pid: number | null } | null = null const conn = withOwnBuildLaunchEnv(transport, () => ownBuildVersion) // Acquire a holder before launch or WSL can idle-stop orcad's distro. const holder = options.holder ?? createWslDistroHolder({ userDataPath: userDataPath() }) @@ -264,8 +261,11 @@ async function launchOverTransport( return { outcome: 'deferred', distro: name, ...deferredUpdate } } clearOrcadUpdateFailedThisLaunch(failureHost) - const probe = () => - probeManagedOrcadReadiness(context, localOrcadDir, deployed.fullVersion, signal) + const probe = async () => { + const answer = await probeWslActiveOrcad(context, localOrcadDir, deployed.fullVersion, signal) + probed = { fullVersion: deployed.fullVersion, pid: answer.pid } + return answer.readiness + } let readiness: ServeReadiness if (deployed.outcome === 'already-active') { // A stopped distro ended the active slot with it; the one-fence deploy already woke it. @@ -298,22 +298,18 @@ async function launchOverTransport( fullVersion: string readiness: ServeReadiness }): Promise> { - const pidFile = joinRemotePath( - context.host, - managedOrcadInstallDir(context, serving.fullVersion), - ORCAD_PID_FILENAME - ) - const pid = Number.parseInt( - await execCommand(conn, `cat ${quotePosixShell(pidFile)} 2>/dev/null || true`, { signal }), - 10 - ) + // The serving probe already read it; an admission that serves an incumbent did not. + const pid = + probed?.fullVersion === serving.fullVersion + ? probed.pid + : await readWslOrcadPid(context, serving.fullVersion, signal) return { outcome: 'ready', distro: name, distroId: await distroId, port: orcadBoundPort(serving.readiness) ?? ORCAD_MANAGED_REMOTE_PORT, fullVersion: serving.fullVersion, - pid: Number.isSafeInteger(pid) && pid > 0 ? pid : null, + pid, readiness: serving.readiness, holder: held.outcome, ...(adoption.outcome === 'adopted' diff --git a/src/main/wsl/wsl-orcad-instance-adoption.test.ts b/src/main/wsl/wsl-orcad-instance-adoption.test.ts index c6f94976ba0..116fd04a641 100644 --- a/src/main/wsl/wsl-orcad-instance-adoption.test.ts +++ b/src/main/wsl/wsl-orcad-instance-adoption.test.ts @@ -248,6 +248,26 @@ describe('adopting an orcad the activation record does not name', () => { expect(host.writes).toEqual([]) }) + it("takes the connect snapshot's holder identity only for the pid the lock names", async () => { + const guest = { distroInstance: `${BOOT_ID}:1`, orcad: null } + // The snapshot saw that pid gone: no identity exec, and nothing to adopt. + const gone = { state: 'present' as const, raw: lockFor(540, OLD), holder: { pid: 540, guest } } + host.lock = lockFor(540, OLD) + host.alive.add(540) + await expect(adoptUnrecordedWslOrcad(slot, record(), () => NOW, gone)).resolves.toEqual({ + outcome: 'none' + }) + expect(host.commands.some((command) => command.includes('/proc/'))).toBe(false) + // An identity read for another pid is ignored; the guest is asked about this one. + const other = { ...gone, holder: { pid: 541, guest } } + host.stopTarget = stopTargetFor(540, OLD) + await expect(adoptUnrecordedWslOrcad(slot, record(), () => NOW, other)).resolves.toMatchObject({ + outcome: 'adopted', + pid: 540 + }) + expect(host.commands.some((command) => command.includes('/proc/540/stat'))).toBe(true) + }) + it('matches a lock that recorded its boot id in upper case', async () => { host.lock = JSON.stringify({ ...JSON.parse(lockFor(530, OLD)), bootId: BOOT_ID.toUpperCase() }) host.alive.add(530) diff --git a/src/main/wsl/wsl-orcad-instance-adoption.ts b/src/main/wsl/wsl-orcad-instance-adoption.ts index 6649b4a0afd..ae23b40646f 100644 --- a/src/main/wsl/wsl-orcad-instance-adoption.ts +++ b/src/main/wsl/wsl-orcad-instance-adoption.ts @@ -34,7 +34,11 @@ import { } from '../managed-server/install/remote-install-model' import { quotePosixShell } from '../../shared/shell-argument-quoting' import { joinRemotePath } from '../ssh/ssh-remote-platform' -import { readWslGuestIdentity, wslGuestOrcadIncarnation } from './wsl-guest-process-identity' +import { + readWslGuestIdentity, + wslGuestOrcadIncarnation, + type WslGuestIdentity +} from './wsl-guest-process-identity' import type { WslOrcadInstanceLockRead } from './wsl-orcad-connect-state-snapshot' const LOCK_RECORD_MAX_BYTES = 64 * 1024 @@ -67,7 +71,10 @@ async function readLiveLockHolder( return null } // Adopt only proven-live holders; otherwise let the host's instance lock arbitrate launch. - return (await lockHolderLiveness(slot, lock)) === 'live' + // The snapshot's identity counts only if it was read for this same pid. + const known = + preRead?.state === 'present' && preRead.holder?.pid === lock.pid ? preRead.holder.guest : null + return (await lockHolderLiveness(slot, lock, known)) === 'live' ? { pid: lock.pid, version: lock.version } : null } @@ -77,9 +84,10 @@ type LockHolderLiveness = 'live' | 'unverifiable' | 'exited' /** A PID is no identity after a distro restart reuses it: the lock's boot id and start ticks are. */ async function lockHolderLiveness( slot: OrcadSlotOptions, - lock: OrcadLockRecord + lock: OrcadLockRecord, + known: WslGuestIdentity | null = null ): Promise { - const guest = await readWslGuestIdentity(slot.conn, lock.pid) + const guest = known ?? (await readWslGuestIdentity(slot.conn, lock.pid)) if (!guest) { return 'unverifiable' } diff --git a/src/main/wsl/wsl-orcad-real-distro-idle-release-lane.test.ts b/src/main/wsl/wsl-orcad-real-distro-idle-release-lane.test.ts index eb51513714d..0796dacf0f5 100644 --- a/src/main/wsl/wsl-orcad-real-distro-idle-release-lane.test.ts +++ b/src/main/wsl/wsl-orcad-real-distro-idle-release-lane.test.ts @@ -66,6 +66,8 @@ const RECEIPT_DIR = process.env.ORCA_WSL_ORCAD_LANE_RECEIPTS ?? join(WORK_DIR, ' const IDLE_TIMEOUT_MS = 15_000 // Idle period, one holder poll, then WSL's own stop of a distro with no client left. const RELEASE_BUDGET_MS = IDLE_TIMEOUT_MS + WSL_DISTRO_HOLDER_POLL_SECONDS * 1000 + 90_000 +// Round-trip counts are exact; five runs keep the ms medians readable on a noisy runner. +const RESTART_RUNS = 5 const HELD_WITH_TERMINAL_MS = IDLE_TIMEOUT_MS + WSL_DISTRO_HOLDER_POLL_SECONDS * 1000 + 45_000 type Receipt = Record @@ -541,7 +543,7 @@ async function profileOrcadRestart(repoPath: string): Promise { } /** - * Distro restart (VM warm) to first command output, median of 3 each, alternating so neither path + * Distro restart (VM warm) to first command output, median of 5 each, alternating so neither path * always runs on a warmer VM. */ async function distroRestartBaseline(): Promise { @@ -551,13 +553,13 @@ async function distroRestartBaseline(): Promise { const host: Awaited>[] = [] const bridge: Receipt[] = [] const conpty: Receipt[] = [] - for (let i = 0; i < 3; i++) { + for (let i = 0; i < RESTART_RUNS; i++) { host.push(await restartThroughHost(repoPath)) bridge.push(await restartThroughBridgeExec()) conpty.push(await restartThroughBridgeConpty()) } const median = (runs: Receipt[], key: string): number => - runs.map((run) => Number(run[key])).toSorted((a, b) => a - b)[1] ?? 0 + runs.map((run) => Number(run[key])).toSorted((a, b) => a - b)[(runs.length - 1) >> 1] ?? 0 receipt.distroRestartToFirstOutput = { host, bridgeExecLowerBound: bridge, @@ -598,7 +600,7 @@ describe.runIf(RUN && process.platform === 'win32')('WSL distro idle release', ( RELEASE_BUDGET_MS + 2 * HELD_WITH_TERMINAL_MS + 6 * 60_000 ) - // After the idle phase: six distro restarts, a warm-up and one profiled restart, each bounded. + // After the idle phase: fifteen distro restarts, a warm-up and one profiled restart, each bounded. it( 'measures distro restart to first output for the WSL host and an old-bridge exec lower bound', distroRestartBaseline, diff --git a/src/main/wsl/wsl-orcad-update-move-hold-driver.test.ts b/src/main/wsl/wsl-orcad-update-move-hold-driver.test.ts index 7f06e298ef9..a6bb05f9bf8 100644 --- a/src/main/wsl/wsl-orcad-update-move-hold-driver.test.ts +++ b/src/main/wsl/wsl-orcad-update-move-hold-driver.test.ts @@ -13,6 +13,7 @@ import { ORCAD_UPDATE_WSL_MOVE_RUNNING_CODE } from '../../shared/wsl-host-types' import type { WslMoveResult } from '../../shared/wsl-move/wsl-move-driver-types' import type { OrcadActivationRecord } from '../managed-server/activation/orcad-activation-record' import { resetOrcadUpdateFailuresForTests } from '../managed-server/lifecycle/orcad-managed-update-failure-memory' +import type * as ActiveProbe from './wsl-orcad-active-probe' import type * as ManagedRuntimeContext from '../managed-server/lifecycle/orcad-managed-runtime-context' import type * as RemoteVersionedInstall from '../managed-server/install/remote-versioned-install' import { testState } from '../persistence/persistence-test-harness' @@ -95,6 +96,14 @@ vi.mock('../managed-server/lifecycle/orcad-managed-runtime-context', async (impo probeManagedOrcadReadiness: host.probe })) +vi.mock('./wsl-orcad-active-probe', async (importOriginal) => ({ + ...(await importOriginal()), + probeWslActiveOrcad: async (...args: unknown[]) => ({ + readiness: await host.probe(...args), + pid: 4242 + }) +})) + import { deployAndLaunchWslOrcad } from './wsl-orcad-deploy' const DISTRO = WSL_MOVE_RIG_REQUEST.distro