From ea7af2dd7cbfd149b76720b46dcb4e81c76c0234 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:55:04 -0700 Subject: [PATCH] fix(ssh): prune superseded staged server builds when an update defers (#35881) * fix(ssh): prune superseded staged server builds when an update defers GC ran only after an activation, so a host whose updates kept deferring over live terminals kept one staged orcad slot per release. Prune on deferral too, keeping the active, previous, live-daemon and newly staged builds. Refs #34149 Co-authored-by: Claude Opus 5.5 * fix(ssh): keep recently staged builds; route and harden the deferred-slot e2e Desktops on different releases can share a host, so the deferral prune keeps complete builds staged within three days instead of deleting a peer's candidate and forcing it to re-upload. Windows hosts, which cannot date their builds, keep them all. Refs #34149 Co-authored-by: Claude Opus 5.5 * fix(ssh): read a build's staging age as unknown when find cannot date it A find without -mtime used to read as an old build and let the deferral prune remove a peer's candidate. Report an error instead, which keeps every candidate; a build without .install-complete reads as not recent. Co-authored-by: Claude Opus 5.5 * test(ssh): pin that an undatable staged build keeps every build Co-authored-by: Claude Opus 5.5 * test(ssh): fold the deferred-slot e2e into the update-anyway spec Keeps the routing table under its size limit and runs the check on the same deferred-update sources. Also drop the unused readiness argument from pruneSupersededOrcadCandidates. Co-authored-by: Claude Opus 5.5 * fix(ssh): report GC as unfinished when it kept a recently staged build Callers that memoize a clean pass would otherwise never prune the build once it ages. Co-authored-by: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- config/scripts/pr-e2e-source-routing.mjs | 2 +- .../install/orcad-remote-gc.test.ts | 40 ++++++++++ .../managed-server/install/orcad-remote-gc.ts | 67 +++++++++++++++- .../lifecycle/orcad-managed-version-gc.ts | 45 ++++++++++- src/main/ssh/orcad-runtime-maintenance.ts | 6 +- .../wsl/wsl-orcad-deploy-app-update.test.ts | 3 +- src/main/wsl/wsl-orcad-deploy.test.ts | 3 +- src/main/wsl/wsl-orcad-deploy.ts | 2 + tests/e2e/ssh-orcad-update-anyway.spec.ts | 77 ++++++++++++++++++- 9 files changed, 237 insertions(+), 8 deletions(-) diff --git a/config/scripts/pr-e2e-source-routing.mjs b/config/scripts/pr-e2e-source-routing.mjs index 72c0efbf516..043bafa0d47 100644 --- a/config/scripts/pr-e2e-source-routing.mjs +++ b/config/scripts/pr-e2e-source-routing.mjs @@ -190,7 +190,7 @@ export const PR_E2E_SOURCE_ROUTES = [ matches: (file) => MANAGED_SSH_HARNESS.test(file) || (isProductSource(file) && - /^src\/(?:main\/(?:ssh\/orcad-(?:update-plan|runtime-maintenance|installed-activation|remote-deploy)|managed-server\/lifecycle\/orcad-(?:managed-update-|managed-auto-update|runtime-status))|renderer\/src\/components\/settings\/(?:managed-server-|use-managed-server-lifecycle))/.test( + /^src\/(?:main\/(?:ssh\/orcad-(?:update-plan|runtime-maintenance|installed-activation|remote-deploy)|managed-server\/lifecycle\/orcad-(?:managed-update-|managed-auto-update|runtime-status|managed-version-gc)|managed-server\/install\/(?:orcad-remote-gc|remote-install-gc)|wsl\/wsl-orcad-deploy\.ts)|renderer\/src\/components\/settings\/(?:managed-server-|use-managed-server-lifecycle))/.test( file )) }, diff --git a/src/main/managed-server/install/orcad-remote-gc.test.ts b/src/main/managed-server/install/orcad-remote-gc.test.ts index 531cdb30911..932a43be0ce 100644 --- a/src/main/managed-server/install/orcad-remote-gc.test.ts +++ b/src/main/managed-server/install/orcad-remote-gc.test.ts @@ -43,8 +43,21 @@ function scriptHost(options: { /** Absent by default; raw journal or error overrides the read. */ journal?: string | Error fenceHeld?: boolean + /** Builds staged within the keep window; undefined leaves the answer incomplete. */ + recentlyStaged?: string[] + /** The host could not date some build. */ + ageUnknown?: boolean }): void { mockExec.mockImplementation(async (_conn, command: string) => { + if (command.includes('ORCAD_RECENT_STAGED_DONE')) { + return options.recentlyStaged === undefined + ? '' + : [ + ...options.recentlyStaged.map((name) => `RECENT ${name}`), + ...(options.ageUnknown ? ['ERR'] : []), + 'ORCAD_RECENT_STAGED_DONE' + ].join('\n') + } if (command.includes('__ORCAD_RECORD_PRESENT__') && command.includes('transaction.json')) { if (options.journal instanceof Error) { throw options.journal @@ -246,6 +259,33 @@ describe('orcad GC', () => { expect(inventories()).toBe(1) }) + it('keeps builds another desktop staged recently, and all of them when it cannot tell', async () => { + const args = { + conn, + host, + remoteHome: '/home/u', + currentDirAbsPath: '/home/u/.orca-remote/orcad-0.4.0+dd0', + record: { ...emptyOrcadActivationRecord(), active: '0.1.0+01d' }, + keepStagedWithinDays: 3 + } + const listing = ['orcad-0.1.0+01d', 'orcad-0.2.0+9ee0', 'orcad-0.3.0+cc0', 'orcad-0.4.0+dd0'] + const removed: string[] = [] + scriptHost({ listing, removed, recentlyStaged: ['orcad-0.3.0+cc0'] }) + expect(await gcOldOrcadVersions(args)).toBe(false) + expect(removed).toEqual(['orcad-0.2.0+9ee0']) + + const kept: string[] = [] + scriptHost({ listing, removed: kept }) + await gcOldOrcadVersions(args) + expect(kept).toEqual([]) + + // An undatable build may be another desktop's fresh candidate. + const undated: string[] = [] + scriptHost({ listing, removed: undated, recentlyStaged: [], ageUnknown: true }) + await gcOldOrcadVersions(args) + expect(undated).toEqual([]) + }) + describe('with an activation transaction in flight', () => { const listing = ['orcad-0.1.0+01d', 'orcad-0.2.0+9ee0', 'orcad-0.3.0+cc0'] const record = { ...emptyOrcadActivationRecord(), active: '0.3.0+cc0' } diff --git a/src/main/managed-server/install/orcad-remote-gc.ts b/src/main/managed-server/install/orcad-remote-gc.ts index f9d4bac6069..e0c12e7aea8 100644 --- a/src/main/managed-server/install/orcad-remote-gc.ts +++ b/src/main/managed-server/install/orcad-remote-gc.ts @@ -24,6 +24,8 @@ import { } from '../../ssh/ssh-remote-platform' import { RELAY_REMOTE_DIR } from '../../ssh/remote-exec/remote-frame-protocol' import { orcadWindowsHomeOpCommand } from './orcad-remote-windows-node' +import { quotePosixShell } from '../../../shared/shell-argument-quoting' +import { ORCAD_INSTALL_COMPLETE_FILENAME } from '../../../shared/orcad-artifacts' import { ORCAD_WINDOWS_LIVENESS_MANY_MARKER } from '../host-state/orcad-windows-host-script' type OrcadGcOptions = { @@ -36,16 +38,24 @@ type OrcadGcOptions = { liveDaemonVersion?: string | null /** Current executable hashes both pin runtimes and authorize runtime-store GC. */ nodeRuntimePins?: readonly string[] + /** Keep complete builds staged this recently; another desktop's deferred update may still want its own. */ + keepStagedWithinDays?: number signal?: AbortSignal } /** True when no stale version dir was left behind (skipped, kept or unlistable ones make it false). */ export async function gcOldOrcadVersions(options: OrcadGcOptions): Promise { + const keepDays = options.keepStagedWithinDays + if (keepDays !== undefined && isWindowsRemoteHost(options.host)) { + // The Windows host script cannot date staged builds, so keep them all. + return false + } const transaction = await readOrcadGcTransactionPins(options) if (transaction.state === 'keep-all') { console.warn('[orcad-gc] An activation transaction is unreadable or unjournaled; skipping GC.') return false } + let keptRecent = false const versionsClean = await gcOldRemoteInstallVersions( options.conn, ORCAD_INSTALL_MODEL, @@ -60,7 +70,15 @@ export async function gcOldOrcadVersions(options: OrcadGcOptions): Promise windowsLiveCandidates(options, candidates) } - : {}), + : keepDays !== undefined + ? { + resolveExtraPinnedDirNames: async (candidates) => { + const recent = await recentlyStagedCandidates(options, candidates, keepDays) + keptRecent = recent === null || recent.length > 0 + return recent + } + } + : {}), isDirLive: async (dir) => { try { const probe = await execCommand( @@ -89,7 +107,8 @@ export async function gcOldOrcadVersions(options: OrcadGcOptions): Promise states[index] !== 'DEAD' && states[index] !== ORCAD_NEVER_LAUNCHED ) } + +/** Null on any failure or unanswerable date, which keeps every candidate. */ +async function recentlyStagedCandidates( + options: OrcadGcOptions, + candidates: readonly string[], + days: number +): Promise { + if (!Number.isInteger(days) || days < 1) { + return null + } + const checks = candidates.map((name) => { + const marker = quotePosixShell( + joinRemotePath( + options.host, + options.remoteHome, + RELAY_REMOTE_DIR, + name, + ORCAD_INSTALL_COMPLETE_FILENAME + ) + ) + // A find without -mtime must read as unknown, not as an old build. + return ( + `if [ ! -e ${marker} ]; then :; elif recent=$(find ${marker} -mtime -${days} 2>/dev/null); then ` + + `if [ -n "$recent" ]; then echo RECENT ${quotePosixShell(name)}; fi; else echo ERR; fi;` + ) + }) + try { + const output = await execCommand(options.conn, `${checks.join(' ')} echo ${RECENT_DONE}`, { + signal: options.signal + }) + const lines = output.split('\n').map((line) => line.trim()) + if (!lines.includes(RECENT_DONE) || lines.includes('ERR')) { + return null + } + return candidates.filter((name) => lines.includes(`RECENT ${name}`)) + } catch (error) { + if (isUnconfirmedSshCommandTermination(error)) { + throw error + } + return null + } +} + +const RECENT_DONE = 'ORCAD_RECENT_STAGED_DONE' diff --git a/src/main/managed-server/lifecycle/orcad-managed-version-gc.ts b/src/main/managed-server/lifecycle/orcad-managed-version-gc.ts index a600cd3d54a..12deef0d82b 100644 --- a/src/main/managed-server/lifecycle/orcad-managed-version-gc.ts +++ b/src/main/managed-server/lifecycle/orcad-managed-version-gc.ts @@ -1,7 +1,12 @@ import type { NodeRuntimeTarget } from '../../../shared/node-runtime-pin' import type { ServeReadiness } from '../../server/serve-readiness' import { readOrcadActivationRecord } from '../activation/orcad-activation-record-store' -import { orcadSlotDir, type OrcadSlotOptions } from '../../ssh/orcad-recovery-slot' +import { + orcadSlotDir, + resolveOrcadSlotIdentity, + type OrcadSlotOptions +} from '../../ssh/orcad-recovery-slot' +import { probeActiveOrcadReadiness } from '../../ssh/orcad-active-readiness' import { gcOldOrcadVersions } from '../install/orcad-remote-gc' import { nodeRuntimeStorePins } from '../../ssh/pinned-runtime-host-target' @@ -19,6 +24,7 @@ export type ManagedOrcadPruneArgs = { serverTarget: NodeRuntimeTarget activeVersion: string readiness: ServeReadiness + keepStagedWithinDays?: number } /** True only when GC ran and left no stale version dir behind. */ @@ -38,6 +44,9 @@ export async function pruneManagedOrcadVersions(args: ManagedOrcadPruneArgs): Pr liveDaemonVersion, // Default and compat builds share one runtime store. nodeRuntimePins: nodeRuntimeStorePins(args.serverTarget), + ...(args.keepStagedWithinDays !== undefined + ? { keepStagedWithinDays: args.keepStagedWithinDays } + : {}), signal: args.slot.signal }) } catch (error) { @@ -46,3 +55,37 @@ export async function pruneManagedOrcadVersions(args: ManagedOrcadPruneArgs): Pr return false } } + +/** GC otherwise runs only after an activation, so each deferred release left its staged slot behind (#34149). */ +export async function pruneSupersededOrcadCandidates( + slot: OrcadSlotOptions, + serverTarget: NodeRuntimeTarget, + candidateVersion: string +): Promise { + let readiness: ServeReadiness + try { + const active = (await readOrcadActivationRecord(slot)).active + if (!active) { + return + } + const identity = await resolveOrcadSlotIdentity(slot, active) + readiness = await probeActiveOrcadReadiness( + { ...slot, remoteInstallDir: identity.remoteDir }, + { buildHash: identity.buildHash, fullVersion: identity.version } + ) + } catch (error) { + // Without the incumbent's daemon version a staged tree may still be executing. + console.warn('[orcad-gc] Skipped pruning staged orcad versions:', error) + return + } + await pruneManagedOrcadVersions({ + slot, + serverTarget, + activeVersion: candidateVersion, + readiness, + keepStagedWithinDays: STAGED_BUILD_KEEP_DAYS + }) +} + +// Desktops on different releases share hosts; without a grace period each re-uploads what the other pruned. +const STAGED_BUILD_KEEP_DAYS = 3 diff --git a/src/main/ssh/orcad-runtime-maintenance.ts b/src/main/ssh/orcad-runtime-maintenance.ts index 150026dd0af..8e66e956cfa 100644 --- a/src/main/ssh/orcad-runtime-maintenance.ts +++ b/src/main/ssh/orcad-runtime-maintenance.ts @@ -35,7 +35,10 @@ import { readRemoteOrcadBuildHash } from '../managed-server/install/orcad-remote import { deployOrcad, type OrcadDeployOptions } from './orcad-remote-deploy' import type { OrcadUpdateForce } from './orcad-update-plan' import { ORCAD_ACTIVATION_POLICY_REFUSED_CODE } from './orcad-installed-activation' -import { pruneManagedOrcadVersions } from '../managed-server/lifecycle/orcad-managed-version-gc' +import { + pruneManagedOrcadVersions, + pruneSupersededOrcadCandidates +} from '../managed-server/lifecycle/orcad-managed-version-gc' import { clearOrcadUpdateFailedThisLaunch } from '../managed-server/lifecycle/orcad-managed-update-failure-memory' import { admitOrcadRecordForIntent } from '../managed-server/lifecycle/orcad-managed-update-decision' import { rollbackOrcad } from '../managed-server/install/orcad-remote-rollback' @@ -156,6 +159,7 @@ export async function runManagedOrcadUpdate( if (result.code !== ORCAD_ACTIVATION_POLICY_REFUSED_CODE) { recordManagedOrcadUpdateDeferral(environment.id, deferral) } + await pruneSupersededOrcadCandidates(slot, context.serverTarget, result.fullVersion) return deferral } clearManagedOrcadUpdateDeferral(environment.id) diff --git a/src/main/wsl/wsl-orcad-deploy-app-update.test.ts b/src/main/wsl/wsl-orcad-deploy-app-update.test.ts index 172223f30c9..c692f4b70b6 100644 --- a/src/main/wsl/wsl-orcad-deploy-app-update.test.ts +++ b/src/main/wsl/wsl-orcad-deploy-app-update.test.ts @@ -67,7 +67,8 @@ vi.mock('./wsl-orcad-active-probe', async (importOriginal) => ({ })) vi.mock('../managed-server/lifecycle/orcad-managed-version-gc', () => ({ provenLiveDaemonVersion: () => null, - pruneManagedOrcadVersions: mocks.prune + pruneManagedOrcadVersions: mocks.prune, + pruneSupersededOrcadCandidates: async () => {} })) vi.mock('../managed-server/lifecycle/orcad-managed-wake', () => ({ wakeStoppedManagedOrcad: mocks.wake diff --git a/src/main/wsl/wsl-orcad-deploy.test.ts b/src/main/wsl/wsl-orcad-deploy.test.ts index a8e103d28e7..afd73d74f24 100644 --- a/src/main/wsl/wsl-orcad-deploy.test.ts +++ b/src/main/wsl/wsl-orcad-deploy.test.ts @@ -65,7 +65,8 @@ vi.mock('./wsl-orcad-active-probe', async (importOriginal) => ({ })) vi.mock('../managed-server/lifecycle/orcad-managed-version-gc', () => ({ provenLiveDaemonVersion: () => null, - pruneManagedOrcadVersions: mocks.prune + pruneManagedOrcadVersions: mocks.prune, + pruneSupersededOrcadCandidates: async () => {} })) vi.mock('../managed-server/lifecycle/orcad-managed-wake', () => ({ wakeStoppedManagedOrcad: mocks.wake diff --git a/src/main/wsl/wsl-orcad-deploy.ts b/src/main/wsl/wsl-orcad-deploy.ts index 1c2ea97f011..3fb6098f5fa 100644 --- a/src/main/wsl/wsl-orcad-deploy.ts +++ b/src/main/wsl/wsl-orcad-deploy.ts @@ -21,6 +21,7 @@ import type { OrcadUpdateIntent } from '../managed-server/lifecycle/orcad-manage import type { OrcadActivationRecord } from '../managed-server/activation/orcad-activation-record' import { errorMessage } from '../../shared/error-message' import { pruneWslOrcadVersionsUntilClean } from './wsl-orcad-version-gc-memo' +import { pruneSupersededOrcadCandidates } from '../managed-server/lifecycle/orcad-managed-version-gc' import { wakeAlreadyActiveOrcad } from '../ssh/orcad-already-active-wake' import { OrcadActiveReadinessError } from '../ssh/orcad-active-readiness' import { WslDistroNotRunningError } from './wsl-orcad-connection-errors' @@ -258,6 +259,7 @@ async function launchOverTransport( fullVersion: incumbentVersion, readiness: incumbent.readiness } + await pruneSupersededOrcadCandidates(slot, context.serverTarget, deployed.fullVersion) return { ...(await readyLaunch(serving)), deferredUpdate } } return { outcome: 'deferred', distro: name, ...deferredUpdate } diff --git a/tests/e2e/ssh-orcad-update-anyway.spec.ts b/tests/e2e/ssh-orcad-update-anyway.spec.ts index ed488ebdaaa..b1bf0272fff 100644 --- a/tests/e2e/ssh-orcad-update-anyway.spec.ts +++ b/tests/e2e/ssh-orcad-update-anyway.spec.ts @@ -1,4 +1,4 @@ -/** Prove Update defers on a running process and Update anyway preserves that process, its output, and keyboard input across restart. */ +/** Prove Update defers on a running process, Update anyway preserves it across restart, and repeated deferrals keep one staged build (#34149). */ import { cpSync, mkdtempSync, rmSync } from 'node:fs' import os from 'node:os' import path from 'node:path' @@ -121,3 +121,78 @@ test('Update anyway moves a managed host to the new build and keeps its running rmSync(scratch, { recursive: true, force: true }) } }) + +function slotVersions(exec: (command: string) => string): string[] { + return exec('cd /root/.orca-remote && ls -d orcad-0* 2>/dev/null || true') + .split('\n') + .map((line) => line.trim().replace(/^orcad-/, '')) + .filter(Boolean) + .sort() +} + +function deferredCandidate(page: Page, environmentId: string): Promise { + return page.evaluate( + async (selector) => + (await window.api.runtimeEnvironments.managedOrcad?.getStatus({ selector }))?.deferredUpdate + ?.candidateVersion ?? null, + environmentId + ) +} + +test('a release deferred over running terminals replaces the build the last one staged', async (// oxlint-disable-next-line no-empty-pattern -- Owns its app launch through a restart session. +{}, testInfo) => { + // Deploys a server, then relaunches through two app updates that each stage a build. + test.setTimeout(20 * 60_000) + const scratch = mkdtempSync(path.join(os.tmpdir(), 'orca-deferred-slots-')) + const templateDir = path.join(scratch, 'template') + cpSync(ORCAD_TEMPLATE!, templateDir, { recursive: true }) + try { + await withManagedHost( + testInfo, + async (fixture) => { + const exec = fixture.host.exec! + const workspace = await launchManagedWorkspace(fixture) + const { environmentId } = workspace + const tab = await openTerminalTab(workspace.page, workspace.seeded.worktreeId) + await typeInTerminal(workspace.page, tab, `bash -c 'while :; do echo TICK; sleep 1; done'`) + await expectPaneText(workspace.page, tab, 'TICK') + const deployed = readHostOrcadActivation(exec).active! + + makeOrcadTemplateVariant(templateDir, 'B') + let page = await relaunchManagedApp(fixture) + await expect + .poll(() => deferredCandidate(page, environmentId), { timeout: 4 * 60_000 }) + .not.toBeNull() + const staged = (await deferredCandidate(page, environmentId))! + expect(slotVersions(exec)).toEqual([deployed, staged].sort()) + + // Releases come days apart; a build staged this recently could still be another desktop's. + exec(`touch -d '5 days ago' /root/.orca-remote/orcad-${staged}/.install-complete`) + makeOrcadTemplateVariant(templateDir, 'C') + page = await relaunchManagedApp(fixture) + let next: string | null = null + await expect + .poll( + async () => { + const candidate = await deferredCandidate(page, environmentId) + next = candidate !== staged ? candidate : null + return next + }, + { timeout: 4 * 60_000 } + ) + .not.toBeNull() + expect(readHostOrcadActivation(exec).active).toBe(deployed) + await expect + .poll(() => slotVersions(exec), { timeout: 60_000 }) + .toEqual([deployed, next!].sort()) + // The prune must not touch the running terminal the deferral protected. + expect( + exec("pgrep -f '[w]hile :; do echo TICK' >/dev/null && echo alive || true") + ).toContain('alive') + }, + templateDir + ) + } finally { + rmSync(scratch, { recursive: true, force: true }) + } +})