mirror of
https://github.com/stablyai/orca.git
synced 2026-11-18 11:54:46 +00:00
fix(ssh): prune superseded staged server builds when an update defers (#35881)
* fix(ssh): prune superseded staged server builds when an update defers GC ran only after an activation, so a host whose updates kept deferring over live terminals kept one staged orcad slot per release. Prune on deferral too, keeping the active, previous, live-daemon and newly staged builds. Refs #34149 Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> * fix(ssh): keep recently staged builds; route and harden the deferred-slot e2e Desktops on different releases can share a host, so the deferral prune keeps complete builds staged within three days instead of deleting a peer's candidate and forcing it to re-upload. Windows hosts, which cannot date their builds, keep them all. Refs #34149 Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> * fix(ssh): read a build's staging age as unknown when find cannot date it A find without -mtime used to read as an old build and let the deferral prune remove a peer's candidate. Report an error instead, which keeps every candidate; a build without .install-complete reads as not recent. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> * test(ssh): pin that an undatable staged build keeps every build Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> * test(ssh): fold the deferred-slot e2e into the update-anyway spec Keeps the routing table under its size limit and runs the check on the same deferred-update sources. Also drop the unused readiness argument from pruneSupersededOrcadCandidates. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> * fix(ssh): report GC as unfinished when it kept a recently staged build Callers that memoize a clean pass would otherwise never prune the build once it ages. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -190,7 +190,7 @@ export const PR_E2E_SOURCE_ROUTES = [
|
||||
matches: (file) =>
|
||||
MANAGED_SSH_HARNESS.test(file) ||
|
||||
(isProductSource(file) &&
|
||||
/^src\/(?:main\/(?:ssh\/orcad-(?:update-plan|runtime-maintenance|installed-activation|remote-deploy)|managed-server\/lifecycle\/orcad-(?:managed-update-|managed-auto-update|runtime-status))|renderer\/src\/components\/settings\/(?:managed-server-|use-managed-server-lifecycle))/.test(
|
||||
/^src\/(?:main\/(?:ssh\/orcad-(?:update-plan|runtime-maintenance|installed-activation|remote-deploy)|managed-server\/lifecycle\/orcad-(?:managed-update-|managed-auto-update|runtime-status|managed-version-gc)|managed-server\/install\/(?:orcad-remote-gc|remote-install-gc)|wsl\/wsl-orcad-deploy\.ts)|renderer\/src\/components\/settings\/(?:managed-server-|use-managed-server-lifecycle))/.test(
|
||||
file
|
||||
))
|
||||
},
|
||||
|
||||
@@ -43,8 +43,21 @@ function scriptHost(options: {
|
||||
/** Absent by default; raw journal or error overrides the read. */
|
||||
journal?: string | Error
|
||||
fenceHeld?: boolean
|
||||
/** Builds staged within the keep window; undefined leaves the answer incomplete. */
|
||||
recentlyStaged?: string[]
|
||||
/** The host could not date some build. */
|
||||
ageUnknown?: boolean
|
||||
}): void {
|
||||
mockExec.mockImplementation(async (_conn, command: string) => {
|
||||
if (command.includes('ORCAD_RECENT_STAGED_DONE')) {
|
||||
return options.recentlyStaged === undefined
|
||||
? ''
|
||||
: [
|
||||
...options.recentlyStaged.map((name) => `RECENT ${name}`),
|
||||
...(options.ageUnknown ? ['ERR'] : []),
|
||||
'ORCAD_RECENT_STAGED_DONE'
|
||||
].join('\n')
|
||||
}
|
||||
if (command.includes('__ORCAD_RECORD_PRESENT__') && command.includes('transaction.json')) {
|
||||
if (options.journal instanceof Error) {
|
||||
throw options.journal
|
||||
@@ -246,6 +259,33 @@ describe('orcad GC', () => {
|
||||
expect(inventories()).toBe(1)
|
||||
})
|
||||
|
||||
it('keeps builds another desktop staged recently, and all of them when it cannot tell', async () => {
|
||||
const args = {
|
||||
conn,
|
||||
host,
|
||||
remoteHome: '/home/u',
|
||||
currentDirAbsPath: '/home/u/.orca-remote/orcad-0.4.0+dd0',
|
||||
record: { ...emptyOrcadActivationRecord(), active: '0.1.0+01d' },
|
||||
keepStagedWithinDays: 3
|
||||
}
|
||||
const listing = ['orcad-0.1.0+01d', 'orcad-0.2.0+9ee0', 'orcad-0.3.0+cc0', 'orcad-0.4.0+dd0']
|
||||
const removed: string[] = []
|
||||
scriptHost({ listing, removed, recentlyStaged: ['orcad-0.3.0+cc0'] })
|
||||
expect(await gcOldOrcadVersions(args)).toBe(false)
|
||||
expect(removed).toEqual(['orcad-0.2.0+9ee0'])
|
||||
|
||||
const kept: string[] = []
|
||||
scriptHost({ listing, removed: kept })
|
||||
await gcOldOrcadVersions(args)
|
||||
expect(kept).toEqual([])
|
||||
|
||||
// An undatable build may be another desktop's fresh candidate.
|
||||
const undated: string[] = []
|
||||
scriptHost({ listing, removed: undated, recentlyStaged: [], ageUnknown: true })
|
||||
await gcOldOrcadVersions(args)
|
||||
expect(undated).toEqual([])
|
||||
})
|
||||
|
||||
describe('with an activation transaction in flight', () => {
|
||||
const listing = ['orcad-0.1.0+01d', 'orcad-0.2.0+9ee0', 'orcad-0.3.0+cc0']
|
||||
const record = { ...emptyOrcadActivationRecord(), active: '0.3.0+cc0' }
|
||||
|
||||
@@ -24,6 +24,8 @@ import {
|
||||
} from '../../ssh/ssh-remote-platform'
|
||||
import { RELAY_REMOTE_DIR } from '../../ssh/remote-exec/remote-frame-protocol'
|
||||
import { orcadWindowsHomeOpCommand } from './orcad-remote-windows-node'
|
||||
import { quotePosixShell } from '../../../shared/shell-argument-quoting'
|
||||
import { ORCAD_INSTALL_COMPLETE_FILENAME } from '../../../shared/orcad-artifacts'
|
||||
import { ORCAD_WINDOWS_LIVENESS_MANY_MARKER } from '../host-state/orcad-windows-host-script'
|
||||
|
||||
type OrcadGcOptions = {
|
||||
@@ -36,16 +38,24 @@ type OrcadGcOptions = {
|
||||
liveDaemonVersion?: string | null
|
||||
/** Current executable hashes both pin runtimes and authorize runtime-store GC. */
|
||||
nodeRuntimePins?: readonly string[]
|
||||
/** Keep complete builds staged this recently; another desktop's deferred update may still want its own. */
|
||||
keepStagedWithinDays?: number
|
||||
signal?: AbortSignal
|
||||
}
|
||||
|
||||
/** True when no stale version dir was left behind (skipped, kept or unlistable ones make it false). */
|
||||
export async function gcOldOrcadVersions(options: OrcadGcOptions): Promise<boolean> {
|
||||
const keepDays = options.keepStagedWithinDays
|
||||
if (keepDays !== undefined && isWindowsRemoteHost(options.host)) {
|
||||
// The Windows host script cannot date staged builds, so keep them all.
|
||||
return false
|
||||
}
|
||||
const transaction = await readOrcadGcTransactionPins(options)
|
||||
if (transaction.state === 'keep-all') {
|
||||
console.warn('[orcad-gc] An activation transaction is unreadable or unjournaled; skipping GC.')
|
||||
return false
|
||||
}
|
||||
let keptRecent = false
|
||||
const versionsClean = await gcOldRemoteInstallVersions(
|
||||
options.conn,
|
||||
ORCAD_INSTALL_MODEL,
|
||||
@@ -60,7 +70,15 @@ export async function gcOldOrcadVersions(options: OrcadGcOptions): Promise<boole
|
||||
// Screen Windows candidates together, then recheck dead candidates individually under the GC claim.
|
||||
...(isWindowsRemoteHost(options.host)
|
||||
? { resolveExtraPinnedDirNames: (candidates) => windowsLiveCandidates(options, candidates) }
|
||||
: {}),
|
||||
: keepDays !== undefined
|
||||
? {
|
||||
resolveExtraPinnedDirNames: async (candidates) => {
|
||||
const recent = await recentlyStagedCandidates(options, candidates, keepDays)
|
||||
keptRecent = recent === null || recent.length > 0
|
||||
return recent
|
||||
}
|
||||
}
|
||||
: {}),
|
||||
isDirLive: async (dir) => {
|
||||
try {
|
||||
const probe = await execCommand(
|
||||
@@ -89,7 +107,8 @@ export async function gcOldOrcadVersions(options: OrcadGcOptions): Promise<boole
|
||||
signal: options.signal
|
||||
})
|
||||
}
|
||||
return versionsClean
|
||||
// A recent build kept now still needs pruning once it ages.
|
||||
return versionsClean && !keptRecent
|
||||
}
|
||||
|
||||
/** Null means unknown host state and keeps every candidate; otherwise pin anything not proven dead. */
|
||||
@@ -125,3 +144,47 @@ async function windowsLiveCandidates(
|
||||
(_name, index) => states[index] !== 'DEAD' && states[index] !== ORCAD_NEVER_LAUNCHED
|
||||
)
|
||||
}
|
||||
|
||||
/** Null on any failure or unanswerable date, which keeps every candidate. */
|
||||
async function recentlyStagedCandidates(
|
||||
options: OrcadGcOptions,
|
||||
candidates: readonly string[],
|
||||
days: number
|
||||
): Promise<readonly string[] | null> {
|
||||
if (!Number.isInteger(days) || days < 1) {
|
||||
return null
|
||||
}
|
||||
const checks = candidates.map((name) => {
|
||||
const marker = quotePosixShell(
|
||||
joinRemotePath(
|
||||
options.host,
|
||||
options.remoteHome,
|
||||
RELAY_REMOTE_DIR,
|
||||
name,
|
||||
ORCAD_INSTALL_COMPLETE_FILENAME
|
||||
)
|
||||
)
|
||||
// A find without -mtime must read as unknown, not as an old build.
|
||||
return (
|
||||
`if [ ! -e ${marker} ]; then :; elif recent=$(find ${marker} -mtime -${days} 2>/dev/null); then ` +
|
||||
`if [ -n "$recent" ]; then echo RECENT ${quotePosixShell(name)}; fi; else echo ERR; fi;`
|
||||
)
|
||||
})
|
||||
try {
|
||||
const output = await execCommand(options.conn, `${checks.join(' ')} echo ${RECENT_DONE}`, {
|
||||
signal: options.signal
|
||||
})
|
||||
const lines = output.split('\n').map((line) => line.trim())
|
||||
if (!lines.includes(RECENT_DONE) || lines.includes('ERR')) {
|
||||
return null
|
||||
}
|
||||
return candidates.filter((name) => lines.includes(`RECENT ${name}`))
|
||||
} catch (error) {
|
||||
if (isUnconfirmedSshCommandTermination(error)) {
|
||||
throw error
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
const RECENT_DONE = 'ORCAD_RECENT_STAGED_DONE'
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
import type { NodeRuntimeTarget } from '../../../shared/node-runtime-pin'
|
||||
import type { ServeReadiness } from '../../server/serve-readiness'
|
||||
import { readOrcadActivationRecord } from '../activation/orcad-activation-record-store'
|
||||
import { orcadSlotDir, type OrcadSlotOptions } from '../../ssh/orcad-recovery-slot'
|
||||
import {
|
||||
orcadSlotDir,
|
||||
resolveOrcadSlotIdentity,
|
||||
type OrcadSlotOptions
|
||||
} from '../../ssh/orcad-recovery-slot'
|
||||
import { probeActiveOrcadReadiness } from '../../ssh/orcad-active-readiness'
|
||||
import { gcOldOrcadVersions } from '../install/orcad-remote-gc'
|
||||
import { nodeRuntimeStorePins } from '../../ssh/pinned-runtime-host-target'
|
||||
|
||||
@@ -19,6 +24,7 @@ export type ManagedOrcadPruneArgs = {
|
||||
serverTarget: NodeRuntimeTarget
|
||||
activeVersion: string
|
||||
readiness: ServeReadiness
|
||||
keepStagedWithinDays?: number
|
||||
}
|
||||
|
||||
/** True only when GC ran and left no stale version dir behind. */
|
||||
@@ -38,6 +44,9 @@ export async function pruneManagedOrcadVersions(args: ManagedOrcadPruneArgs): Pr
|
||||
liveDaemonVersion,
|
||||
// Default and compat builds share one runtime store.
|
||||
nodeRuntimePins: nodeRuntimeStorePins(args.serverTarget),
|
||||
...(args.keepStagedWithinDays !== undefined
|
||||
? { keepStagedWithinDays: args.keepStagedWithinDays }
|
||||
: {}),
|
||||
signal: args.slot.signal
|
||||
})
|
||||
} catch (error) {
|
||||
@@ -46,3 +55,37 @@ export async function pruneManagedOrcadVersions(args: ManagedOrcadPruneArgs): Pr
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/** GC otherwise runs only after an activation, so each deferred release left its staged slot behind (#34149). */
|
||||
export async function pruneSupersededOrcadCandidates(
|
||||
slot: OrcadSlotOptions,
|
||||
serverTarget: NodeRuntimeTarget,
|
||||
candidateVersion: string
|
||||
): Promise<void> {
|
||||
let readiness: ServeReadiness
|
||||
try {
|
||||
const active = (await readOrcadActivationRecord(slot)).active
|
||||
if (!active) {
|
||||
return
|
||||
}
|
||||
const identity = await resolveOrcadSlotIdentity(slot, active)
|
||||
readiness = await probeActiveOrcadReadiness(
|
||||
{ ...slot, remoteInstallDir: identity.remoteDir },
|
||||
{ buildHash: identity.buildHash, fullVersion: identity.version }
|
||||
)
|
||||
} catch (error) {
|
||||
// Without the incumbent's daemon version a staged tree may still be executing.
|
||||
console.warn('[orcad-gc] Skipped pruning staged orcad versions:', error)
|
||||
return
|
||||
}
|
||||
await pruneManagedOrcadVersions({
|
||||
slot,
|
||||
serverTarget,
|
||||
activeVersion: candidateVersion,
|
||||
readiness,
|
||||
keepStagedWithinDays: STAGED_BUILD_KEEP_DAYS
|
||||
})
|
||||
}
|
||||
|
||||
// Desktops on different releases share hosts; without a grace period each re-uploads what the other pruned.
|
||||
const STAGED_BUILD_KEEP_DAYS = 3
|
||||
|
||||
@@ -35,7 +35,10 @@ import { readRemoteOrcadBuildHash } from '../managed-server/install/orcad-remote
|
||||
import { deployOrcad, type OrcadDeployOptions } from './orcad-remote-deploy'
|
||||
import type { OrcadUpdateForce } from './orcad-update-plan'
|
||||
import { ORCAD_ACTIVATION_POLICY_REFUSED_CODE } from './orcad-installed-activation'
|
||||
import { pruneManagedOrcadVersions } from '../managed-server/lifecycle/orcad-managed-version-gc'
|
||||
import {
|
||||
pruneManagedOrcadVersions,
|
||||
pruneSupersededOrcadCandidates
|
||||
} from '../managed-server/lifecycle/orcad-managed-version-gc'
|
||||
import { clearOrcadUpdateFailedThisLaunch } from '../managed-server/lifecycle/orcad-managed-update-failure-memory'
|
||||
import { admitOrcadRecordForIntent } from '../managed-server/lifecycle/orcad-managed-update-decision'
|
||||
import { rollbackOrcad } from '../managed-server/install/orcad-remote-rollback'
|
||||
@@ -156,6 +159,7 @@ export async function runManagedOrcadUpdate(
|
||||
if (result.code !== ORCAD_ACTIVATION_POLICY_REFUSED_CODE) {
|
||||
recordManagedOrcadUpdateDeferral(environment.id, deferral)
|
||||
}
|
||||
await pruneSupersededOrcadCandidates(slot, context.serverTarget, result.fullVersion)
|
||||
return deferral
|
||||
}
|
||||
clearManagedOrcadUpdateDeferral(environment.id)
|
||||
|
||||
@@ -67,7 +67,8 @@ vi.mock('./wsl-orcad-active-probe', async (importOriginal) => ({
|
||||
}))
|
||||
vi.mock('../managed-server/lifecycle/orcad-managed-version-gc', () => ({
|
||||
provenLiveDaemonVersion: () => null,
|
||||
pruneManagedOrcadVersions: mocks.prune
|
||||
pruneManagedOrcadVersions: mocks.prune,
|
||||
pruneSupersededOrcadCandidates: async () => {}
|
||||
}))
|
||||
vi.mock('../managed-server/lifecycle/orcad-managed-wake', () => ({
|
||||
wakeStoppedManagedOrcad: mocks.wake
|
||||
|
||||
@@ -65,7 +65,8 @@ vi.mock('./wsl-orcad-active-probe', async (importOriginal) => ({
|
||||
}))
|
||||
vi.mock('../managed-server/lifecycle/orcad-managed-version-gc', () => ({
|
||||
provenLiveDaemonVersion: () => null,
|
||||
pruneManagedOrcadVersions: mocks.prune
|
||||
pruneManagedOrcadVersions: mocks.prune,
|
||||
pruneSupersededOrcadCandidates: async () => {}
|
||||
}))
|
||||
vi.mock('../managed-server/lifecycle/orcad-managed-wake', () => ({
|
||||
wakeStoppedManagedOrcad: mocks.wake
|
||||
|
||||
@@ -21,6 +21,7 @@ import type { OrcadUpdateIntent } from '../managed-server/lifecycle/orcad-manage
|
||||
import type { OrcadActivationRecord } from '../managed-server/activation/orcad-activation-record'
|
||||
import { errorMessage } from '../../shared/error-message'
|
||||
import { pruneWslOrcadVersionsUntilClean } from './wsl-orcad-version-gc-memo'
|
||||
import { pruneSupersededOrcadCandidates } from '../managed-server/lifecycle/orcad-managed-version-gc'
|
||||
import { wakeAlreadyActiveOrcad } from '../ssh/orcad-already-active-wake'
|
||||
import { OrcadActiveReadinessError } from '../ssh/orcad-active-readiness'
|
||||
import { WslDistroNotRunningError } from './wsl-orcad-connection-errors'
|
||||
@@ -258,6 +259,7 @@ async function launchOverTransport(
|
||||
fullVersion: incumbentVersion,
|
||||
readiness: incumbent.readiness
|
||||
}
|
||||
await pruneSupersededOrcadCandidates(slot, context.serverTarget, deployed.fullVersion)
|
||||
return { ...(await readyLaunch(serving)), deferredUpdate }
|
||||
}
|
||||
return { outcome: 'deferred', distro: name, ...deferredUpdate }
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
/** Prove Update defers on a running process and Update anyway preserves that process, its output, and keyboard input across restart. */
|
||||
/** Prove Update defers on a running process, Update anyway preserves it across restart, and repeated deferrals keep one staged build (#34149). */
|
||||
import { cpSync, mkdtempSync, rmSync } from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
@@ -121,3 +121,78 @@ test('Update anyway moves a managed host to the new build and keeps its running
|
||||
rmSync(scratch, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
function slotVersions(exec: (command: string) => string): string[] {
|
||||
return exec('cd /root/.orca-remote && ls -d orcad-0* 2>/dev/null || true')
|
||||
.split('\n')
|
||||
.map((line) => line.trim().replace(/^orcad-/, ''))
|
||||
.filter(Boolean)
|
||||
.sort()
|
||||
}
|
||||
|
||||
function deferredCandidate(page: Page, environmentId: string): Promise<string | null> {
|
||||
return page.evaluate(
|
||||
async (selector) =>
|
||||
(await window.api.runtimeEnvironments.managedOrcad?.getStatus({ selector }))?.deferredUpdate
|
||||
?.candidateVersion ?? null,
|
||||
environmentId
|
||||
)
|
||||
}
|
||||
|
||||
test('a release deferred over running terminals replaces the build the last one staged', async (// oxlint-disable-next-line no-empty-pattern -- Owns its app launch through a restart session.
|
||||
{}, testInfo) => {
|
||||
// Deploys a server, then relaunches through two app updates that each stage a build.
|
||||
test.setTimeout(20 * 60_000)
|
||||
const scratch = mkdtempSync(path.join(os.tmpdir(), 'orca-deferred-slots-'))
|
||||
const templateDir = path.join(scratch, 'template')
|
||||
cpSync(ORCAD_TEMPLATE!, templateDir, { recursive: true })
|
||||
try {
|
||||
await withManagedHost(
|
||||
testInfo,
|
||||
async (fixture) => {
|
||||
const exec = fixture.host.exec!
|
||||
const workspace = await launchManagedWorkspace(fixture)
|
||||
const { environmentId } = workspace
|
||||
const tab = await openTerminalTab(workspace.page, workspace.seeded.worktreeId)
|
||||
await typeInTerminal(workspace.page, tab, `bash -c 'while :; do echo TICK; sleep 1; done'`)
|
||||
await expectPaneText(workspace.page, tab, 'TICK')
|
||||
const deployed = readHostOrcadActivation(exec).active!
|
||||
|
||||
makeOrcadTemplateVariant(templateDir, 'B')
|
||||
let page = await relaunchManagedApp(fixture)
|
||||
await expect
|
||||
.poll(() => deferredCandidate(page, environmentId), { timeout: 4 * 60_000 })
|
||||
.not.toBeNull()
|
||||
const staged = (await deferredCandidate(page, environmentId))!
|
||||
expect(slotVersions(exec)).toEqual([deployed, staged].sort())
|
||||
|
||||
// Releases come days apart; a build staged this recently could still be another desktop's.
|
||||
exec(`touch -d '5 days ago' /root/.orca-remote/orcad-${staged}/.install-complete`)
|
||||
makeOrcadTemplateVariant(templateDir, 'C')
|
||||
page = await relaunchManagedApp(fixture)
|
||||
let next: string | null = null
|
||||
await expect
|
||||
.poll(
|
||||
async () => {
|
||||
const candidate = await deferredCandidate(page, environmentId)
|
||||
next = candidate !== staged ? candidate : null
|
||||
return next
|
||||
},
|
||||
{ timeout: 4 * 60_000 }
|
||||
)
|
||||
.not.toBeNull()
|
||||
expect(readHostOrcadActivation(exec).active).toBe(deployed)
|
||||
await expect
|
||||
.poll(() => slotVersions(exec), { timeout: 60_000 })
|
||||
.toEqual([deployed, next!].sort())
|
||||
// The prune must not touch the running terminal the deferral protected.
|
||||
expect(
|
||||
exec("pgrep -f '[w]hile :; do echo TICK' >/dev/null && echo alive || true")
|
||||
).toContain('alive')
|
||||
},
|
||||
templateDir
|
||||
)
|
||||
} finally {
|
||||
rmSync(scratch, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user