feat(ssh): stage, commit and abort a dormant migration against its destination (#16741 T8-c3) (#24523)

* feat(ssh): stage, commit and abort a dormant migration against its destination (#16741 T8-c3)

The coordinator re-checks before every stage and commit that the fenced source
still exports the journaled manifest, carries no untransferable state and
started no terminal. A lost answer is re-read from the destination's catalog
state; only a committed read whose receipt matches the journal advances it,
and the journal is on disk before anything returns. Abort releases the fence
only on proof the destination holds nothing, or on an unsupported destination
before anything was staged, and never once the destination committed. Codes
against T6-9's catalog client through an injected interface. Inert until
T8-c4/T6-10.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

* refactor(ssh): import the T6-9 client's unsupported refusal instead of mirroring it

---------

Co-authored-by: m4air <m4air@Mac.localdomain>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
OrcaWin
2026-10-02 02:22:41 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 m4air
parent d6925c93c9
commit ef01867ae4
7 changed files with 589 additions and 0 deletions
@@ -0,0 +1,23 @@
import type { OrcadMigrationCatalogState } from '../../shared/orcad-migration-manifest'
export async function resolveDurableOrcadCatalogMutation(
mutate: () => Promise<OrcadMigrationCatalogState>,
read: () => Promise<OrcadMigrationCatalogState>,
accepted: (state: OrcadMigrationCatalogState) => boolean
) {
try {
return await mutate()
} catch (mutationError) {
let observed: OrcadMigrationCatalogState
try {
observed = await read()
} catch {
throw mutationError
}
if (!accepted(observed)) {
throw mutationError
}
}
// State reads may reflect unflushed mutations; an idempotent acknowledgment proves durability.
return mutate()
}
@@ -0,0 +1,284 @@
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { getManagedOrcadOwnerEnvironmentId } from '../../shared/managed-orcad-ssh-owner'
import type {
OrcadMigrationCatalogState,
OrcadMigrationManifest
} from '../../shared/orcad-migration-manifest'
import type { SshTarget } from '../../shared/ssh-types'
import { closeTestStores, createSqliteTestStore } from '../persistence-test-harness'
import { Store } from '../persistence/loading-store/store'
import {
abortOrcadMigrationCutover,
commitOrcadMigrationDestination,
stageOrcadMigrationDestination,
type OrcadMigrationCutoverContext
} from './orcad-migration-cutover-coordinator'
import { ORCAD_MIGRATION_DESTINATION_UNSUPPORTED } from './orcad-migration-catalog-client'
import { listOrcadMigrationSourceCutovers } from './orcad-migration-cutover-journal'
import { fenceOrcadMigrationSource } from './orcad-migration-source-fence'
import { SshTargetOrcadClaims } from './ssh-target-orcad-claims'
const TARGET: SshTarget = {
id: 'ssh-prod',
label: 'Production',
host: 'prod.example.com',
port: 22,
username: 'deploy',
generation: 2
}
const directories: string[] = []
afterEach(async () => {
await closeTestStores()
for (const directory of directories.splice(0)) {
rmSync(directory, { recursive: true, force: true })
}
})
/** An in-memory destination with the T6-9 semantics: idempotent stage, receipt-keyed commit. */
function fakeDestination() {
let state: 'absent' | 'staged' | 'committed' = 'absent'
const view = (manifest: OrcadMigrationManifest): OrcadMigrationCatalogState => {
const base = { migrationId: manifest.migrationId, manifestSha256: manifest.manifestSha256 }
if (state === 'committed') {
return {
...base,
state,
receipt: {
version: 1,
migrationId: manifest.migrationId,
manifestSha256: manifest.manifestSha256,
source: manifest.source,
importedAt: '2026-10-01T00:00:00.000Z',
repositoryIds: [],
projectGroupIds: [],
folderWorkspaceIds: []
}
}
}
return state === 'staged'
? { ...base, state, stagedAt: '2026-10-01T00:00:00.000Z', snapshotUploads: [] }
: { ...base, state }
}
const destination = {
commits: 0,
readState: vi.fn(async (manifest: OrcadMigrationManifest) => view(manifest)),
stage: vi.fn(async (manifest: OrcadMigrationManifest) => {
if (state === 'absent') {
state = 'staged'
}
return view(manifest)
}),
commit: vi.fn(async (manifest: OrcadMigrationManifest) => {
if (state === 'staged') {
state = 'committed'
destination.commits += 1
}
return view(manifest)
}),
abort: vi.fn(async (manifest: OrcadMigrationManifest) => {
const aborted = state === 'staged'
if (aborted) {
state = 'absent'
}
return { ...view(manifest), aborted, ...(aborted ? {} : { durableAbsent: true as const }) }
}),
stageChunk: vi.fn()
}
return destination
}
async function setup() {
const userDataPath = mkdtempSync(join(tmpdir(), 'orcad-cutover-coordinator-'))
directories.push(userDataPath)
const store = createSqliteTestStore(Store, { dataFile: join(userDataPath, 'orca-data.json') })
store.addSshTarget(TARGET)
store.addRepo({
id: 'repo-1',
path: '/srv/app',
displayName: 'App',
badgeColor: '#737373',
addedAt: 1,
kind: 'git',
connectionId: TARGET.id
})
const claims = new SshTargetOrcadClaims(store)
const fenced = await fenceOrcadMigrationSource({
userDataPath,
store,
claims,
targetId: TARGET.id,
destinationEnvironmentId: 'env-1',
destinationName: 'Managed',
terminalProof: { verdict: 'exited', provenPtyIds: [] },
hasDirectSshAuthority: () => false
})
if (fenced.outcome !== 'fenced') {
throw new Error('expected a fence')
}
const destination = fakeDestination()
const context: OrcadMigrationCutoverContext = { userDataPath, store, claims, destination }
const journal = () => listOrcadMigrationSourceCutovers(userDataPath)[0]
const owner = () => getManagedOrcadOwnerEnvironmentId(store.getSshTarget(TARGET.id)?.owner)
return { context, destination, store, migrationId: fenced.cutover.migrationId, journal, owner }
}
describe('migration cutover coordinator', () => {
it('stages then commits once, journaling each phase the destination proves', async () => {
const h = await setup()
await expect(stageOrcadMigrationDestination(h.context, h.migrationId)).resolves.toMatchObject({
phase: 'destination-staged'
})
expect(h.journal()?.phase).toBe('destination-staged')
await expect(commitOrcadMigrationDestination(h.context, h.migrationId)).resolves.toMatchObject({
phase: 'destination-committed'
})
expect(h.journal()?.phase).toBe('destination-committed')
await commitOrcadMigrationDestination(h.context, h.migrationId)
expect(h.destination.commits).toBe(1)
})
it('reads a lost commit reply back from the destination instead of failing or retrying blind', async () => {
const h = await setup()
await stageOrcadMigrationDestination(h.context, h.migrationId)
const commit = h.destination.commit.getMockImplementation()!
h.destination.commit.mockImplementationOnce(async (manifest) => {
await commit(manifest)
throw new Error('socket closed')
})
await expect(commitOrcadMigrationDestination(h.context, h.migrationId)).resolves.toMatchObject({
phase: 'destination-committed'
})
expect(h.destination.commits).toBe(1)
})
it('keeps the journal at staged when the commit reply and the re-read are both lost', async () => {
const h = await setup()
await stageOrcadMigrationDestination(h.context, h.migrationId)
h.destination.commit.mockRejectedValueOnce(new Error('socket closed'))
h.destination.readState.mockRejectedValueOnce(new Error('socket closed'))
await expect(commitOrcadMigrationDestination(h.context, h.migrationId)).rejects.toThrow(
'socket closed'
)
expect(h.journal()?.phase).toBe('destination-staged')
expect(h.owner()).toBe('env-1')
})
it.each([
[
'a source row changed',
(store: Store) => store.updateRepo('repo-1', { displayName: 'Renamed' }),
'orcad_migration_source_changed'
],
[
'a terminal started on the source',
(store: Store) =>
store.upsertSshRemotePtyLease({ targetId: TARGET.id, ptyId: 'late', state: 'attached' }),
'orcad_migration_source_terminals_live'
]
])('refuses to stage or commit after %s', async (_label, change, code) => {
const h = await setup()
change(h.store)
await expect(stageOrcadMigrationDestination(h.context, h.migrationId)).rejects.toThrow(code)
expect(h.destination.stage).not.toHaveBeenCalled()
})
it('refuses a commit whose source changed after staging', async () => {
const h = await setup()
await stageOrcadMigrationDestination(h.context, h.migrationId)
h.store.updateRepo('repo-1', { displayName: 'Renamed' })
await expect(commitOrcadMigrationDestination(h.context, h.migrationId)).rejects.toThrow(
'orcad_migration_source_changed'
)
expect(h.destination.commit).not.toHaveBeenCalled()
})
it('aborts a stage and releases the fence only once the destination proves it absent', async () => {
const h = await setup()
await stageOrcadMigrationDestination(h.context, h.migrationId)
await expect(abortOrcadMigrationCutover(h.context, h.migrationId)).resolves.toEqual({
outcome: 'released',
evidence: 'catalog-absent'
})
expect(h.owner()).toBeNull()
expect(h.journal()).toBeUndefined()
})
it('keeps the fence when the abort answer is lost and the destination cannot be read', async () => {
const h = await setup()
await stageOrcadMigrationDestination(h.context, h.migrationId)
h.destination.abort.mockRejectedValueOnce(new Error('socket closed'))
h.destination.readState.mockResolvedValueOnce({
state: 'absent',
migrationId: h.migrationId,
manifestSha256: h.journal()!.manifestSha256
})
h.destination.readState.mockRejectedValueOnce(new Error('socket closed'))
h.destination.abort.mockRejectedValueOnce(new Error('socket closed'))
await expect(abortOrcadMigrationCutover(h.context, h.migrationId)).rejects.toThrow(
'socket closed'
)
expect(h.owner()).toBe('env-1')
expect(h.journal()?.phase).toBe('destination-staged')
})
it('never releases a committed destination, even when the journal lags behind it', async () => {
const h = await setup()
await stageOrcadMigrationDestination(h.context, h.migrationId)
await h.destination.commit(h.journal()!.manifest)
await expect(abortOrcadMigrationCutover(h.context, h.migrationId)).resolves.toMatchObject({
outcome: 'refused',
code: 'orcad_migration_committed_source_cannot_be_released'
})
expect(h.journal()?.phase).toBe('destination-committed')
expect(h.owner()).toBe('env-1')
await expect(abortOrcadMigrationCutover(h.context, h.migrationId)).resolves.toMatchObject({
outcome: 'refused'
})
})
it('releases on an unsupported destination only before anything was staged', async () => {
const unsupported = new Error(ORCAD_MIGRATION_DESTINATION_UNSUPPORTED)
const fresh = await setup()
fresh.destination.readState.mockRejectedValueOnce(unsupported)
await expect(abortOrcadMigrationCutover(fresh.context, fresh.migrationId)).resolves.toEqual({
outcome: 'released',
evidence: 'destination-unsupported'
})
expect(fresh.owner()).toBeNull()
const staged = await setup()
await stageOrcadMigrationDestination(staged.context, staged.migrationId)
staged.destination.readState.mockRejectedValueOnce(unsupported)
await expect(abortOrcadMigrationCutover(staged.context, staged.migrationId)).rejects.toThrow(
ORCAD_MIGRATION_DESTINATION_UNSUPPORTED
)
expect(staged.owner()).toBe('env-1')
})
it('treats any other read failure at abort as unverifiable and keeps the fence', async () => {
const h = await setup()
h.destination.readState.mockRejectedValueOnce(new Error('socket closed'))
await expect(abortOrcadMigrationCutover(h.context, h.migrationId)).rejects.toThrow(
'socket closed'
)
expect(h.owner()).toBe('env-1')
})
it('refuses a destination answer for another manifest', async () => {
const h = await setup()
h.destination.stage.mockResolvedValueOnce({
state: 'staged',
migrationId: h.migrationId,
manifestSha256: 'f'.repeat(64),
stagedAt: '2026-10-01T00:00:00.000Z'
})
await expect(stageOrcadMigrationDestination(h.context, h.migrationId)).rejects.toThrow(
'orcad_migration_destination_state_mismatch'
)
expect(h.journal()?.phase).toBe('source-fenced')
})
})
@@ -0,0 +1,241 @@
/**
* Staging, committing and aborting a dormant migration on its destination orcad.
*
* The source stays authoritative until the destination proves a commit: before every stage and
* commit the fenced source must still export the journaled manifest and carry no state the
* manifest cannot. A lost answer is never read as success or as absence; the destination's
* catalog state is re-read, and only that observation moves the journal, which is on disk before
* anything is returned. A committed destination is never released.
*/
import type {
OrcadMigrationCatalogAbortResult,
OrcadMigrationCatalogState,
OrcadMigrationManifest
} from '../../shared/orcad-migration-manifest'
import type {
OrcadMigrationSnapshotChunkRequest,
OrcadMigrationSnapshotChunkResult
} from '../../shared/orcad-migration-scrollback'
import type {
OrcadMigrationSourceCutover,
OrcadMigrationSourceCutoverPhase
} from '../../shared/orcad-migration-source-cutover'
import { resolveDurableOrcadCatalogMutation } from './orcad-catalog-durable-mutation'
import { ORCAD_MIGRATION_DESTINATION_UNSUPPORTED } from './orcad-migration-catalog-client'
import {
listOrcadMigrationSourceCutovers,
removeOrcadMigrationSourceCutover,
writeOrcadMigrationSourceCutover
} from './orcad-migration-cutover-journal'
import {
transferOrcadMigrationSnapshots,
type OrcadMigrationSnapshotSource
} from './orcad-migration-snapshot-coordinator'
import { assertOrcadMigrationSourceUnchanged } from './orcad-migration-source-assertions'
import type { OrcadMigrationPreflightStore } from './ssh-target-orcad-preflight'
import type { SshTargetOrcadClaims } from './ssh-target-orcad-claims'
/** The destination's catalog operations, served by the T6-9 orcad.migration.* client. */
export type OrcadMigrationDestinationCatalog = {
readState: (manifest: OrcadMigrationManifest) => Promise<OrcadMigrationCatalogState>
stage: (manifest: OrcadMigrationManifest) => Promise<OrcadMigrationCatalogState>
commit: (manifest: OrcadMigrationManifest) => Promise<OrcadMigrationCatalogState>
abort: (manifest: OrcadMigrationManifest) => Promise<OrcadMigrationCatalogAbortResult>
stageChunk: (
request: OrcadMigrationSnapshotChunkRequest
) => Promise<OrcadMigrationSnapshotChunkResult>
}
export type OrcadMigrationCutoverContext = {
userDataPath: string
store: OrcadMigrationPreflightStore & OrcadMigrationSnapshotSource
claims: SshTargetOrcadClaims
destination: OrcadMigrationDestinationCatalog
now?: () => Date
}
export async function stageOrcadMigrationDestination(
context: OrcadMigrationCutoverContext,
migrationId: string
): Promise<OrcadMigrationSourceCutover> {
let cutover = requireCutover(context.userDataPath, migrationId)
if (cutover.phase === 'destination-committed' || cutover.phase === 'source-retired') {
return cutover
}
assertOrcadMigrationSourceUnchanged(context, cutover)
const { destination } = context
let state = await destination.readState(cutover.manifest)
if (state.state !== 'committed') {
state = await resolveDurableOrcadCatalogMutation(
() => destination.stage(cutover.manifest),
() => destination.readState(cutover.manifest),
(observed) => observed.state !== 'absent'
)
}
cutover = recordObservedState(context, cutover, state)
if (state.state === 'staged') {
await transferOrcadMigrationSnapshots({
source: context.store,
manifest: cutover.manifest,
state,
destination: { readState: destination.readState, stageChunk: destination.stageChunk }
})
}
return cutover
}
export async function commitOrcadMigrationDestination(
context: OrcadMigrationCutoverContext,
migrationId: string
): Promise<OrcadMigrationSourceCutover> {
const staged = await stageOrcadMigrationDestination(context, migrationId)
if (staged.phase !== 'destination-staged') {
return staged
}
assertOrcadMigrationSourceUnchanged(context, staged)
const { destination } = context
// Why re-read: a lost commit reply may hide a commit that landed; the read decides.
const state = await resolveDurableOrcadCatalogMutation(
() => destination.commit(staged.manifest),
() => destination.readState(staged.manifest),
(observed) => observed.state === 'committed'
)
if (state.state !== 'committed') {
throw new Error(`orcad_migration_commit_not_committed:${state.state}`)
}
return recordObservedState(context, staged, state)
}
export type OrcadMigrationAbortResult =
| { outcome: 'released'; evidence: 'catalog-absent' | 'destination-unsupported' }
| { outcome: 'refused'; code: string; reason: string }
/** Releases the source fence only on proof the destination holds nothing of this migration. */
export async function abortOrcadMigrationCutover(
context: OrcadMigrationCutoverContext,
migrationId: string
): Promise<OrcadMigrationAbortResult> {
const cutover = requireCutover(context.userDataPath, migrationId)
if (cutover.phase === 'destination-committed' || cutover.phase === 'source-retired') {
return committedRefusal()
}
const { destination } = context
let state: OrcadMigrationCatalogState
try {
state = await destination.readState(cutover.manifest)
} catch (error) {
// Unsupported means nothing could have been staged, but only if nothing ever was.
if (isDestinationUnsupported(error) && cutover.phase === 'source-fenced') {
await releaseFence(context, cutover)
return { outcome: 'released', evidence: 'destination-unsupported' }
}
throw error
}
if (state.state !== 'committed') {
state = await abortWithRecovery(destination, cutover.manifest)
}
if (state.state === 'committed') {
recordObservedState(context, cutover, state)
return committedRefusal()
}
if (state.state !== 'absent') {
throw new Error(`orcad_migration_abort_not_absent:${state.state}`)
}
await releaseFence(context, cutover)
return { outcome: 'released', evidence: 'catalog-absent' }
}
/** Moves the journal to what the destination reported; durable before it returns. */
function recordObservedState(
context: OrcadMigrationCutoverContext,
cutover: OrcadMigrationSourceCutover,
state: OrcadMigrationCatalogState
): OrcadMigrationSourceCutover {
if (state.state === 'absent') {
throw new Error('orcad_migration_destination_catalog_absent')
}
if (
state.migrationId !== cutover.migrationId ||
state.manifestSha256 !== cutover.manifestSha256
) {
throw new Error('orcad_migration_destination_state_mismatch')
}
if (
state.state === 'committed' &&
(state.receipt.migrationId !== cutover.migrationId ||
state.receipt.manifestSha256 !== cutover.manifestSha256)
) {
throw new Error('orcad_migration_destination_receipt_mismatch')
}
const phase: OrcadMigrationSourceCutoverPhase =
state.state === 'committed' ? 'destination-committed' : 'destination-staged'
if (phase === cutover.phase) {
return cutover
}
if (cutover.phase === 'destination-committed' || cutover.phase === 'source-retired') {
// Why: a journal never moves back from a proven commit.
return cutover
}
const next: OrcadMigrationSourceCutover = {
...cutover,
phase,
updatedAt: (context.now ?? (() => new Date()))().toISOString()
}
writeOrcadMigrationSourceCutover(context.userDataPath, next)
return next
}
async function abortWithRecovery(
destination: OrcadMigrationDestinationCatalog,
manifest: OrcadMigrationManifest
): Promise<OrcadMigrationCatalogState> {
try {
return await destination.abort(manifest)
} catch (abortError) {
try {
const observed = await destination.readState(manifest)
if (observed.state === 'committed') {
return observed
}
if (observed.state === 'absent') {
// An absent read may predate the flush; a repeated abort proves it durable.
return await destination.abort(manifest)
}
} catch {
// Still unverifiable: keep the first failure and the fence.
}
throw abortError
}
}
/** Owner first, then the journal: a crash between leaves a stale journal, never an orphan owner. */
async function releaseFence(
context: OrcadMigrationCutoverContext,
cutover: OrcadMigrationSourceCutover
): Promise<void> {
context.claims.release(cutover.sshTargetId, cutover.destinationEnvironmentId)
await context.claims.flush()
removeOrcadMigrationSourceCutover(context.userDataPath, cutover.migrationId)
}
function requireCutover(userDataPath: string, migrationId: string): OrcadMigrationSourceCutover {
const cutover = listOrcadMigrationSourceCutovers(userDataPath).find(
(entry) => entry.migrationId === migrationId
)
if (!cutover) {
throw new Error('orcad_migration_source_cutover_not_found')
}
return cutover
}
function committedRefusal(): OrcadMigrationAbortResult {
return {
outcome: 'refused',
code: 'orcad_migration_committed_source_cannot_be_released',
reason: 'The managed server already holds this migration; it can only move forward.'
}
}
function isDestinationUnsupported(error: unknown): boolean {
return error instanceof Error && error.message === ORCAD_MIGRATION_DESTINATION_UNSUPPORTED
}
@@ -40,6 +40,7 @@ function cutover(migrationId = 'migration-1', sshTargetId = 'ssh-1'): OrcadMigra
sshTargetId,
sshTargetGeneration: 2,
manifestSha256: manifest.manifestSha256,
provenPtyIds: [],
manifest
}
}
@@ -0,0 +1,37 @@
/** Checks run before every stage and commit: the fenced source is still what the journal says. */
import type { OrcadMigrationSourceCutover } from '../../shared/orcad-migration-source-cutover'
import { createOrcadMigrationManifest } from './orcad-migration-manifest-export'
import { resolveOrcadMigrationFence } from './orcad-migration-source-fence'
import { confirmOrcadMigrationTerminalsUnderFence } from './orcad-migration-terminal-gate'
import { collectUntransferredDependentBlockers } from './ssh-target-orcad-dependents'
import type { OrcadMigrationPreflightStore } from './ssh-target-orcad-preflight'
export function assertOrcadMigrationSourceUnchanged(
context: { userDataPath: string; store: OrcadMigrationPreflightStore },
cutover: OrcadMigrationSourceCutover
): void {
const target = context.store.getSshTarget(cutover.sshTargetId)
const fence = target ? resolveOrcadMigrationFence(context.userDataPath, target) : null
if (!target || fence?.state !== 'fenced' || fence.cutover.migrationId !== cutover.migrationId) {
throw new Error('orcad_migration_source_fence_lost')
}
// Same id, time and destination as the journaled export: any difference is a source change.
const current = createOrcadMigrationManifest(context.store, target, {
migrationId: cutover.migrationId,
destinationEnvironmentId: cutover.destinationEnvironmentId,
now: () => new Date(cutover.manifest.createdAt)
})
if (current.manifestSha256 !== cutover.manifestSha256) {
throw new Error('orcad_migration_source_changed')
}
if (collectUntransferredDependentBlockers(context.store, cutover.manifest).length > 0) {
throw new Error('orcad_migration_source_dependencies_present')
}
const terminals = confirmOrcadMigrationTerminalsUnderFence(context.store, target.id, {
verdict: 'exited',
provenPtyIds: cutover.provenPtyIds
})
if (terminals.verdict !== 'exited') {
throw new Error(`orcad_migration_source_terminals_${terminals.verdict}`)
}
}
@@ -137,6 +137,7 @@ export async function fenceOrcadMigrationSource(args: {
sshTargetId: target.id,
sshTargetGeneration: generation,
manifestSha256: manifest.manifestSha256,
provenPtyIds: args.terminalProof.provenPtyIds,
manifest
}
writeOrcadMigrationSourceCutover(args.userDataPath, cutover)
@@ -34,6 +34,8 @@ const CutoverRecordSchema = z
sshTargetId: z.string().min(1).max(1_024),
sshTargetGeneration: z.number().int().positive(),
manifestSha256: z.string().regex(/^[a-f0-9]{64}$/),
// Terminals proven exited before the fence; any other live lease later means work started.
provenPtyIds: z.array(z.string().min(1).max(256)).max(10_000),
manifest: z.unknown()
})
.strict()