Files
orca/src/main/ssh/runtime-ssh-access.test.ts
T
NeilandFrank-zhu2000 92c8f3cbdc fix(ssh): stop floating terminal metadata from blocking SSH access links (#31963)
* fix(ssh): stop local-only workspace metadata from blocking SSH access links

Unstamped floating and setup terminal metadata rows were projected onto every
execution host, so the dependent-state census refused SSH access links and
empty-host claims for every SSH target with orcad_migration_dependent_state.

- Local-only workspace ids project only onto the local host, are stamped local
  on load, and their metadata writes always land on the local host.
- The census counts an unstamped legacy row only when its repo is not
  registered solely on other hosts.
- The SSH access error now names the blocking state.

Builds on the approach in #31862 by Frank-zhu2000.

Fixes #31498

Co-authored-by: Frank-zhu2000 <Frank-zhu2000@users.noreply.github.com>

* fix(ssh): rehome remote-stamped local-only metadata, skip orphan rows, and word the SSH access error for a link

* fix(persistence): keep a local or unstamped floating row over a remote copy when rehoming

* fix(ssh): make the SSH access blocker copy switch exhaustive

---------

Co-authored-by: Frank-zhu2000 <Frank-zhu2000@users.noreply.github.com>
2026-10-22 03:39:54 -07:00

407 lines
16 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { pairedRuntimeConnectionFromOffer } from '../../shared/paired-runtime-connection'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { encodePairingOffer } from '../../shared/pairing'
import {
addEnvironmentFromPairingCode,
resolveEnvironment
} from '../../shared/runtime-environment-store'
import { readPersistedEnvironmentStore } from '../../shared/runtime-environment-store-file'
import { writeRuntimeEnvironmentSidecarEntry } from '../../shared/runtime-environment-sidecar'
import type { SshTarget } from '../../shared/ssh-types'
const mocks = vi.hoisted(() => ({
infrastructure: vi.fn(),
connect: vi.fn(),
disconnect: vi.fn(async () => undefined),
start: vi.fn(),
close: vi.fn(),
ensure: vi.fn(),
verify: vi.fn(),
flush: vi.fn(),
preflight: vi.fn()
}))
vi.mock('../managed-server/lifecycle/orcad-managed-runtime-context', () => ({
requireManagedOrcadInfrastructure: mocks.infrastructure
}))
vi.mock('../managed-server/lifecycle/orcad-managed-tunnel', () => ({
startOrcadManagedTunnel: mocks.start,
closeOrcadManagedTunnel: mocks.close,
ensureOrcadManagedTunnel: mocks.ensure
}))
vi.mock('./runtime-ssh-access-verification', () => ({
verifyRuntimeEnvironmentSshTunnel: mocks.verify
}))
import { linkRuntimeSshAccess, unlinkRuntimeSshAccess } from './runtime-ssh-access'
import { fingerprintRuntimeSshTarget } from './runtime-ssh-target-fingerprint'
const pairing = {
v: 2 as const,
endpoint: 'wss://example.test/runtime',
publicKeyB64: Buffer.alloc(32, 1).toString('base64'),
deviceToken: 'secret',
pairedDeviceId: 'client'
}
describe('independent runtime SSH access coordinator', () => {
let userDataPath: string
let environmentId: string
let target: SshTarget
let events: string[]
const request = () => ({
selector: environmentId,
requestId: 'request',
sshTargetId: 'target',
remotePort: 6768
})
const current = () => resolveEnvironment(userDataPath, environmentId)
const unlink = (requestId = 'unlink') =>
unlinkRuntimeSshAccess(
userDataPath,
{ selector: environmentId, requestId },
{
invalidateTransport: () => {
events.push('invalidate')
expect(current().sshAccess).toBeUndefined()
}
}
)
beforeEach(() => {
vi.resetAllMocks()
events = []
userDataPath = mkdtempSync(join(tmpdir(), 'runtime-access-test-'))
environmentId = addEnvironmentFromPairingCode(userDataPath, {
name: 'server',
pairingCode: encodePairingOffer(pairing)
}).id
target = {
id: 'target',
label: 'host',
host: 'host.example',
port: 22,
username: 'user',
source: 'manual',
generation: 3
}
mocks.preflight.mockReturnValue({ blockers: [] })
mocks.flush.mockImplementation(async () => {
expect(current().pendingSshAccessOperation).toBeDefined()
events.push('flush')
})
mocks.infrastructure.mockReturnValue({
connectionManager: { connect: mocks.connect, disconnect: mocks.disconnect },
claims: {
listTargets: () => [target],
preflight: mocks.preflight,
flush: mocks.flush,
claim: () => {
expect(current().pendingSshAccessOperation?.operation).toBe('link')
events.push('claim')
target = { ...target, orcadFence: { environmentId: environmentId } }
return target
},
release: () => {
expect(current().pendingSshAccessOperation?.operation).toBe('unlink')
events.push('release')
target = { ...target, orcadFence: undefined }
return target
}
}
})
mocks.connect.mockImplementation(async () => {
events.push('connect')
return {}
})
mocks.start.mockImplementation(async () => {
events.push('tunnel')
return 41000
})
mocks.close.mockImplementation(async () => {
events.push('close')
})
mocks.verify.mockImplementation(async () => {
events.push('verify')
return {
verifiedRuntimeId: 'runtime',
verifiedConnection: pairedRuntimeConnectionFromOffer({
...pairing,
endpoint: 'ws://127.0.0.1:41000'
}),
runtimeStatus: {}
}
})
})
afterEach(() => rmSync(userDataPath, { recursive: true, force: true }))
it.each(['link', 'unlink'])(
'refuses %s before side effects during host reconciliation',
async (operation) => {
const environments = readPersistedEnvironmentStore(userDataPath).environments
const persisted = environments.find((entry) => entry.id === environmentId)
if (!persisted) {
throw new Error('missing environment')
}
writeRuntimeEnvironmentSidecarEntry(userDataPath, environments, persisted, {
reconciliation: {
version: 1,
stage: 'prepared',
requestId: 'reconciliation',
canonicalEnvironmentId: environmentId,
runtimeId: 'host-runtime',
preparedAt: 1,
registrations: [
{ environmentId, authorityDigest: 'a'.repeat(64) },
{ environmentId: 'peer', authorityDigest: 'b'.repeat(64) }
]
}
})
await expect(
operation === 'link' ? linkRuntimeSshAccess(userDataPath, request()) : unlink()
).rejects.toThrow('reconciliation')
expect(mocks.connect).not.toHaveBeenCalled()
expect(mocks.start).not.toHaveBeenCalled()
expect(mocks.close).not.toHaveBeenCalled()
expect(mocks.flush).not.toHaveBeenCalled()
}
)
it('persists intent then claim before connecting and publishes redacted authenticated access', async () => {
const result = await linkRuntimeSshAccess(userDataPath, request())
expect(events).toEqual(['flush', 'claim', 'flush', 'connect', 'tunnel', 'verify'])
expect(result.sshAccess?.sshTargetId).toBe('target')
expect(result.orcadDeployment).toBeUndefined()
expect(JSON.stringify(result)).not.toContain('secret')
expect(current().pendingSshAccessOperation).toBeUndefined()
expect(mocks.preflight).toHaveBeenCalledWith('target')
})
it('names the saved dependency when dependent state blocks an SSH access link', async () => {
mocks.preflight.mockReturnValue({
blockers: [
{
code: 'orcad_migration_dependent_state',
category: 'client-owned-state',
dependencies: [{ kind: 'worktree-metadata', count: 1, names: ['repo-1::/srv/app'] }]
}
]
})
await expect(linkRuntimeSshAccess(userDataPath, request())).rejects.toThrow(
/orcad_migration_dependent_state\. [\s\S]*worktree-metadata ×1 \(repo-1::\/srv\/app\)/
)
expect(mocks.connect).not.toHaveBeenCalled()
expect(mocks.flush).not.toHaveBeenCalled()
expect(events).not.toContain('claim')
})
it('cancels a pending connect and starts no tunnel when the link is aborted', async () => {
const controller = new AbortController()
mocks.connect.mockImplementationOnce(
() =>
new Promise((_resolve, fail) => {
mocks.disconnect.mockImplementationOnce(async () => {
fail(new Error('cancelled'))
return undefined
})
})
)
const link = linkRuntimeSshAccess(userDataPath, request(), { signal: controller.signal })
await vi.waitFor(() => expect(mocks.connect).toHaveBeenCalled())
controller.abort()
await expect(link).rejects.toThrow('cancelled')
expect(mocks.disconnect).toHaveBeenCalledWith('target')
expect(mocks.start).not.toHaveBeenCalled()
})
it('handles lost link responses without connecting or verifying again', async () => {
const first = await linkRuntimeSshAccess(userDataPath, request())
expect(await linkRuntimeSshAccess(userDataPath, request())).toEqual(first)
expect(mocks.verify).toHaveBeenCalledTimes(1)
expect(mocks.ensure).toHaveBeenCalledWith(userDataPath, environmentId)
})
it('invalidates old transport only after authenticated access is published', async () => {
const invalidateTransport = vi.fn(() => {
expect(current().sshAccess).toBeDefined()
expect(current().pendingSshAccessOperation).toBeUndefined()
})
await linkRuntimeSshAccess(userDataPath, request(), { invalidateTransport })
await linkRuntimeSshAccess(userDataPath, request(), { invalidateTransport })
expect(invalidateTransport).toHaveBeenCalledTimes(2)
})
it('rejects changed request tuple after a completed link', async () => {
await linkRuntimeSshAccess(userDataPath, request())
await expect(
linkRuntimeSshAccess(userDataPath, { ...request(), remotePort: 6769 })
).rejects.toThrow('different')
expect(mocks.verify).toHaveBeenCalledTimes(1)
})
it('preserves the committed tunnel when transport refresh fails and retries the same link', async () => {
const invalidateTransport = vi.fn().mockRejectedValueOnce(new Error('refresh failed'))
await expect(
linkRuntimeSshAccess(userDataPath, request(), { invalidateTransport })
).rejects.toThrow('refresh failed')
const committed = current()
expect(committed.sshAccess?.requestId).toBe('request')
expect(committed.pendingSshAccessOperation).toBeUndefined()
expect(target?.orcadFence).toEqual({ environmentId: environmentId })
expect(mocks.close).not.toHaveBeenCalled()
await linkRuntimeSshAccess(userDataPath, request(), { invalidateTransport })
expect(current()).toEqual(committed)
expect(mocks.ensure).toHaveBeenCalledExactlyOnceWith(userDataPath, environmentId)
expect(mocks.connect).toHaveBeenCalledTimes(1)
expect(mocks.verify).toHaveBeenCalledTimes(1)
expect(mocks.start).toHaveBeenCalledTimes(1)
expect(invalidateTransport).toHaveBeenCalledTimes(2)
expect(mocks.close).not.toHaveBeenCalled()
})
it('fences connection configuration changes before opening a forward', async () => {
mocks.connect.mockImplementationOnce(async () => {
target = { ...target, username: 'other' }
return {}
})
await expect(linkRuntimeSshAccess(userDataPath, request())).rejects.toThrow('changed')
expect(mocks.start).not.toHaveBeenCalled()
expect(current().pendingSshAccessOperation?.operation).toBe('link')
})
it('restores pairing before invalidating and closes before releasing the exact claim', async () => {
await linkRuntimeSshAccess(userDataPath, request())
events = []
await unlink()
expect(events).toEqual(['invalidate', 'close', 'release', 'flush'])
expect(current().endpoints[0].endpoint).toBe(pairing.endpoint)
expect(current().sshAccess).toBeUndefined()
expect(target.orcadFence).toBeUndefined()
})
it('retains claim and pending intent on wrong-host proof, then permits cancellation', async () => {
mocks.verify.mockRejectedValueOnce(new Error('wrong host'))
await expect(linkRuntimeSshAccess(userDataPath, request())).rejects.toThrow('wrong host')
expect(current().pendingSshAccessOperation?.operation).toBe('link')
expect(current().sshAccess).toBeUndefined()
expect(target.orcadFence).toBeDefined()
expect(mocks.close).toHaveBeenCalledTimes(1)
await unlink('request')
expect(target.orcadFence).toBeUndefined()
})
it('retries verification failure with the same durable intent', async () => {
mocks.verify.mockRejectedValueOnce(new Error('unverifiable'))
await expect(linkRuntimeSshAccess(userDataPath, request())).rejects.toThrow()
await linkRuntimeSshAccess(userDataPath, request())
expect(current().sshAccess).toBeDefined()
})
it.each(['host', 'generation', 'owner'] as const)(
'fences %s reassignment during authenticated proof',
async (field) => {
mocks.verify.mockImplementationOnce(async () => {
target = {
...target,
...(field === 'host'
? { host: 'elsewhere' }
: field === 'generation'
? { generation: 4 }
: { orcadFence: { environmentId: 'other' } })
}
return {
verifiedRuntimeId: 'runtime',
verifiedConnection: pairedRuntimeConnectionFromOffer(pairing)
}
})
await expect(linkRuntimeSshAccess(userDataPath, request())).rejects.toThrow('changed')
expect(current().sshAccess).toBeUndefined()
expect(mocks.close).toHaveBeenCalled()
await expect(unlink('request')).rejects.toThrow('changed')
expect(events).not.toContain('release')
}
)
it.each([1, 2])('never connects when durable flush %s fails', async (flushNumber) => {
if (flushNumber === 2) {
mocks.flush.mockResolvedValueOnce(undefined)
}
mocks.flush.mockRejectedValueOnce(new Error('disk full'))
await expect(linkRuntimeSshAccess(userDataPath, request())).rejects.toThrow('disk full')
expect(mocks.connect).not.toHaveBeenCalled()
expect(current().pendingSshAccessOperation).toBeDefined()
expect(!!target.orcadFence).toBe(flushNumber === 2)
})
it('resumes unlink after release flush failed without releasing another owner', async () => {
await linkRuntimeSshAccess(userDataPath, request())
mocks.flush.mockRejectedValueOnce(new Error('disk full'))
await expect(unlink()).rejects.toThrow('disk full')
expect(target.orcadFence).toBeUndefined()
await unlink()
expect(events.filter((event) => event === 'release')).toHaveLength(1)
expect(current().pendingSshAccessOperation).toBeUndefined()
})
it('does not release a new owner when retrying an interrupted unlink', async () => {
await linkRuntimeSshAccess(userDataPath, request())
mocks.flush.mockRejectedValueOnce(new Error('disk full'))
await expect(unlink()).rejects.toThrow('disk full')
target = { ...target, orcadFence: { environmentId: 'other' } }
await expect(unlink()).rejects.toThrow('changed')
expect(events.filter((event) => event === 'release')).toHaveLength(1)
expect(current().pendingSshAccessOperation?.operation).toBe('unlink')
})
it('refuses direct authority even on an already claimed retry', async () => {
mocks.verify.mockRejectedValueOnce(new Error('offline'))
await expect(linkRuntimeSshAccess(userDataPath, request())).rejects.toThrow()
mocks.preflight.mockReturnValue({
blockers: [{ code: 'orcad_migration_direct_ssh_repositories' }]
})
await expect(linkRuntimeSshAccess(userDataPath, request())).rejects.toThrow(
'direct SSH authority'
)
expect(mocks.connect).toHaveBeenCalledTimes(1)
})
it('rejects provisioning and missing durable generations without connecting', async () => {
target.orcadProvisioning = { name: 'host', requestId: 'provision' }
await expect(linkRuntimeSshAccess(userDataPath, request())).rejects.toThrow('provisioning')
target.orcadProvisioning = undefined
target.generation = undefined
await expect(linkRuntimeSshAccess(userDataPath, request())).rejects.toThrow('generation')
expect(mocks.connect).not.toHaveBeenCalled()
})
it('treats a lost successful unlink response as an idempotent no-op', async () => {
await linkRuntimeSshAccess(userDataPath, request())
const unlinked = await unlink()
const before = [...events]
expect(await unlink()).toEqual(unlinked)
expect(events).toEqual(before)
})
it('hashes connection identity, not display state or owner', () => {
const hash = fingerprintRuntimeSshTarget(target)
expect(
fingerprintRuntimeSshTarget({
...target,
label: 'renamed',
lastRequiredPassphrase: true,
orcadFence: { environmentId: 'other' }
})
).toBe(hash)
expect(fingerprintRuntimeSshTarget({ ...target, proxyCommand: 'other-proxy' })).not.toBe(hash)
})
it('strictly validates ports and request IDs', () => {
expect(() => linkRuntimeSshAccess(userDataPath, { ...request(), remotePort: 0 })).toThrow()
expect(() =>
linkRuntimeSshAccess(userDataPath, { ...request(), requestId: '../other' })
).toThrow()
expect(mocks.infrastructure).not.toHaveBeenCalled()
})
})