mirror of
https://github.com/stablyai/orca.git
synced 2026-10-30 16:47:34 +00:00
fix(ssh): move a managed tunnel off an unbindable saved local port (#33271)
* fix(ssh): move a managed tunnel off an unbindable saved local port When the saved local port of a managed SSH server's tunnel can no longer be bound (EADDRINUSE from another listener, or EACCES from a Windows excluded port range), pick a free port, save it with the pairing and the deployment link together, and build the tunnel again. The pairing, server identity, remote port and projects are unchanged. A move interrupted between its sidecar and store writes reads the moved port from the stored endpoint, so the deployment link is never dropped. Fixes #33182 Co-authored-by: SahilZ0810 <sahil@zamp.ai> * test(e2e): drop squatter connections so teardown never waits on them * fix(ssh): move the tunnel port before pairing repair and retire old transports Move outside buildWithPairingRepair so a refusal on the moved port is repaired against the revision the move saved, and retire transports and status owners still dialing the old port, as a repair does. * fix(ssh): refresh maintenance pairings on the current tunnel port Update, rollback and restart reread the deployment's local port when they save the new pairing, since the census before them may have moved it. * fix(ssh): let concurrent tunnel builds follow a port another caller moved --------- Co-authored-by: SahilZ0810 <sahil@zamp.ai>
This commit is contained in:
@@ -288,6 +288,17 @@ export const PR_E2E_SOURCE_ROUTES = [
|
||||
file
|
||||
))
|
||||
},
|
||||
{
|
||||
// A saved managed tunnel port that can no longer be bound moves to a free one (#33182).
|
||||
id: 'ssh.orcad-unbindable-tunnel-port',
|
||||
specs: ['tests/e2e/ssh-orcad-unbindable-tunnel-port.spec.ts'],
|
||||
matches: (file) =>
|
||||
MANAGED_SSH_HARNESS.test(file) ||
|
||||
(isProductSource(file) &&
|
||||
/^src\/(?:main\/managed-server\/lifecycle\/orcad-managed-tunnel|main\/ssh\/system-transport\/system-ssh-forward-port-reservation|shared\/runtime-environment-(?:sidecar|managed-orcad-store))/.test(
|
||||
file
|
||||
))
|
||||
},
|
||||
{
|
||||
// Why: hand-kept routes missed #30130 and #30120; scoped to managed-host code to bound lane cost.
|
||||
id: 'ssh.orcad-remote-sources',
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { createServer, type Server } from 'node:net'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { encodePairingOffer } from '../../../shared/pairing'
|
||||
import { addManagedOrcadEnvironment } from '../../../shared/runtime-environment-managed-orcad-store'
|
||||
import { resolveEnvironment } from '../../../shared/runtime-environment-store'
|
||||
import { getRuntimeSshAccess } from '../../../shared/runtime-environments'
|
||||
import type { SshTarget } from '../../../shared/ssh-types'
|
||||
import type { SshConnection } from '../../ssh/connection/ssh-connection'
|
||||
import type { SshConnectionManager } from '../../ssh/connection/ssh-connection-manager'
|
||||
import type { SshConnectionStore } from '../../ssh/connection/ssh-connection-store'
|
||||
import type { SshPortForwardManager } from '../../ssh/port-forwarding/ssh-port-forward'
|
||||
import { reserveLocalForwardPort } from '../../ssh/system-transport/system-ssh-forward-port-reservation'
|
||||
import { OrcadManagedTunnelManager } from './orcad-managed-tunnel-manager'
|
||||
import { buildOnBindableLocalPort } from './orcad-managed-tunnel-local-port'
|
||||
import { setManagedOrcadPairingRepairedListener } from './orcad-managed-pairing-repaired-listener'
|
||||
|
||||
function listen(port = 0): Promise<Server> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = createServer()
|
||||
server.once('error', reject)
|
||||
server.listen(port, '127.0.0.1', () => resolve(server))
|
||||
})
|
||||
}
|
||||
|
||||
function portOf(server: Server): number {
|
||||
const address = server.address()
|
||||
if (!address || typeof address === 'string') {
|
||||
throw new Error('listener has no port')
|
||||
}
|
||||
return address.port
|
||||
}
|
||||
|
||||
describe('managed tunnel on an unbindable saved local port', () => {
|
||||
let userDataPath: string
|
||||
let occupied: Server
|
||||
let savedPort: number
|
||||
|
||||
beforeEach(async () => {
|
||||
userDataPath = mkdtempSync(join(tmpdir(), 'orca-managed-tunnel-port-'))
|
||||
// Stands in for a Windows excluded range or another app's listener on the saved port.
|
||||
occupied = await listen()
|
||||
savedPort = portOf(occupied)
|
||||
addManagedOrcadEnvironment(userDataPath, {
|
||||
id: 'environment-1',
|
||||
name: 'Builder',
|
||||
pairingCode: encodePairingOffer({
|
||||
v: 2,
|
||||
endpoint: `ws://127.0.0.1:${savedPort}`,
|
||||
deviceToken: 'device-token',
|
||||
publicKeyB64: Buffer.alloc(32, 1).toString('base64'),
|
||||
pairedDeviceId: 'device-1'
|
||||
}),
|
||||
orcadDeployment: {
|
||||
sshTargetId: 'ssh-1',
|
||||
sshTargetGeneration: 7,
|
||||
localPort: savedPort,
|
||||
remotePort: 6_768
|
||||
},
|
||||
now: 100
|
||||
})
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
setManagedOrcadPairingRepairedListener(() => {})
|
||||
await new Promise((resolve) => occupied.close(resolve))
|
||||
rmSync(userDataPath, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
function setup(bindError?: Error, connectDelayMs = 0) {
|
||||
const target: SshTarget = {
|
||||
id: 'ssh-1',
|
||||
label: 'Builder',
|
||||
host: 'example.com',
|
||||
port: 22,
|
||||
username: 'deploy',
|
||||
generation: 7,
|
||||
orcadFence: { environmentId: 'environment-1' }
|
||||
}
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a test double for the members the tunnel manager calls.
|
||||
const connection = { getConnectGeneration: () => 1 } as unknown as SshConnection
|
||||
const addForward = vi.fn(
|
||||
async (connectionId: string, _conn: SshConnection, localPort: number) => {
|
||||
if (bindError && localPort === savedPort) {
|
||||
throw bindError
|
||||
}
|
||||
// The real transport reserves fixed ports before it forwards.
|
||||
await reserveLocalForwardPort(localPort)
|
||||
return {
|
||||
id: `forward-${localPort}`,
|
||||
connectionId,
|
||||
localPort,
|
||||
remoteHost: '',
|
||||
remotePort: 1
|
||||
}
|
||||
}
|
||||
)
|
||||
const manager = new OrcadManagedTunnelManager({
|
||||
getConnectionManager: () =>
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the tunnel manager reads only connect.
|
||||
({
|
||||
connect: async () => {
|
||||
await new Promise((resolve) => setTimeout(resolve, connectDelayMs))
|
||||
return connection
|
||||
}
|
||||
}) as unknown as SshConnectionManager,
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the tunnel manager reads only getTarget.
|
||||
getTargetStore: () => ({ getTarget: () => target }) as unknown as SshConnectionStore,
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a test double for the members the tunnel manager calls.
|
||||
forwardManager: {
|
||||
setCallbacks: vi.fn(),
|
||||
addForward,
|
||||
removeForwardAndWait: vi.fn(),
|
||||
dispose: vi.fn()
|
||||
} as unknown as SshPortForwardManager,
|
||||
ensureServing: vi.fn().mockResolvedValue({ state: 'serving' })
|
||||
})
|
||||
const resolve = () => resolveEnvironment(userDataPath, 'environment-1')
|
||||
const retired = vi.fn()
|
||||
setManagedOrcadPairingRepairedListener(retired)
|
||||
const ensure = () =>
|
||||
buildOnBindableLocalPort(userDataPath, 'environment-1', () =>
|
||||
manager.ensure(resolve(), resolve)
|
||||
)
|
||||
return { addForward, ensure, resolve, retired }
|
||||
}
|
||||
|
||||
it.each([
|
||||
['held by another listener', undefined],
|
||||
[
|
||||
'in a Windows excluded range',
|
||||
new Error('Local port is not available: listen EACCES', { cause: 'EACCES' })
|
||||
]
|
||||
])('moves to a free port and saves it with the same pairing when %s', async (_case, error) => {
|
||||
const { addForward, ensure, resolve, retired } = setup(error)
|
||||
const before = resolve()
|
||||
|
||||
await ensure()
|
||||
|
||||
const moved = resolve()
|
||||
const movedPort = getRuntimeSshAccess(moved)?.localPort
|
||||
expect(movedPort).not.toBe(savedPort)
|
||||
expect(addForward).toHaveBeenLastCalledWith(
|
||||
'ssh-1',
|
||||
expect.anything(),
|
||||
movedPort,
|
||||
'127.0.0.1',
|
||||
6_768,
|
||||
'Orca on Builder'
|
||||
)
|
||||
expect(moved.endpoints[0]?.endpoint).toBe(`ws://127.0.0.1:${movedPort}/`)
|
||||
expect(moved.endpoints[0]?.deviceToken).toBe(before.endpoints[0]?.deviceToken)
|
||||
expect(moved.endpoints[0]?.publicKeyB64).toBe(before.endpoints[0]?.publicKeyB64)
|
||||
expect(moved.pairedDeviceId).toBe('device-1')
|
||||
expect(moved.runtimeId).toBe(before.runtimeId)
|
||||
expect(moved.pairingRevision).toBeGreaterThan(before.pairingRevision ?? 0)
|
||||
expect(getRuntimeSshAccess(moved)?.remotePort).toBe(6_768)
|
||||
// Status owners and transports still dialing the old port are retired, as after a repair.
|
||||
expect(retired).toHaveBeenCalledWith('environment-1')
|
||||
})
|
||||
|
||||
it('moves once and lets every concurrent caller build on the moved port', async () => {
|
||||
const { ensure, resolve, retired } = setup(undefined, 50)
|
||||
|
||||
await expect(Promise.all([ensure(), ensure(), ensure()])).resolves.toHaveLength(3)
|
||||
|
||||
expect(getRuntimeSshAccess(resolve())?.localPort).not.toBe(savedPort)
|
||||
expect(retired).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('keeps the saved port when the bind fails for another reason', async () => {
|
||||
const { ensure, resolve, retired } = setup(new Error('ssh exited', { cause: 'ECONNRESET' }))
|
||||
|
||||
await expect(ensure()).rejects.toThrow('ssh exited')
|
||||
|
||||
expect(getRuntimeSshAccess(resolve())?.localPort).toBe(savedPort)
|
||||
expect(retired).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,48 @@
|
||||
// A saved tunnel port can become unbindable: a Windows excluded port range or another app's listener.
|
||||
import { moveManagedOrcadLocalPort } from '../../../shared/runtime-environment-managed-orcad-store'
|
||||
import { resolveEnvironment } from '../../../shared/runtime-environment-store'
|
||||
import { getOrcadSshDeployment } from '../../../shared/runtime-environments'
|
||||
import { reserveLocalForwardPort } from '../../ssh/system-transport/system-ssh-forward-port-reservation'
|
||||
import { notifyManagedOrcadPairingRepaired } from './orcad-managed-pairing-repaired-listener'
|
||||
|
||||
// Reservation reports the bind error code as `cause`; Windows excluded ranges answer EACCES.
|
||||
const UNBINDABLE_LOCAL_PORT_CODES = new Set<unknown>(['EADDRINUSE', 'EACCES'])
|
||||
|
||||
/**
|
||||
* Builds once more on a free local port when the saved one cannot be bound. The port is local to
|
||||
* this desktop, so the move keeps the pairing, the server's identity and its projects.
|
||||
*/
|
||||
export async function buildOnBindableLocalPort<T>(
|
||||
userDataPath: string,
|
||||
environmentId: string,
|
||||
build: () => Promise<T>
|
||||
): Promise<T> {
|
||||
const savedPort = () =>
|
||||
getOrcadSshDeployment(resolveEnvironment(userDataPath, environmentId))?.localPort
|
||||
const saved = savedPort()
|
||||
try {
|
||||
return await build()
|
||||
} catch (error) {
|
||||
if (saved === undefined) {
|
||||
throw error
|
||||
}
|
||||
// A concurrent build moved this server, superseding or failing ours; build on what it saved.
|
||||
if (savedPort() !== saved) {
|
||||
return build()
|
||||
}
|
||||
if (!(error instanceof Error) || !UNBINDABLE_LOCAL_PORT_CODES.has(error.cause)) {
|
||||
throw error
|
||||
}
|
||||
const localPort = await reserveLocalForwardPort(0)
|
||||
if (savedPort() !== saved) {
|
||||
return build()
|
||||
}
|
||||
console.warn(
|
||||
`[ssh] Moving the tunnel to this host’s Orca server from local port ${saved} to ${localPort}: ${error.message}`
|
||||
)
|
||||
moveManagedOrcadLocalPort(userDataPath, environmentId, localPort)
|
||||
// Like a repair, the rebind retires transports and status owners still dialing the old port.
|
||||
await notifyManagedOrcadPairingRepaired(environmentId)
|
||||
return build()
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import type { SshConnection } from '../../ssh/connection/ssh-connection'
|
||||
import { probeManagedOrcadTunnel } from './orcad-managed-tunnel-resume'
|
||||
import { ensureManagedOrcadServing } from './orcad-managed-serving'
|
||||
import { OrcadManagedTunnelManager } from './orcad-managed-tunnel-manager'
|
||||
import { buildOnBindableLocalPort } from './orcad-managed-tunnel-local-port'
|
||||
import { getRemoteRuntimeSharedControlAnsweredAt } from '../../ipc/runtime-environments/runtime-environment-shared-control-answers'
|
||||
import {
|
||||
getSshConnectionManager,
|
||||
@@ -60,8 +61,10 @@ export async function ensureOrcadManagedTunnel(
|
||||
selector: string
|
||||
): Promise<KnownRuntimeEnvironment> {
|
||||
const { id } = resolveEnvironment(userDataPath, selector)
|
||||
await buildWithPairingRepair(pairingRecovery(userDataPath), id, (environment) =>
|
||||
managedTunnels.ensure(environment, () => resolveEnvironmentOrNull(userDataPath, id))
|
||||
await buildOnBindableLocalPort(userDataPath, id, () =>
|
||||
buildWithPairingRepair(pairingRecovery(userDataPath), id, (environment) =>
|
||||
managedTunnels.ensure(environment, () => resolveEnvironmentOrNull(userDataPath, id))
|
||||
)
|
||||
)
|
||||
return resolveEnvironment(userDataPath, id)
|
||||
}
|
||||
@@ -71,8 +74,10 @@ export async function rebuildOrcadManagedTunnel(
|
||||
selector: string
|
||||
): Promise<'built' | 'repaired'> {
|
||||
const { id } = resolveEnvironment(userDataPath, selector)
|
||||
return buildWithPairingRepair(pairingRecovery(userDataPath), id, (environment) =>
|
||||
managedTunnels.rebuild(environment, () => resolveEnvironmentOrNull(userDataPath, id))
|
||||
return buildOnBindableLocalPort(userDataPath, id, () =>
|
||||
buildWithPairingRepair(pairingRecovery(userDataPath), id, (environment) =>
|
||||
managedTunnels.rebuild(environment, () => resolveEnvironmentOrNull(userDataPath, id))
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
import { refreshManagedOrcadPairing } from '../../shared/runtime-environment-managed-orcad-store'
|
||||
import { resolveEnvironment } from '../../shared/runtime-environment-store'
|
||||
import {
|
||||
getOrcadSshDeployment,
|
||||
getPreferredPairingOffer,
|
||||
type KnownRuntimeEnvironment
|
||||
} from '../../shared/runtime-environments'
|
||||
import type { ServeReadiness } from '../server/serve-readiness'
|
||||
import { tunneledOrcadPairingCode } from './orcad-tunneled-pairing'
|
||||
|
||||
/** Saves the pairing a managed server published after an update, rollback or restart. */
|
||||
export function refreshMaintainedOrcadPairing(
|
||||
userDataPath: string,
|
||||
environmentId: string,
|
||||
readiness: ServeReadiness
|
||||
): KnownRuntimeEnvironment {
|
||||
// Why the enrolled grant: a restarted orcad offers a new device once this one has paired, and
|
||||
// taking it would change who this desktop is to the host (as WSL's #29042 found). Re-read: the
|
||||
// census may have repaired a refused pairing, or moved an unbindable tunnel port, meanwhile.
|
||||
const current = resolveEnvironment(userDataPath, environmentId)
|
||||
const localPort = getOrcadSshDeployment(current)?.localPort
|
||||
if (localPort === undefined) {
|
||||
throw new Error('This server is no longer managed over SSH.')
|
||||
}
|
||||
const enrolled = getPreferredPairingOffer(current)
|
||||
const pairingCode = tunneledOrcadPairingCode(readiness, localPort, enrolled)
|
||||
return refreshManagedOrcadPairing(userDataPath, environmentId, pairingCode)
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { listEnvironments } from '../../shared/runtime-environment-store'
|
||||
import { ORCAD_UPDATE_ADMISSION_FENCE_MS } from '../../shared/orcad-terminal-census'
|
||||
import {
|
||||
moveManagedOrcadLocalPort,
|
||||
recordManagedOrcadMigration,
|
||||
refreshManagedOrcadPairing
|
||||
} from '../../shared/runtime-environment-managed-orcad-store'
|
||||
@@ -271,6 +272,21 @@ describe('updateManagedOrcadEnvironment', () => {
|
||||
await updateManagedOrcadEnvironment(harness.userDataPath, { selector: 'Managed' })
|
||||
expectEnrolledPairing()
|
||||
})
|
||||
|
||||
it('saves the new pairing on the tunnel port the census moved to', async () => {
|
||||
mocks.census.mockImplementationOnce(async () => {
|
||||
moveManagedOrcadLocalPort(harness.userDataPath, 'environment-1', MANAGED_LOCAL_PORT + 1)
|
||||
return idle
|
||||
})
|
||||
mocks.deploy.mockResolvedValueOnce({ outcome: 'installed-and-activated', fullVersion: 'v3' })
|
||||
mocks.probe.mockResolvedValueOnce(managedReadiness())
|
||||
|
||||
await updateManagedOrcadEnvironment(harness.userDataPath, { selector: 'Managed' })
|
||||
|
||||
const [environment] = listEnvironments(harness.userDataPath)
|
||||
expect(environment?.orcadDeployment?.localPort).toBe(MANAGED_LOCAL_PORT + 1)
|
||||
expect(environment?.endpoints[0]?.endpoint).toBe(`ws://127.0.0.1:${MANAGED_LOCAL_PORT + 1}/`)
|
||||
})
|
||||
})
|
||||
|
||||
describe('updateManagedOrcadEnvironment on a host already running this build', () => {
|
||||
|
||||
@@ -4,11 +4,8 @@
|
||||
* is unverifiable, never zero, so an update over live or uncounted terminals defers (D7).
|
||||
*/
|
||||
import { getAppEnvironment } from '../../shared/app-environment'
|
||||
import { refreshManagedOrcadPairing } from '../../shared/runtime-environment-managed-orcad-store'
|
||||
import { assertRuntimeEnvironmentNotReconciling } from '../../shared/runtime-environment-reconciliation-record'
|
||||
import { resolveEnvironment } from '../../shared/runtime-environment-store'
|
||||
import {
|
||||
getPreferredPairingOffer,
|
||||
redactRuntimeEnvironment,
|
||||
type KnownRuntimeEnvironment
|
||||
} from '../../shared/runtime-environments'
|
||||
@@ -18,7 +15,6 @@ import type {
|
||||
OrcadManagedRollbackResult
|
||||
} from '../../shared/orcad-managed-runtime'
|
||||
import { runTargetLifecycle } from '../ipc/ssh/ssh-target-lifecycle-queue'
|
||||
import type { ServeReadiness } from '../server/serve-readiness'
|
||||
import { recoverInterruptedOrcadActivation } from '../managed-server/activation/orcad-activation-recovery'
|
||||
import { probeActiveOrcadReadiness } from './orcad-active-readiness'
|
||||
import { materializeOrcadArtifact } from '../orcad/orcad-artifact-materializer'
|
||||
@@ -49,7 +45,7 @@ import { collectManagedTerminalCensus } from './orcad-terminal-census-client'
|
||||
import { updateTerminalCounts } from './orcad-fenced-stop'
|
||||
import { findIncompleteManagedOrcadMigration } from '../managed-server/lifecycle/orcad-managed-migration-status'
|
||||
import { latestOrcadMigrationInto } from './orcad-migration-rollback-mark'
|
||||
import { tunneledOrcadPairingCode } from './orcad-tunneled-pairing'
|
||||
import { refreshMaintainedOrcadPairing } from './orcad-maintenance-pairing-refresh'
|
||||
|
||||
type LifecycleArgs = { selector: string; signal?: AbortSignal }
|
||||
|
||||
@@ -67,20 +63,6 @@ export function withManagedOrcadLifecycle<T>(
|
||||
})
|
||||
}
|
||||
|
||||
function refreshPairing(
|
||||
userDataPath: string,
|
||||
environment: KnownRuntimeEnvironment,
|
||||
readiness: ServeReadiness,
|
||||
localPort: number
|
||||
): KnownRuntimeEnvironment {
|
||||
// Why the enrolled grant: a restarted orcad offers a new device once this one has paired, and
|
||||
// taking it would change who this desktop is to the host (as WSL's #29042 found). Re-read: the
|
||||
// census may have repaired a refused pairing since `environment` was read.
|
||||
const enrolled = getPreferredPairingOffer(resolveEnvironment(userDataPath, environment.id))
|
||||
const pairingCode = tunneledOrcadPairingCode(readiness, localPort, enrolled)
|
||||
return refreshManagedOrcadPairing(userDataPath, environment.id, pairingCode)
|
||||
}
|
||||
|
||||
/** A launched slot may bind another port than the forward in use, so forward anew. */
|
||||
export async function followLaunchedServerPort(
|
||||
userDataPath: string,
|
||||
@@ -184,7 +166,7 @@ export async function runManagedOrcadUpdate(
|
||||
activeVersion: result.fullVersion,
|
||||
readiness
|
||||
})
|
||||
const updated = refreshPairing(userDataPath, environment, readiness, deployment.localPort)
|
||||
const updated = refreshMaintainedOrcadPairing(userDataPath, environment.id, readiness)
|
||||
if (started || result.outcome !== 'already-active') {
|
||||
await followLaunchedServerPort(userDataPath, environment.id)
|
||||
}
|
||||
@@ -255,7 +237,7 @@ export function rollbackManagedOrcadEnvironment(
|
||||
{ ...slot, remoteInstallDir: targetDir },
|
||||
{ buildHash: targetBuildHash, fullVersion: result.target }
|
||||
)
|
||||
const updated = refreshPairing(userDataPath, environment, readiness, deployment.localPort)
|
||||
const updated = refreshMaintainedOrcadPairing(userDataPath, environment.id, readiness)
|
||||
await followLaunchedServerPort(userDataPath, environment.id)
|
||||
return {
|
||||
outcome: 'rolled-back',
|
||||
@@ -285,7 +267,7 @@ export function recoverManagedOrcadEnvironment(
|
||||
return result
|
||||
}
|
||||
const updated = result.readiness
|
||||
? refreshPairing(userDataPath, environment, result.readiness, deployment.localPort)
|
||||
? refreshMaintainedOrcadPairing(userDataPath, environment.id, result.readiness)
|
||||
: environment
|
||||
if (result.activeVersion) {
|
||||
await followLaunchedServerPort(userDataPath, environment.id)
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
} from './runtime-environment-store'
|
||||
import {
|
||||
addManagedOrcadEnvironment,
|
||||
moveManagedOrcadLocalPort,
|
||||
recordManagedOrcadMigration,
|
||||
refreshManagedOrcadPairing,
|
||||
removeManagedOrcadEnvironment
|
||||
@@ -160,6 +161,35 @@ describe('managed orcad environment store', () => {
|
||||
expect(listEnvironments(userDataPath)[0]?.orcadDeployment, 'store written').toEqual(deployment)
|
||||
})
|
||||
|
||||
it('keeps a moved tunnel port consistent at every point of its two-file write', () => {
|
||||
const before = add()
|
||||
const store = readPersistedEnvironmentStore(userDataPath)
|
||||
const existing = store.environments[0]!
|
||||
const { binding: _binding, ...state } = readCurrentRuntimeEnvironmentSidecarEntry(
|
||||
userDataPath,
|
||||
existing
|
||||
)!
|
||||
const movedEndpoint = { ...existing.endpoints[0]!, endpoint: 'ws://127.0.0.1:50001/' }
|
||||
const next = { ...existing, pairingRevision: 500, endpoints: [movedEndpoint] }
|
||||
// The first write of a move, then a store that landed on the moved endpoint.
|
||||
writeRuntimeEnvironmentSidecarEntry(userDataPath, store.environments, existing, state, next)
|
||||
expect(listEnvironments(userDataPath)[0]?.orcadDeployment, 'store not yet written').toEqual(
|
||||
deployment
|
||||
)
|
||||
shippedBuildRewrite(userDataPath, (environments) => {
|
||||
environments[0] = next
|
||||
})
|
||||
expect(listEnvironments(userDataPath)[0]?.orcadDeployment, 'store written').toEqual({
|
||||
...deployment,
|
||||
localPort: 50_001
|
||||
})
|
||||
|
||||
const moved = moveManagedOrcadLocalPort(userDataPath, 'environment-1', 50_002, 600)
|
||||
expect(moved.orcadDeployment).toEqual({ ...deployment, localPort: 50_002 })
|
||||
expect(moved.endpoints[0]?.deviceToken).toBe(before.endpoints[0]?.deviceToken)
|
||||
expect(listEnvironments(userDataPath)[0]).toEqual(moved)
|
||||
})
|
||||
|
||||
it('keeps the re-paired device identity when a reply from the previous pairing lands late', () => {
|
||||
const offer = (deviceToken: string, pairedDeviceId: string) =>
|
||||
encodePairingOffer({
|
||||
|
||||
@@ -112,6 +112,37 @@ export function refreshManagedOrcadPairing(
|
||||
if (!offer) {
|
||||
throw new RuntimeEnvironmentStoreError('invalid_argument', 'Invalid managed pairing code.')
|
||||
}
|
||||
return rebindManagedOrcadPairing(userDataPath, environmentId, () => offer, null, now)
|
||||
}
|
||||
|
||||
/** Moves a managed server's tunnel to a new local port, keeping its pairing and identity. */
|
||||
export function moveManagedOrcadLocalPort(
|
||||
userDataPath: string,
|
||||
environmentId: string,
|
||||
localPort: number,
|
||||
now = Date.now()
|
||||
): KnownRuntimeEnvironment {
|
||||
return rebindManagedOrcadPairing(
|
||||
userDataPath,
|
||||
environmentId,
|
||||
(current) => {
|
||||
const offer = getPreferredPairingOffer(current)
|
||||
const endpoint = new URL(offer.endpoint)
|
||||
endpoint.port = String(localPort)
|
||||
return { ...offer, endpoint: endpoint.toString() }
|
||||
},
|
||||
localPort,
|
||||
now
|
||||
)
|
||||
}
|
||||
|
||||
function rebindManagedOrcadPairing(
|
||||
userDataPath: string,
|
||||
environmentId: string,
|
||||
nextOffer: (current: KnownRuntimeEnvironment) => PairingOffer,
|
||||
localPort: number | null,
|
||||
now: number
|
||||
): KnownRuntimeEnvironment {
|
||||
const current = resolveEnvironmentFromStore(readEnvironmentStore(userDataPath), environmentId)
|
||||
if (!current.orcadDeployment) {
|
||||
throw new RuntimeEnvironmentStoreError('invalid_argument', 'This server is not managed.')
|
||||
@@ -123,6 +154,8 @@ export function refreshManagedOrcadPairing(
|
||||
'This server is not managed over SSH.'
|
||||
)
|
||||
}
|
||||
const offer = nextOffer(current)
|
||||
const nextDeployment = { ...deployment, localPort: localPort ?? deployment.localPort }
|
||||
// Why skip: orcad keeps its device registry across versions and callers pass the enrolled grant,
|
||||
// so an unchanged offer needs no rewrite, and every rewrite re-binds the sidecar entry.
|
||||
if (samePairing(getPreferredPairingOffer(current), offer)) {
|
||||
@@ -144,7 +177,7 @@ export function refreshManagedOrcadPairing(
|
||||
pairingRevision: Math.max(now, (existing.pairingRevision ?? existing.createdAt) + 1),
|
||||
lastUsedAt: existing.lastUsedAt
|
||||
})
|
||||
if (getPreferredLoopbackRuntimePort(next) !== deployment.localPort) {
|
||||
if (getPreferredLoopbackRuntimePort(next) !== nextDeployment.localPort) {
|
||||
throw new RuntimeEnvironmentStoreError(
|
||||
'invalid_argument',
|
||||
'The managed pairing does not point at its SSH tunnel.'
|
||||
@@ -154,12 +187,21 @@ export function refreshManagedOrcadPairing(
|
||||
candidate.id === existing.id ? next : candidate
|
||||
)
|
||||
const { binding: _binding, ...rest } = entry ?? {}
|
||||
const state = { ...rest, orcadDeployment: deployment }
|
||||
// Why three writes: the entry stays current for both store states, so a failed or interrupted
|
||||
// store write can never strand the deployment link.
|
||||
writeRuntimeEnvironmentSidecarEntry(userDataPath, store.environments, existing, state, next)
|
||||
// store write can never strand the deployment link. In between, a moved port is read from the
|
||||
// stored endpoint.
|
||||
writeRuntimeEnvironmentSidecarEntry(
|
||||
userDataPath,
|
||||
store.environments,
|
||||
existing,
|
||||
{ ...rest, orcadDeployment: deployment },
|
||||
next
|
||||
)
|
||||
writeEnvironmentStore(userDataPath, { version: 1, environments })
|
||||
writeRuntimeEnvironmentSidecarEntry(userDataPath, environments, next, state)
|
||||
writeRuntimeEnvironmentSidecarEntry(userDataPath, environments, next, {
|
||||
...rest,
|
||||
orcadDeployment: nextDeployment
|
||||
})
|
||||
return resolveEnvironmentFromStore(readEnvironmentStore(userDataPath), environmentId)
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import { readNodeFileSyncWithinLimit } from './node-bounded-file-reader'
|
||||
import { RuntimeEnvironmentReconciliationRecordSchema } from './runtime-environment-reconciliation-record'
|
||||
import { readEnvironmentStoreRecords } from './runtime-environment-store-records'
|
||||
import {
|
||||
getPreferredLoopbackRuntimePort,
|
||||
KnownRuntimeEnvironmentSchema,
|
||||
RuntimeAccessEndpointSchema,
|
||||
RuntimeSshAccessLinkSchema,
|
||||
@@ -124,12 +125,30 @@ function isCurrentEntry(
|
||||
environment: PersistedRuntimeEnvironment,
|
||||
entry: RuntimeEnvironmentSidecarEntry
|
||||
): boolean {
|
||||
return currentEntry(environment, entry) !== undefined
|
||||
}
|
||||
|
||||
/** The entry as of `environment`, or none when it is bound to a different store state. */
|
||||
function currentEntry(
|
||||
environment: PersistedRuntimeEnvironment,
|
||||
entry: RuntimeEnvironmentSidecarEntry
|
||||
): RuntimeEnvironmentSidecarEntry | undefined {
|
||||
const binding = runtimeEnvironmentSidecarBinding(environment)
|
||||
const matches = (candidate: z.infer<typeof SidecarBindingSchema> | undefined): boolean =>
|
||||
candidate?.createdAt === binding.createdAt &&
|
||||
candidate.pairingRevision === binding.pairingRevision &&
|
||||
candidate.preferredEndpointId === binding.preferredEndpointId
|
||||
return matches(entry.binding) || matches(entry.pendingBinding)
|
||||
if (matches(entry.binding)) {
|
||||
return entry
|
||||
}
|
||||
if (!matches(entry.pendingBinding)) {
|
||||
return undefined
|
||||
}
|
||||
// A rebind stopped after its store write: the stored endpoint already names the new tunnel port.
|
||||
const localPort = getPreferredLoopbackRuntimePort(environment)
|
||||
return entry.orcadDeployment && localPort !== null
|
||||
? { ...entry, orcadDeployment: { ...entry.orcadDeployment, localPort } }
|
||||
: entry
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -141,21 +160,22 @@ export function overlayRuntimeEnvironmentSidecar(
|
||||
entry: RuntimeEnvironmentSidecarEntry | undefined
|
||||
): KnownRuntimeEnvironment {
|
||||
const base = KnownRuntimeEnvironmentSchema.parse(environment)
|
||||
if (!entry || !isCurrentEntry(environment, entry)) {
|
||||
const current = entry && currentEntry(environment, entry)
|
||||
if (!current) {
|
||||
return base
|
||||
}
|
||||
const basePairingRevision = environment.pairingRevision ?? environment.createdAt
|
||||
const access = entry.sshAccess
|
||||
const access = current.sshAccess
|
||||
const pairingRevision = Math.max(
|
||||
basePairingRevision,
|
||||
entry.pairingRevisionFloor ?? basePairingRevision
|
||||
current.pairingRevisionFloor ?? basePairingRevision
|
||||
)
|
||||
const { endpoint, ...link } = access ?? {}
|
||||
const overlaid = KnownRuntimeEnvironmentSchema.safeParse({
|
||||
...environment,
|
||||
...(pairingRevision !== basePairingRevision ? { pairingRevision } : {}),
|
||||
...(entry.runtimeId !== undefined
|
||||
? { runtimeId: environment.runtimeId ?? entry.runtimeId }
|
||||
...(current.runtimeId !== undefined
|
||||
? { runtimeId: environment.runtimeId ?? current.runtimeId }
|
||||
: {}),
|
||||
...(access && endpoint
|
||||
? {
|
||||
@@ -165,12 +185,12 @@ export function overlayRuntimeEnvironmentSidecar(
|
||||
preferredEndpointId: endpoint.id
|
||||
}
|
||||
: {}),
|
||||
...(entry.pendingSshAccessOperation
|
||||
? { pendingSshAccessOperation: entry.pendingSshAccessOperation }
|
||||
...(current.pendingSshAccessOperation
|
||||
? { pendingSshAccessOperation: current.pendingSshAccessOperation }
|
||||
: {}),
|
||||
...(entry.orcadDeployment ? { orcadDeployment: entry.orcadDeployment } : {}),
|
||||
...(entry.orcadMigratedAt ? { orcadMigratedAt: entry.orcadMigratedAt } : {}),
|
||||
...(entry.reconciliation ? { reconciliation: entry.reconciliation } : {})
|
||||
...(current.orcadDeployment ? { orcadDeployment: current.orcadDeployment } : {}),
|
||||
...(current.orcadMigratedAt ? { orcadMigratedAt: current.orcadMigratedAt } : {}),
|
||||
...(current.reconciliation ? { reconciliation: current.reconciliation } : {})
|
||||
})
|
||||
return overlaid.success ? overlaid.data : base
|
||||
}
|
||||
@@ -254,5 +274,5 @@ export function readCurrentRuntimeEnvironmentSidecarEntry(
|
||||
)
|
||||
}
|
||||
const entry = sidecar.entries[environment.id]
|
||||
return entry && isCurrentEntry(environment, entry) ? entry : undefined
|
||||
return entry && currentEntry(environment, entry)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
/**
|
||||
* A managed SSH host whose saved local tunnel port can no longer be bound when Orca relaunches
|
||||
* (a Windows excluded port range, or another app listening there) is reached on a fresh port,
|
||||
* with the same pairing, and its pane answers again (#33182).
|
||||
*/
|
||||
import { createServer, type Server } from 'node:net'
|
||||
import type { Page } from '@stablyai/playwright-test'
|
||||
import { expect, test } from './helpers/orca-app'
|
||||
import { waitForSessionReady, waitForStartupWorktreeRefresh } from './helpers/store'
|
||||
import {
|
||||
activateManagedWorkspace,
|
||||
echoMarker,
|
||||
expectTerminalAnswers,
|
||||
launchManagedWorkspace,
|
||||
openTerminalTab,
|
||||
selectTerminalTab,
|
||||
skipUnlessManagedDockerHost,
|
||||
withManagedHost
|
||||
} from './helpers/orcad-managed-workspace'
|
||||
|
||||
skipUnlessManagedDockerHost()
|
||||
test.skip(process.platform === 'win32', 'The Docker SSH host uses POSIX tooling')
|
||||
|
||||
async function managedTunnel(page: Page, environmentId: string) {
|
||||
const environment = (await page.evaluate(() => window.api.runtimeEnvironments.list())).find(
|
||||
(entry) => entry.id === environmentId
|
||||
)
|
||||
return {
|
||||
localPort: environment?.orcadDeployment?.localPort ?? null,
|
||||
pairedDeviceId: environment?.pairedDeviceId ?? null
|
||||
}
|
||||
}
|
||||
|
||||
function occupy(port: number): Promise<Server> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Holds the port only; open connections would keep close() waiting through teardown.
|
||||
const server = createServer((socket) => socket.destroy())
|
||||
server.once('error', reject)
|
||||
server.listen(port, '127.0.0.1', () => resolve(server))
|
||||
})
|
||||
}
|
||||
|
||||
test('a relaunch whose saved tunnel port is taken moves to a free one and the pane answers', async (// oxlint-disable-next-line no-empty-pattern -- Owns its app launch through a restart session.
|
||||
{}, testInfo) => {
|
||||
test.setTimeout(15 * 60_000)
|
||||
const squatters: Server[] = []
|
||||
try {
|
||||
await withManagedHost(testInfo, async (fixture) => {
|
||||
const { page, seeded, environmentId } = await launchManagedWorkspace(fixture)
|
||||
const tabId = await openTerminalTab(page, seeded.worktreeId)
|
||||
await echoMarker(page, tabId, 'BEFORE_SQUAT')
|
||||
const before = await managedTunnel(page, environmentId)
|
||||
expect(before.localPort).toBeTruthy()
|
||||
|
||||
await fixture.session.close(fixture.app!)
|
||||
fixture.app = null
|
||||
// The quit releases the forward's listener; take its port before Orca starts again.
|
||||
await expect(async () => {
|
||||
squatters.push(await occupy(before.localPort!))
|
||||
}).toPass({ timeout: 30_000 })
|
||||
const relaunched = await fixture.session.launch()
|
||||
fixture.app = relaunched.app
|
||||
await waitForSessionReady(relaunched.page)
|
||||
await waitForStartupWorktreeRefresh(relaunched.page)
|
||||
|
||||
await activateManagedWorkspace(relaunched.page, seeded.worktreeId, environmentId)
|
||||
await selectTerminalTab(relaunched.page, tabId)
|
||||
await expectTerminalAnswers(relaunched.page, tabId, 4 * 60_000)
|
||||
const after = await managedTunnel(relaunched.page, environmentId)
|
||||
expect(after.localPort).not.toBe(before.localPort)
|
||||
expect(after.pairedDeviceId).toBe(before.pairedDeviceId)
|
||||
await relaunched.page.screenshot({ path: testInfo.outputPath('moved-tunnel-port.png') })
|
||||
})
|
||||
} finally {
|
||||
await Promise.all(squatters.map((server) => new Promise((resolve) => server.close(resolve))))
|
||||
}
|
||||
})
|
||||
Reference in New Issue
Block a user