Files
orca/src/shared/runtime-host-status-owner.ts
T
Neil ba6599eabb fix(ssh): pair again over SSH when a managed server drops this desktop, instead of reconnecting forever (#30775)
* fix(ssh): say the host dropped this desktop's pairing, not that another Orca holds the port

* fix(ssh): pair again over SSH when our own managed server refuses this desktop's token

* fix(ssh): key re-pairs by the refused revision, cap attempts, retry on explicit connect

* fix(ssh): keep a newer pairing stored mid-repair, count only completed reads, and offer Connect, not an access link, for managed servers

* fix(ssh): judge a re-pair only against the pairing still stored; Check connection retries a managed server's pairing

* fix(ssh): a pane whose managed server refuses this desktop says why and offers Reconnect, instead of spinning

* test(e2e): live failure path for a refused managed pairing

* fix(ssh): a Connect that re-paired answers from the new pairing, not the retired one

* fix(ssh): only an explicit Connect lifts a managed re-pair's hold and cap; say SSH access can pair again

Startup restore reaches ssh:connect with automatic: true and went through the same
ensureTunnel that cleared the hold, so repeated automatic reconnects could bypass the
three-attempt cap. The reset now lives in connectTarget's explicit branch.

The Shared Server Access footnote now says revoking there doesn't stop someone with
SSH access to this computer from pairing again.

* ci(e2e): route the managed pairing-repair spec with the managed-host remote sources

* fix(ssh): the repair gate decides whether a managed tunnel builds, not the build

A refused shared-control open rebuilt the tunnel (SSH execs, forward, handshake, wake)
before the held failure or the attempt cap was consulted, so reopens on a blocked
server rebuilt without limit. buildWithPairingRepair now asks the gate first: a failure
stands for the refused pairing revision for 15 s, or through the window once the cap is
spent, and every caller gets it without reaching the host.

- A connect during an in-flight repair keeps its reset (per-environment reset epoch).
- An unpaired answer fails at once; it is not a moved port, so no re-read or wake.
- A refusal that only rebuilt the tunnel no longer reports "pairing changed".
- WSL and cloud VMs no longer say Orca tried to pair over SSH, and Check connection
  only re-pairs servers managed over SSH (runtimePairingRepairRoute).
- Translate the new strings; route the pane-banner files to the managed-SSH e2e lane.

* fix(ssh): fence a status re-pair follow-up to the same server, and hold a refused repair under its own revision

- getRuntimeEnvironmentStatus followed any pairing change mid-probe, past a caller's
  expectedEnvironmentPairingRevision fence. It now follows only for an unpinned caller and only
  a new pairing of the same SSH-managed server under its pinned host key.
- recordRefusedAfterRepair stood under whatever revision was stored now, so a newer pairing
  stored mid-refusal was blocked untried; it stands under the repaired revision.
- The WSL restart test expects the refused-pairing classification main's WSL path now reaches.

* fix(ssh): a re-pair hands its callers the new credentials, and a failed one judges only the pairing it tried

- ensureOrcadManagedTunnel returns the environment as stored once the tunnel stands. The
  terminal census and a deploy's already-current result use it, so after a re-pair they
  no longer authenticate with the refused token, which read as an unverifiable census and
  refused restarts, updates and stops.
- The shared-control refusal handler blocks the status owner only while the failure stands
  for the pairing stored now (managedOrcadPairingFailureStands). A newer pairing stored
  meanwhile keeps its owner and retries.

* fix(ssh): migrations commit with the pairing the tunnel stands on, and a Connect mid-handshake stays reset

- Conversion binds the environment ensureOrcadManagedTunnel returns, so the destination, the
  server project reads and the result use the re-paired credentials (the migration mark is kept).
- A delta move gets its destination only from connectDestination, which opens the tunnel and
  builds the destination from what it returns; a pre-ensure destination can no longer exist. A
  tunnel that cannot open keeps the delta journaled and the host marked for the next move.
- buildWithPairingRepair takes the reset epoch when it starts, so a refusal after the re-pair
  cannot reinstate a failure that a Connect pressed during the handshake cleared.

* refactor(ssh): one module owns a shared control socket's environment policy, keeping request-connections under its line limit
2026-10-19 22:45:25 -07:00

284 lines
8.1 KiB
TypeScript

import {
runtimeHostStatusBlockedReason,
runtimeHostStatusError,
runtimeHostStatusFailure,
type RuntimeHostStatusResponse,
type RuntimeHostStatusSnapshot
} from './runtime-host-status'
const RETRY_DELAYS_MS = [3_000, 6_000, 12_000, 30_000, 60_000]
const REQUEST_TIMEOUT_MS = 15_000
let publicationSequence = 0
type Waiter = {
resolve: (response: RuntimeHostStatusResponse) => void
cleanup: () => void
}
type StatusOwnerOptions = {
environmentId: string
pairingRevision: number
persistent?: boolean
request: (signal: AbortSignal) => Promise<RuntimeHostStatusResponse>
verified: (response: Extract<RuntimeHostStatusResponse, { ok: true }>, active: boolean) => boolean
publish: (snapshot: RuntimeHostStatusSnapshot) => void
}
/** One verification and one retry slot, shared by all readers of this connection. */
export class RuntimeHostStatusOwner {
private active = false
private disposed = false
private persistent: boolean
private attempt = 0
private retry: ReturnType<typeof setTimeout> | null = null
private request: AbortController | null = null
private readonly waiters = new Set<Waiter>()
private response: RuntimeHostStatusResponse = runtimeHostStatusFailure(
'runtime_unavailable',
'Status has not been checked.'
)
private snapshot: RuntimeHostStatusSnapshot
constructor(private readonly options: StatusOwnerOptions) {
this.persistent = options.persistent ?? false
this.snapshot = {
environmentId: options.environmentId,
pairingRevision: options.pairingRevision,
sequence: ++publicationSequence,
checkedAt: 0,
status: null,
verification: 'checking',
transport: 'unknown'
}
}
read(): RuntimeHostStatusSnapshot {
return this.snapshot
}
activate(): void {
if (this.active || this.disposed) {
return
}
this.active = true
this.startRequest()
}
acceptVerified(response: Extract<RuntimeHostStatusResponse, { ok: true }>): void {
if (this.disposed) {
return
}
this.active = true
this.retireRequest()
this.clearRetry()
this.complete(response)
}
refresh(
options: { timeoutMs?: number; observeOnly?: true; reconnect?: true; signal?: AbortSignal } = {}
): Promise<RuntimeHostStatusResponse> {
if (options.signal?.aborted) {
return Promise.reject(options.signal.reason)
}
if (this.disposed) {
return Promise.resolve(this.response)
}
if (!options.observeOnly) {
this.active = true
}
if (options.reconnect) {
this.attempt = 0
this.update({ verification: 'checking' })
}
if (this.snapshot.verification === 'blocked') {
return Promise.resolve(this.response)
}
const result = new Promise<RuntimeHostStatusResponse>((resolve, reject) => {
const release = (): void => {
waiter.cleanup()
this.waiters.delete(waiter)
if (!this.active && this.waiters.size === 0) {
this.retireRequest()
}
}
const abort = (): void => {
release()
reject(options.signal?.reason)
}
const timer = setTimeout(() => {
release()
resolve(
runtimeHostStatusFailure(
'runtime_unavailable',
this.snapshot.transport === 'ready'
? 'Status request timed out.'
: 'Timed out waiting for the remote Orca runtime.'
)
)
}, options.timeoutMs ?? REQUEST_TIMEOUT_MS)
const waiter: Waiter = {
resolve,
cleanup: () => {
clearTimeout(timer)
options.signal?.removeEventListener('abort', abort)
}
}
this.waiters.add(waiter)
options.signal?.addEventListener('abort', abort, { once: true })
})
this.startRequest()
return result
}
connectionChanged(
transport: RuntimeHostStatusSnapshot['transport'],
remoteControl?: RuntimeHostStatusSnapshot['remoteControl']
): void {
if (this.disposed) {
return
}
const previous = this.snapshot.transport
this.update({ transport, ...(remoteControl !== undefined ? { remoteControl } : {}) })
if (transport === previous) {
return
}
if (previous === 'ready') {
this.retireRequest()
this.clearRetry()
if (this.snapshot.verification !== 'blocked') {
this.update({ verification: 'unavailable' })
}
}
if (transport === 'ready' && this.snapshot.verification !== 'blocked') {
// A pre-reconnect answer cannot verify the new socket's runtime.
this.retireRequest()
if (this.active || this.waiters.size > 0) {
this.startRequest()
}
}
}
authenticationRejected(message = 'Pair this client again.'): void {
if (this.disposed) {
return
}
this.retireRequest()
this.clearRetry()
this.complete(runtimeHostStatusFailure('unauthorized', message))
}
dispose(): void {
if (this.disposed) {
return
}
this.disposed = true
this.active = false
this.retireRequest()
this.clearRetry()
this.response = runtimeHostStatusFailure(
'runtime_manually_disconnected',
'Runtime environment was disconnected or replaced.'
)
this.update({ retired: true, transport: 'disconnected', verification: 'blocked' })
this.settleWaiters()
}
private startRequest(): void {
if (this.disposed || this.request || this.snapshot.verification === 'blocked') {
return
}
this.clearRetry()
const controller = new AbortController()
this.request = controller
if (this.snapshot.verification !== 'verified') {
this.update({ verification: 'checking' })
}
void this.verify(controller)
}
private async verify(controller: AbortController): Promise<void> {
let response: RuntimeHostStatusResponse
try {
response = await this.options.request(controller.signal)
} catch (error) {
response = runtimeHostStatusError(error)
if (error instanceof TypeError || error instanceof SyntaxError) {
console.error('Runtime status verification failed:', error)
response = runtimeHostStatusFailure('invalid_runtime_response', error.message)
}
}
if (this.request !== controller || this.disposed) {
return
}
this.request = null
this.complete(response)
}
private complete(response: RuntimeHostStatusResponse): void {
this.response = response
if (response.ok) {
this.attempt = 0
this.update({ status: response.result, checkedAt: Date.now(), verification: 'verified' })
this.persistent = this.options.verified(response, this.active)
} else {
const blockedReason = runtimeHostStatusBlockedReason(response)
this.update({
checkedAt: Date.now(),
verification: blockedReason ? 'blocked' : 'unavailable',
...(blockedReason ? { blockedReason } : {})
})
this.scheduleRetry()
}
this.settleWaiters()
}
private scheduleRetry(): void {
if (
!this.active ||
this.disposed ||
this.snapshot.verification === 'blocked' ||
(this.persistent && this.snapshot.transport !== 'ready')
) {
return
}
const delay = RETRY_DELAYS_MS[Math.min(this.attempt++, RETRY_DELAYS_MS.length - 1)]
this.retry = setTimeout(() => {
this.retry = null
this.startRequest()
}, delay)
}
private settleWaiters(): void {
for (const waiter of this.waiters) {
waiter.cleanup()
waiter.resolve(this.response)
}
this.waiters.clear()
}
private retireRequest(): void {
const request = this.request
this.request = null
request?.abort()
}
private clearRetry(): void {
if (this.retry) {
clearTimeout(this.retry)
}
this.retry = null
}
private update(patch: Partial<RuntimeHostStatusSnapshot>): void {
const { blockedReason, ...next } = { ...this.snapshot, ...patch }
// Why: a retired owner is a disconnect, not a refusal, so it never carries a reason.
const keepReason = blockedReason && next.verification === 'blocked' && !next.retired
this.snapshot = {
...next,
...(keepReason ? { blockedReason } : {}),
sequence: ++publicationSequence
}
this.options.publish(this.snapshot)
}
}