fix(cloud): a slow wake no longer leaves the woken pane dead (calls and streams wait for the in-flight wake instead of being refused at their call timeout; waking refusals retry soon; pane layout push retries); the pill waits until the pane takes input and fails visibly at 8s; no reconnect banner during a wake, cloud copy names the workspace; pill sits below the pane tab strip; earlier-started wakes show at once

This commit is contained in:
Neil
2026-11-10 00:44:42 -08:00
parent ebf1d8c7bc
commit 5f82993ac6
26 changed files with 706 additions and 110 deletions
@@ -43,7 +43,11 @@ import { wslHostPhoneRefusal } from '../../wsl/host-lifecycle/wsl-host-phone-rea
import { broadcastSshState } from '../ssh/ssh-renderer-broadcast'
import { restoreManagedHostPortForwards } from '../ssh/ssh-managed-port-forward-restore'
import { recipeVmWakeIntentFor } from '../../../shared/remote-runtime/recipe-vm-wake'
import { prepareRecipeVm } from './runtime-environment-recipe-vm-wake'
import {
RECIPE_VM_IN_FLIGHT_WAKE_HOLD_MS,
awaitRecipeVmInFlightWake,
prepareRecipeVm
} from './runtime-environment-recipe-vm-wake'
/** `wslStart: 'never'` serves a WSL host only while it runs: a paired phone never boots a distro.
* `recipeVmHoldMs` bounds how long an open of a sleeping recipe VM workspace is held. */
@@ -61,6 +65,30 @@ export async function resolveManagedRuntimeEnvironment(
.environment
}
/**
* A stream (tabs, statuses) opened while the VM is already waking waits for that wake, as a call
* does, instead of being refused and left to the renderer's backoff (which can outlast the wake
* and leave the woken pane dead). Bounded by the in-flight hold, not the setup timeout: a stream
* is long-lived and starts once the server answers.
*/
export async function resolveSubscribedRuntimeEnvironment(
userDataPath: string,
selector: string,
preparation: ManagedRuntimeCallPreparation | undefined,
method: string,
signal?: AbortSignal
): Promise<ReturnType<typeof resolveEnvironment>> {
if (recipeVmWakeIntentFor(method) === 'none') {
await awaitRecipeVmInFlightWake(
userDataPath,
resolveEnvironment(userDataPath, selector),
Math.min(RECIPE_VM_IN_FLIGHT_WAKE_HOLD_MS, preparation?.recipeVmHoldMs ?? Infinity),
signal
)
}
return resolveManagedRuntimeEnvironment(userDataPath, selector, preparation, method, signal)
}
/** The server prepared for a request, with the record from before preparation. */
export async function prepareManagedRuntimeRequest(
userDataPath: string,
@@ -142,13 +142,14 @@ describe('a request to a recipe VM environment', () => {
awaitWake,
openEnvironment: async () => 'up'
})
await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list', 5_000)
expect(awaitWake).toHaveBeenCalledWith('workspace-of-vm-1', false, 5_000, undefined)
await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list')
// Not bounded by the call's own timeout (that starts once dispatched): the in-flight maximum.
expect(awaitWake).toHaveBeenCalledWith('workspace-of-vm-1', false, 60_000, undefined)
// Bounded by the caller's hold (0 = never wait) and never for an open (that wakes in its slot).
await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list', 5_000, {
await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list', {
preparation: { recipeVmHoldMs: 0 }
})
await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'worktree.activate', 5_000)
await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'worktree.activate')
expect(awaitWake).toHaveBeenCalledTimes(1)
expect(holdOpen).not.toHaveBeenCalled()
// Inside the slot the request is still answered at once from the owner's state.
@@ -165,10 +166,10 @@ describe('a request to a recipe VM environment', () => {
awaitWake,
openEnvironment: async () => 'up'
})
await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list', 600_000)
await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list')
expect(awaitWake).not.toHaveBeenCalled()
ownerState = 'waking'
await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list', 600_000)
await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list')
expect(awaitWake).toHaveBeenCalledWith('workspace-of-vm-1', false, 60_000, undefined)
})
@@ -186,7 +187,7 @@ describe('a request to a recipe VM environment', () => {
openEnvironment: async () => 'up'
})
await expect(
holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list', 5_000)
holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list')
).resolves.toEqual({ before: VM_ENVIRONMENT, wokenPairingRevision: 9 })
})
@@ -197,14 +197,18 @@ function refusal(outcome: Exclude<RecipeVmOpening, 'up'> | 'asleep'): RemoteRunt
}
}
/** A request that does not open the workspace waits (bounded by its own timeout and hold) for a wake
* already in flight. Called before the request takes a per-environment call slot, so held
* background requests never block an open. */
/**
* A request that does not open the workspace waits for a wake already in flight (bounded by the
* in-flight hold and its caller's preparation), then gets its own timeout once dispatched. Not
* bounded by that call timeout: a wake slower than it would refuse every request made meanwhile,
* and several (terminal.resolvePane, session.tabs.activate, the pane layout) are not retried, which
* left the woken pane dead. A caller that gives up aborts its signal, which ends the hold. Called
* before the request takes a per-environment call slot, so held requests never block an open.
*/
export function holdForInFlightRecipeVmWake<E extends RecipeVmEnvironment>(
userDataPath: string,
environment: E,
method: string,
timeoutMs: number | undefined,
options?: { preparation?: { recipeVmHoldMs?: number }; signal?: AbortSignal }
): Promise<HeldRecipeVmWake<E> | null> {
if (recipeVmWakeIntentFor(method) !== 'none') {
@@ -212,11 +216,7 @@ export function holdForInFlightRecipeVmWake<E extends RecipeVmEnvironment>(
}
const holdMs = Math.min(
RECIPE_VM_IN_FLIGHT_WAKE_HOLD_MS,
options?.preparation?.recipeVmHoldMs ?? Infinity,
timeoutMs ?? DEFAULT_CALL_TIMEOUT_MS
options?.preparation?.recipeVmHoldMs ?? Infinity
)
return awaitRecipeVmInFlightWake(userDataPath, environment, holdMs, options?.signal)
}
/** The transport's default call timeout, which also bounds the hold for calls that set none. */
const DEFAULT_CALL_TIMEOUT_MS = 15_000
@@ -0,0 +1,63 @@
// A stream (tabs, statuses) opened while the recipe VM is already waking waits for that wake, as a
// call does, instead of being refused at once and left to the renderer's backoff (a slow wake then
// left the woken pane dead).
import { beforeEach, describe, expect, it, vi } from 'vitest'
const ENVIRONMENT = { id: 'env-vm', pairingRevision: 1, createdAt: 0 }
const wake = vi.hoisted(() => ({
awaitInFlight: vi.fn(async () => null),
prepare: vi.fn(async () => ({ wokenPairingRevision: null }))
}))
vi.mock('../../../shared/remote-runtime/host-store/runtime-environment-store', () => ({
resolveEnvironment: () => ENVIRONMENT
}))
vi.mock('../../../shared/remote-runtime/host-catalog/runtime-environment-host-kind', () => ({
classifyRuntimeEnvironmentHost: () => ({ kind: 'paired' })
}))
vi.mock('../../wsl/host-lifecycle/wsl-host-gate', () => ({ assertWslHostReachable: () => {} }))
vi.mock('./runtime-environment-recipe-vm-wake', () => ({
RECIPE_VM_IN_FLIGHT_WAKE_HOLD_MS: 60_000,
awaitRecipeVmInFlightWake: wake.awaitInFlight,
prepareRecipeVm: wake.prepare
}))
import { resolveSubscribedRuntimeEnvironment } from './runtime-environment-managed-tunnel'
describe('a subscription opened while the VM wakes', () => {
beforeEach(() => {
wake.awaitInFlight.mockClear()
wake.prepare.mockClear()
})
it('waits for the in-flight wake (the in-flight maximum, or less if its caller says so) first', async () => {
await resolveSubscribedRuntimeEnvironment(
'/user-data',
'env-vm',
undefined,
'session.tabs.subscribe'
)
expect(wake.awaitInFlight).toHaveBeenCalledWith('/user-data', ENVIRONMENT, 60_000, undefined)
expect(wake.awaitInFlight.mock.invocationCallOrder[0]).toBeLessThan(
wake.prepare.mock.invocationCallOrder[0]!
)
await resolveSubscribedRuntimeEnvironment(
'/user-data',
'env-vm',
{ recipeVmHoldMs: 2_000 },
'session.tabs.subscribe'
)
expect(wake.awaitInFlight).toHaveBeenLastCalledWith('/user-data', ENVIRONMENT, 2_000, undefined)
})
it('does not hold a stream that opens the workspace (that one wakes it in its own preparation)', async () => {
await resolveSubscribedRuntimeEnvironment(
'/user-data',
'env-vm',
undefined,
'worktree.activate'
)
expect(wake.awaitInFlight).not.toHaveBeenCalled()
expect(wake.prepare).toHaveBeenCalled()
})
})
@@ -12,7 +12,7 @@ import { resolveEnvironment } from '../../../shared/remote-runtime/host-store/ru
import { prepareFencedRuntimeRequest } from './runtime-environment-request-preparation'
import {
requireWslHostWithoutStart,
resolveManagedRuntimeEnvironment,
resolveSubscribedRuntimeEnvironment,
type ManagedRuntimeCallPreparation
} from './runtime-environment-managed-tunnel'
import { markEnvironmentUsedFromResponse } from './runtime-environment-usage-marking'
@@ -85,13 +85,7 @@ export async function callRuntimeEnvironment(
)
}
const environment = resolveEnvironment(userDataPath, selector)
const held = await holdForInFlightRecipeVmWake(
userDataPath,
environment,
method,
timeoutMs,
options
)
const held = await holdForInFlightRecipeVmWake(userDataPath, environment, method, options)
// Connect failures reject; retain the dispatched endpoint for hints because queued re-pairing can change it.
let endpoint = getPreferredPairedRuntimeConnection(environment).endpoint
try {
@@ -213,7 +207,7 @@ export async function subscribeRuntimeEnvironment(
crewAttempt?.assertCurrent()
const environment = crewAttempt
? resolveEnvironment(userDataPath, selector)
: await resolveManagedRuntimeEnvironment(userDataPath, selector, preparation, method, signal)
: await resolveSubscribedRuntimeEnvironment(userDataPath, selector, preparation, method, signal)
crewAttempt?.assertCurrent()
if (crewAttempt) {
assertCrewSourceCurrent(environment)
@@ -7,6 +7,7 @@ import { useNow } from '@/hooks/use-now'
import { useTranslate } from '@/i18n/use-translate'
import { cn } from '@/lib/utils'
import { wakeElapsedSeconds } from '@/lib/orca-cloud/orca-cloud-workspace-state'
import { TERMINAL_CONTENT_INSET_PX, useTerminalContentInset } from './use-terminal-content-inset'
import type { CloudWakeProgressEvent } from '../../../../shared/hosted-sandbox/cloud-workspace-api'
/** After this long the pill says it's slower than usual; after STUCK_S it offers recovery inline. */
@@ -66,6 +67,7 @@ export function CloudWakingOverlay({
autoFocus = true,
typeThrough = false,
pendingInput = 0,
terminalStalled = false,
onClearPending,
onRetry,
onRestart,
@@ -78,6 +80,8 @@ export function CloudWakingOverlay({
typeThrough?: boolean
/** Characters held for the woken terminal (a count, never the text: it may be a password). */
pendingInput?: number
/** The server woke but the terminal in front never took input again: a reconnect card. */
terminalStalled?: boolean
onClearPending?: () => void
/** Move focus into the overlay (only when it covers the pane that had focus). */
autoFocus?: boolean
@@ -126,6 +130,8 @@ export function CloudWakingOverlay({
// Announced once each (the text changes only when crossing WAKE_SLOW_S and WAKE_STUCK_S).
const slowAnnouncement = stuck ? hint : slow ? hint : ''
const passThrough = typeThrough && !failed
// Over the terminal content, below the pane's tab strip: never over Run, split or close.
const inset = useTerminalContentInset(rootRef, passThrough)
return (
<div
ref={rootRef}
@@ -133,7 +139,7 @@ export function CloudWakingOverlay({
className={cn(
passThrough
? // Top-right, clear of the prompt line and what is being typed into it.
'pointer-events-none absolute inset-x-0 top-3 flex justify-end pr-3 outline-none'
'pointer-events-none absolute inset-0 outline-none'
: 'absolute inset-0 bg-background outline-none',
'motion-safe:animate-in motion-safe:fade-in-0 motion-safe:duration-150',
'motion-safe:transition-opacity motion-safe:duration-200',
@@ -160,9 +166,18 @@ export function CloudWakingOverlay({
<div
className={
passThrough
? 'pointer-events-auto flex flex-col items-end gap-1.5'
? 'pointer-events-auto absolute flex flex-col items-end gap-1.5'
: 'absolute inset-0 flex items-center justify-center p-4'
}
style={
passThrough
? {
top: inset?.top ?? TERMINAL_CONTENT_INSET_PX,
right: inset?.right ?? TERMINAL_CONTENT_INSET_PX
}
: undefined
}
data-cloud-waking-anchor={passThrough ? 'terminal' : undefined}
>
{failed ? (
<div
@@ -176,7 +191,14 @@ export function CloudWakingOverlay({
/>
<div className="min-w-0 flex-1 space-y-1">
<div className="text-sm font-medium">
{t('orcaCloudWorkspace.overlay.failedTitle', 'Couldn’t wake {{name}}', { name })}
{terminalStalled
? t(
'orcaCloudWorkspace.overlay.terminalStalledTitle',
'Couldn’t reconnect the terminal'
)
: t('orcaCloudWorkspace.overlay.failedTitle', 'Couldn’t wake {{name}}', {
name
})}
</div>
{wake.message ? (
<p className="break-words text-xs text-foreground">{wake.message}</p>
@@ -196,9 +218,11 @@ export function CloudWakingOverlay({
</Button>
</div>
<div className="flex justify-end gap-2">
<Button size="sm" variant="ghost" onClick={onRestart}>
{t('orcaCloudWorkspace.restartSandbox', 'Restart sandbox')}
</Button>
{terminalStalled ? null : (
<Button size="sm" variant="ghost" onClick={onRestart}>
{t('orcaCloudWorkspace.restartSandbox', 'Restart sandbox')}
</Button>
)}
<Button ref={retryRef} size="sm" onClick={onRetry}>
{t('orcaCloudWorkspace.overlay.tryAgain', 'Try again')}
</Button>
@@ -183,9 +183,7 @@ describe('Orca Cloud surfaces', () => {
'12 keys waiting · they go to the terminal when it’s up'
)
// Still the pill (no skeleton cover), and recovery only once stuck.
expect(container.querySelector('[data-cloud-waking-overlay]')?.className).toContain(
'justify-end'
)
expect(container.querySelector('[data-cloud-waking-anchor="terminal"]')).not.toBeNull()
expect(queryByRole('button', { name: 'Try again' })).toBeNull()
rerender(
<CloudWakingOverlay
@@ -128,9 +128,10 @@ function useWakeOverlay(
setShown({ wake: effective, at: shown.at })
return undefined
}
// Failures show at once (and wakes the person is already typing into); others only if they
// Failures show at once (and wakes the person is already typing into, and a wake that
// started earlier, e.g. on hover, and has already outlasted the delay); others only if they
// outlast the show delay.
if (failed || immediate) {
if (failed || immediate || Date.now() - effective.startedAt >= SHOW_DELAY_MS) {
setShown({ wake: effective, at: Date.now() })
return undefined
}
@@ -227,7 +228,7 @@ function CloudSurface({ workspaceId }: { workspaceId: string }): React.JSX.Eleme
// Ready means the server answers; the pill stays ("Connecting…") until the pane in front has
// re-attached and can take keys.
const attachHold = useWakeAttachHold(workspaceId, view.wake)
const shownWake = attachHold ?? view.wake
const shownWake = attachHold.wake ?? view.wake
const now = useNow(1_000, shownWake !== undefined)
// Typing is accepted while a wake is in flight, so the terminal stays visible and typeable.
const wakeInput = useWakeInput(workspaceId)
@@ -382,9 +383,14 @@ function CloudSurface({ workspaceId }: { workspaceId: string }): React.JSX.Eleme
typeThrough={typeThrough}
pendingInput={wakeInput.pending}
onClearPending={() => clearWakeInput(workspaceId)}
onRetry={wake}
terminalStalled={attachHold.terminalStalled}
onRetry={attachHold.terminalStalled ? attachHold.retry : wake}
onRestart={restart}
onDismiss={() => dismissCloudWake(workspaceId)}
onDismiss={
attachHold.terminalStalled
? attachHold.dismiss
: () => dismissCloudWake(workspaceId)
}
/>
) : layer === 'asleep' && offline.snapshot ? (
<div className="absolute inset-0 flex justify-center overflow-hidden bg-background">
@@ -87,3 +87,23 @@ export const wakingFromPredictive = defineFixture({
width: 640,
render: () => <TypeThrough value={startedAgo(3_000)} />
})
// The sandbox woke but the pane in front never took input within the cap: a reconnect card,
// never a silent dead pane.
export const wakingTerminalStalled = defineFixture({
width: 640,
render: () => (
<Pane>
<CloudWakingOverlay
wake={startedAgo(26_000, { phase: 'failed' })}
workspaceName="fix-auth-redirect"
autoFocus={false}
typeThrough
terminalStalled
onRetry={noop}
onRestart={noop}
onDismiss={noop}
/>
</Pane>
)
})
@@ -0,0 +1,77 @@
import { useLayoutEffect, useState, type RefObject } from 'react'
/** Gap between the waking pill and the edges of the terminal it sits over. */
export const TERMINAL_CONTENT_INSET_PX = 8
export type TerminalContentInset = { top: number; right: number }
/** The terminal the person is in (it holds focus), else the first one showing. */
function frontTerminal(scope: HTMLElement): HTMLElement | null {
const focused = document.activeElement?.closest?.('.xterm')
if (focused instanceof HTMLElement && scope.contains(focused)) {
return focused
}
for (const element of Array.from(scope.querySelectorAll('.xterm'))) {
if (element instanceof HTMLElement && element.getBoundingClientRect().width > 0) {
return element
}
}
return null
}
/** Where the pill goes, relative to `anchor`'s parent: inside the terminal content area, below
* the pane's tab strip (Run, split, close), so it never covers those controls. */
export function measureTerminalContentInset(anchor: HTMLElement): TerminalContentInset | null {
const scope = anchor.parentElement?.parentElement ?? anchor.parentElement
if (!scope) {
return null
}
const terminal = frontTerminal(scope)
if (!terminal) {
return null
}
const box = anchor.getBoundingClientRect()
const content = terminal.getBoundingClientRect()
return {
top: Math.max(0, Math.round(content.top - box.top)) + TERMINAL_CONTENT_INSET_PX,
right: Math.max(0, Math.round(box.right - content.right)) + TERMINAL_CONTENT_INSET_PX
}
}
/**
* Keeps the inset current while `enabled`: on mount and whenever the anchor's area resizes (a
* split, the sidebar). Null until measured or when no terminal is showing (callers fall back to
* a plain top-right inset).
*/
export function useTerminalContentInset(
anchor: RefObject<HTMLElement | null>,
enabled: boolean
): TerminalContentInset | null {
const [inset, setInset] = useState<TerminalContentInset | null>(null)
useLayoutEffect(() => {
const element = anchor.current
if (!enabled || !element) {
return
}
const update = (): void => {
const next = measureTerminalContentInset(element)
setInset((current) =>
current?.top === next?.top && current?.right === next?.right ? current : next
)
}
update()
const scope = element.parentElement?.parentElement ?? element.parentElement
if (!scope || typeof ResizeObserver === 'undefined') {
return
}
const observer = new ResizeObserver(update)
observer.observe(scope)
// Focus moving to another pane's terminal moves the pill with it.
document.addEventListener('focusin', update)
return () => {
observer.disconnect()
document.removeEventListener('focusin', update)
}
}, [anchor, enabled])
return inset
}
@@ -6,7 +6,8 @@ import type { CloudWakeProgressEvent } from '../../../../shared/hosted-sandbox/c
import {
isActiveWakePaneAttached,
registerWakePane,
resetWakePanesForTests
resetWakePanesForTests,
retryActiveWakePane
} from '@/components/terminal-pane/input/wake-pane-attach'
import {
WAKE_ATTACH_HOLD_MAX_MS,
@@ -47,6 +48,8 @@ describe('wake pane registry', () => {
expect(isActiveWakePaneAttached('ws')).toBe(false)
frontReady = true
expect(isActiveWakePaneAttached('ws')).toBe(true)
// Try again reaches the pane in front only.
expect(retryActiveWakePane('ws')).toBe(false)
// A remount registers again under the same leaf; the old session's removal leaves it alone.
registerWakePane('ws', 'front', { isReady: () => false, isActive: () => true })
offFront()
@@ -56,53 +59,70 @@ describe('wake pane registry', () => {
})
describe('useWakeAttachHold', () => {
it('holds a ready wake as Connecting… until the pane in front re-attaches, keeping its start', () => {
let attached: boolean | null = false
const { result, rerender } = renderHook(
const hook = (read: () => boolean | null, retry = vi.fn(() => true)) => {
const isAttached = (): boolean | null => read()
return renderHook(
({ wake }: { wake: CloudWakeProgressEvent | undefined }) =>
useWakeAttachHold('ws', wake, () => attached),
useWakeAttachHold('ws', wake, isAttached, retry),
{ initialProps: start }
)
expect(result.current).toBeUndefined()
}
it('holds a ready wake as Connecting… until the pane in front takes input, keeping its start', () => {
let attached: boolean | null = false
const { result, rerender } = hook(() => attached)
expect(result.current.wake).toBeUndefined()
rerender({ wake: { ...waking, phase: 'ready', at: T0 + 1_000 } })
expect(result.current).toMatchObject({ phase: 'connecting', startedAt: waking.startedAt })
expect(result.current.wake).toMatchObject({ phase: 'connecting', startedAt: waking.startedAt })
// The store drops the settled wake; the hold stays.
rerender({ wake: undefined })
act(() => vi.advanceTimersByTime(WAKE_ATTACH_POLL_MS * 5))
expect(result.current?.phase).toBe('connecting')
expect(result.current.wake?.phase).toBe('connecting')
attached = true
act(() => vi.advanceTimersByTime(WAKE_ATTACH_POLL_MS))
expect(result.current).toBeUndefined()
expect(result.current.wake).toBeUndefined()
expect(result.current.terminalStalled).toBe(false)
})
it('does not hold when the pane is already attached or none is known, nor after a failure', () => {
let attached: boolean | null = true
const { result, rerender } = renderHook(
({ wake }: { wake: CloudWakeProgressEvent | undefined }) =>
useWakeAttachHold('ws', wake, () => attached),
{ initialProps: start }
)
const { result, rerender } = hook(() => attached)
rerender({ wake: { ...waking, phase: 'ready' } })
expect(result.current).toBeUndefined()
expect(result.current.wake).toBeUndefined()
attached = null
rerender({ wake: { ...waking, startedAt: T0 } })
rerender({ wake: { ...waking, startedAt: T0, phase: 'ready' } })
expect(result.current).toBeUndefined()
expect(result.current.wake).toBeUndefined()
attached = false
rerender({ wake: { ...waking, startedAt: T0 + 1 } })
rerender({ wake: { ...waking, startedAt: T0 + 1, phase: 'failed' } })
expect(result.current).toBeUndefined()
expect(result.current.wake).toBeUndefined()
})
it('gives up on a pane that never re-attaches', () => {
const { result, rerender } = renderHook(
({ wake }: { wake: CloudWakeProgressEvent | undefined }) =>
useWakeAttachHold('ws', wake, () => false),
{ initialProps: start }
)
it('a pane that never takes input becomes "Couldn’t reconnect the terminal"; Try again reconnects it', () => {
let attached: boolean | null = false
const retry = vi.fn(() => true)
const { result, rerender } = hook(() => attached, retry)
rerender({ wake: { ...waking, phase: 'ready' } })
expect(result.current?.phase).toBe('connecting')
expect(result.current.wake?.phase).toBe('connecting')
act(() => vi.advanceTimersByTime(WAKE_ATTACH_HOLD_MAX_MS + WAKE_ATTACH_POLL_MS))
expect(result.current).toBeUndefined()
expect(result.current.terminalStalled).toBe(true)
expect(result.current.wake?.phase).toBe('failed')
// Try again: the pane reconnects and the pill waits for it once more.
act(() => result.current.retry())
expect(retry).toHaveBeenCalledWith('ws')
expect(result.current.terminalStalled).toBe(false)
expect(result.current.wake?.phase).toBe('connecting')
attached = true
act(() => vi.advanceTimersByTime(WAKE_ATTACH_POLL_MS))
expect(result.current.wake).toBeUndefined()
})
it('Dismiss clears the reconnect card', () => {
const { result, rerender } = hook(() => false)
rerender({ wake: { ...waking, phase: 'ready' } })
act(() => vi.advanceTimersByTime(WAKE_ATTACH_HOLD_MAX_MS + WAKE_ATTACH_POLL_MS))
act(() => result.current.dismiss())
expect(result.current.wake).toBeUndefined()
})
})
@@ -1,11 +1,14 @@
import { useEffect, useState } from 'react'
import { isActiveWakePaneAttached } from '@/components/terminal-pane/input/wake-pane-attach'
import { useCallback, useEffect, useState } from 'react'
import {
isActiveWakePaneAttached,
retryActiveWakePane
} from '@/components/terminal-pane/input/wake-pane-attach'
import { installWindowVisibilityInterval } from '@/lib/window-visibility-interval'
import type { CloudWakeProgressEvent } from '../../../../shared/hosted-sandbox/cloud-workspace-api'
/** How often the hold checks the pane in front while it waits. */
export const WAKE_ATTACH_POLL_MS = 100
/** A pane that never re-attaches (closed, broken) stops holding the pill after this. */
/** A pane that hasn't re-attached by now gets the "Couldn't reconnect the terminal" card. */
export const WAKE_ATTACH_HOLD_MAX_MS = 8_000
type Track = {
@@ -13,30 +16,54 @@ type Track = {
last: CloudWakeProgressEvent | undefined
/** The wake shown as still connecting while the pane in front re-attaches. */
held: CloudWakeProgressEvent | undefined
/** Bumped per hold (a Try again starts a new one), so each gets its own clock. */
attempt: number
/** The pane never re-attached within the cap: shown as a failed card, not a dead pane. */
stalled: CloudWakeProgressEvent | undefined
}
const IDLE: Track = { last: undefined, held: undefined, attempt: 0, stalled: undefined }
export type WakeAttachHold = {
/** What the overlay shows in place of the store's wake, if anything. */
wake: CloudWakeProgressEvent | undefined
/** True when `wake` is the "Couldn't reconnect the terminal" failure. */
terminalStalled: boolean
/** Try again: reconnect the pane in front and wait for it once more. */
retry: () => void
dismiss: () => void
}
/**
* A wake reports ready once the server answers; the pane in front re-attaches to its terminal a
* moment later. Until it has, this returns the wake as still `connecting` (same start time and
* source, so the pill keeps counting from the wake's start), then `undefined` hands back to the
* store's wake. Bounded, so a pane that never attaches can't hold the pill forever.
* moment later. Until it takes input (attached, with nothing it was given still unsent), this
* returns the wake as still `connecting` (same start time and source, so the pill keeps counting
* from the wake's start). A pane that hasn't by the cap becomes a failed card with Try again,
* never a silent dead pane. Then `undefined` hands back to the store's wake.
*/
export function useWakeAttachHold(
workspaceId: string,
wake: CloudWakeProgressEvent | undefined,
isAttached: (workspaceId: string) => boolean | null = isActiveWakePaneAttached
): CloudWakeProgressEvent | undefined {
const [track, setTrack] = useState<Track>({ last: undefined, held: undefined })
isAttached: (workspaceId: string) => boolean | null = isActiveWakePaneAttached,
retryPane: (workspaceId: string) => boolean = retryActiveWakePane
): WakeAttachHold {
const [track, setTrack] = useState<Track>(IDLE)
// Adjusted during render (not in an effect), so the pill never shows a stale frame.
if (wake && (wake.phase === 'waking' || wake.phase === 'connecting')) {
const last = track.last
if (!last || last.startedAt !== wake.startedAt || last.phase !== wake.phase || track.held) {
setTrack({ last: wake, held: undefined })
if (
!last ||
last.startedAt !== wake.startedAt ||
last.phase !== wake.phase ||
track.held ||
track.stalled
) {
setTrack({ ...IDLE, attempt: track.attempt, last: wake })
}
} else if (wake?.phase === 'failed') {
if (track.last || track.held) {
setTrack({ last: undefined, held: undefined })
if (track.last || track.held || track.stalled) {
setTrack({ ...IDLE, attempt: track.attempt })
}
} else if (track.last && (!wake || wake.startedAt === track.last.startedAt)) {
// Ready, or the settled entry already dropped: hold only a wake that was in flight here.
@@ -44,28 +71,64 @@ export function useWakeAttachHold(
isAttached(workspaceId) === false
? { ...track.last, phase: 'connecting' as const }
: undefined
setTrack({ last: undefined, held })
setTrack({ ...IDLE, attempt: track.attempt + 1, held })
}
// Release once the pane is attached, or after the cap. The store dropping the settled wake
// doesn't cancel this: it is keyed on the held wake only.
const heldStart = track.held?.startedAt
// Release once the pane takes input; at the cap, fail visibly. The store dropping the settled
// wake doesn't cancel this: it is keyed on the hold only.
const holding = track.held !== undefined
const attempt = track.attempt
useEffect(() => {
if (heldStart === undefined) {
if (!holding) {
return
}
const since = Date.now()
return installWindowVisibilityInterval({
run: () => {
if (isAttached(workspaceId) !== false || Date.now() - since >= WAKE_ATTACH_HOLD_MAX_MS) {
setTrack((current) =>
current.held?.startedAt === heldStart ? { ...current, held: undefined } : current
)
const attached = isAttached(workspaceId)
const timedOut = Date.now() - since >= WAKE_ATTACH_HOLD_MAX_MS
if (attached === false && !timedOut) {
return
}
setTrack((current) => {
if (current.attempt !== attempt || !current.held) {
return current
}
return attached === false
? {
...current,
held: undefined,
stalled: { ...current.held, phase: 'failed', at: Date.now() }
}
: { ...current, held: undefined }
})
},
intervalMs: WAKE_ATTACH_POLL_MS
})
}, [heldStart, workspaceId, isAttached])
}, [holding, attempt, workspaceId, isAttached])
return track.held
const retry = useCallback(() => {
retryPane(workspaceId)
setTrack((current) =>
current.stalled
? {
...current,
attempt: current.attempt + 1,
held: { ...current.stalled, phase: 'connecting' },
stalled: undefined
}
: current
)
}, [workspaceId, retryPane])
const dismiss = useCallback(
() => setTrack((current) => (current.stalled ? { ...current, stalled: undefined } : current)),
[]
)
return {
wake: track.stalled ?? track.held,
terminalStalled: track.stalled !== undefined,
retry,
dismiss
}
}
@@ -4,8 +4,10 @@
// never disappears over a pane that can't take a keystroke yet.
export type WakePaneEntry = {
/** Attached to the woken terminal and taking input. */
/** Taking input: attached to a terminal, connected, nothing it was given still unsent. */
isReady: () => boolean
/** Reconnect this pane's terminal (Try again on "Couldn't reconnect the terminal"). */
retry?: () => boolean
/** The pane in front: its tab is the active, visible one and it is that tab's active pane. */
isActive: () => boolean
}
@@ -61,6 +63,16 @@ export function isActiveWakePaneAttached(workspaceId: string): boolean | null {
return null
}
/** Try again: reconnects the pane in front. False when there is none, or it has no way to. */
export function retryActiveWakePane(workspaceId: string): boolean {
for (const entry of panes.get(workspaceId)?.values() ?? []) {
if (safely(entry.isActive)) {
return entry.retry ? safely(entry.retry) : false
}
}
return false
}
/** Tests. */
export function resetWakePanesForTests(): void {
panes.clear()
@@ -175,7 +175,21 @@ export function installPtyInputForward(session: ConnectPanePtySession): void {
rebindWakeInputTarget(session.deps.worktreeId, wakeInputKey, forwardUserInput, wakeInputReady)
// The waking pill stays until the pane in front is attached again (it can take keys then).
const unregisterWakePane = registerWakePane(session.deps.worktreeId, wakeInputKey, {
isReady: wakeInputReady,
// Input reaches the terminal: attached to one, and nothing typed into it still unsent (a pane
// that looks connected but drops input stays "Connecting…" and then fails visibly).
isReady: () => {
const transport = session.deps.paneTransportsRef.current.get(session.pane.id)
return (
wakeInputReady() &&
Boolean(transport?.getPtyId()) &&
transport?.isConnectPending?.() !== true &&
transport?.hasUndeliveredInput?.() !== true
)
},
retry: () => {
const transport = session.deps.paneTransportsRef.current.get(session.pane.id)
return Boolean(transport?.retryRecovery?.() || transport?.reopenAfterHostReturned?.())
},
isActive: () =>
!session.disposed &&
Boolean(session.deps.isActiveRef.current) &&
@@ -12,10 +12,13 @@ import { useTranslate } from '@/i18n/use-translate'
/** Show one shared server-outage notice; its reveal state keeps pane recovery banners quiet. */
export function RemoteServerOutageNotice({
environmentId,
reveal
reveal,
cloudWorkspaceName = null
}: {
environmentId: string
reveal: CalmStatusView
/** An Orca Cloud workspace's name: the notice names it, never the sandbox. */
cloudWorkspaceName?: string | null
}): React.JSX.Element | null {
const t = useTranslate()
const serverName = useAppStore((state) => selectRuntimeServerName(state, environmentId))
@@ -26,7 +29,7 @@ export function RemoteServerOutageNotice({
if (!reveal.visible) {
return null
}
const copy = remoteRuntimeRecoveryCopy(cause, serverName, t)
const copy = remoteRuntimeRecoveryCopy(cause, serverName, t, cloudWorkspaceName)
return (
<RemoteRuntimeRecoveryCard
icon="retrying"
@@ -18,6 +18,7 @@ import { WslHostStoppedCard } from '../../recovery/wsl/WslHostStoppedCard'
import { useRetryPanesWhenWslHostReturns } from '../../recovery/wsl/use-wsl-host-return-retry'
import { useCalmStatusReveal } from '@/hooks/use-calm-status-reveal'
import { useAppStore } from '@/store/index'
import { useCloudRecoveryContext } from '@/lib/orca-cloud/use-cloud-recovery-context'
import { paneShowsWslHostStoppedCard } from '@/runtime/hosts/wsl/wsl-host-stopped-status'
import { withoutWslHostStoppedLines } from '../../../../../../../shared/wsl-host-stopped-message'
import { mapPaneTerminalErrors } from '../../../errors/terminal-error-accumulation'
@@ -106,6 +107,9 @@ export function TerminalPaneRecoveryPortals({
clearStoppedHostErrors
)
const retryingOutageEnvironmentId = useRetryingOutageEnvironmentId(controller.worktreeId)
// An Orca Cloud workspace mid-wake already shows its waking pill; "Reconnecting" would be noise.
const cloud = useCloudRecoveryContext(controller.worktreeId)
const cloudWaking = cloud?.waking === true
const isolated = controller.isolatedPaneKey !== null
// Why: a pane waiting for SSH sign-in says so itself; "reconnecting" over it would be false.
const awaitingSignInHere = managedPanes.some(
@@ -120,7 +124,8 @@ export function TerminalPaneRecoveryPortals({
!wslHostUnavailable &&
// Why: "retrying automatically" would be false; each pane offers Start instead.
stoppedWslDistro === null &&
pairingRefusedEnvironmentId === null
pairingRefusedEnvironmentId === null &&
!cloudWaking
const serverDown = retryingOutageEnvironmentId !== null
const notice = useRemoteServerNoticeClaim(ownerEnvironmentId, serverDown, canSpeak)
const [noticeEnvironmentId, setNoticeEnvironmentId] = useState<string | null>(null)
@@ -150,6 +155,7 @@ export function TerminalPaneRecoveryPortals({
key={noticeEnvironmentId}
environmentId={noticeEnvironmentId}
reveal={noticeReveal}
cloudWorkspaceName={cloud?.workspaceName ?? null}
/>
) : null}
{managedPanes.map((pane) => {
@@ -195,13 +201,15 @@ export function TerminalPaneRecoveryPortals({
environmentId={paneEnvironmentId}
// Why stopped panes too: a card flashing over the notice's hold is the overlap this avoids.
heldByServer={
!isolated &&
!awaitingSignIn &&
!wslHostUnavailable &&
pairingRefusedEnvironmentId === null &&
notice.quiet &&
paneEnvironmentId === ownerEnvironmentId
cloudWaking ||
(!isolated &&
!awaitingSignIn &&
!wslHostUnavailable &&
pairingRefusedEnvironmentId === null &&
notice.quiet &&
paneEnvironmentId === ownerEnvironmentId)
}
cloudWorkspaceName={cloud?.workspaceName ?? null}
onReconnect={onReconnect}
wslHostUnavailable={wslHostUnavailable}
pairingRefused={pairingRefusedEnvironmentId !== null}
@@ -31,6 +31,8 @@ type BannerProps = {
onOpenReplacedServerSettings?: (() => void) | null
/** Main refused because the SSH host is waiting for the user to sign in; it resumes after. */
awaitingSignIn?: boolean
/** An Orca Cloud workspace's name: notices name it, never the sandbox. */
cloudWorkspaceName?: string | null
}
function reconnectAction(
@@ -55,7 +57,8 @@ export function TerminalRemoteRuntimeReconnectBanner({
wslHostUnavailable = null,
pairingRefused = false,
onOpenReplacedServerSettings = null,
awaitingSignIn = false
awaitingSignIn = false,
cloudWorkspaceName = null
}: BannerProps): React.JSX.Element {
const t = useTranslate()
if (wslHostUnavailable) {
@@ -143,7 +146,7 @@ export function TerminalRemoteRuntimeReconnectBanner({
)
}
const cause = paneRecoveryCause(recovery)
const copy = remoteRuntimeRecoveryCopy(cause, serverName, t)
const copy = remoteRuntimeRecoveryCopy(cause, serverName, t, cloudWorkspaceName)
return (
<RemoteRuntimeRecoveryCard
icon={copy.retrying ? 'retrying' : 'waiting'}
@@ -0,0 +1,32 @@
import { describe, expect, it } from 'vitest'
import { remoteRuntimeRecoveryCopy } from './remote-runtime-recovery-copy'
describe('reconnect notice copy', () => {
it('names an Orca Cloud workspace as the person knows it, never its sandbox', () => {
const copy = remoteRuntimeRecoveryCopy(
'connection-lost',
'hello sandbox a062c18e',
undefined,
'fix-auth'
)
expect(copy.title).toBe('Reconnecting to fix-auth…')
expect(copy.body).toBe(
'The connection dropped. Orca is retrying; your files and chats are safe.'
)
expect(`${copy.title} ${copy.body}`).not.toContain('a062c18e')
const other = remoteRuntimeRecoveryCopy(
'reattaching',
'hello sandbox a062c18e',
undefined,
'fix-auth'
)
expect(other.body).toContain('fix-auth')
expect(other.body).not.toContain('a062c18e')
})
it('keeps the server wording for everything else', () => {
expect(remoteRuntimeRecoveryCopy('connection-lost', 'devbox').title).toBe(
'Reconnecting to devbox'
)
})
})
@@ -33,9 +33,27 @@ export function paneRecoveryCause(
export function remoteRuntimeRecoveryCopy(
cause: RemoteRuntimeRecoveryCause,
serverName: string,
t: Translate = translate
t: Translate = translate,
/** An Orca Cloud workspace: named as the person knows it, never by its sandbox's id. */
cloudWorkspaceName: string | null = null
): RemoteRuntimeRecoveryCopy {
const server = { server: serverName }
const server = { server: cloudWorkspaceName ?? serverName }
if (cloudWorkspaceName !== null && cause === 'connection-lost') {
return {
retrying: true,
title: t(
'orcaCloudWorkspace.recovery.reconnectingTitle',
'Reconnecting to {{workspaceName}}…',
{
workspaceName: cloudWorkspaceName
}
),
body: t(
'orcaCloudWorkspace.recovery.reconnectingBody',
'The connection dropped. Orca is retrying; your files and chats are safe.'
)
}
}
switch (cause) {
case 'connection-lost':
return {
+6 -1
View File
@@ -26479,6 +26479,7 @@
"unarchiving": "Restoring from archive…",
"waking": "Waking…",
"connecting": "Connecting…",
"terminalStalledTitle": "Couldn’t reconnect the terminal",
"stuck": "This is taking much longer than usual.",
"slow": "Taking longer than usual. Your files and chats are safe.",
"messageQueued": "Your message sends as soon as it’s up.",
@@ -26579,7 +26580,11 @@
"moveHere": "Move here",
"openLocal": "Open local workspace",
"copiedOn": "Moved to this computer · {{date}}",
"moveAgain": "Move again"
"moveAgain": "Move again",
"recovery": {
"reconnectingTitle": "Reconnecting to {{workspaceName}}…",
"reconnectingBody": "The connection dropped. Orca is retrying; your files and chats are safe."
}
},
"cloudAgentSetup": {
"loading": "Checking what agents can use…",
@@ -0,0 +1,25 @@
import { useAppStore } from '@/store'
import { findWorktreeById } from '@/store/github/refresh/worktree-refresh'
import { useCloudWorkspace, useIsCloudWorkspace } from './orca-cloud-workspace-store'
export type CloudRecoveryContext = {
/** A wake is in flight: the waking pill already says what's happening, so no reconnect banner. */
waking: boolean
/** What the person calls it (never the sandbox's server name or id). */
workspaceName: string | null
}
/** For a terminal's recovery notices: null for a workspace that isn't on Orca Cloud. */
export function useCloudRecoveryContext(
worktreeId: string | null | undefined
): CloudRecoveryContext | null {
const isCloud = useIsCloudWorkspace(worktreeId)
const phase = useCloudWorkspace((s) => (worktreeId ? s.wake[worktreeId]?.phase : undefined))
const workspaceName = useAppStore((state) =>
worktreeId ? (findWorktreeById(state, worktreeId)?.displayName ?? null) : null
)
if (!isCloud) {
return null
}
return { waking: phase === 'waking' || phase === 'connecting', workspaceName }
}
@@ -0,0 +1,64 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { RUNTIME_ENVIRONMENT_WAKING_MESSAGE } from '../../../../../shared/remote-runtime/recipe-vm-wake'
const call = vi.hoisted(() => vi.fn())
vi.mock('../../../store/index', () => ({ useAppStore: { getState: () => ({}) } }))
vi.mock('../../../lib/workspace/ownership/resolve-owner', () => ({
runtimeEnvironmentIdUnlessTwin: () => 'env-1'
}))
vi.mock('../../web-session-tabs-sync/identity/web-runtime-session-environment', () => ({
isWebRuntimeSessionActive: () => true,
captureRuntimeEnvironmentCall: () => call
}))
vi.mock('../../web-session-tabs-sync/frame-drawn-server-workspace-set', () => ({
isServerFrameDrawnWorkspace: () => false
}))
import {
PANE_LAYOUT_WAKING_SENDS,
updateWebRuntimePaneLayout
} from './web-runtime-pane-layout-update'
const waking = new Error(`RemoteRuntimeClientError: ${RUNTIME_ENVIRONMENT_WAKING_MESSAGE}`)
const ok = { ok: true, result: { updated: true }, _meta: { runtimeId: 'r' } }
const push = (tabId = 'tab-1') =>
updateWebRuntimePaneLayout({ worktreeId: 'wt', tabId, root: null, expandedLeafId: null })
describe('pane layout push while the VM wakes', () => {
beforeEach(() => {
call.mockReset()
vi.spyOn(console, 'warn').mockImplementation(() => {})
})
it('sends again when refused as still waking, until it lands', async () => {
call.mockRejectedValueOnce(waking).mockRejectedValueOnce(waking).mockResolvedValue(ok)
await expect(push()).resolves.toBe(true)
expect(call).toHaveBeenCalledTimes(3)
})
it('gives up after a bounded number of sends, and never retries other failures', async () => {
call.mockRejectedValue(waking)
await expect(push()).resolves.toBe(false)
expect(call).toHaveBeenCalledTimes(PANE_LAYOUT_WAKING_SENDS)
call.mockReset()
call.mockRejectedValue(new Error('forbidden'))
await expect(push()).resolves.toBe(false)
expect(call).toHaveBeenCalledTimes(1)
})
it('drops a retry a newer push for the same tab has replaced', async () => {
let releaseFirst: (value: unknown) => void = () => {}
call
.mockImplementationOnce(
() => new Promise((_resolve, reject) => (releaseFirst = () => reject(waking)))
)
.mockResolvedValue(ok)
const first = push()
await expect(push()).resolves.toBe(true)
releaseFirst(undefined)
await expect(first).resolves.toBe(false)
// The older layout was not sent again over the newer one.
expect(call).toHaveBeenCalledTimes(2)
})
})
@@ -13,6 +13,14 @@ import {
isWebRuntimeSessionActive
} from '../../web-session-tabs-sync/identity/web-runtime-session-environment'
import { isServerFrameDrawnWorkspace } from '../../web-session-tabs-sync/frame-drawn-server-workspace-set'
import { isRecipeVmWakingRefusal } from '../../../../../shared/remote-runtime/recipe-vm-wake'
/** A push refused because the VM is still waking is sent again (each send waits for the wake in
* the host first), up to this many sends in all. */
export const PANE_LAYOUT_WAKING_SENDS = 4
// The newest push per tab: a retry never sends a layout a later push already replaced.
const latestPush = new Map<string, number>()
let pushSeq = 0
// Push remote geometry to its host or the next authoritative snapshot reverts it.
export async function updateWebRuntimePaneLayout(args: {
@@ -41,6 +49,41 @@ export async function updateWebRuntimePaneLayout(args: {
const hostTabId = isWebTerminalSurfaceTabId(args.tabId)
? toHostSessionTabId(args.tabId)
: args.tabId
const pushKey = `${args.worktreeId}\u0000${args.tabId}`
const seq = ++pushSeq
latestPush.set(pushKey, seq)
for (let send = 1; ; send++) {
const sent = await sendPaneLayout(callEnvironment, args, hostTabId)
const newest = latestPush.get(pushKey) === seq
// Still waking after the host's hold: the same push lands once it is up, unless a newer one
// for this tab has replaced it meanwhile.
if (
sent !== true &&
isRecipeVmWakingRefusal(sent) &&
send < PANE_LAYOUT_WAKING_SENDS &&
newest
) {
continue
}
if (newest) {
latestPush.delete(pushKey)
}
if (sent === true) {
return true
}
console.warn(
'[web-runtime-session] failed to update pane layout:',
sent instanceof Error ? sent.message : String(sent)
)
return false
}
}
async function sendPaneLayout(
callEnvironment: ReturnType<typeof captureRuntimeEnvironmentCall>,
args: Parameters<typeof updateWebRuntimePaneLayout>[0],
hostTabId: string
): Promise<true | unknown> {
try {
const response = await callEnvironment({
method: 'session.tabs.updatePaneLayout',
@@ -58,10 +101,6 @@ export async function updateWebRuntimePaneLayout(args: {
unwrapRuntimeRpcResult(response as RuntimeRpcResponse<{ updated: true }>)
return true
} catch (error) {
console.warn(
'[web-runtime-session] failed to update pane layout:',
error instanceof Error ? error.message : String(error)
)
return false
return error
}
}
@@ -0,0 +1,57 @@
// @vitest-environment happy-dom
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { resetStaleDocumentVisibilityForTesting } from '@/components/terminal-pane/viewport/visibility/stale-document-visibility'
import { RUNTIME_ENVIRONMENT_WAKING_MESSAGE } from '../../../../../shared/remote-runtime/recipe-vm-wake'
import {
WINDOW_VISIBILITY_SUBSCRIPTION_RETRY_INITIAL_MS,
installWindowVisibilitySubscriptionParking
} from './window-visibility-subscription-parking'
async function settle(): Promise<void> {
for (let i = 0; i < 4; i++) {
await Promise.resolve()
}
}
describe('a subscribe refused because the VM is still waking', () => {
beforeEach(() => {
vi.useFakeTimers()
vi.spyOn(Math, 'random').mockReturnValue(0)
})
afterEach(() => {
resetStaleDocumentVisibilityForTesting()
vi.useRealTimers()
vi.restoreAllMocks()
})
it('retries at the first delay, not on a backoff grown by earlier failures', async () => {
// As the IPC hop leaves it: the code is gone, the message stays.
const waking = new Error(
`Error invoking remote method 'runtimeEnvironments:subscribe': RemoteRuntimeClientError: ${RUNTIME_ENVIRONMENT_WAKING_MESSAGE}`
)
const subscribe = vi
.fn()
.mockRejectedValueOnce(new Error('Could not connect'))
.mockRejectedValueOnce(new Error('Could not connect'))
.mockRejectedValueOnce(new Error('Could not connect'))
.mockRejectedValueOnce(waking)
.mockResolvedValue({ unsubscribe: vi.fn() })
const onSubscribeError = vi.fn()
const dispose = installWindowVisibilitySubscriptionParking([{ subscribe, onSubscribeError }])
await settle()
// Three ordinary failures: 1 s, 2 s, 4 s.
for (const ms of [1_000, 2_000, 4_000]) {
await vi.advanceTimersByTimeAsync(ms)
await settle()
}
expect(subscribe).toHaveBeenCalledTimes(4)
expect(onSubscribeError).toHaveBeenLastCalledWith(waking)
// The waking refusal retries after the first delay (not 8 s).
await vi.advanceTimersByTimeAsync(WINDOW_VISIBILITY_SUBSCRIPTION_RETRY_INITIAL_MS)
await settle()
expect(subscribe).toHaveBeenCalledTimes(5)
dispose()
})
})
@@ -3,6 +3,7 @@ import {
subscribeWindowParkVisibility,
WINDOW_HIDE_PARK_GRACE_MS
} from '@/lib/window-park-visibility'
import { isRecipeVmWakingRefusal } from '../../../../../shared/remote-runtime/recipe-vm-wake'
import {
createWindowVisibilitySubscriptionEntry,
type SubscriptionEntry
@@ -179,6 +180,11 @@ export function installWindowVisibilitySubscriptionParking(
entry.pending = null
if (isCurrent()) {
spec.onSubscribeError?.(error)
// The VM is still waking: the same subscribe succeeds once it is up, so retry soon rather
// than on a backoff that can outlast the wake and leave the woken pane without its stream.
if (isRecipeVmWakingRefusal(error)) {
entry.retryAttempt = 0
}
scheduleRetry(entry, spec)
return
}
@@ -54,6 +54,22 @@ export const RUNTIME_ENVIRONMENT_SLEPT_MESSAGE =
export const RUNTIME_ENVIRONMENT_ASLEEP_MESSAGE =
"This workspace's VM is asleep. Open the workspace to wake it."
/**
* The host refused because the VM is still waking (it held the request as long as it would).
* Matches the error itself, a refused RPC response's error, and the message an IPC hop leaves.
*/
export function isRecipeVmWakingRefusal(error: unknown): boolean {
if (!error || typeof error !== 'object') {
return false
}
const code = 'code' in error ? error.code : undefined
const message = 'message' in error ? error.message : undefined
return (
code === RUNTIME_ENVIRONMENT_WAKING_ERROR_CODE ||
(typeof message === 'string' && message.includes(RUNTIME_ENVIRONMENT_WAKING_MESSAGE))
)
}
/** Every refusal a request to a recipe VM's server can get from the host. */
export const RECIPE_VM_REFUSAL_CODES = [
RUNTIME_ENVIRONMENT_WAKING_ERROR_CODE,