W8: phone forgets cloud workspaces that are gone (confirmed by the registry), never on transient errors (#154)

* fix(w8): phone forgets cloud workspaces that are gone (missing from the complete registry list, or wake says not_found); never on transient errors

* fix(w8): prune only on the registry's own not_found for that workspace (lists are capped and skip hidden rows; status 404s are owner-only); restore a link whose host removal failed

* perf(w8): host prune is single-flight and remembers a confirmed-present workspace for 5 minutes
This commit is contained in:
Neil
2026-11-09 19:52:06 -08:00
committed by GitHub
parent e5e0ef5cc7
commit 5e9fdffc41
8 changed files with 440 additions and 9 deletions
+34 -3
View File
@@ -115,11 +115,20 @@ function fromSandbox(row: z.infer<typeof SandboxViewSchema>): CloudWorkspace {
// Lists archived rows as `archived` rather than folding them into `suspended` (W2 lifecycle).
const FEATURES = { 'x-orca-sandbox-features': 'archive' }
/**
* `complete`: this is the team registry, so a workspace this phone can open that is missing from it
* is gone. The owner-only fallback omits teammates' workspaces and never proves that.
*/
export type CloudWorkspaceList = { workspaces: CloudWorkspace[]; complete: boolean }
// A destroyed workspace is gone, whatever list still carries it.
const isLive = (row: { status: string }): boolean => row.status !== 'destroyed'
/** The team registry (titles, repos, cost) first; the owner-only lifecycle list until it ships. */
export async function listCloudWorkspaces(
plane: CloudControlPlane,
signal?: AbortSignal
): Promise<CloudWorkspace[]> {
): Promise<CloudWorkspaceList> {
try {
const body = await request(
plane,
@@ -128,7 +137,7 @@ export async function listCloudWorkspaces(
CloudWorkspaceListResponseSchema,
{ signal }
)
return body.workspaces.map(fromRegistry)
return { workspaces: body.workspaces.filter(isLive).map(fromRegistry), complete: true }
} catch (error) {
if (!(error instanceof CloudRequestError) || error.status !== 404) {
throw error
@@ -141,7 +150,7 @@ export async function listCloudWorkspaces(
z.object({ sandboxes: z.array(SandboxViewSchema) }),
{ signal, headers: FEATURES }
)
return body.sandboxes.map(fromSandbox)
return { workspaces: body.sandboxes.filter(isLive).map(fromSandbox), complete: false }
}
export type CloudWakeState = HostedWakeResponse['state']
@@ -250,6 +259,28 @@ export async function renameCloudWorkspace(
).catch(() => undefined)
}
/**
* True only when the team registry says this workspace does not exist for this phone (destroyed,
* or no longer shared with it). Any other answer or failure is not proof.
*/
export async function isCloudWorkspaceGoneFromRegistry(
plane: CloudControlPlane,
sandboxId: string
): Promise<boolean> {
try {
await request(
plane,
'GET',
`/v1/team/workspaces/${encodeURIComponent(sandboxId)}`,
z.unknown(),
{ timeoutMs: 10_000 }
)
return false
} catch (error) {
return error instanceof CloudRequestError && error.status === 404 && error.code === 'not_found'
}
}
export type CloudRole = 'owner' | 'admin' | 'member'
const MeSchema = z.object({ role: z.enum(['owner', 'admin', 'member']) }).passthrough()
+212
View File
@@ -0,0 +1,212 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
const store = vi.hoisted(() => new Map<string, string>())
vi.mock('@react-native-async-storage/async-storage', () => ({
default: {
getItem: async (key: string) => store.get(key) ?? null,
setItem: async (key: string, value: string) => void store.set(key, value),
removeItem: async (key: string) => void store.delete(key)
}
}))
vi.mock('../transport/hosts/host-store', () => ({ loadHostCatalog: async () => [] }))
vi.mock('./cloud-workspace-attach', () => ({
CloudAttachError: class CloudAttachError extends Error {
constructor(readonly code: string) {
super(code)
}
}
}))
import { isCloudWorkspaceGoneFromRegistry } from './cloud-control-plane-client'
import { CloudRequestError } from './cloud-control-plane-request'
import {
isCloudWorkspaceGone,
pruneCloudHostsMissingFrom,
pruneGoneCloudHost
} from './cloud-host-pruning'
import { enqueueCloudPrompt, listQueuedCloudPrompts } from './cloud-prompt-queue'
import { CloudAttachError } from './cloud-workspace-attach'
import {
cloudWorkspaceHostIds,
cloudWorkspaceProjectRoot,
recordCloudWorkspaceHost
} from './cloud-workspace-links'
const LINKS_KEY = 'orca:cloud-workspace-hosts:v1'
type SeedLink = { hostId?: string; pairedAt?: number; updatedAt?: number; projectRoot?: string }
function seedLinks(links: Record<string, SeedLink>) {
store.set(LINKS_KEY, JSON.stringify(links))
}
const goneOnly =
(...ids: string[]) =>
async (sandboxId: string) =>
ids.includes(sandboxId)
describe('pruning hosts of gone cloud workspaces', () => {
beforeEach(() => {
store.clear()
})
it('forgets the host, link and queued prompts of a missing workspace the registry confirms is gone', async () => {
seedLinks({
sbx_gone: { hostId: 'host-gone', pairedAt: 1_000, updatedAt: 1_000 },
sbx_live: { hostId: 'host-live', pairedAt: 1_000, updatedAt: 1_000 }
})
await enqueueCloudPrompt({
sandboxId: 'sbx_gone',
text: 'never delivered',
agent: 'claude',
operationId: 'op-1'
})
await enqueueCloudPrompt({
sandboxId: 'sbx_live',
text: 'still wanted',
agent: 'claude',
operationId: 'op-2'
})
const removed: string[] = []
const confirmGone = vi.fn(goneOnly('sbx_gone'))
const pruned = await pruneCloudHostsMissingFrom(
new Set(['sbx_live']),
5_000,
confirmGone,
async (hostId) => {
removed.push(hostId)
}
)
expect(pruned).toEqual(['sbx_gone'])
// Only the workspace the list lacks is checked by id.
expect(confirmGone.mock.calls).toEqual([['sbx_gone']])
expect(removed).toEqual(['host-gone'])
expect([...(await cloudWorkspaceHostIds())]).toEqual(['host-live'])
expect(await listQueuedCloudPrompts('sbx_gone')).toEqual([])
expect(await listQueuedCloudPrompts('sbx_live')).toHaveLength(1)
})
it('keeps a missing workspace the registry still serves (capped list, hidden row, teammate share) or cannot answer for', async () => {
seedLinks({
sbx_hidden: { hostId: 'host-hidden', pairedAt: 1_000 },
sbx_unknown: { hostId: 'host-unknown', pairedAt: 1_000 }
})
const removeHost = vi.fn(async () => undefined)
const confirmGone = async (sandboxId: string) => {
if (sandboxId === 'sbx_unknown') {
throw new Error('offline')
}
return false
}
// An empty list (a server bug, or a capped page) proves nothing on its own.
expect(await pruneCloudHostsMissingFrom(new Set(), 5_000, confirmGone, removeHost)).toEqual([])
expect(removeHost).not.toHaveBeenCalled()
expect([...(await cloudWorkspaceHostIds())].sort()).toEqual(['host-hidden', 'host-unknown'])
})
it('never checks a link written after the list request started (created or paired meanwhile)', async () => {
seedLinks({ sbx_new: { hostId: 'host-new', pairedAt: 6_000, updatedAt: 6_000 } })
const confirmGone = vi.fn(async () => true)
expect(
await pruneCloudHostsMissingFrom(new Set(), 5_000, confirmGone, async () => undefined)
).toEqual([])
expect(confirmGone).not.toHaveBeenCalled()
expect([...(await cloudWorkspaceHostIds())]).toEqual(['host-new'])
})
it('puts the link back when the host cannot be removed, so the next confirmed miss retries', async () => {
seedLinks({
sbx_gone: { hostId: 'host-gone', pairedAt: 1_000, projectRoot: '/vercel/sandbox/project' }
})
await enqueueCloudPrompt({
sandboxId: 'sbx_gone',
text: 'kept with its link',
agent: 'claude',
operationId: 'op-3'
})
const pruned = await pruneCloudHostsMissingFrom(
new Set(),
5_000,
goneOnly('sbx_gone'),
async () => {
throw new Error('host store busy')
}
)
expect(pruned).toEqual([])
expect([...(await cloudWorkspaceHostIds())]).toEqual(['host-gone'])
expect(await cloudWorkspaceProjectRoot('sbx_gone')).toBe('/vercel/sandbox/project')
expect(await listQueuedCloudPrompts('sbx_gone')).toHaveLength(1)
})
it('forgets exactly one workspace on its own confirmed not_found', async () => {
await recordCloudWorkspaceHost('sbx_gone', 'host-gone')
await recordCloudWorkspaceHost('sbx_other', 'host-other')
const removed: string[] = []
expect(
await pruneGoneCloudHost('sbx_gone', async (hostId) => {
removed.push(hostId)
})
).toEqual(['sbx_gone'])
expect(removed).toEqual(['host-gone'])
expect([...(await cloudWorkspaceHostIds())]).toEqual(['host-other'])
})
it('asks about a still-present missing workspace once per few minutes, and shares one run between overlapping polls', async () => {
seedLinks({ sbx_capped: { hostId: 'host-capped', pairedAt: 1_000 } })
const confirmGone = vi.fn(async () => false)
const removeHost = async () => undefined
const [first, second] = await Promise.all([
pruneCloudHostsMissingFrom(new Set(), 5_000, confirmGone, removeHost),
pruneCloudHostsMissingFrom(new Set(), 5_000, confirmGone, removeHost)
])
expect(first).toEqual([])
expect(second).toEqual([])
await pruneCloudHostsMissingFrom(new Set(), 5_000, confirmGone, removeHost)
expect(confirmGone).toHaveBeenCalledTimes(1)
expect([...(await cloudWorkspaceHostIds())]).toEqual(['host-capped'])
})
it('treats only confirmed answers as gone, never transient failures', () => {
expect(isCloudWorkspaceGone(new CloudAttachError('not_found'))).toBe(true)
expect(isCloudWorkspaceGone(new CloudRequestError('not_found', 404))).toBe(true)
expect(isCloudWorkspaceGone(new CloudRequestError('network_error', 0))).toBe(false)
expect(isCloudWorkspaceGone(new CloudRequestError('timeout', 0))).toBe(false)
expect(isCloudWorkspaceGone(new CloudRequestError('provider_unavailable', 502))).toBe(false)
expect(isCloudWorkspaceGone(new CloudRequestError('unauthorized', 401))).toBe(false)
expect(isCloudWorkspaceGone(new CloudAttachError('wake_timeout'))).toBe(false)
expect(isCloudWorkspaceGone(new Error('offline'))).toBe(false)
})
})
describe('isCloudWorkspaceGoneFromRegistry', () => {
const plane = (reply: () => Promise<Response>) => ({
baseUrl: 'https://cp.test',
token: () => 'tok',
fetch: async () => reply()
})
it('is true only for the registry’s own 404 not_found', async () => {
const answer = (status: number, body: unknown) => async () => Response.json(body, { status })
expect(
await isCloudWorkspaceGoneFromRegistry(plane(answer(404, { error: 'not_found' })), 'sbx')
).toBe(true)
expect(
await isCloudWorkspaceGoneFromRegistry(plane(answer(200, { workspace: {} })), 'sbx')
).toBe(false)
expect(
await isCloudWorkspaceGoneFromRegistry(plane(answer(500, { error: 'internal' })), 'sbx')
).toBe(false)
expect(
await isCloudWorkspaceGoneFromRegistry(plane(answer(404, { error: 'route_missing' })), 'sbx')
).toBe(false)
expect(
await isCloudWorkspaceGoneFromRegistry(
plane(async () => {
throw new TypeError('Network request failed')
}),
'sbx'
)
).toBe(false)
})
})
+114
View File
@@ -0,0 +1,114 @@
import { CloudRequestError } from './cloud-control-plane-request'
import { removeCloudPromptsFor } from './cloud-prompt-queue'
import { CloudAttachError } from './cloud-workspace-attach'
import {
cloudWorkspaceLinkEntries,
restoreCloudWorkspaceLinks,
takeCloudWorkspaceLinks,
type TakenCloudWorkspaceLink
} from './cloud-workspace-links'
// A paired cloud workspace whose VM is gone would otherwise be retried by the phone forever. A
// workspace is forgotten only on a confirmed answer that it no longer exists for this phone: the
// team registry's own GET for it says not_found, or wake says not_found. Missing from a list only
// nominates it (lists are capped and skip hidden rows). Network errors, timeouts, server errors and
// 401s (the session's own path) never prune.
/** The control plane's confirmed answer that this workspace no longer exists for this phone. */
export function isCloudWorkspaceGone(error: unknown): boolean {
if (error instanceof CloudAttachError) {
return error.code === 'not_found'
}
return error instanceof CloudRequestError && error.status === 404 && error.code === 'not_found'
}
async function forget(
taken: readonly TakenCloudWorkspaceLink[],
removeHost: (hostId: string) => Promise<void>
): Promise<string[]> {
const kept: TakenCloudWorkspaceLink[] = []
for (const link of taken) {
if (link.hostId) {
try {
await removeHost(link.hostId)
} catch {
// A host still in the catalog with no link would be retried forever and unreachable by a
// later prune or sign-out: put the link back so the next confirmed miss tries again.
kept.push(link)
}
}
}
if (kept.length > 0) {
await restoreCloudWorkspaceLinks(kept)
}
const ids = taken.filter((link) => !kept.includes(link)).map((link) => link.sandboxId)
if (ids.length > 0) {
await removeCloudPromptsFor(new Set(ids))
}
return ids
}
/**
* After a successful list: every linked workspace the list lacks is checked by id (`confirmGone`),
* and forgotten only if that says it is gone. A link written after the list request started (a
* workspace created or paired meanwhile) is never a candidate.
*/
export function pruneCloudHostsMissingFrom(
listedIds: ReadonlySet<string>,
listStartedAt: number,
confirmGone: (sandboxId: string) => Promise<boolean>,
removeHost: (hostId: string) => Promise<void>
): Promise<string[]> {
// Single-flight: list polls overlap, and the next poll runs again with its own list anyway.
inFlight ??= prune(listedIds, listStartedAt, confirmGone, removeHost).finally(() => {
inFlight = null
})
return inFlight
}
// A workspace the registry still serves while the list lacks it (hidden row, past the list cap)
// would otherwise cost one GET per poll forever: its "present" answer holds for a while.
const PRESENT_TTL_MS = 5 * 60_000
const presentUntil = new Map<string, number>()
let inFlight: Promise<string[]> | null = null
async function prune(
listedIds: ReadonlySet<string>,
listStartedAt: number,
confirmGone: (sandboxId: string) => Promise<boolean>,
removeHost: (hostId: string) => Promise<void>
): Promise<string[]> {
const isCandidate = (sandboxId: string, link: { updatedAt?: number; pairedAt?: number }) =>
!listedIds.has(sandboxId) && (link.updatedAt ?? link.pairedAt ?? 0) < listStartedAt
const now = Date.now()
const candidates = (await cloudWorkspaceLinkEntries()).filter(
([id, link]) => isCandidate(id, link) && (presentUntil.get(id) ?? 0) <= now
)
const gone = new Set<string>()
for (const [sandboxId] of candidates) {
const answer = await confirmGone(sandboxId).catch(() => null)
if (answer === true) {
gone.add(sandboxId)
} else if (answer === false) {
// Only a real "still there" is remembered; a failed check is retried next poll.
presentUntil.set(sandboxId, now + PRESENT_TTL_MS)
}
}
if (gone.size === 0) {
return []
}
// Re-check inside the write: a pairing recorded during the confirmation is kept.
const taken = await takeCloudWorkspaceLinks(
(sandboxId, link) => gone.has(sandboxId) && isCandidate(sandboxId, link)
)
return forget(taken, removeHost)
}
/** One workspace answered not_found to wake: forget its pairing now. */
export async function pruneGoneCloudHost(
sandboxId: string,
removeHost: (hostId: string) => Promise<void>
): Promise<string[]> {
const taken = await takeCloudWorkspaceLinks((id) => id === sandboxId)
return forget(taken, removeHost)
}
+8
View File
@@ -152,6 +152,14 @@ export function removeCloudPrompt(id: string): Promise<void> {
return mutate((queue) => ({ next: queue.filter((entry) => entry.id !== id), result: undefined }))
}
/** The workspaces are gone: nothing queued for them can ever be delivered. */
export function removeCloudPromptsFor(sandboxIds: ReadonlySet<string>): Promise<void> {
return mutate((queue) => ({
next: queue.filter((entry) => !sandboxIds.has(entry.sandboxId)),
result: undefined
}))
}
export function subscribeCloudPromptQueue(listener: () => void): () => void {
listeners.add(listener)
return () => listeners.delete(listener)
+5
View File
@@ -39,6 +39,11 @@ export function useCloudHostRemoval(): CloudHostRemoval {
return useMemo(() => ({ forgetHostClient, queryClient }), [forgetHostClient, queryClient])
}
/** Unpairs one cloud VM host and closes its connection. */
export function cloudHostRemover(removal: CloudHostRemoval): (hostId: string) => Promise<void> {
return (hostId) => removeHostAndCloseClient(hostId, removal.forgetHostClient, removal.queryClient)
}
/** Ends everything Orca Cloud gave this phone: its VM pairings first, then the session and key. */
export async function forgetAllCloudAccess(removal: CloudHostRemoval): Promise<void> {
for (const hostId of await cloudWorkspaceHostIds()) {
+41 -4
View File
@@ -9,7 +9,9 @@ const LinkSchema = z.object({
hostId: z.string().optional(),
pairedAt: z.number().optional(),
/** From the create response; the VM's checkout of the workspace's repo. */
projectRoot: z.string().startsWith('/').optional()
projectRoot: z.string().startsWith('/').optional(),
/** Last write: a list that started before it cannot prove the workspace is gone. */
updatedAt: z.number().optional()
})
const LinksSchema = z.record(z.string(), LinkSchema)
type Links = z.infer<typeof LinksSchema>
@@ -50,9 +52,44 @@ function updateLinks(change: (links: Links) => void): Promise<void> {
export function recordCloudWorkspaceHost(sandboxId: string, hostId: string | null): Promise<void> {
return updateLinks((links) => {
const current = links[sandboxId] ?? {}
const now = Date.now()
links[sandboxId] = hostId
? { ...current, hostId, pairedAt: Date.now() }
: { projectRoot: current.projectRoot }
? { ...current, hostId, pairedAt: now, updatedAt: now }
: { projectRoot: current.projectRoot, updatedAt: now }
})
}
type Link = Links[string]
export type TakenCloudWorkspaceLink = { sandboxId: string; hostId: string | null; link: Link }
export async function cloudWorkspaceLinkEntries(): Promise<[string, Link][]> {
return Object.entries(await readLinks())
}
/** Removes, in one write, every link `matches` picks; returns what was removed. */
export async function takeCloudWorkspaceLinks(
matches: (sandboxId: string, link: Link) => boolean
): Promise<TakenCloudWorkspaceLink[]> {
const taken: TakenCloudWorkspaceLink[] = []
await updateLinks((links) => {
for (const [sandboxId, link] of Object.entries(links)) {
if (matches(sandboxId, link)) {
taken.push({ sandboxId, hostId: link.hostId ?? null, link })
delete links[sandboxId]
}
}
})
return taken
}
/** Puts back taken links whose cleanup failed; a link written since then wins. */
export function restoreCloudWorkspaceLinks(
restored: readonly TakenCloudWorkspaceLink[]
): Promise<void> {
return updateLinks((links) => {
for (const entry of restored) {
links[entry.sandboxId] ??= entry.link
}
})
}
@@ -64,7 +101,7 @@ export function recordCloudWorkspaceProjectRoot(
return Promise.resolve()
}
return updateLinks((links) => {
links[sandboxId] = { ...links[sandboxId], projectRoot }
links[sandboxId] = { ...links[sandboxId], projectRoot, updatedAt: Date.now() }
})
}
+10 -1
View File
@@ -3,7 +3,7 @@ import { useCallback, useEffect, useRef, useState } from 'react'
import { AccessibilityInfo } from 'react-native'
import { useMobileTranslation } from '../i18n/use-mobile-translation'
import { useHostClient } from '../transport/connection/host-client-hooks'
import type { CloudWorkspace } from './cloud-control-plane-client'
import { isCloudWorkspaceGoneFromRegistry, type CloudWorkspace } from './cloud-control-plane-client'
import { cloudDeviceLabel } from './cloud-device-label'
import {
deliverQueuedCloudPrompts,
@@ -11,7 +11,9 @@ import {
markWaitingCloudPromptsFailed,
requeueUnsentCloudPrompts
} from './cloud-prompt-queue'
import { isCloudWorkspaceGone, pruneGoneCloudHost } from './cloud-host-pruning'
import {
cloudHostRemover,
forgetRevokedCloudSession,
loadCloudControlPlane,
useCloudHostRemoval
@@ -250,6 +252,13 @@ export function useCloudWorkspaceRun(sandboxId: string, workspace: CloudWorkspac
return
}
await forgetRevokedCloudSession(error, removal)
if (
isCloudWorkspaceGone(error) &&
(await isCloudWorkspaceGoneFromRegistry(plane, sandboxId).catch(() => false))
) {
// Destroyed: its pairing and queued prompts go, so nothing retries a VM that no longer exists.
await pruneGoneCloudHost(sandboxId, cloudHostRemover(removal)).catch(() => undefined)
}
// Prompts that were going out with this attach now say so, instead of "waiting".
await markWaitingCloudPromptsFailed(sandboxId, 'not_connected').catch(() => undefined)
const role = await roleForCloudError(error, plane, controller.signal)
+16 -1
View File
@@ -2,6 +2,7 @@ import { useQuery, useQueryClient } from '@tanstack/react-query'
import { useFocusEffect } from 'expo-router'
import { useCallback, useEffect, useState } from 'react'
import {
isCloudWorkspaceGoneFromRegistry,
listCloudWorkspaces,
readCloudWorkspaceStatus,
type CloudWorkspace
@@ -11,7 +12,9 @@ import {
subscribeCloudDeviceIdentity,
type CloudEnrollment
} from './cloud-device-identity'
import { pruneCloudHostsMissingFrom } from './cloud-host-pruning'
import {
cloudHostRemover,
forgetRevokedCloudSession,
loadCloudControlPlane,
useCloudHostRemoval
@@ -70,7 +73,19 @@ export function useCloudWorkspaces(enrollment: CloudEnrollment | null | undefine
return []
}
try {
return await listCloudWorkspaces(plane, signal)
const startedAt = Date.now()
const list = await listCloudWorkspaces(plane, signal)
if (list.complete) {
// A linked workspace the list lacks is checked by id; one the registry says is gone
// (destroyed, or no longer shared) stops being retried.
void pruneCloudHostsMissingFrom(
new Set(list.workspaces.map((workspace) => workspace.id)),
startedAt,
(sandboxId) => isCloudWorkspaceGoneFromRegistry(plane, sandboxId),
cloudHostRemover(removal)
).catch(() => undefined)
}
return list.workspaces
} catch (error) {
await forgetRevokedCloudSession(error, removal)
throw error