diff --git a/mobile/src/cloud/cloud-control-plane-client.ts b/mobile/src/cloud/cloud-control-plane-client.ts index 9a73f1b2e9a..6eef33f6281 100644 --- a/mobile/src/cloud/cloud-control-plane-client.ts +++ b/mobile/src/cloud/cloud-control-plane-client.ts @@ -115,11 +115,20 @@ function fromSandbox(row: z.infer): CloudWorkspace { // Lists archived rows as `archived` rather than folding them into `suspended` (W2 lifecycle). const FEATURES = { 'x-orca-sandbox-features': 'archive' } +/** + * `complete`: this is the team registry, so a workspace this phone can open that is missing from it + * is gone. The owner-only fallback omits teammates' workspaces and never proves that. + */ +export type CloudWorkspaceList = { workspaces: CloudWorkspace[]; complete: boolean } + +// A destroyed workspace is gone, whatever list still carries it. +const isLive = (row: { status: string }): boolean => row.status !== 'destroyed' + /** The team registry (titles, repos, cost) first; the owner-only lifecycle list until it ships. */ export async function listCloudWorkspaces( plane: CloudControlPlane, signal?: AbortSignal -): Promise { +): Promise { try { const body = await request( plane, @@ -128,7 +137,7 @@ export async function listCloudWorkspaces( CloudWorkspaceListResponseSchema, { signal } ) - return body.workspaces.map(fromRegistry) + return { workspaces: body.workspaces.filter(isLive).map(fromRegistry), complete: true } } catch (error) { if (!(error instanceof CloudRequestError) || error.status !== 404) { throw error @@ -141,7 +150,7 @@ export async function listCloudWorkspaces( z.object({ sandboxes: z.array(SandboxViewSchema) }), { signal, headers: FEATURES } ) - return body.sandboxes.map(fromSandbox) + return { workspaces: body.sandboxes.filter(isLive).map(fromSandbox), complete: false } } export type CloudWakeState = HostedWakeResponse['state'] @@ -250,6 +259,28 @@ export async function renameCloudWorkspace( ).catch(() => undefined) } +/** + * True only when the team registry says this workspace does not exist for this phone (destroyed, + * or no longer shared with it). Any other answer or failure is not proof. + */ +export async function isCloudWorkspaceGoneFromRegistry( + plane: CloudControlPlane, + sandboxId: string +): Promise { + try { + await request( + plane, + 'GET', + `/v1/team/workspaces/${encodeURIComponent(sandboxId)}`, + z.unknown(), + { timeoutMs: 10_000 } + ) + return false + } catch (error) { + return error instanceof CloudRequestError && error.status === 404 && error.code === 'not_found' + } +} + export type CloudRole = 'owner' | 'admin' | 'member' const MeSchema = z.object({ role: z.enum(['owner', 'admin', 'member']) }).passthrough() diff --git a/mobile/src/cloud/cloud-host-pruning.test.ts b/mobile/src/cloud/cloud-host-pruning.test.ts new file mode 100644 index 00000000000..d3ffd9e428e --- /dev/null +++ b/mobile/src/cloud/cloud-host-pruning.test.ts @@ -0,0 +1,212 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const store = vi.hoisted(() => new Map()) +vi.mock('@react-native-async-storage/async-storage', () => ({ + default: { + getItem: async (key: string) => store.get(key) ?? null, + setItem: async (key: string, value: string) => void store.set(key, value), + removeItem: async (key: string) => void store.delete(key) + } +})) +vi.mock('../transport/hosts/host-store', () => ({ loadHostCatalog: async () => [] })) +vi.mock('./cloud-workspace-attach', () => ({ + CloudAttachError: class CloudAttachError extends Error { + constructor(readonly code: string) { + super(code) + } + } +})) + +import { isCloudWorkspaceGoneFromRegistry } from './cloud-control-plane-client' +import { CloudRequestError } from './cloud-control-plane-request' +import { + isCloudWorkspaceGone, + pruneCloudHostsMissingFrom, + pruneGoneCloudHost +} from './cloud-host-pruning' +import { enqueueCloudPrompt, listQueuedCloudPrompts } from './cloud-prompt-queue' +import { CloudAttachError } from './cloud-workspace-attach' +import { + cloudWorkspaceHostIds, + cloudWorkspaceProjectRoot, + recordCloudWorkspaceHost +} from './cloud-workspace-links' + +const LINKS_KEY = 'orca:cloud-workspace-hosts:v1' +type SeedLink = { hostId?: string; pairedAt?: number; updatedAt?: number; projectRoot?: string } + +function seedLinks(links: Record) { + store.set(LINKS_KEY, JSON.stringify(links)) +} + +const goneOnly = + (...ids: string[]) => + async (sandboxId: string) => + ids.includes(sandboxId) + +describe('pruning hosts of gone cloud workspaces', () => { + beforeEach(() => { + store.clear() + }) + + it('forgets the host, link and queued prompts of a missing workspace the registry confirms is gone', async () => { + seedLinks({ + sbx_gone: { hostId: 'host-gone', pairedAt: 1_000, updatedAt: 1_000 }, + sbx_live: { hostId: 'host-live', pairedAt: 1_000, updatedAt: 1_000 } + }) + await enqueueCloudPrompt({ + sandboxId: 'sbx_gone', + text: 'never delivered', + agent: 'claude', + operationId: 'op-1' + }) + await enqueueCloudPrompt({ + sandboxId: 'sbx_live', + text: 'still wanted', + agent: 'claude', + operationId: 'op-2' + }) + const removed: string[] = [] + const confirmGone = vi.fn(goneOnly('sbx_gone')) + + const pruned = await pruneCloudHostsMissingFrom( + new Set(['sbx_live']), + 5_000, + confirmGone, + async (hostId) => { + removed.push(hostId) + } + ) + + expect(pruned).toEqual(['sbx_gone']) + // Only the workspace the list lacks is checked by id. + expect(confirmGone.mock.calls).toEqual([['sbx_gone']]) + expect(removed).toEqual(['host-gone']) + expect([...(await cloudWorkspaceHostIds())]).toEqual(['host-live']) + expect(await listQueuedCloudPrompts('sbx_gone')).toEqual([]) + expect(await listQueuedCloudPrompts('sbx_live')).toHaveLength(1) + }) + + it('keeps a missing workspace the registry still serves (capped list, hidden row, teammate share) or cannot answer for', async () => { + seedLinks({ + sbx_hidden: { hostId: 'host-hidden', pairedAt: 1_000 }, + sbx_unknown: { hostId: 'host-unknown', pairedAt: 1_000 } + }) + const removeHost = vi.fn(async () => undefined) + const confirmGone = async (sandboxId: string) => { + if (sandboxId === 'sbx_unknown') { + throw new Error('offline') + } + return false + } + // An empty list (a server bug, or a capped page) proves nothing on its own. + expect(await pruneCloudHostsMissingFrom(new Set(), 5_000, confirmGone, removeHost)).toEqual([]) + expect(removeHost).not.toHaveBeenCalled() + expect([...(await cloudWorkspaceHostIds())].sort()).toEqual(['host-hidden', 'host-unknown']) + }) + + it('never checks a link written after the list request started (created or paired meanwhile)', async () => { + seedLinks({ sbx_new: { hostId: 'host-new', pairedAt: 6_000, updatedAt: 6_000 } }) + const confirmGone = vi.fn(async () => true) + expect( + await pruneCloudHostsMissingFrom(new Set(), 5_000, confirmGone, async () => undefined) + ).toEqual([]) + expect(confirmGone).not.toHaveBeenCalled() + expect([...(await cloudWorkspaceHostIds())]).toEqual(['host-new']) + }) + + it('puts the link back when the host cannot be removed, so the next confirmed miss retries', async () => { + seedLinks({ + sbx_gone: { hostId: 'host-gone', pairedAt: 1_000, projectRoot: '/vercel/sandbox/project' } + }) + await enqueueCloudPrompt({ + sandboxId: 'sbx_gone', + text: 'kept with its link', + agent: 'claude', + operationId: 'op-3' + }) + const pruned = await pruneCloudHostsMissingFrom( + new Set(), + 5_000, + goneOnly('sbx_gone'), + async () => { + throw new Error('host store busy') + } + ) + expect(pruned).toEqual([]) + expect([...(await cloudWorkspaceHostIds())]).toEqual(['host-gone']) + expect(await cloudWorkspaceProjectRoot('sbx_gone')).toBe('/vercel/sandbox/project') + expect(await listQueuedCloudPrompts('sbx_gone')).toHaveLength(1) + }) + + it('forgets exactly one workspace on its own confirmed not_found', async () => { + await recordCloudWorkspaceHost('sbx_gone', 'host-gone') + await recordCloudWorkspaceHost('sbx_other', 'host-other') + const removed: string[] = [] + expect( + await pruneGoneCloudHost('sbx_gone', async (hostId) => { + removed.push(hostId) + }) + ).toEqual(['sbx_gone']) + expect(removed).toEqual(['host-gone']) + expect([...(await cloudWorkspaceHostIds())]).toEqual(['host-other']) + }) + + it('asks about a still-present missing workspace once per few minutes, and shares one run between overlapping polls', async () => { + seedLinks({ sbx_capped: { hostId: 'host-capped', pairedAt: 1_000 } }) + const confirmGone = vi.fn(async () => false) + const removeHost = async () => undefined + const [first, second] = await Promise.all([ + pruneCloudHostsMissingFrom(new Set(), 5_000, confirmGone, removeHost), + pruneCloudHostsMissingFrom(new Set(), 5_000, confirmGone, removeHost) + ]) + expect(first).toEqual([]) + expect(second).toEqual([]) + await pruneCloudHostsMissingFrom(new Set(), 5_000, confirmGone, removeHost) + expect(confirmGone).toHaveBeenCalledTimes(1) + expect([...(await cloudWorkspaceHostIds())]).toEqual(['host-capped']) + }) + + it('treats only confirmed answers as gone, never transient failures', () => { + expect(isCloudWorkspaceGone(new CloudAttachError('not_found'))).toBe(true) + expect(isCloudWorkspaceGone(new CloudRequestError('not_found', 404))).toBe(true) + expect(isCloudWorkspaceGone(new CloudRequestError('network_error', 0))).toBe(false) + expect(isCloudWorkspaceGone(new CloudRequestError('timeout', 0))).toBe(false) + expect(isCloudWorkspaceGone(new CloudRequestError('provider_unavailable', 502))).toBe(false) + expect(isCloudWorkspaceGone(new CloudRequestError('unauthorized', 401))).toBe(false) + expect(isCloudWorkspaceGone(new CloudAttachError('wake_timeout'))).toBe(false) + expect(isCloudWorkspaceGone(new Error('offline'))).toBe(false) + }) +}) + +describe('isCloudWorkspaceGoneFromRegistry', () => { + const plane = (reply: () => Promise) => ({ + baseUrl: 'https://cp.test', + token: () => 'tok', + fetch: async () => reply() + }) + + it('is true only for the registry’s own 404 not_found', async () => { + const answer = (status: number, body: unknown) => async () => Response.json(body, { status }) + expect( + await isCloudWorkspaceGoneFromRegistry(plane(answer(404, { error: 'not_found' })), 'sbx') + ).toBe(true) + expect( + await isCloudWorkspaceGoneFromRegistry(plane(answer(200, { workspace: {} })), 'sbx') + ).toBe(false) + expect( + await isCloudWorkspaceGoneFromRegistry(plane(answer(500, { error: 'internal' })), 'sbx') + ).toBe(false) + expect( + await isCloudWorkspaceGoneFromRegistry(plane(answer(404, { error: 'route_missing' })), 'sbx') + ).toBe(false) + expect( + await isCloudWorkspaceGoneFromRegistry( + plane(async () => { + throw new TypeError('Network request failed') + }), + 'sbx' + ) + ).toBe(false) + }) +}) diff --git a/mobile/src/cloud/cloud-host-pruning.ts b/mobile/src/cloud/cloud-host-pruning.ts new file mode 100644 index 00000000000..3e327e1f210 --- /dev/null +++ b/mobile/src/cloud/cloud-host-pruning.ts @@ -0,0 +1,114 @@ +import { CloudRequestError } from './cloud-control-plane-request' +import { removeCloudPromptsFor } from './cloud-prompt-queue' +import { CloudAttachError } from './cloud-workspace-attach' +import { + cloudWorkspaceLinkEntries, + restoreCloudWorkspaceLinks, + takeCloudWorkspaceLinks, + type TakenCloudWorkspaceLink +} from './cloud-workspace-links' + +// A paired cloud workspace whose VM is gone would otherwise be retried by the phone forever. A +// workspace is forgotten only on a confirmed answer that it no longer exists for this phone: the +// team registry's own GET for it says not_found, or wake says not_found. Missing from a list only +// nominates it (lists are capped and skip hidden rows). Network errors, timeouts, server errors and +// 401s (the session's own path) never prune. + +/** The control plane's confirmed answer that this workspace no longer exists for this phone. */ +export function isCloudWorkspaceGone(error: unknown): boolean { + if (error instanceof CloudAttachError) { + return error.code === 'not_found' + } + return error instanceof CloudRequestError && error.status === 404 && error.code === 'not_found' +} + +async function forget( + taken: readonly TakenCloudWorkspaceLink[], + removeHost: (hostId: string) => Promise +): Promise { + const kept: TakenCloudWorkspaceLink[] = [] + for (const link of taken) { + if (link.hostId) { + try { + await removeHost(link.hostId) + } catch { + // A host still in the catalog with no link would be retried forever and unreachable by a + // later prune or sign-out: put the link back so the next confirmed miss tries again. + kept.push(link) + } + } + } + if (kept.length > 0) { + await restoreCloudWorkspaceLinks(kept) + } + const ids = taken.filter((link) => !kept.includes(link)).map((link) => link.sandboxId) + if (ids.length > 0) { + await removeCloudPromptsFor(new Set(ids)) + } + return ids +} + +/** + * After a successful list: every linked workspace the list lacks is checked by id (`confirmGone`), + * and forgotten only if that says it is gone. A link written after the list request started (a + * workspace created or paired meanwhile) is never a candidate. + */ +export function pruneCloudHostsMissingFrom( + listedIds: ReadonlySet, + listStartedAt: number, + confirmGone: (sandboxId: string) => Promise, + removeHost: (hostId: string) => Promise +): Promise { + // Single-flight: list polls overlap, and the next poll runs again with its own list anyway. + inFlight ??= prune(listedIds, listStartedAt, confirmGone, removeHost).finally(() => { + inFlight = null + }) + return inFlight +} + +// A workspace the registry still serves while the list lacks it (hidden row, past the list cap) +// would otherwise cost one GET per poll forever: its "present" answer holds for a while. +const PRESENT_TTL_MS = 5 * 60_000 +const presentUntil = new Map() +let inFlight: Promise | null = null + +async function prune( + listedIds: ReadonlySet, + listStartedAt: number, + confirmGone: (sandboxId: string) => Promise, + removeHost: (hostId: string) => Promise +): Promise { + const isCandidate = (sandboxId: string, link: { updatedAt?: number; pairedAt?: number }) => + !listedIds.has(sandboxId) && (link.updatedAt ?? link.pairedAt ?? 0) < listStartedAt + const now = Date.now() + const candidates = (await cloudWorkspaceLinkEntries()).filter( + ([id, link]) => isCandidate(id, link) && (presentUntil.get(id) ?? 0) <= now + ) + const gone = new Set() + for (const [sandboxId] of candidates) { + const answer = await confirmGone(sandboxId).catch(() => null) + if (answer === true) { + gone.add(sandboxId) + } else if (answer === false) { + // Only a real "still there" is remembered; a failed check is retried next poll. + presentUntil.set(sandboxId, now + PRESENT_TTL_MS) + } + } + if (gone.size === 0) { + return [] + } + // Re-check inside the write: a pairing recorded during the confirmation is kept. + const taken = await takeCloudWorkspaceLinks( + (sandboxId, link) => gone.has(sandboxId) && isCandidate(sandboxId, link) + ) + return forget(taken, removeHost) +} + +/** One workspace answered not_found to wake: forget its pairing now. */ +export async function pruneGoneCloudHost( + sandboxId: string, + removeHost: (hostId: string) => Promise +): Promise { + const taken = await takeCloudWorkspaceLinks((id) => id === sandboxId) + return forget(taken, removeHost) +} diff --git a/mobile/src/cloud/cloud-prompt-queue.ts b/mobile/src/cloud/cloud-prompt-queue.ts index 00f397373ef..18d4fbfda26 100644 --- a/mobile/src/cloud/cloud-prompt-queue.ts +++ b/mobile/src/cloud/cloud-prompt-queue.ts @@ -152,6 +152,14 @@ export function removeCloudPrompt(id: string): Promise { return mutate((queue) => ({ next: queue.filter((entry) => entry.id !== id), result: undefined })) } +/** The workspaces are gone: nothing queued for them can ever be delivered. */ +export function removeCloudPromptsFor(sandboxIds: ReadonlySet): Promise { + return mutate((queue) => ({ + next: queue.filter((entry) => !sandboxIds.has(entry.sandboxId)), + result: undefined + })) +} + export function subscribeCloudPromptQueue(listener: () => void): () => void { listeners.add(listener) return () => listeners.delete(listener) diff --git a/mobile/src/cloud/cloud-session.ts b/mobile/src/cloud/cloud-session.ts index 54625a48990..364cf2b0fd9 100644 --- a/mobile/src/cloud/cloud-session.ts +++ b/mobile/src/cloud/cloud-session.ts @@ -39,6 +39,11 @@ export function useCloudHostRemoval(): CloudHostRemoval { return useMemo(() => ({ forgetHostClient, queryClient }), [forgetHostClient, queryClient]) } +/** Unpairs one cloud VM host and closes its connection. */ +export function cloudHostRemover(removal: CloudHostRemoval): (hostId: string) => Promise { + return (hostId) => removeHostAndCloseClient(hostId, removal.forgetHostClient, removal.queryClient) +} + /** Ends everything Orca Cloud gave this phone: its VM pairings first, then the session and key. */ export async function forgetAllCloudAccess(removal: CloudHostRemoval): Promise { for (const hostId of await cloudWorkspaceHostIds()) { diff --git a/mobile/src/cloud/cloud-workspace-links.ts b/mobile/src/cloud/cloud-workspace-links.ts index 999fa21da0e..1477593d3c6 100644 --- a/mobile/src/cloud/cloud-workspace-links.ts +++ b/mobile/src/cloud/cloud-workspace-links.ts @@ -9,7 +9,9 @@ const LinkSchema = z.object({ hostId: z.string().optional(), pairedAt: z.number().optional(), /** From the create response; the VM's checkout of the workspace's repo. */ - projectRoot: z.string().startsWith('/').optional() + projectRoot: z.string().startsWith('/').optional(), + /** Last write: a list that started before it cannot prove the workspace is gone. */ + updatedAt: z.number().optional() }) const LinksSchema = z.record(z.string(), LinkSchema) type Links = z.infer @@ -50,9 +52,44 @@ function updateLinks(change: (links: Links) => void): Promise { export function recordCloudWorkspaceHost(sandboxId: string, hostId: string | null): Promise { return updateLinks((links) => { const current = links[sandboxId] ?? {} + const now = Date.now() links[sandboxId] = hostId - ? { ...current, hostId, pairedAt: Date.now() } - : { projectRoot: current.projectRoot } + ? { ...current, hostId, pairedAt: now, updatedAt: now } + : { projectRoot: current.projectRoot, updatedAt: now } + }) +} + +type Link = Links[string] +export type TakenCloudWorkspaceLink = { sandboxId: string; hostId: string | null; link: Link } + +export async function cloudWorkspaceLinkEntries(): Promise<[string, Link][]> { + return Object.entries(await readLinks()) +} + +/** Removes, in one write, every link `matches` picks; returns what was removed. */ +export async function takeCloudWorkspaceLinks( + matches: (sandboxId: string, link: Link) => boolean +): Promise { + const taken: TakenCloudWorkspaceLink[] = [] + await updateLinks((links) => { + for (const [sandboxId, link] of Object.entries(links)) { + if (matches(sandboxId, link)) { + taken.push({ sandboxId, hostId: link.hostId ?? null, link }) + delete links[sandboxId] + } + } + }) + return taken +} + +/** Puts back taken links whose cleanup failed; a link written since then wins. */ +export function restoreCloudWorkspaceLinks( + restored: readonly TakenCloudWorkspaceLink[] +): Promise { + return updateLinks((links) => { + for (const entry of restored) { + links[entry.sandboxId] ??= entry.link + } }) } @@ -64,7 +101,7 @@ export function recordCloudWorkspaceProjectRoot( return Promise.resolve() } return updateLinks((links) => { - links[sandboxId] = { ...links[sandboxId], projectRoot } + links[sandboxId] = { ...links[sandboxId], projectRoot, updatedAt: Date.now() } }) } diff --git a/mobile/src/cloud/use-cloud-workspace-run.ts b/mobile/src/cloud/use-cloud-workspace-run.ts index b184fb2d512..bf4076f22c5 100644 --- a/mobile/src/cloud/use-cloud-workspace-run.ts +++ b/mobile/src/cloud/use-cloud-workspace-run.ts @@ -3,7 +3,7 @@ import { useCallback, useEffect, useRef, useState } from 'react' import { AccessibilityInfo } from 'react-native' import { useMobileTranslation } from '../i18n/use-mobile-translation' import { useHostClient } from '../transport/connection/host-client-hooks' -import type { CloudWorkspace } from './cloud-control-plane-client' +import { isCloudWorkspaceGoneFromRegistry, type CloudWorkspace } from './cloud-control-plane-client' import { cloudDeviceLabel } from './cloud-device-label' import { deliverQueuedCloudPrompts, @@ -11,7 +11,9 @@ import { markWaitingCloudPromptsFailed, requeueUnsentCloudPrompts } from './cloud-prompt-queue' +import { isCloudWorkspaceGone, pruneGoneCloudHost } from './cloud-host-pruning' import { + cloudHostRemover, forgetRevokedCloudSession, loadCloudControlPlane, useCloudHostRemoval @@ -250,6 +252,13 @@ export function useCloudWorkspaceRun(sandboxId: string, workspace: CloudWorkspac return } await forgetRevokedCloudSession(error, removal) + if ( + isCloudWorkspaceGone(error) && + (await isCloudWorkspaceGoneFromRegistry(plane, sandboxId).catch(() => false)) + ) { + // Destroyed: its pairing and queued prompts go, so nothing retries a VM that no longer exists. + await pruneGoneCloudHost(sandboxId, cloudHostRemover(removal)).catch(() => undefined) + } // Prompts that were going out with this attach now say so, instead of "waiting". await markWaitingCloudPromptsFailed(sandboxId, 'not_connected').catch(() => undefined) const role = await roleForCloudError(error, plane, controller.signal) diff --git a/mobile/src/cloud/use-cloud-workspaces.ts b/mobile/src/cloud/use-cloud-workspaces.ts index b1a6ea659aa..3ef74125e1f 100644 --- a/mobile/src/cloud/use-cloud-workspaces.ts +++ b/mobile/src/cloud/use-cloud-workspaces.ts @@ -2,6 +2,7 @@ import { useQuery, useQueryClient } from '@tanstack/react-query' import { useFocusEffect } from 'expo-router' import { useCallback, useEffect, useState } from 'react' import { + isCloudWorkspaceGoneFromRegistry, listCloudWorkspaces, readCloudWorkspaceStatus, type CloudWorkspace @@ -11,7 +12,9 @@ import { subscribeCloudDeviceIdentity, type CloudEnrollment } from './cloud-device-identity' +import { pruneCloudHostsMissingFrom } from './cloud-host-pruning' import { + cloudHostRemover, forgetRevokedCloudSession, loadCloudControlPlane, useCloudHostRemoval @@ -70,7 +73,19 @@ export function useCloudWorkspaces(enrollment: CloudEnrollment | null | undefine return [] } try { - return await listCloudWorkspaces(plane, signal) + const startedAt = Date.now() + const list = await listCloudWorkspaces(plane, signal) + if (list.complete) { + // A linked workspace the list lacks is checked by id; one the registry says is gone + // (destroyed, or no longer shared) stops being retried. + void pruneCloudHostsMissingFrom( + new Set(list.workspaces.map((workspace) => workspace.id)), + startedAt, + (sandboxId) => isCloudWorkspaceGoneFromRegistry(plane, sandboxId), + cloudHostRemover(removal) + ).catch(() => undefined) + } + return list.workspaces } catch (error) { await forgetRevokedCloudSession(error, removal) throw error