From a319237abfc2c0bc2d0f833fcc9d0d32f59121f2 Mon Sep 17 00:00:00 2001 From: Neil Date: Fri, 13 Nov 2026 04:33:18 -0800 Subject: [PATCH] fix(cloud): EP-5 a cancelled create never leaks its sandbox or claims it was deleted --- cloud/apps/sandbox-control/src/index.ts | 8 +- .../test/ep5-cancel-create.test.ts | 73 ++++++++++++++++ .../cloud-create-from-github.test.ts | 48 +++++++++++ .../cloud-create-from-github.ts | 82 ++++++++++++++---- src/main/hosted-sandbox/hosted-account-ipc.ts | 10 ++- .../hosted-sandbox-create-reclaim.test.ts | 75 ++++++++++++++++- .../hosted-sandbox-create-reclaim.ts | 59 +++++++++++-- .../hosted-sandbox/hosted-sandbox-driver.ts | 48 ++++++++--- .../api/orca-cloud-workspace-bridge.ts | 1 + .../src/lib/orca-cloud-cancel-cleanup.test.ts | 64 +++++++++++++- .../src/lib/orca-cloud-cancel-cleanup.ts | 62 +++++++++----- ...cloud-workspace-from-github.cancel.test.ts | 84 ++++++++++++++++++- .../start-cloud-workspace-from-github.ts | 23 ++++- .../hosted-sandbox/cloud-workspace-api.ts | 9 ++ 14 files changed, 579 insertions(+), 67 deletions(-) create mode 100644 cloud/apps/sandbox-control/test/ep5-cancel-create.test.ts diff --git a/cloud/apps/sandbox-control/src/index.ts b/cloud/apps/sandbox-control/src/index.ts index eb90ff06277..4109ba47a7b 100644 --- a/cloud/apps/sandbox-control/src/index.ts +++ b/cloud/apps/sandbox-control/src/index.ts @@ -1778,7 +1778,12 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise { @@ -1794,6 +1799,7 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise `${phase};dur=${ms}`) diff --git a/cloud/apps/sandbox-control/test/ep5-cancel-create.test.ts b/cloud/apps/sandbox-control/test/ep5-cancel-create.test.ts new file mode 100644 index 00000000000..0e75ddc1427 --- /dev/null +++ b/cloud/apps/sandbox-control/test/ep5-cancel-create.test.ts @@ -0,0 +1,73 @@ +// EP-5: a desktop that cancels a create mid-request disconnects. The create used to die with the +// request (Workers cancel it), leaving the row `creating` and the Idempotency-Key pending, so every +// cancel replay answered 409 until the 5 min sweep. The create now outlives its request +// (ctx.waitUntil), so the replay gets the sandbox to delete; a delete that lands while the create +// is still running wins (the create frees its VM instead of finishing); and a create that was cut +// off anyway is swept (create-speed.test.ts covers the 5 min sweep). +import { describe, expect, it } from 'vitest' +import { HostedSandboxReadyResponseSchema } from '../../../../src/shared/hosted-sandbox/hosted-sandbox-wire' +import { createHarness } from './harness' + +const create = { workspaceId: 'ws-cancel', machineSize: 'standard' } +const key = { 'idempotency-key': 'press-0123456789' } + +async function setup() { + const background: Promise[] = [] + const h = await createHarness({}, { waitUntil: (work) => void background.push(work) }) + // Hold the provider's VM create so the request is still mid-create when the client goes away. + let open = (): void => {} + const held = new Promise((resolve) => (open = resolve)) + const createVm = h.vercel.createSandbox.bind(h.vercel) + h.vercel.createSandbox = async (params) => { + await held + return createVm(params) + } + const row = () => h.raw.prepare(`SELECT id, status, provider_name FROM sandboxes WHERE workspace_id = ? ORDER BY rowid DESC`).get(create.workspaceId) as any + const untilCreating = async () => { + for (let i = 0; i < 200 && row()?.status !== 'creating'; i++) { + await new Promise((resolve) => setTimeout(resolve, 10)) + } + expect(row()?.status).toBe('creating') + } + const liveVms = () => [...(h.vercel as any).boxes.values()].filter((box: any) => box.params.tags?.orcaSandbox === row()?.id) + return { h, background, open, row, untilCreating, liveVms } +} + +describe('EP-5: cancelling a create mid-request', () => { + it('the create outlives its request, so the replay gets the sandbox instead of 409 in progress', async () => { + const t = await setup() + const first = t.h.call('POST', '/v1/sandboxes', { body: create, headers: key }) + await t.untilCreating() + expect(t.background.length).toBeGreaterThan(0) + // While it runs, a replay is told to wait (the desktop retries). + expect((await t.h.call('POST', '/v1/sandboxes', { body: create, headers: key })).status).toBe(409) + // The client is gone: only the work handed to waitUntil carries on, and it finishes. + t.open() + await Promise.all(t.background.splice(0)) + await first + expect(t.row().status).toBe('running') + const replay = await t.h.call('POST', '/v1/sandboxes', { body: create, headers: key }) + expect(replay.status).toBe(200) + const id = HostedSandboxReadyResponseSchema.parse(replay.body).sandbox.id + expect(id).toBe(t.row().id) + // ... and the cancel deletes it for real. + expect((await t.h.call('DELETE', `/v1/sandboxes/${id}`)).status).toBe(200) + expect(t.row().status).toBe('destroyed') + expect(t.liveVms()).toEqual([]) + }) + + it('a delete while the VM is still being made wins: the create frees its VM instead of finishing', async () => { + const t = await setup() + const first = t.h.call('POST', '/v1/sandboxes', { body: create, headers: key }) + await t.untilCreating() + const id = t.row().id + const deleted = await t.h.call('DELETE', `/v1/sandboxes/${id}`) + expect(deleted.status).toBe(200) + // The VM arrives after the delete: the create sees the row is no longer its own and frees it. + t.open() + await Promise.all(t.background.splice(0)) + expect((await first).status).not.toBe(200) + expect(t.row().status).toBe('destroyed') + expect(t.liveVms()).toEqual([]) + }) +}) diff --git a/src/main/hosted-sandbox/cloud-create-from-github.test.ts b/src/main/hosted-sandbox/cloud-create-from-github.test.ts index 1783d99d019..327c99ecb0a 100644 --- a/src/main/hosted-sandbox/cloud-create-from-github.test.ts +++ b/src/main/hosted-sandbox/cloud-create-from-github.test.ts @@ -1,9 +1,11 @@ import { describe, expect, it, vi } from 'vitest' import { HostedEntitlementDeniedError } from './hosted-environment-provider' import { EnvironmentProviderError } from '../environment-providers/environment-provider-driver' +import { CANCEL_CLEANUP_UNCONFIRMED } from '../../shared/hosted-sandbox/cloud-workspace-api' import { createCloudWorkspaceFromGitHub, githubWorkspaceId, + retryCancelledGitHubCreate, type CreateFromGitHubClient } from './cloud-create-from-github' @@ -26,6 +28,7 @@ function client(overrides: Partial = {}): CreateFromGitH bootPlan: vi.fn(async () => 'clone' as const), cloneStatus: vi.fn(async () => ({ state: 'done' as const })), destroy: vi.fn(async (id: string) => ({ id, status: 'destroyed' as const })), + list: vi.fn(async () => []), ...overrides } } @@ -129,3 +132,48 @@ describe('GitHub-only create on the build sheet (GH-CREATE-SHEET)', () => { expect(api.destroy).toHaveBeenCalledWith('sbx_1') }) }) + +describe('EP-5: a cancel whose sandbox was not confirmed deleted', () => { + it('answers reason cancelCleanupUnconfirmed (never a plain cancel), and Try again deletes it', async () => { + const controller = new AbortController() + let down = true + const api = client({ + create: vi.fn(async (_body, signal) => { + if (signal) { + controller.abort() + throw new EnvironmentProviderError('aborted', 'Cancelled.') + } + if (down) { + throw new EnvironmentProviderError('unavailable', 'down') + } + return READY + }) + }) + vi.useFakeTimers() + let result: Awaited> + try { + const pending = createCloudWorkspaceFromGitHub(api, INPUT, { signal: controller.signal }) + await vi.runAllTimersAsync() + result = await pending + } finally { + vi.useRealTimers() + } + expect(result).toMatchObject({ ok: false, reason: CANCEL_CLEANUP_UNCONFIRMED }) + expect(api.destroy).not.toHaveBeenCalled() + + // Try again, once the control plane answers: the same body and key find the sandbox; it is deleted. + down = false + expect( + await retryCancelledGitHubCreate(api, INPUT.provisionId, { waitMs: 0, log: () => {} }) + ).toBe(true) + expect(api.create).toHaveBeenLastCalledWith( + expect.objectContaining({ workspaceId: githubWorkspaceId(INPUT.slug, INPUT.provisionId) }), + undefined, + INPUT.idempotencyKey + ) + expect(api.destroy).toHaveBeenCalledWith('sbx_1') + // Done: nothing left to retry. + expect(await retryCancelledGitHubCreate(api, INPUT.provisionId)).toBe(true) + expect(api.destroy).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/main/hosted-sandbox/cloud-create-from-github.ts b/src/main/hosted-sandbox/cloud-create-from-github.ts index 23dca09b989..0b3bf34b9cf 100644 --- a/src/main/hosted-sandbox/cloud-create-from-github.ts +++ b/src/main/hosted-sandbox/cloud-create-from-github.ts @@ -7,8 +7,17 @@ import type { CloudWorkspaceErrorCode, CloudWorkspaceResult } from '../../shared/hosted-sandbox/cloud-workspace-api' +import { + EnvironmentProviderError, + type EnvironmentCreateSpec +} from '../environment-providers/environment-provider-driver' import type { HostedSandboxControlPlaneClient } from './hosted-sandbox-control-plane-client' -import { createSandboxWithProgress } from './hosted-sandbox-driver' +import { reclaimCancelledCreate } from './hosted-sandbox-create-reclaim' +import { + CANCEL_CLEANUP_UNCONFIRMED, + createSandboxWithProgress, + hostedCreateBody +} from './hosted-sandbox-driver' import { hostedCreateFailure, hostedProgressLine } from './hosted-sandbox-recipe' /** The server's Idempotency-Key format. */ @@ -27,7 +36,7 @@ export type CreateFromGitHubInput = { export type CreateFromGitHubClient = Pick< HostedSandboxControlPlaneClient, - 'create' | 'bootPlan' | 'cloneStatus' | 'destroy' + 'create' | 'bootPlan' | 'cloneStatus' | 'destroy' | 'list' > /** @@ -53,28 +62,69 @@ const CODE_BY_FAILURE: Partial> = { * Starts the sandbox and waits for its clone, reporting each step through `onLog`. A failure comes * back with the failure code as `reason` (e.g. `cloudPlanRequired`) after its reason line was logged. */ +function createSpec(input: CreateFromGitHubInput): EnvironmentCreateSpec { + return { + workspaceId: githubWorkspaceId(input.slug, input.provisionId), + machineSize: 'standard', + repo: { url: `https://github.com/${input.slug}.git` }, + ...(input.title ? { workspaceName: input.title } : {}), + ...(input.idempotencyKey ? { idempotencyKey: input.idempotencyKey } : {}) + } +} + +/** EP-5: cancelled creates whose sandbox wasn't confirmed deleted, by creation id, for Try again. */ +const unconfirmedCancels = new Map() + +/** + * Try again for a cancelled create the first cleanup couldn't confirm: replays it by its key (or + * finds it by its workspace id) and deletes it. True once deleted, or when there is nothing to do. + */ +export async function retryCancelledGitHubCreate( + client: Pick, + provisionId: string, + options: Parameters[3] = {} +): Promise { + const input = unconfirmedCancels.get(provisionId) + if (!input?.idempotencyKey) { + return true + } + const spec = createSpec(input) + const ok = await reclaimCancelledCreate( + client, + hostedCreateBody(spec), + input.idempotencyKey, + options + ) + if (ok) { + unconfirmedCancels.delete(provisionId) + } + return ok +} + export async function createCloudWorkspaceFromGitHub( client: CreateFromGitHubClient, input: CreateFromGitHubInput, io: { signal?: AbortSignal; onLog?: (chunk: string) => void } = {} ): Promise> { try { - const ready = await createSandboxWithProgress( - client, - { - workspaceId: githubWorkspaceId(input.slug, input.provisionId), - machineSize: 'standard', - repo: { url: `https://github.com/${input.slug}.git` }, - ...(input.title ? { workspaceName: input.title } : {}), - ...(input.idempotencyKey ? { idempotencyKey: input.idempotencyKey } : {}) - }, - { - ...(io.signal ? { signal: io.signal } : {}), - onProgress: (progress) => io.onLog?.(hostedProgressLine(progress)) - } - ) + const ready = await createSandboxWithProgress(client, createSpec(input), { + ...(io.signal ? { signal: io.signal } : {}), + onProgress: (progress) => io.onLog?.(hostedProgressLine(progress)) + }) return { ok: true, value: { sandboxId: ready.sandbox.id } } } catch (error) { + if (error instanceof EnvironmentProviderError && error.reason === CANCEL_CLEANUP_UNCONFIRMED) { + // The window must not say "Sandbox deleted" for this one; Try again retries the delete. + if (input.provisionId) { + unconfirmedCancels.set(input.provisionId, input) + } + return { + ok: false, + code: 'failed', + message: 'Cancelled.', + reason: CANCEL_CLEANUP_UNCONFIRMED + } + } const result = hostedCreateFailure(error) if (result.stderr) { io.onLog?.(result.stderr) diff --git a/src/main/hosted-sandbox/hosted-account-ipc.ts b/src/main/hosted-sandbox/hosted-account-ipc.ts index 0811c502970..901afa2b853 100644 --- a/src/main/hosted-sandbox/hosted-account-ipc.ts +++ b/src/main/hosted-sandbox/hosted-account-ipc.ts @@ -15,7 +15,8 @@ import type { GitHubInstallRights } from '../../shared/hosted-sandbox/cloud-work import { CLOUD_CREATE_IDEMPOTENCY_KEY, CLOUD_CREATE_PROVISION_ID, - createCloudWorkspaceFromGitHub + createCloudWorkspaceFromGitHub, + retryCancelledGitHubCreate } from './cloud-create-from-github' import { activeProvisionControllers } from '../ipc/provision-controllers' import { @@ -33,6 +34,7 @@ export const HOSTED_ACCOUNT_CHANNELS = [ CH.gitHubInstallRights, CH.listGitHubRepositories, CH.createFromGitHub, + CH.retryCancelledCreate, ...HOSTED_GITHUB_USER_CHANNELS ] as const @@ -196,8 +198,14 @@ export function registerHostedAccountHandlers( } } }) + ipcMain.handle(CH.retryCancelledCreate, async (_event, args?: RetryCancelledCreateArgs) => { + const provisionId = matching(args?.provisionId, CLOUD_CREATE_PROVISION_ID) + return provisionId ? retryCancelledGitHubCreate(client, provisionId) : false + }) } +type RetryCancelledCreateArgs = { provisionId?: unknown } | undefined + type CreateFromGitHubArgs = | { repoUrl?: unknown; title?: unknown; provisionId?: unknown; idempotencyKey?: unknown } | undefined diff --git a/src/main/hosted-sandbox/hosted-sandbox-create-reclaim.test.ts b/src/main/hosted-sandbox/hosted-sandbox-create-reclaim.test.ts index b5686f8a2ee..2080f1eec3e 100644 --- a/src/main/hosted-sandbox/hosted-sandbox-create-reclaim.test.ts +++ b/src/main/hosted-sandbox/hosted-sandbox-create-reclaim.test.ts @@ -4,7 +4,7 @@ import { describe, expect, it, vi } from 'vitest' import { EnvironmentProviderError } from '../environment-providers/environment-provider-driver' import { HostedEntitlementDeniedError } from './hosted-environment-provider' import { reclaimCancelledCreate } from './hosted-sandbox-create-reclaim' -import { createSandboxWithProgress } from './hosted-sandbox-driver' +import { CANCEL_CLEANUP_UNCONFIRMED, createSandboxWithProgress } from './hosted-sandbox-driver' const READY = { sandbox: { @@ -26,7 +26,8 @@ function client() { create: vi.fn(async () => READY), bootPlan: vi.fn(async () => 'clone' as const), cloneStatus: vi.fn(async () => ({ state: 'done' as const })), - destroy: vi.fn(async (id: string) => ({ id, status: 'destroyed' as const })) + destroy: vi.fn(async (id: string) => ({ id, status: 'destroyed' as const })), + list: vi.fn(async (): Promise<(typeof READY)['sandbox'][]> => []) } } @@ -55,6 +56,40 @@ describe('reclaimCancelledCreate', () => { expect(api.destroy).not.toHaveBeenCalled() }) + it('EP-5: while the first create still runs (409), deletes the sandbox it is making by workspace id', async () => { + const api = client() + api.create.mockRejectedValue( + new EnvironmentProviderError('conflict', 'idempotency_in_progress') + ) + api.list.mockResolvedValue([ + { ...READY.sandbox, id: 'sbx_other', workspaceId: 'other-1' }, + { ...READY.sandbox, status: 'creating' as never } + ]) + expect(await reclaimCancelledCreate(api, BODY, 'create:k1', quiet)).toBe(true) + expect(api.destroy).toHaveBeenCalledTimes(1) + expect(api.destroy).toHaveBeenCalledWith('sbx_1') + expect(api.create).toHaveBeenCalledTimes(1) + }) + + it('EP-5: a delete the control plane does not answer with destroyed is not confirmed', async () => { + const api = client() + api.destroy.mockImplementation(async (id: string) => ({ id, status: 'destroying' as never })) + expect(await reclaimCancelledCreate(api, BODY, 'create:k1', { ...quiet, attempts: 2 })).toBe( + false + ) + }) + + it('EP-5: 409 with nothing listed yet keeps replaying until the create answers', async () => { + const api = client() + api.create.mockRejectedValueOnce( + new EnvironmentProviderError('conflict', 'idempotency_in_progress') + ) + expect(await reclaimCancelledCreate(api, BODY, 'create:k1', quiet)).toBe(true) + expect(api.list).toHaveBeenCalledTimes(1) + expect(api.create).toHaveBeenCalledTimes(2) + expect(api.destroy).toHaveBeenCalledWith('sbx_1') + }) + it('gives up (false) when it can never tell', async () => { const api = client() api.create.mockRejectedValue(new EnvironmentProviderError('unavailable', 'down')) @@ -96,6 +131,42 @@ describe('createSandboxWithProgress cancel', () => { expect(api.create).toHaveBeenCalledTimes(1) }) + it('EP-5: a cancel whose sandbox could not be confirmed deleted says so (reason), not a plain cancel', async () => { + const api = client() + const controller = new AbortController() + api.create.mockImplementation(async (_body, signal) => { + if (signal) { + controller.abort() + throw new EnvironmentProviderError('aborted', 'Cancelled.') + } + throw new EnvironmentProviderError('unavailable', 'down') + }) + vi.useFakeTimers() + try { + const outcome = expect( + createSandboxWithProgress(api, spec, { signal: controller.signal }) + ).rejects.toMatchObject({ code: 'aborted', reason: CANCEL_CLEANUP_UNCONFIRMED }) + // Every replay (2 min of them) fails: the retry waits run out. + await vi.runAllTimersAsync() + await outcome + } finally { + vi.useRealTimers() + } + }) + + it('EP-5: cancelled as it answered and the delete failed: not confirmed', async () => { + const api = client() + const controller = new AbortController() + api.create.mockImplementationOnce(async () => { + controller.abort() + return READY + }) + api.destroy.mockRejectedValueOnce(new EnvironmentProviderError('unavailable', 'down')) + await expect( + createSandboxWithProgress(api, spec, { signal: controller.signal }) + ).rejects.toMatchObject({ code: 'aborted', reason: CANCEL_CLEANUP_UNCONFIRMED }) + }) + it('not cancelled: no replay, no delete', async () => { const api = client() await expect(createSandboxWithProgress(api, spec, {})).resolves.toBe(READY) diff --git a/src/main/hosted-sandbox/hosted-sandbox-create-reclaim.ts b/src/main/hosted-sandbox/hosted-sandbox-create-reclaim.ts index 343360ff400..9619bf95bf4 100644 --- a/src/main/hosted-sandbox/hosted-sandbox-create-reclaim.ts +++ b/src/main/hosted-sandbox/hosted-sandbox-create-reclaim.ts @@ -3,12 +3,18 @@ // and body: sending the same create again hands back the sandbox the first one made (or makes one, if // the first never arrived), and that sandbox is deleted. Shared by the owned create (the recipe) and // the GitHub-only create, which both go through createSandboxWithProgress. +// EP-5: while the first create still runs, the replay answers 409; the sandbox it is making is found +// by its workspace id and deleted directly (the control plane accepts a delete mid-create and the +// create then frees its VM), so a cancel doesn't wait out the create. Only a confirmed delete (or +// a refused create, which made nothing) counts as cleaned up. import { isEnvironmentProviderError } from '../environment-providers/environment-provider-driver' import type { HostedSandboxControlPlaneClient } from './hosted-sandbox-control-plane-client' type CreateBody = Parameters[0] +export type ReclaimClient = Pick -const RECLAIM_ATTEMPTS = 8 +/** 2 min at 5 s: past a create's longest run, under the cancel toast's 3 min limit. */ +const RECLAIM_ATTEMPTS = 24 const RECLAIM_WAIT_MS = 5_000 /** The first create may still be running (409 in progress) or the service briefly unreachable. */ @@ -26,7 +32,7 @@ function worthRetrying(error: unknown): boolean { * the create, so nothing was made); false when it still couldn't tell after every attempt. */ export async function reclaimCancelledCreate( - client: Pick, + client: ReclaimClient, body: CreateBody, idempotencyKey: string, { @@ -48,17 +54,56 @@ export async function reclaimCancelledCreate( log( `[orca-cloud] cancelled create: replay ${attempt}/${attempts} failed (${isEnvironmentProviderError(error) ? error.code : 'error'})` ) + if ( + isEnvironmentProviderError(error, 'conflict') && + (await deleteByWorkspace(client, body.workspaceId, log)) + ) { + return true + } await new Promise((resolve) => setTimeout(resolve, waitMs)) continue } - try { - await client.destroy(sandboxId) - log(`[orca-cloud] cancelled create: deleted sandbox ${sandboxId}`) + if (await deleted(client, sandboxId, log)) { return true - } catch { - await new Promise((resolve) => setTimeout(resolve, waitMs)) } + await new Promise((resolve) => setTimeout(resolve, waitMs)) } log('[orca-cloud] cancelled create: could not confirm the sandbox was deleted') return false } + +/** True once the control plane answers the delete with the sandbox destroyed. */ +async function deleted( + client: ReclaimClient, + sandboxId: string, + log: (message: string) => void +): Promise { + try { + const gone = await client.destroy(sandboxId) + if (gone.status === 'destroyed') { + log(`[orca-cloud] cancelled create: deleted sandbox ${sandboxId}`) + return true + } + } catch { + // Retried with the next attempt. + } + return false +} + +/** The create still running: delete the sandbox it is making, found by the create's workspace id. */ +async function deleteByWorkspace( + client: ReclaimClient, + workspaceId: string, + log: (message: string) => void +): Promise { + const sandboxes = await client.list().catch(() => null) + const live = sandboxes?.filter( + (sandbox) => sandbox.workspaceId === workspaceId && sandbox.status !== 'destroyed' + ) + if (!live || live.length === 0) { + // Not listed yet (or the list failed): the next replay tries again. + return false + } + const results = await Promise.all(live.map((sandbox) => deleted(client, sandbox.id, log))) + return results.every(Boolean) +} diff --git a/src/main/hosted-sandbox/hosted-sandbox-driver.ts b/src/main/hosted-sandbox/hosted-sandbox-driver.ts index 088f92f5b89..9f382ac6e11 100644 --- a/src/main/hosted-sandbox/hosted-sandbox-driver.ts +++ b/src/main/hosted-sandbox/hosted-sandbox-driver.ts @@ -1,5 +1,6 @@ // Orca Cloud workspaces: an EnvironmentProviderDriver that talks only to Orca's control plane. // Which provider runs the VM (Vercel today) is a control-plane detail the desktop never sees. +import { CANCEL_CLEANUP_UNCONFIRMED } from '../../shared/hosted-sandbox/cloud-workspace-api' import { HOSTED_SANDBOX_MACHINE_SIZES, ORCA_CLOUD_IDLE_SLEEP_MINUTES, @@ -128,12 +129,31 @@ export function createHostedSandboxDriver( * create, then git's clone or update until it is done. Shared by the owned create (the recipe) and * the GitHub-only create (GH-CREATE-SHEET), so both sheets read the same phases. */ +export { CANCEL_CLEANUP_UNCONFIRMED } + +/** The create body for `spec`; a replay must send exactly this (the server keys replays by body). */ +export function hostedCreateBody( + spec: EnvironmentCreateSpec +): Parameters[0] { + return { + workspaceId: spec.workspaceId, + machineSize: toMachineSize(spec.machineSize), + // Named rows: Home shows the workspace's name, never an untitled sandbox (PC-8). + ...(spec.workspaceName?.trim() ? { title: spec.workspaceName.trim().slice(0, 120) } : {}), + ...(spec.region ? { region: spec.region } : {}), + ...(spec.repo ? { repo: spec.repo } : {}) + } +} + export async function createSandboxWithProgress( - client: Pick, + client: Pick< + HostedSandboxControlPlaneClient, + 'create' | 'bootPlan' | 'cloneStatus' | 'destroy' | 'list' + >, spec: EnvironmentCreateSpec, ctx?: EnvironmentProviderCallContext ): Promise { - const machineSize = toMachineSize(spec.machineSize) + const body = hostedCreateBody(spec) ctx?.onProgress?.({ phase: 'provisioning' }) // Names the repository step before the create answers (no relabel mid-run). Best effort: an // older control plane answers 404 and the step is named when the create answers. @@ -145,14 +165,6 @@ export async function createSandboxWithProgress( ) .catch(() => undefined) } - const body = { - workspaceId: spec.workspaceId, - machineSize, - // Named rows: Home shows the workspace's name, never an untitled sandbox (PC-8). - ...(spec.workspaceName?.trim() ? { title: spec.workspaceName.trim().slice(0, 120) } : {}), - ...(spec.region ? { region: spec.region } : {}), - ...(spec.repo ? { repo: spec.repo } : {}) - } let ready: HostedSandboxReadyResponse try { ready = await client.create(body, ctx?.signal, spec.idempotencyKey) @@ -160,7 +172,10 @@ export async function createSandboxWithProgress( // GH-CREATE-CANCEL: cancelled in flight, the control plane may still have made it: find it by // its key and delete it before reporting the cancel. if (ctx?.signal?.aborted && spec.idempotencyKey) { - await reclaimCancelledCreate(client, body, spec.idempotencyKey) + // EP-5: the cancel may report "deleted" only when the control plane confirmed it. + if (!(await reclaimCancelledCreate(client, body, spec.idempotencyKey))) { + throw new EnvironmentProviderError('aborted', 'Cancelled.', CANCEL_CLEANUP_UNCONFIRMED) + } } throw error } @@ -170,8 +185,15 @@ export async function createSandboxWithProgress( } if (ctx?.signal?.aborted) { // Cancelled just as it answered: nothing will use it, so it must not keep running (and billing). - await client.destroy(ready.sandbox.id).catch(() => undefined) - throw new EnvironmentProviderError('aborted', 'Cancelled.') + const gone = await client + .destroy(ready.sandbox.id) + .then((sandbox) => sandbox.status === 'destroyed') + .catch(() => false) + throw new EnvironmentProviderError( + 'aborted', + 'Cancelled.', + gone ? null : CANCEL_CLEANUP_UNCONFIRMED + ) } ctx?.onProgress?.({ phase: 'connecting' }) return ready diff --git a/src/preload/api/orca-cloud-workspace-bridge.ts b/src/preload/api/orca-cloud-workspace-bridge.ts index a86f6382db4..e7d661a9068 100644 --- a/src/preload/api/orca-cloud-workspace-bridge.ts +++ b/src/preload/api/orca-cloud-workspace-bridge.ts @@ -28,6 +28,7 @@ export const orcaCloudWorkspaceApi = { gitHubInstallRights: (args) => ipcRenderer.invoke(CH.gitHubInstallRights, args), listGitHubRepositories: (args) => ipcRenderer.invoke(CH.listGitHubRepositories, args), createFromGitHub: (args) => ipcRenderer.invoke(CH.createFromGitHub, args), + retryCancelledCreate: (args) => ipcRenderer.invoke(CH.retryCancelledCreate, args), onWakeProgress: (callback) => { const listener = (_event: Electron.IpcRendererEvent, event: CloudWakeProgressEvent): void => callback(event) diff --git a/src/renderer/src/lib/orca-cloud-cancel-cleanup.test.ts b/src/renderer/src/lib/orca-cloud-cancel-cleanup.test.ts index b166e028ef3..7ff772d0de2 100644 --- a/src/renderer/src/lib/orca-cloud-cancel-cleanup.test.ts +++ b/src/renderer/src/lib/orca-cloud-cancel-cleanup.test.ts @@ -7,7 +7,11 @@ const toasts = vi.hoisted(() => ({ })) vi.mock('@/lib/orca-cloud-session-toasts', () => toasts) -import { beginOrcaCloudCancelCleanup, trackOrcaCloudCreateStep } from './orca-cloud-cancel-cleanup' +import { + beginOrcaCloudCancelCleanup, + setOrcaCloudCancelRetry, + trackOrcaCloudCreateStep +} from './orca-cloud-cancel-cleanup' function deferred(): { promise: Promise; resolve: (value: T) => void } { let resolve!: (value: T) => void @@ -131,4 +135,62 @@ describe('Orca Cloud cancel cleanup', () => { expect(cleanupCalls).toEqual(['mine']) expect(toasts.showOrcaCloudCleanupStartedToast).toHaveBeenCalledTimes(2) }) + + it('EP-5: a create step whose result says its sandbox was not confirmed deleted fails the cancel', async () => { + const run = deferred<{ ok: boolean; reason?: string }>() + void trackOrcaCloudCreateStep( + 'c-ep5', + run.promise, + (result) => result.ok || result.reason !== 'unconfirmed' + ) + beginOrcaCloudCancelCleanup('c-ep5', [], { createStartedAt: 0, canceledAt: 1000 }) + run.resolve({ ok: false, reason: 'unconfirmed' }) + await flush() + expect(toasts.showOrcaCloudCleanupDoneToast).not.toHaveBeenCalled() + expect(toasts.showOrcaCloudCleanupFailedToast).toHaveBeenCalledWith( + 'c-ep5', + expect.any(Function) + ) + }) + + it('EP-5: a plain cancelled create (nothing left running) still reads as deleted', async () => { + const run = deferred<{ ok: boolean; reason?: string }>() + void trackOrcaCloudCreateStep( + 'c-ep5b', + run.promise, + (result) => result.ok || result.reason !== 'unconfirmed' + ) + beginOrcaCloudCancelCleanup('c-ep5b', [], { createStartedAt: 0, canceledAt: 1000 }) + run.resolve({ ok: false, reason: null as never }) + await flush() + expect(toasts.showOrcaCloudCleanupDoneToast).toHaveBeenCalledWith('c-ep5b') + }) + + it("EP-5: Try again runs the create's own retry (no runtime records) and reads deleted only if it confirms", async () => { + const retry = vi + .fn<() => Promise>() + .mockResolvedValueOnce(false) + .mockResolvedValueOnce(true) + setOrcaCloudCancelRetry('c-ep5c', retry) + beginOrcaCloudCancelCleanup('c-ep5c', [Promise.resolve(false)], { + createStartedAt: 0, + canceledAt: 1000 + }) + await flush() + const tryAgain = () => toasts.showOrcaCloudCleanupFailedToast.mock.calls.at(-1)![1]() + expect(toasts.showOrcaCloudCleanupFailedToast).toHaveBeenCalledTimes(1) + + tryAgain() + await flush() + await flush() + expect(retry).toHaveBeenCalledTimes(1) + expect(toasts.showOrcaCloudCleanupFailedToast).toHaveBeenCalledTimes(2) + expect(toasts.showOrcaCloudCleanupDoneToast).not.toHaveBeenCalled() + + tryAgain() + await flush() + await flush() + expect(retry).toHaveBeenCalledTimes(2) + expect(toasts.showOrcaCloudCleanupDoneToast).toHaveBeenCalledWith('c-ep5c') + }) }) diff --git a/src/renderer/src/lib/orca-cloud-cancel-cleanup.ts b/src/renderer/src/lib/orca-cloud-cancel-cleanup.ts index 8495436da46..bd218a0acab 100644 --- a/src/renderer/src/lib/orca-cloud-cancel-cleanup.ts +++ b/src/renderer/src/lib/orca-cloud-cancel-cleanup.ts @@ -30,37 +30,52 @@ type Cleanup = { // A provision that never settles must not leave "Deleting the sandbox…" spinning forever. const CLEANUP_TIMEOUT_MS = 3 * 60_000 -// Orca Cloud creates still running, by creation id; after a cancel each ends once its own cleanup has. -const inFlight = new Map>>() +// Orca Cloud creates still running, by creation id, as "left nothing running" outcomes; after a +// cancel each ends once its own cleanup has. +const inFlight = new Map>() const cleanups = new Map() +// EP-5: a step that couldn't confirm its sandbox deleted leaves its own Try again here. +const retries = new Map Promise>() export function isOrcaCloudCreateRequest(request: WorktreeCreationRequest): boolean { return request.ephemeralVmRecipe?.recipeId === ORCA_CLOUD_RECIPE_ID } -/** A create run a cancel must wait for: it may still bring the sandbox up or tear it down. */ -export function trackOrcaCloudCreateStep(creationId: string, step: Promise): Promise { +/** + * A create run a cancel must wait for: it may still bring the sandbox up or tear it down. + * `leftNothingRunning` reads its result (EP-5): false when it couldn't confirm its sandbox deleted, + * so the cancel never says "Sandbox deleted" for it. A rejected step counts as not confirmed. + */ +export function trackOrcaCloudCreateStep( + creationId: string, + step: Promise, + leftNothingRunning: (value: T) => boolean = () => true +): Promise { const steps = inFlight.get(creationId) ?? new Set() inFlight.set(creationId, steps) - steps.add(step) + const outcome = step.then(leftNothingRunning) + // Read by a cancel's cleanup if there is one; never an unhandled rejection otherwise. + outcome.catch(() => undefined) + steps.add(outcome) const settled = (): void => { - steps.delete(step) + steps.delete(outcome) if (steps.size === 0 && inFlight.get(creationId) === steps) { inFlight.delete(creationId) } } - step.then(settled, settled) + outcome.then(settled, settled) const cleanup = cleanups.get(creationId) if (cleanup) { - waitForStep( - creationId, - cleanup, - step.then(() => true) - ) + waitForStep(creationId, cleanup, outcome) } return step } +/** EP-5: how Try again deletes what a step of this create couldn't confirm deleted. */ +export function setOrcaCloudCancelRetry(creationId: string, retry: () => Promise): void { + retries.set(creationId, retry) +} + /** The user canceled an Orca Cloud create; `steps` are the cleanups the cancel itself started. */ export function beginOrcaCloudCancelCleanup( creationId: string, @@ -76,11 +91,7 @@ export function beginOrcaCloudCancelCleanup( cleanups.set(creationId, cleanup) showOrcaCloudCleanupStartedToast(creationId) for (const step of inFlight.get(creationId) ?? []) { - waitForStep( - creationId, - cleanup, - step.then(() => true) - ) + waitForStep(creationId, cleanup, step) } for (const step of steps) { waitForStep(creationId, cleanup, step) @@ -120,6 +131,7 @@ function finish(creationId: string, cleanup: Cleanup, timedOut: boolean): void { if (failed) { showOrcaCloudCleanupFailedToast(creationId, () => retryCleanup(creationId, cleanup.scope)) } else { + retries.delete(creationId) showOrcaCloudCleanupDoneToast(creationId) } }) @@ -166,7 +178,19 @@ function retryCleanup(creationId: string, scope: OrcaCloudCancelScope): void { if (cleanups.has(creationId)) { return } - const step = leftoverRuntimeIds(scope, { timedOut: false, failedOnly: true }).then((ids) => + const retry = retries.get(creationId) + // A create with no runtime records (GitHub-only) retries its own delete instead. + const own = retry + ? retry() + .catch(() => false) + .then((ok) => { + if (ok && retries.get(creationId) === retry) { + retries.delete(creationId) + } + return ok + }) + : Promise.resolve(true) + const records = leftoverRuntimeIds(scope, { timedOut: false, failedOnly: true }).then((ids) => ids === null ? false : Promise.all( @@ -178,5 +202,5 @@ function retryCleanup(creationId: string, scope: OrcaCloudCancelScope): void { ) ).then((results) => results.every(Boolean)) ) - beginOrcaCloudCancelCleanup(creationId, [step], scope, true) + beginOrcaCloudCancelCleanup(creationId, [own, records], scope, true) } diff --git a/src/renderer/src/lib/orca-cloud/start-cloud-workspace-from-github.cancel.test.ts b/src/renderer/src/lib/orca-cloud/start-cloud-workspace-from-github.cancel.test.ts index 216791fa2c3..79ba3985bc9 100644 --- a/src/renderer/src/lib/orca-cloud/start-cloud-workspace-from-github.cancel.test.ts +++ b/src/renderer/src/lib/orca-cloud/start-cloud-workspace-from-github.cancel.test.ts @@ -10,7 +10,16 @@ type OpenOutcome = { ok: true; worktreeId: string | null; hostId?: string } const mocks = vi.hoisted(() => ({ open: vi.fn<(workspace: unknown, options?: OpenOptions) => Promise>(), - launch: vi.fn() + launch: vi.fn(), + cleanupDone: vi.fn(), + cleanupFailed: vi.fn<(creationId: string, tryAgain: () => void) => void>() +})) + +vi.mock('@/lib/orca-cloud-session-toasts', async (importOriginal) => ({ + ...(await importOriginal()), + showOrcaCloudCleanupStartedToast: vi.fn(), + showOrcaCloudCleanupDoneToast: mocks.cleanupDone, + showOrcaCloudCleanupFailedToast: mocks.cleanupFailed })) vi.mock('@/components/cloud-workspaces/open-cloud-workspace', () => ({ @@ -24,7 +33,10 @@ import { startCloudWorkspaceFromGitHub } from './start-cloud-workspace-from-gith import { resetGitHubCreateKeysForTest } from './github-cloud-create-key' const createFromGitHub = vi.fn<(args: unknown) => Promise>() -const destroy = vi.fn(async () => ({ ok: true, value: undefined })) +const destroy = vi.fn( + async (_id: string) => ({ ok: true, value: undefined }) as { ok: boolean; value?: undefined } +) +const retryCancelledCreate = vi.fn(async (_args: { provisionId: string }) => true) beforeEach(() => { resetGitHubCreateKeysForTest() @@ -34,7 +46,7 @@ beforeEach(() => { Object.assign(window, { api: { ...window.api, - orcaCloudWorkspace: { createFromGitHub }, + orcaCloudWorkspace: { createFromGitHub, retryCancelledCreate }, cloudWorkspaces: { get: vi.fn(async (id: string) => ({ ok: true, value: { id } })), destroy, @@ -100,6 +112,72 @@ describe('Cancel leaves no workspace (GH-CREATE-CANCEL)', () => { }) }) +const settle = async (): Promise => { + for (let i = 0; i < 5; i++) { + await new Promise((resolve) => setTimeout(resolve, 0)) + } +} + +describe('EP-5: Cancel says "Sandbox deleted" only once the control plane confirmed it', () => { + // Earlier tests' cleanups can still finish late: every check is scoped to this test's creation. + let id = '' + // As in the app: the cancel lands while the create's steps are already tracked (a tick in). + const cancelHere = async (): Promise => { + await Promise.resolve() + id = creationId() + cancel() + } + + it('cancelled in flight and main reclaimed it: Sandbox deleted', async () => { + createFromGitHub.mockImplementationOnce(async () => { + await cancelHere() + return { ok: false, code: 'failed', message: 'Cancelled.' } + }) + await startCloudWorkspaceFromGitHub(REQUEST) + await settle() + expect(mocks.cleanupDone).toHaveBeenCalledWith(id) + expect(mocks.cleanupFailed).not.toHaveBeenCalledWith(id, expect.anything()) + }) + + it('cancelled in flight and main could not confirm the delete: not deleted; Try again retries it', async () => { + createFromGitHub.mockImplementationOnce(async () => { + await cancelHere() + return { + ok: false, + code: 'failed', + message: 'Cancelled.', + reason: 'cancelCleanupUnconfirmed' + } + }) + await startCloudWorkspaceFromGitHub(REQUEST) + await settle() + expect(mocks.cleanupDone).not.toHaveBeenCalledWith(id) + const failed = mocks.cleanupFailed.mock.calls.filter(([c]) => c === id) + expect(failed).toHaveLength(1) + failed[0]![1]() + await settle() + expect(retryCancelledCreate).toHaveBeenCalledWith({ provisionId: id }) + expect(mocks.cleanupDone).toHaveBeenCalledWith(id) + }) + + it('cancelled after it started and the delete failed: not deleted; Try again deletes it again', async () => { + createFromGitHub.mockImplementationOnce(async () => { + await cancelHere() + return { ok: true, value: { sandboxId: 'sbx_1' } } + }) + destroy.mockResolvedValueOnce({ ok: false }) + await startCloudWorkspaceFromGitHub(REQUEST) + await settle() + expect(mocks.cleanupDone).not.toHaveBeenCalledWith(id) + const failed = mocks.cleanupFailed.mock.calls.filter(([c]) => c === id) + expect(failed).toHaveLength(1) + failed[0]![1]() + await settle() + expect(destroy).toHaveBeenCalledTimes(2) + expect(mocks.cleanupDone).toHaveBeenCalledWith(id) + }) +}) + describe('pending row hand-off (GH-CREATE-NO-ROW)', () => { it('records the paired server while opening, and the hand-off drops the pending row', async () => { let pairingHostIdDuringOpen: string | undefined diff --git a/src/renderer/src/lib/orca-cloud/start-cloud-workspace-from-github.ts b/src/renderer/src/lib/orca-cloud/start-cloud-workspace-from-github.ts index dd3fdd7000e..b71097c3f5c 100644 --- a/src/renderer/src/lib/orca-cloud/start-cloud-workspace-from-github.ts +++ b/src/renderer/src/lib/orca-cloud/start-cloud-workspace-from-github.ts @@ -15,7 +15,8 @@ import { isTuiAgent } from '../../../../shared/agent-launch/catalog/tui-agent-co import { ORCA_CLOUD_RECIPE_ID } from '../../../../shared/hosted-sandbox/orca-cloud-recipe-id' import type { WorktreeCreationRequest } from '@/lib/workspace/creation/pending-worktree-creation' import { appendProvisioningLog } from '@/lib/workspace/creation/pending-creation-provisioning-log' -import { trackOrcaCloudCreateStep } from '@/lib/orca-cloud-cancel-cleanup' +import { setOrcaCloudCancelRetry, trackOrcaCloudCreateStep } from '@/lib/orca-cloud-cancel-cleanup' +import { CANCEL_CLEANUP_UNCONFIRMED } from '../../../../shared/hosted-sandbox/cloud-workspace-api' import { launchFirstCloudAgent } from './github-cloud-first-agent' import { forgetGitHubCreateKey, @@ -158,8 +159,15 @@ async function startSandbox(creationId: string): Promise { provisionId: creationId, idempotencyKey: githubCreateIdempotencyKey(creationId) }) - .catch(() => null) + .catch(() => null), + // EP-5: a cancel whose sandbox the control plane didn't confirm deleted is not "Sandbox deleted". + (result) => result?.ok !== false || result.reason !== CANCEL_CLEANUP_UNCONFIRMED ).finally(() => unsubscribe?.()) + if (created && !created.ok && created.reason === CANCEL_CLEANUP_UNCONFIRMED) { + setOrcaCloudCancelRetry(creationId, () => + window.api.orcaCloudWorkspace.retryCancelledCreate({ provisionId: creationId }) + ) + } noteGitHubCreateOutcome( creationId, !created || @@ -244,9 +252,16 @@ async function runGitHubCloudCreate(creationId: string): Promise { async function discardSandbox(creationId: string, sandboxId: string): Promise { startedSandboxes.delete(creationId) forgetGitHubCreateKey(creationId) - const result = await window.api.cloudWorkspaces.destroy(sandboxId).catch(() => null) - if (!result?.ok) { + const destroy = (): Promise => + window.api.cloudWorkspaces + .destroy(sandboxId) + .then((result) => result.ok) + .catch(() => false) + // EP-5: the cancel's toast waits for this delete and says "deleted" only if it went through. + const ok = await trackOrcaCloudCreateStep(creationId, destroy(), (deleted) => deleted) + if (!ok) { console.warn('[orca-cloud] cancelled GitHub create: could not delete sandbox', sandboxId) + setOrcaCloudCancelRetry(creationId, destroy) } void refreshCloudWorktreeBindings().catch(() => undefined) } diff --git a/src/shared/hosted-sandbox/cloud-workspace-api.ts b/src/shared/hosted-sandbox/cloud-workspace-api.ts index 8c61e6f6a57..d69a0fcd4ec 100644 --- a/src/shared/hosted-sandbox/cloud-workspace-api.ts +++ b/src/shared/hosted-sandbox/cloud-workspace-api.ts @@ -22,6 +22,9 @@ export type CloudWorkspaceErrorCode = | 'not_found' | 'failed' +/** EP-5: the `reason` on a cancelled create whose sandbox the control plane didn't confirm deleted. */ +export const CANCEL_CLEANUP_UNCONFIRMED = 'cancelCleanupUnconfirmed' + export type CloudWorkspaceResult = | { ok: true; value: T } | { @@ -143,6 +146,11 @@ export type OrcaCloudWorkspaceApi = { /** One per Create press; a retry after a lost answer reuses it and gets the same sandbox. */ idempotencyKey?: string }) => Promise> + /** + * EP-5: Try again for a cancelled GitHub-only create whose sandbox wasn't confirmed deleted + * (`reason: 'cancelCleanupUnconfirmed'`). True once the control plane confirms the delete. + */ + retryCancelledCreate: (args: { provisionId: string }) => Promise onWakeProgress: (callback: (event: CloudWakeProgressEvent) => void) => () => void onOom: (callback: (event: CloudOomEvent) => void) => () => void } @@ -164,6 +172,7 @@ export const ORCA_CLOUD_WORKSPACE_CHANNELS = { gitHubInstallRights: 'orcaCloudWorkspace:gitHubInstallRights', listGitHubRepositories: 'orcaCloudWorkspace:listGitHubRepositories', createFromGitHub: 'orcaCloudWorkspace:createFromGitHub', + retryCancelledCreate: 'orcaCloudWorkspace:retryCancelledCreate', wakeProgress: 'orcaCloudWorkspace:wakeProgress', oom: 'orcaCloudWorkspace:oom' } as const