fix(cloud): EP-5 a cancelled create never leaks its sandbox or claims it was deleted

This commit is contained in:
Neil
2026-11-13 04:33:18 -08:00
parent b6453d9dc9
commit a319237abf
14 changed files with 579 additions and 67 deletions
+7 -1
View File
@@ -1778,7 +1778,12 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise<Res
// with the sandbox the first one made, for 24 h, instead of starting another (PC-8).
let timings: Ready['timings']
try {
const response = await withIdempotency(
// EP-5: a create outlives its request (ctx.waitUntil), like a wake. A desktop that cancels mid-create
// disconnects; the create still finishes (or fails and frees its VM) and stores its answer, so the
// cancel's replay gets the sandbox to delete instead of 409 `idempotency_in_progress` for minutes.
const response = await outlivingRequest(
life,
withIdempotency(
life.db,
{ accountId: caller.accountId, scope: `sandbox.create:${caller.userId}`, key: readIdempotencyKey(request), body: raw, now: life.now() },
async () => {
@@ -1794,6 +1799,7 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise<Res
timings = ready.timings
return { status: 200, body: readyView(ready.row, ready.connection, ready.cloning, features.includes('repo-update')) }
}
)
)
const timing = Object.entries(timings ?? {})
.map(([phase, ms]) => `${phase};dur=${ms}`)
@@ -0,0 +1,73 @@
// EP-5: a desktop that cancels a create mid-request disconnects. The create used to die with the
// request (Workers cancel it), leaving the row `creating` and the Idempotency-Key pending, so every
// cancel replay answered 409 until the 5 min sweep. The create now outlives its request
// (ctx.waitUntil), so the replay gets the sandbox to delete; a delete that lands while the create
// is still running wins (the create frees its VM instead of finishing); and a create that was cut
// off anyway is swept (create-speed.test.ts covers the 5 min sweep).
import { describe, expect, it } from 'vitest'
import { HostedSandboxReadyResponseSchema } from '../../../../src/shared/hosted-sandbox/hosted-sandbox-wire'
import { createHarness } from './harness'
const create = { workspaceId: 'ws-cancel', machineSize: 'standard' }
const key = { 'idempotency-key': 'press-0123456789' }
async function setup() {
const background: Promise<unknown>[] = []
const h = await createHarness({}, { waitUntil: (work) => void background.push(work) })
// Hold the provider's VM create so the request is still mid-create when the client goes away.
let open = (): void => {}
const held = new Promise<void>((resolve) => (open = resolve))
const createVm = h.vercel.createSandbox.bind(h.vercel)
h.vercel.createSandbox = async (params) => {
await held
return createVm(params)
}
const row = () => h.raw.prepare(`SELECT id, status, provider_name FROM sandboxes WHERE workspace_id = ? ORDER BY rowid DESC`).get(create.workspaceId) as any
const untilCreating = async () => {
for (let i = 0; i < 200 && row()?.status !== 'creating'; i++) {
await new Promise((resolve) => setTimeout(resolve, 10))
}
expect(row()?.status).toBe('creating')
}
const liveVms = () => [...(h.vercel as any).boxes.values()].filter((box: any) => box.params.tags?.orcaSandbox === row()?.id)
return { h, background, open, row, untilCreating, liveVms }
}
describe('EP-5: cancelling a create mid-request', () => {
it('the create outlives its request, so the replay gets the sandbox instead of 409 in progress', async () => {
const t = await setup()
const first = t.h.call('POST', '/v1/sandboxes', { body: create, headers: key })
await t.untilCreating()
expect(t.background.length).toBeGreaterThan(0)
// While it runs, a replay is told to wait (the desktop retries).
expect((await t.h.call('POST', '/v1/sandboxes', { body: create, headers: key })).status).toBe(409)
// The client is gone: only the work handed to waitUntil carries on, and it finishes.
t.open()
await Promise.all(t.background.splice(0))
await first
expect(t.row().status).toBe('running')
const replay = await t.h.call('POST', '/v1/sandboxes', { body: create, headers: key })
expect(replay.status).toBe(200)
const id = HostedSandboxReadyResponseSchema.parse(replay.body).sandbox.id
expect(id).toBe(t.row().id)
// ... and the cancel deletes it for real.
expect((await t.h.call('DELETE', `/v1/sandboxes/${id}`)).status).toBe(200)
expect(t.row().status).toBe('destroyed')
expect(t.liveVms()).toEqual([])
})
it('a delete while the VM is still being made wins: the create frees its VM instead of finishing', async () => {
const t = await setup()
const first = t.h.call('POST', '/v1/sandboxes', { body: create, headers: key })
await t.untilCreating()
const id = t.row().id
const deleted = await t.h.call('DELETE', `/v1/sandboxes/${id}`)
expect(deleted.status).toBe(200)
// The VM arrives after the delete: the create sees the row is no longer its own and frees it.
t.open()
await Promise.all(t.background.splice(0))
expect((await first).status).not.toBe(200)
expect(t.row().status).toBe('destroyed')
expect(t.liveVms()).toEqual([])
})
})
@@ -1,9 +1,11 @@
import { describe, expect, it, vi } from 'vitest'
import { HostedEntitlementDeniedError } from './hosted-environment-provider'
import { EnvironmentProviderError } from '../environment-providers/environment-provider-driver'
import { CANCEL_CLEANUP_UNCONFIRMED } from '../../shared/hosted-sandbox/cloud-workspace-api'
import {
createCloudWorkspaceFromGitHub,
githubWorkspaceId,
retryCancelledGitHubCreate,
type CreateFromGitHubClient
} from './cloud-create-from-github'
@@ -26,6 +28,7 @@ function client(overrides: Partial<CreateFromGitHubClient> = {}): CreateFromGitH
bootPlan: vi.fn(async () => 'clone' as const),
cloneStatus: vi.fn(async () => ({ state: 'done' as const })),
destroy: vi.fn(async (id: string) => ({ id, status: 'destroyed' as const })),
list: vi.fn(async () => []),
...overrides
}
}
@@ -129,3 +132,48 @@ describe('GitHub-only create on the build sheet (GH-CREATE-SHEET)', () => {
expect(api.destroy).toHaveBeenCalledWith('sbx_1')
})
})
describe('EP-5: a cancel whose sandbox was not confirmed deleted', () => {
it('answers reason cancelCleanupUnconfirmed (never a plain cancel), and Try again deletes it', async () => {
const controller = new AbortController()
let down = true
const api = client({
create: vi.fn(async (_body, signal) => {
if (signal) {
controller.abort()
throw new EnvironmentProviderError('aborted', 'Cancelled.')
}
if (down) {
throw new EnvironmentProviderError('unavailable', 'down')
}
return READY
})
})
vi.useFakeTimers()
let result: Awaited<ReturnType<typeof createCloudWorkspaceFromGitHub>>
try {
const pending = createCloudWorkspaceFromGitHub(api, INPUT, { signal: controller.signal })
await vi.runAllTimersAsync()
result = await pending
} finally {
vi.useRealTimers()
}
expect(result).toMatchObject({ ok: false, reason: CANCEL_CLEANUP_UNCONFIRMED })
expect(api.destroy).not.toHaveBeenCalled()
// Try again, once the control plane answers: the same body and key find the sandbox; it is deleted.
down = false
expect(
await retryCancelledGitHubCreate(api, INPUT.provisionId, { waitMs: 0, log: () => {} })
).toBe(true)
expect(api.create).toHaveBeenLastCalledWith(
expect.objectContaining({ workspaceId: githubWorkspaceId(INPUT.slug, INPUT.provisionId) }),
undefined,
INPUT.idempotencyKey
)
expect(api.destroy).toHaveBeenCalledWith('sbx_1')
// Done: nothing left to retry.
expect(await retryCancelledGitHubCreate(api, INPUT.provisionId)).toBe(true)
expect(api.destroy).toHaveBeenCalledTimes(1)
})
})
@@ -7,8 +7,17 @@ import type {
CloudWorkspaceErrorCode,
CloudWorkspaceResult
} from '../../shared/hosted-sandbox/cloud-workspace-api'
import {
EnvironmentProviderError,
type EnvironmentCreateSpec
} from '../environment-providers/environment-provider-driver'
import type { HostedSandboxControlPlaneClient } from './hosted-sandbox-control-plane-client'
import { createSandboxWithProgress } from './hosted-sandbox-driver'
import { reclaimCancelledCreate } from './hosted-sandbox-create-reclaim'
import {
CANCEL_CLEANUP_UNCONFIRMED,
createSandboxWithProgress,
hostedCreateBody
} from './hosted-sandbox-driver'
import { hostedCreateFailure, hostedProgressLine } from './hosted-sandbox-recipe'
/** The server's Idempotency-Key format. */
@@ -27,7 +36,7 @@ export type CreateFromGitHubInput = {
export type CreateFromGitHubClient = Pick<
HostedSandboxControlPlaneClient,
'create' | 'bootPlan' | 'cloneStatus' | 'destroy'
'create' | 'bootPlan' | 'cloneStatus' | 'destroy' | 'list'
>
/**
@@ -53,28 +62,69 @@ const CODE_BY_FAILURE: Partial<Record<string, CloudWorkspaceErrorCode>> = {
* Starts the sandbox and waits for its clone, reporting each step through `onLog`. A failure comes
* back with the failure code as `reason` (e.g. `cloudPlanRequired`) after its reason line was logged.
*/
function createSpec(input: CreateFromGitHubInput): EnvironmentCreateSpec {
return {
workspaceId: githubWorkspaceId(input.slug, input.provisionId),
machineSize: 'standard',
repo: { url: `https://github.com/${input.slug}.git` },
...(input.title ? { workspaceName: input.title } : {}),
...(input.idempotencyKey ? { idempotencyKey: input.idempotencyKey } : {})
}
}
/** EP-5: cancelled creates whose sandbox wasn't confirmed deleted, by creation id, for Try again. */
const unconfirmedCancels = new Map<string, CreateFromGitHubInput>()
/**
* Try again for a cancelled create the first cleanup couldn't confirm: replays it by its key (or
* finds it by its workspace id) and deletes it. True once deleted, or when there is nothing to do.
*/
export async function retryCancelledGitHubCreate(
client: Pick<CreateFromGitHubClient, 'create' | 'destroy' | 'list'>,
provisionId: string,
options: Parameters<typeof reclaimCancelledCreate>[3] = {}
): Promise<boolean> {
const input = unconfirmedCancels.get(provisionId)
if (!input?.idempotencyKey) {
return true
}
const spec = createSpec(input)
const ok = await reclaimCancelledCreate(
client,
hostedCreateBody(spec),
input.idempotencyKey,
options
)
if (ok) {
unconfirmedCancels.delete(provisionId)
}
return ok
}
export async function createCloudWorkspaceFromGitHub(
client: CreateFromGitHubClient,
input: CreateFromGitHubInput,
io: { signal?: AbortSignal; onLog?: (chunk: string) => void } = {}
): Promise<CloudWorkspaceResult<{ sandboxId: string }>> {
try {
const ready = await createSandboxWithProgress(
client,
{
workspaceId: githubWorkspaceId(input.slug, input.provisionId),
machineSize: 'standard',
repo: { url: `https://github.com/${input.slug}.git` },
...(input.title ? { workspaceName: input.title } : {}),
...(input.idempotencyKey ? { idempotencyKey: input.idempotencyKey } : {})
},
{
...(io.signal ? { signal: io.signal } : {}),
onProgress: (progress) => io.onLog?.(hostedProgressLine(progress))
}
)
const ready = await createSandboxWithProgress(client, createSpec(input), {
...(io.signal ? { signal: io.signal } : {}),
onProgress: (progress) => io.onLog?.(hostedProgressLine(progress))
})
return { ok: true, value: { sandboxId: ready.sandbox.id } }
} catch (error) {
if (error instanceof EnvironmentProviderError && error.reason === CANCEL_CLEANUP_UNCONFIRMED) {
// The window must not say "Sandbox deleted" for this one; Try again retries the delete.
if (input.provisionId) {
unconfirmedCancels.set(input.provisionId, input)
}
return {
ok: false,
code: 'failed',
message: 'Cancelled.',
reason: CANCEL_CLEANUP_UNCONFIRMED
}
}
const result = hostedCreateFailure(error)
if (result.stderr) {
io.onLog?.(result.stderr)
@@ -15,7 +15,8 @@ import type { GitHubInstallRights } from '../../shared/hosted-sandbox/cloud-work
import {
CLOUD_CREATE_IDEMPOTENCY_KEY,
CLOUD_CREATE_PROVISION_ID,
createCloudWorkspaceFromGitHub
createCloudWorkspaceFromGitHub,
retryCancelledGitHubCreate
} from './cloud-create-from-github'
import { activeProvisionControllers } from '../ipc/provision-controllers'
import {
@@ -33,6 +34,7 @@ export const HOSTED_ACCOUNT_CHANNELS = [
CH.gitHubInstallRights,
CH.listGitHubRepositories,
CH.createFromGitHub,
CH.retryCancelledCreate,
...HOSTED_GITHUB_USER_CHANNELS
] as const
@@ -196,8 +198,14 @@ export function registerHostedAccountHandlers(
}
}
})
ipcMain.handle(CH.retryCancelledCreate, async (_event, args?: RetryCancelledCreateArgs) => {
const provisionId = matching(args?.provisionId, CLOUD_CREATE_PROVISION_ID)
return provisionId ? retryCancelledGitHubCreate(client, provisionId) : false
})
}
type RetryCancelledCreateArgs = { provisionId?: unknown } | undefined
type CreateFromGitHubArgs =
| { repoUrl?: unknown; title?: unknown; provisionId?: unknown; idempotencyKey?: unknown }
| undefined
@@ -4,7 +4,7 @@ import { describe, expect, it, vi } from 'vitest'
import { EnvironmentProviderError } from '../environment-providers/environment-provider-driver'
import { HostedEntitlementDeniedError } from './hosted-environment-provider'
import { reclaimCancelledCreate } from './hosted-sandbox-create-reclaim'
import { createSandboxWithProgress } from './hosted-sandbox-driver'
import { CANCEL_CLEANUP_UNCONFIRMED, createSandboxWithProgress } from './hosted-sandbox-driver'
const READY = {
sandbox: {
@@ -26,7 +26,8 @@ function client() {
create: vi.fn(async () => READY),
bootPlan: vi.fn(async () => 'clone' as const),
cloneStatus: vi.fn(async () => ({ state: 'done' as const })),
destroy: vi.fn(async (id: string) => ({ id, status: 'destroyed' as const }))
destroy: vi.fn(async (id: string) => ({ id, status: 'destroyed' as const })),
list: vi.fn(async (): Promise<(typeof READY)['sandbox'][]> => [])
}
}
@@ -55,6 +56,40 @@ describe('reclaimCancelledCreate', () => {
expect(api.destroy).not.toHaveBeenCalled()
})
it('EP-5: while the first create still runs (409), deletes the sandbox it is making by workspace id', async () => {
const api = client()
api.create.mockRejectedValue(
new EnvironmentProviderError('conflict', 'idempotency_in_progress')
)
api.list.mockResolvedValue([
{ ...READY.sandbox, id: 'sbx_other', workspaceId: 'other-1' },
{ ...READY.sandbox, status: 'creating' as never }
])
expect(await reclaimCancelledCreate(api, BODY, 'create:k1', quiet)).toBe(true)
expect(api.destroy).toHaveBeenCalledTimes(1)
expect(api.destroy).toHaveBeenCalledWith('sbx_1')
expect(api.create).toHaveBeenCalledTimes(1)
})
it('EP-5: a delete the control plane does not answer with destroyed is not confirmed', async () => {
const api = client()
api.destroy.mockImplementation(async (id: string) => ({ id, status: 'destroying' as never }))
expect(await reclaimCancelledCreate(api, BODY, 'create:k1', { ...quiet, attempts: 2 })).toBe(
false
)
})
it('EP-5: 409 with nothing listed yet keeps replaying until the create answers', async () => {
const api = client()
api.create.mockRejectedValueOnce(
new EnvironmentProviderError('conflict', 'idempotency_in_progress')
)
expect(await reclaimCancelledCreate(api, BODY, 'create:k1', quiet)).toBe(true)
expect(api.list).toHaveBeenCalledTimes(1)
expect(api.create).toHaveBeenCalledTimes(2)
expect(api.destroy).toHaveBeenCalledWith('sbx_1')
})
it('gives up (false) when it can never tell', async () => {
const api = client()
api.create.mockRejectedValue(new EnvironmentProviderError('unavailable', 'down'))
@@ -96,6 +131,42 @@ describe('createSandboxWithProgress cancel', () => {
expect(api.create).toHaveBeenCalledTimes(1)
})
it('EP-5: a cancel whose sandbox could not be confirmed deleted says so (reason), not a plain cancel', async () => {
const api = client()
const controller = new AbortController()
api.create.mockImplementation(async (_body, signal) => {
if (signal) {
controller.abort()
throw new EnvironmentProviderError('aborted', 'Cancelled.')
}
throw new EnvironmentProviderError('unavailable', 'down')
})
vi.useFakeTimers()
try {
const outcome = expect(
createSandboxWithProgress(api, spec, { signal: controller.signal })
).rejects.toMatchObject({ code: 'aborted', reason: CANCEL_CLEANUP_UNCONFIRMED })
// Every replay (2 min of them) fails: the retry waits run out.
await vi.runAllTimersAsync()
await outcome
} finally {
vi.useRealTimers()
}
})
it('EP-5: cancelled as it answered and the delete failed: not confirmed', async () => {
const api = client()
const controller = new AbortController()
api.create.mockImplementationOnce(async () => {
controller.abort()
return READY
})
api.destroy.mockRejectedValueOnce(new EnvironmentProviderError('unavailable', 'down'))
await expect(
createSandboxWithProgress(api, spec, { signal: controller.signal })
).rejects.toMatchObject({ code: 'aborted', reason: CANCEL_CLEANUP_UNCONFIRMED })
})
it('not cancelled: no replay, no delete', async () => {
const api = client()
await expect(createSandboxWithProgress(api, spec, {})).resolves.toBe(READY)
@@ -3,12 +3,18 @@
// and body: sending the same create again hands back the sandbox the first one made (or makes one, if
// the first never arrived), and that sandbox is deleted. Shared by the owned create (the recipe) and
// the GitHub-only create, which both go through createSandboxWithProgress.
// EP-5: while the first create still runs, the replay answers 409; the sandbox it is making is found
// by its workspace id and deleted directly (the control plane accepts a delete mid-create and the
// create then frees its VM), so a cancel doesn't wait out the create. Only a confirmed delete (or
// a refused create, which made nothing) counts as cleaned up.
import { isEnvironmentProviderError } from '../environment-providers/environment-provider-driver'
import type { HostedSandboxControlPlaneClient } from './hosted-sandbox-control-plane-client'
type CreateBody = Parameters<HostedSandboxControlPlaneClient['create']>[0]
export type ReclaimClient = Pick<HostedSandboxControlPlaneClient, 'create' | 'destroy' | 'list'>
const RECLAIM_ATTEMPTS = 8
/** 2 min at 5 s: past a create's longest run, under the cancel toast's 3 min limit. */
const RECLAIM_ATTEMPTS = 24
const RECLAIM_WAIT_MS = 5_000
/** The first create may still be running (409 in progress) or the service briefly unreachable. */
@@ -26,7 +32,7 @@ function worthRetrying(error: unknown): boolean {
* the create, so nothing was made); false when it still couldn't tell after every attempt.
*/
export async function reclaimCancelledCreate(
client: Pick<HostedSandboxControlPlaneClient, 'create' | 'destroy'>,
client: ReclaimClient,
body: CreateBody,
idempotencyKey: string,
{
@@ -48,17 +54,56 @@ export async function reclaimCancelledCreate(
log(
`[orca-cloud] cancelled create: replay ${attempt}/${attempts} failed (${isEnvironmentProviderError(error) ? error.code : 'error'})`
)
if (
isEnvironmentProviderError(error, 'conflict') &&
(await deleteByWorkspace(client, body.workspaceId, log))
) {
return true
}
await new Promise((resolve) => setTimeout(resolve, waitMs))
continue
}
try {
await client.destroy(sandboxId)
log(`[orca-cloud] cancelled create: deleted sandbox ${sandboxId}`)
if (await deleted(client, sandboxId, log)) {
return true
} catch {
await new Promise((resolve) => setTimeout(resolve, waitMs))
}
await new Promise((resolve) => setTimeout(resolve, waitMs))
}
log('[orca-cloud] cancelled create: could not confirm the sandbox was deleted')
return false
}
/** True once the control plane answers the delete with the sandbox destroyed. */
async function deleted(
client: ReclaimClient,
sandboxId: string,
log: (message: string) => void
): Promise<boolean> {
try {
const gone = await client.destroy(sandboxId)
if (gone.status === 'destroyed') {
log(`[orca-cloud] cancelled create: deleted sandbox ${sandboxId}`)
return true
}
} catch {
// Retried with the next attempt.
}
return false
}
/** The create still running: delete the sandbox it is making, found by the create's workspace id. */
async function deleteByWorkspace(
client: ReclaimClient,
workspaceId: string,
log: (message: string) => void
): Promise<boolean> {
const sandboxes = await client.list().catch(() => null)
const live = sandboxes?.filter(
(sandbox) => sandbox.workspaceId === workspaceId && sandbox.status !== 'destroyed'
)
if (!live || live.length === 0) {
// Not listed yet (or the list failed): the next replay tries again.
return false
}
const results = await Promise.all(live.map((sandbox) => deleted(client, sandbox.id, log)))
return results.every(Boolean)
}
@@ -1,5 +1,6 @@
// Orca Cloud workspaces: an EnvironmentProviderDriver that talks only to Orca's control plane.
// Which provider runs the VM (Vercel today) is a control-plane detail the desktop never sees.
import { CANCEL_CLEANUP_UNCONFIRMED } from '../../shared/hosted-sandbox/cloud-workspace-api'
import {
HOSTED_SANDBOX_MACHINE_SIZES,
ORCA_CLOUD_IDLE_SLEEP_MINUTES,
@@ -128,12 +129,31 @@ export function createHostedSandboxDriver(
* create, then git's clone or update until it is done. Shared by the owned create (the recipe) and
* the GitHub-only create (GH-CREATE-SHEET), so both sheets read the same phases.
*/
export { CANCEL_CLEANUP_UNCONFIRMED }
/** The create body for `spec`; a replay must send exactly this (the server keys replays by body). */
export function hostedCreateBody(
spec: EnvironmentCreateSpec
): Parameters<HostedSandboxControlPlaneClient['create']>[0] {
return {
workspaceId: spec.workspaceId,
machineSize: toMachineSize(spec.machineSize),
// Named rows: Home shows the workspace's name, never an untitled sandbox (PC-8).
...(spec.workspaceName?.trim() ? { title: spec.workspaceName.trim().slice(0, 120) } : {}),
...(spec.region ? { region: spec.region } : {}),
...(spec.repo ? { repo: spec.repo } : {})
}
}
export async function createSandboxWithProgress(
client: Pick<HostedSandboxControlPlaneClient, 'create' | 'bootPlan' | 'cloneStatus' | 'destroy'>,
client: Pick<
HostedSandboxControlPlaneClient,
'create' | 'bootPlan' | 'cloneStatus' | 'destroy' | 'list'
>,
spec: EnvironmentCreateSpec,
ctx?: EnvironmentProviderCallContext
): Promise<HostedSandboxReadyResponse> {
const machineSize = toMachineSize(spec.machineSize)
const body = hostedCreateBody(spec)
ctx?.onProgress?.({ phase: 'provisioning' })
// Names the repository step before the create answers (no relabel mid-run). Best effort: an
// older control plane answers 404 and the step is named when the create answers.
@@ -145,14 +165,6 @@ export async function createSandboxWithProgress(
)
.catch(() => undefined)
}
const body = {
workspaceId: spec.workspaceId,
machineSize,
// Named rows: Home shows the workspace's name, never an untitled sandbox (PC-8).
...(spec.workspaceName?.trim() ? { title: spec.workspaceName.trim().slice(0, 120) } : {}),
...(spec.region ? { region: spec.region } : {}),
...(spec.repo ? { repo: spec.repo } : {})
}
let ready: HostedSandboxReadyResponse
try {
ready = await client.create(body, ctx?.signal, spec.idempotencyKey)
@@ -160,7 +172,10 @@ export async function createSandboxWithProgress(
// GH-CREATE-CANCEL: cancelled in flight, the control plane may still have made it: find it by
// its key and delete it before reporting the cancel.
if (ctx?.signal?.aborted && spec.idempotencyKey) {
await reclaimCancelledCreate(client, body, spec.idempotencyKey)
// EP-5: the cancel may report "deleted" only when the control plane confirmed it.
if (!(await reclaimCancelledCreate(client, body, spec.idempotencyKey))) {
throw new EnvironmentProviderError('aborted', 'Cancelled.', CANCEL_CLEANUP_UNCONFIRMED)
}
}
throw error
}
@@ -170,8 +185,15 @@ export async function createSandboxWithProgress(
}
if (ctx?.signal?.aborted) {
// Cancelled just as it answered: nothing will use it, so it must not keep running (and billing).
await client.destroy(ready.sandbox.id).catch(() => undefined)
throw new EnvironmentProviderError('aborted', 'Cancelled.')
const gone = await client
.destroy(ready.sandbox.id)
.then((sandbox) => sandbox.status === 'destroyed')
.catch(() => false)
throw new EnvironmentProviderError(
'aborted',
'Cancelled.',
gone ? null : CANCEL_CLEANUP_UNCONFIRMED
)
}
ctx?.onProgress?.({ phase: 'connecting' })
return ready
@@ -28,6 +28,7 @@ export const orcaCloudWorkspaceApi = {
gitHubInstallRights: (args) => ipcRenderer.invoke(CH.gitHubInstallRights, args),
listGitHubRepositories: (args) => ipcRenderer.invoke(CH.listGitHubRepositories, args),
createFromGitHub: (args) => ipcRenderer.invoke(CH.createFromGitHub, args),
retryCancelledCreate: (args) => ipcRenderer.invoke(CH.retryCancelledCreate, args),
onWakeProgress: (callback) => {
const listener = (_event: Electron.IpcRendererEvent, event: CloudWakeProgressEvent): void =>
callback(event)
@@ -7,7 +7,11 @@ const toasts = vi.hoisted(() => ({
}))
vi.mock('@/lib/orca-cloud-session-toasts', () => toasts)
import { beginOrcaCloudCancelCleanup, trackOrcaCloudCreateStep } from './orca-cloud-cancel-cleanup'
import {
beginOrcaCloudCancelCleanup,
setOrcaCloudCancelRetry,
trackOrcaCloudCreateStep
} from './orca-cloud-cancel-cleanup'
function deferred<T>(): { promise: Promise<T>; resolve: (value: T) => void } {
let resolve!: (value: T) => void
@@ -131,4 +135,62 @@ describe('Orca Cloud cancel cleanup', () => {
expect(cleanupCalls).toEqual(['mine'])
expect(toasts.showOrcaCloudCleanupStartedToast).toHaveBeenCalledTimes(2)
})
it('EP-5: a create step whose result says its sandbox was not confirmed deleted fails the cancel', async () => {
const run = deferred<{ ok: boolean; reason?: string }>()
void trackOrcaCloudCreateStep(
'c-ep5',
run.promise,
(result) => result.ok || result.reason !== 'unconfirmed'
)
beginOrcaCloudCancelCleanup('c-ep5', [], { createStartedAt: 0, canceledAt: 1000 })
run.resolve({ ok: false, reason: 'unconfirmed' })
await flush()
expect(toasts.showOrcaCloudCleanupDoneToast).not.toHaveBeenCalled()
expect(toasts.showOrcaCloudCleanupFailedToast).toHaveBeenCalledWith(
'c-ep5',
expect.any(Function)
)
})
it('EP-5: a plain cancelled create (nothing left running) still reads as deleted', async () => {
const run = deferred<{ ok: boolean; reason?: string }>()
void trackOrcaCloudCreateStep(
'c-ep5b',
run.promise,
(result) => result.ok || result.reason !== 'unconfirmed'
)
beginOrcaCloudCancelCleanup('c-ep5b', [], { createStartedAt: 0, canceledAt: 1000 })
run.resolve({ ok: false, reason: null as never })
await flush()
expect(toasts.showOrcaCloudCleanupDoneToast).toHaveBeenCalledWith('c-ep5b')
})
it("EP-5: Try again runs the create's own retry (no runtime records) and reads deleted only if it confirms", async () => {
const retry = vi
.fn<() => Promise<boolean>>()
.mockResolvedValueOnce(false)
.mockResolvedValueOnce(true)
setOrcaCloudCancelRetry('c-ep5c', retry)
beginOrcaCloudCancelCleanup('c-ep5c', [Promise.resolve(false)], {
createStartedAt: 0,
canceledAt: 1000
})
await flush()
const tryAgain = () => toasts.showOrcaCloudCleanupFailedToast.mock.calls.at(-1)![1]()
expect(toasts.showOrcaCloudCleanupFailedToast).toHaveBeenCalledTimes(1)
tryAgain()
await flush()
await flush()
expect(retry).toHaveBeenCalledTimes(1)
expect(toasts.showOrcaCloudCleanupFailedToast).toHaveBeenCalledTimes(2)
expect(toasts.showOrcaCloudCleanupDoneToast).not.toHaveBeenCalled()
tryAgain()
await flush()
await flush()
expect(retry).toHaveBeenCalledTimes(2)
expect(toasts.showOrcaCloudCleanupDoneToast).toHaveBeenCalledWith('c-ep5c')
})
})
@@ -30,37 +30,52 @@ type Cleanup = {
// A provision that never settles must not leave "Deleting the sandbox…" spinning forever.
const CLEANUP_TIMEOUT_MS = 3 * 60_000
// Orca Cloud creates still running, by creation id; after a cancel each ends once its own cleanup has.
const inFlight = new Map<string, Set<Promise<unknown>>>()
// Orca Cloud creates still running, by creation id, as "left nothing running" outcomes; after a
// cancel each ends once its own cleanup has.
const inFlight = new Map<string, Set<CleanupStep>>()
const cleanups = new Map<string, Cleanup>()
// EP-5: a step that couldn't confirm its sandbox deleted leaves its own Try again here.
const retries = new Map<string, () => Promise<boolean>>()
export function isOrcaCloudCreateRequest(request: WorktreeCreationRequest): boolean {
return request.ephemeralVmRecipe?.recipeId === ORCA_CLOUD_RECIPE_ID
}
/** A create run a cancel must wait for: it may still bring the sandbox up or tear it down. */
export function trackOrcaCloudCreateStep<T>(creationId: string, step: Promise<T>): Promise<T> {
/**
* A create run a cancel must wait for: it may still bring the sandbox up or tear it down.
* `leftNothingRunning` reads its result (EP-5): false when it couldn't confirm its sandbox deleted,
* so the cancel never says "Sandbox deleted" for it. A rejected step counts as not confirmed.
*/
export function trackOrcaCloudCreateStep<T>(
creationId: string,
step: Promise<T>,
leftNothingRunning: (value: T) => boolean = () => true
): Promise<T> {
const steps = inFlight.get(creationId) ?? new Set()
inFlight.set(creationId, steps)
steps.add(step)
const outcome = step.then(leftNothingRunning)
// Read by a cancel's cleanup if there is one; never an unhandled rejection otherwise.
outcome.catch(() => undefined)
steps.add(outcome)
const settled = (): void => {
steps.delete(step)
steps.delete(outcome)
if (steps.size === 0 && inFlight.get(creationId) === steps) {
inFlight.delete(creationId)
}
}
step.then(settled, settled)
outcome.then(settled, settled)
const cleanup = cleanups.get(creationId)
if (cleanup) {
waitForStep(
creationId,
cleanup,
step.then(() => true)
)
waitForStep(creationId, cleanup, outcome)
}
return step
}
/** EP-5: how Try again deletes what a step of this create couldn't confirm deleted. */
export function setOrcaCloudCancelRetry(creationId: string, retry: () => Promise<boolean>): void {
retries.set(creationId, retry)
}
/** The user canceled an Orca Cloud create; `steps` are the cleanups the cancel itself started. */
export function beginOrcaCloudCancelCleanup(
creationId: string,
@@ -76,11 +91,7 @@ export function beginOrcaCloudCancelCleanup(
cleanups.set(creationId, cleanup)
showOrcaCloudCleanupStartedToast(creationId)
for (const step of inFlight.get(creationId) ?? []) {
waitForStep(
creationId,
cleanup,
step.then(() => true)
)
waitForStep(creationId, cleanup, step)
}
for (const step of steps) {
waitForStep(creationId, cleanup, step)
@@ -120,6 +131,7 @@ function finish(creationId: string, cleanup: Cleanup, timedOut: boolean): void {
if (failed) {
showOrcaCloudCleanupFailedToast(creationId, () => retryCleanup(creationId, cleanup.scope))
} else {
retries.delete(creationId)
showOrcaCloudCleanupDoneToast(creationId)
}
})
@@ -166,7 +178,19 @@ function retryCleanup(creationId: string, scope: OrcaCloudCancelScope): void {
if (cleanups.has(creationId)) {
return
}
const step = leftoverRuntimeIds(scope, { timedOut: false, failedOnly: true }).then((ids) =>
const retry = retries.get(creationId)
// A create with no runtime records (GitHub-only) retries its own delete instead.
const own = retry
? retry()
.catch(() => false)
.then((ok) => {
if (ok && retries.get(creationId) === retry) {
retries.delete(creationId)
}
return ok
})
: Promise.resolve(true)
const records = leftoverRuntimeIds(scope, { timedOut: false, failedOnly: true }).then((ids) =>
ids === null
? false
: Promise.all(
@@ -178,5 +202,5 @@ function retryCleanup(creationId: string, scope: OrcaCloudCancelScope): void {
)
).then((results) => results.every(Boolean))
)
beginOrcaCloudCancelCleanup(creationId, [step], scope, true)
beginOrcaCloudCancelCleanup(creationId, [own, records], scope, true)
}
@@ -10,7 +10,16 @@ type OpenOutcome = { ok: true; worktreeId: string | null; hostId?: string }
const mocks = vi.hoisted(() => ({
open: vi.fn<(workspace: unknown, options?: OpenOptions) => Promise<OpenOutcome>>(),
launch: vi.fn()
launch: vi.fn(),
cleanupDone: vi.fn(),
cleanupFailed: vi.fn<(creationId: string, tryAgain: () => void) => void>()
}))
vi.mock('@/lib/orca-cloud-session-toasts', async (importOriginal) => ({
...(await importOriginal<object>()),
showOrcaCloudCleanupStartedToast: vi.fn(),
showOrcaCloudCleanupDoneToast: mocks.cleanupDone,
showOrcaCloudCleanupFailedToast: mocks.cleanupFailed
}))
vi.mock('@/components/cloud-workspaces/open-cloud-workspace', () => ({
@@ -24,7 +33,10 @@ import { startCloudWorkspaceFromGitHub } from './start-cloud-workspace-from-gith
import { resetGitHubCreateKeysForTest } from './github-cloud-create-key'
const createFromGitHub = vi.fn<(args: unknown) => Promise<unknown>>()
const destroy = vi.fn(async () => ({ ok: true, value: undefined }))
const destroy = vi.fn(
async (_id: string) => ({ ok: true, value: undefined }) as { ok: boolean; value?: undefined }
)
const retryCancelledCreate = vi.fn(async (_args: { provisionId: string }) => true)
beforeEach(() => {
resetGitHubCreateKeysForTest()
@@ -34,7 +46,7 @@ beforeEach(() => {
Object.assign(window, {
api: {
...window.api,
orcaCloudWorkspace: { createFromGitHub },
orcaCloudWorkspace: { createFromGitHub, retryCancelledCreate },
cloudWorkspaces: {
get: vi.fn(async (id: string) => ({ ok: true, value: { id } })),
destroy,
@@ -100,6 +112,72 @@ describe('Cancel leaves no workspace (GH-CREATE-CANCEL)', () => {
})
})
const settle = async (): Promise<void> => {
for (let i = 0; i < 5; i++) {
await new Promise((resolve) => setTimeout(resolve, 0))
}
}
describe('EP-5: Cancel says "Sandbox deleted" only once the control plane confirmed it', () => {
// Earlier tests' cleanups can still finish late: every check is scoped to this test's creation.
let id = ''
// As in the app: the cancel lands while the create's steps are already tracked (a tick in).
const cancelHere = async (): Promise<void> => {
await Promise.resolve()
id = creationId()
cancel()
}
it('cancelled in flight and main reclaimed it: Sandbox deleted', async () => {
createFromGitHub.mockImplementationOnce(async () => {
await cancelHere()
return { ok: false, code: 'failed', message: 'Cancelled.' }
})
await startCloudWorkspaceFromGitHub(REQUEST)
await settle()
expect(mocks.cleanupDone).toHaveBeenCalledWith(id)
expect(mocks.cleanupFailed).not.toHaveBeenCalledWith(id, expect.anything())
})
it('cancelled in flight and main could not confirm the delete: not deleted; Try again retries it', async () => {
createFromGitHub.mockImplementationOnce(async () => {
await cancelHere()
return {
ok: false,
code: 'failed',
message: 'Cancelled.',
reason: 'cancelCleanupUnconfirmed'
}
})
await startCloudWorkspaceFromGitHub(REQUEST)
await settle()
expect(mocks.cleanupDone).not.toHaveBeenCalledWith(id)
const failed = mocks.cleanupFailed.mock.calls.filter(([c]) => c === id)
expect(failed).toHaveLength(1)
failed[0]![1]()
await settle()
expect(retryCancelledCreate).toHaveBeenCalledWith({ provisionId: id })
expect(mocks.cleanupDone).toHaveBeenCalledWith(id)
})
it('cancelled after it started and the delete failed: not deleted; Try again deletes it again', async () => {
createFromGitHub.mockImplementationOnce(async () => {
await cancelHere()
return { ok: true, value: { sandboxId: 'sbx_1' } }
})
destroy.mockResolvedValueOnce({ ok: false })
await startCloudWorkspaceFromGitHub(REQUEST)
await settle()
expect(mocks.cleanupDone).not.toHaveBeenCalledWith(id)
const failed = mocks.cleanupFailed.mock.calls.filter(([c]) => c === id)
expect(failed).toHaveLength(1)
failed[0]![1]()
await settle()
expect(destroy).toHaveBeenCalledTimes(2)
expect(mocks.cleanupDone).toHaveBeenCalledWith(id)
})
})
describe('pending row hand-off (GH-CREATE-NO-ROW)', () => {
it('records the paired server while opening, and the hand-off drops the pending row', async () => {
let pairingHostIdDuringOpen: string | undefined
@@ -15,7 +15,8 @@ import { isTuiAgent } from '../../../../shared/agent-launch/catalog/tui-agent-co
import { ORCA_CLOUD_RECIPE_ID } from '../../../../shared/hosted-sandbox/orca-cloud-recipe-id'
import type { WorktreeCreationRequest } from '@/lib/workspace/creation/pending-worktree-creation'
import { appendProvisioningLog } from '@/lib/workspace/creation/pending-creation-provisioning-log'
import { trackOrcaCloudCreateStep } from '@/lib/orca-cloud-cancel-cleanup'
import { setOrcaCloudCancelRetry, trackOrcaCloudCreateStep } from '@/lib/orca-cloud-cancel-cleanup'
import { CANCEL_CLEANUP_UNCONFIRMED } from '../../../../shared/hosted-sandbox/cloud-workspace-api'
import { launchFirstCloudAgent } from './github-cloud-first-agent'
import {
forgetGitHubCreateKey,
@@ -158,8 +159,15 @@ async function startSandbox(creationId: string): Promise<string | null> {
provisionId: creationId,
idempotencyKey: githubCreateIdempotencyKey(creationId)
})
.catch(() => null)
.catch(() => null),
// EP-5: a cancel whose sandbox the control plane didn't confirm deleted is not "Sandbox deleted".
(result) => result?.ok !== false || result.reason !== CANCEL_CLEANUP_UNCONFIRMED
).finally(() => unsubscribe?.())
if (created && !created.ok && created.reason === CANCEL_CLEANUP_UNCONFIRMED) {
setOrcaCloudCancelRetry(creationId, () =>
window.api.orcaCloudWorkspace.retryCancelledCreate({ provisionId: creationId })
)
}
noteGitHubCreateOutcome(
creationId,
!created ||
@@ -244,9 +252,16 @@ async function runGitHubCloudCreate(creationId: string): Promise<boolean> {
async function discardSandbox(creationId: string, sandboxId: string): Promise<void> {
startedSandboxes.delete(creationId)
forgetGitHubCreateKey(creationId)
const result = await window.api.cloudWorkspaces.destroy(sandboxId).catch(() => null)
if (!result?.ok) {
const destroy = (): Promise<boolean> =>
window.api.cloudWorkspaces
.destroy(sandboxId)
.then((result) => result.ok)
.catch(() => false)
// EP-5: the cancel's toast waits for this delete and says "deleted" only if it went through.
const ok = await trackOrcaCloudCreateStep(creationId, destroy(), (deleted) => deleted)
if (!ok) {
console.warn('[orca-cloud] cancelled GitHub create: could not delete sandbox', sandboxId)
setOrcaCloudCancelRetry(creationId, destroy)
}
void refreshCloudWorktreeBindings().catch(() => undefined)
}
@@ -22,6 +22,9 @@ export type CloudWorkspaceErrorCode =
| 'not_found'
| 'failed'
/** EP-5: the `reason` on a cancelled create whose sandbox the control plane didn't confirm deleted. */
export const CANCEL_CLEANUP_UNCONFIRMED = 'cancelCleanupUnconfirmed'
export type CloudWorkspaceResult<T> =
| { ok: true; value: T }
| {
@@ -143,6 +146,11 @@ export type OrcaCloudWorkspaceApi = {
/** One per Create press; a retry after a lost answer reuses it and gets the same sandbox. */
idempotencyKey?: string
}) => Promise<CloudWorkspaceResult<{ sandboxId: string }>>
/**
* EP-5: Try again for a cancelled GitHub-only create whose sandbox wasn't confirmed deleted
* (`reason: 'cancelCleanupUnconfirmed'`). True once the control plane confirms the delete.
*/
retryCancelledCreate: (args: { provisionId: string }) => Promise<boolean>
onWakeProgress: (callback: (event: CloudWakeProgressEvent) => void) => () => void
onOom: (callback: (event: CloudOomEvent) => void) => () => void
}
@@ -164,6 +172,7 @@ export const ORCA_CLOUD_WORKSPACE_CHANNELS = {
gitHubInstallRights: 'orcaCloudWorkspace:gitHubInstallRights',
listGitHubRepositories: 'orcaCloudWorkspace:listGitHubRepositories',
createFromGitHub: 'orcaCloudWorkspace:createFromGitHub',
retryCancelledCreate: 'orcaCloudWorkspace:retryCancelledCreate',
wakeProgress: 'orcaCloudWorkspace:wakeProgress',
oom: 'orcaCloudWorkspace:oom'
} as const