mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 19:07:41 +00:00
fix(cloud): EP-5 a cancelled create never leaks its sandbox or claims it was deleted
This commit is contained in:
@@ -1778,7 +1778,12 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise<Res
|
||||
// with the sandbox the first one made, for 24 h, instead of starting another (PC-8).
|
||||
let timings: Ready['timings']
|
||||
try {
|
||||
const response = await withIdempotency(
|
||||
// EP-5: a create outlives its request (ctx.waitUntil), like a wake. A desktop that cancels mid-create
|
||||
// disconnects; the create still finishes (or fails and frees its VM) and stores its answer, so the
|
||||
// cancel's replay gets the sandbox to delete instead of 409 `idempotency_in_progress` for minutes.
|
||||
const response = await outlivingRequest(
|
||||
life,
|
||||
withIdempotency(
|
||||
life.db,
|
||||
{ accountId: caller.accountId, scope: `sandbox.create:${caller.userId}`, key: readIdempotencyKey(request), body: raw, now: life.now() },
|
||||
async () => {
|
||||
@@ -1794,6 +1799,7 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise<Res
|
||||
timings = ready.timings
|
||||
return { status: 200, body: readyView(ready.row, ready.connection, ready.cloning, features.includes('repo-update')) }
|
||||
}
|
||||
)
|
||||
)
|
||||
const timing = Object.entries(timings ?? {})
|
||||
.map(([phase, ms]) => `${phase};dur=${ms}`)
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
// EP-5: a desktop that cancels a create mid-request disconnects. The create used to die with the
|
||||
// request (Workers cancel it), leaving the row `creating` and the Idempotency-Key pending, so every
|
||||
// cancel replay answered 409 until the 5 min sweep. The create now outlives its request
|
||||
// (ctx.waitUntil), so the replay gets the sandbox to delete; a delete that lands while the create
|
||||
// is still running wins (the create frees its VM instead of finishing); and a create that was cut
|
||||
// off anyway is swept (create-speed.test.ts covers the 5 min sweep).
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { HostedSandboxReadyResponseSchema } from '../../../../src/shared/hosted-sandbox/hosted-sandbox-wire'
|
||||
import { createHarness } from './harness'
|
||||
|
||||
const create = { workspaceId: 'ws-cancel', machineSize: 'standard' }
|
||||
const key = { 'idempotency-key': 'press-0123456789' }
|
||||
|
||||
async function setup() {
|
||||
const background: Promise<unknown>[] = []
|
||||
const h = await createHarness({}, { waitUntil: (work) => void background.push(work) })
|
||||
// Hold the provider's VM create so the request is still mid-create when the client goes away.
|
||||
let open = (): void => {}
|
||||
const held = new Promise<void>((resolve) => (open = resolve))
|
||||
const createVm = h.vercel.createSandbox.bind(h.vercel)
|
||||
h.vercel.createSandbox = async (params) => {
|
||||
await held
|
||||
return createVm(params)
|
||||
}
|
||||
const row = () => h.raw.prepare(`SELECT id, status, provider_name FROM sandboxes WHERE workspace_id = ? ORDER BY rowid DESC`).get(create.workspaceId) as any
|
||||
const untilCreating = async () => {
|
||||
for (let i = 0; i < 200 && row()?.status !== 'creating'; i++) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 10))
|
||||
}
|
||||
expect(row()?.status).toBe('creating')
|
||||
}
|
||||
const liveVms = () => [...(h.vercel as any).boxes.values()].filter((box: any) => box.params.tags?.orcaSandbox === row()?.id)
|
||||
return { h, background, open, row, untilCreating, liveVms }
|
||||
}
|
||||
|
||||
describe('EP-5: cancelling a create mid-request', () => {
|
||||
it('the create outlives its request, so the replay gets the sandbox instead of 409 in progress', async () => {
|
||||
const t = await setup()
|
||||
const first = t.h.call('POST', '/v1/sandboxes', { body: create, headers: key })
|
||||
await t.untilCreating()
|
||||
expect(t.background.length).toBeGreaterThan(0)
|
||||
// While it runs, a replay is told to wait (the desktop retries).
|
||||
expect((await t.h.call('POST', '/v1/sandboxes', { body: create, headers: key })).status).toBe(409)
|
||||
// The client is gone: only the work handed to waitUntil carries on, and it finishes.
|
||||
t.open()
|
||||
await Promise.all(t.background.splice(0))
|
||||
await first
|
||||
expect(t.row().status).toBe('running')
|
||||
const replay = await t.h.call('POST', '/v1/sandboxes', { body: create, headers: key })
|
||||
expect(replay.status).toBe(200)
|
||||
const id = HostedSandboxReadyResponseSchema.parse(replay.body).sandbox.id
|
||||
expect(id).toBe(t.row().id)
|
||||
// ... and the cancel deletes it for real.
|
||||
expect((await t.h.call('DELETE', `/v1/sandboxes/${id}`)).status).toBe(200)
|
||||
expect(t.row().status).toBe('destroyed')
|
||||
expect(t.liveVms()).toEqual([])
|
||||
})
|
||||
|
||||
it('a delete while the VM is still being made wins: the create frees its VM instead of finishing', async () => {
|
||||
const t = await setup()
|
||||
const first = t.h.call('POST', '/v1/sandboxes', { body: create, headers: key })
|
||||
await t.untilCreating()
|
||||
const id = t.row().id
|
||||
const deleted = await t.h.call('DELETE', `/v1/sandboxes/${id}`)
|
||||
expect(deleted.status).toBe(200)
|
||||
// The VM arrives after the delete: the create sees the row is no longer its own and frees it.
|
||||
t.open()
|
||||
await Promise.all(t.background.splice(0))
|
||||
expect((await first).status).not.toBe(200)
|
||||
expect(t.row().status).toBe('destroyed')
|
||||
expect(t.liveVms()).toEqual([])
|
||||
})
|
||||
})
|
||||
@@ -1,9 +1,11 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { HostedEntitlementDeniedError } from './hosted-environment-provider'
|
||||
import { EnvironmentProviderError } from '../environment-providers/environment-provider-driver'
|
||||
import { CANCEL_CLEANUP_UNCONFIRMED } from '../../shared/hosted-sandbox/cloud-workspace-api'
|
||||
import {
|
||||
createCloudWorkspaceFromGitHub,
|
||||
githubWorkspaceId,
|
||||
retryCancelledGitHubCreate,
|
||||
type CreateFromGitHubClient
|
||||
} from './cloud-create-from-github'
|
||||
|
||||
@@ -26,6 +28,7 @@ function client(overrides: Partial<CreateFromGitHubClient> = {}): CreateFromGitH
|
||||
bootPlan: vi.fn(async () => 'clone' as const),
|
||||
cloneStatus: vi.fn(async () => ({ state: 'done' as const })),
|
||||
destroy: vi.fn(async (id: string) => ({ id, status: 'destroyed' as const })),
|
||||
list: vi.fn(async () => []),
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
@@ -129,3 +132,48 @@ describe('GitHub-only create on the build sheet (GH-CREATE-SHEET)', () => {
|
||||
expect(api.destroy).toHaveBeenCalledWith('sbx_1')
|
||||
})
|
||||
})
|
||||
|
||||
describe('EP-5: a cancel whose sandbox was not confirmed deleted', () => {
|
||||
it('answers reason cancelCleanupUnconfirmed (never a plain cancel), and Try again deletes it', async () => {
|
||||
const controller = new AbortController()
|
||||
let down = true
|
||||
const api = client({
|
||||
create: vi.fn(async (_body, signal) => {
|
||||
if (signal) {
|
||||
controller.abort()
|
||||
throw new EnvironmentProviderError('aborted', 'Cancelled.')
|
||||
}
|
||||
if (down) {
|
||||
throw new EnvironmentProviderError('unavailable', 'down')
|
||||
}
|
||||
return READY
|
||||
})
|
||||
})
|
||||
vi.useFakeTimers()
|
||||
let result: Awaited<ReturnType<typeof createCloudWorkspaceFromGitHub>>
|
||||
try {
|
||||
const pending = createCloudWorkspaceFromGitHub(api, INPUT, { signal: controller.signal })
|
||||
await vi.runAllTimersAsync()
|
||||
result = await pending
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
expect(result).toMatchObject({ ok: false, reason: CANCEL_CLEANUP_UNCONFIRMED })
|
||||
expect(api.destroy).not.toHaveBeenCalled()
|
||||
|
||||
// Try again, once the control plane answers: the same body and key find the sandbox; it is deleted.
|
||||
down = false
|
||||
expect(
|
||||
await retryCancelledGitHubCreate(api, INPUT.provisionId, { waitMs: 0, log: () => {} })
|
||||
).toBe(true)
|
||||
expect(api.create).toHaveBeenLastCalledWith(
|
||||
expect.objectContaining({ workspaceId: githubWorkspaceId(INPUT.slug, INPUT.provisionId) }),
|
||||
undefined,
|
||||
INPUT.idempotencyKey
|
||||
)
|
||||
expect(api.destroy).toHaveBeenCalledWith('sbx_1')
|
||||
// Done: nothing left to retry.
|
||||
expect(await retryCancelledGitHubCreate(api, INPUT.provisionId)).toBe(true)
|
||||
expect(api.destroy).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -7,8 +7,17 @@ import type {
|
||||
CloudWorkspaceErrorCode,
|
||||
CloudWorkspaceResult
|
||||
} from '../../shared/hosted-sandbox/cloud-workspace-api'
|
||||
import {
|
||||
EnvironmentProviderError,
|
||||
type EnvironmentCreateSpec
|
||||
} from '../environment-providers/environment-provider-driver'
|
||||
import type { HostedSandboxControlPlaneClient } from './hosted-sandbox-control-plane-client'
|
||||
import { createSandboxWithProgress } from './hosted-sandbox-driver'
|
||||
import { reclaimCancelledCreate } from './hosted-sandbox-create-reclaim'
|
||||
import {
|
||||
CANCEL_CLEANUP_UNCONFIRMED,
|
||||
createSandboxWithProgress,
|
||||
hostedCreateBody
|
||||
} from './hosted-sandbox-driver'
|
||||
import { hostedCreateFailure, hostedProgressLine } from './hosted-sandbox-recipe'
|
||||
|
||||
/** The server's Idempotency-Key format. */
|
||||
@@ -27,7 +36,7 @@ export type CreateFromGitHubInput = {
|
||||
|
||||
export type CreateFromGitHubClient = Pick<
|
||||
HostedSandboxControlPlaneClient,
|
||||
'create' | 'bootPlan' | 'cloneStatus' | 'destroy'
|
||||
'create' | 'bootPlan' | 'cloneStatus' | 'destroy' | 'list'
|
||||
>
|
||||
|
||||
/**
|
||||
@@ -53,28 +62,69 @@ const CODE_BY_FAILURE: Partial<Record<string, CloudWorkspaceErrorCode>> = {
|
||||
* Starts the sandbox and waits for its clone, reporting each step through `onLog`. A failure comes
|
||||
* back with the failure code as `reason` (e.g. `cloudPlanRequired`) after its reason line was logged.
|
||||
*/
|
||||
function createSpec(input: CreateFromGitHubInput): EnvironmentCreateSpec {
|
||||
return {
|
||||
workspaceId: githubWorkspaceId(input.slug, input.provisionId),
|
||||
machineSize: 'standard',
|
||||
repo: { url: `https://github.com/${input.slug}.git` },
|
||||
...(input.title ? { workspaceName: input.title } : {}),
|
||||
...(input.idempotencyKey ? { idempotencyKey: input.idempotencyKey } : {})
|
||||
}
|
||||
}
|
||||
|
||||
/** EP-5: cancelled creates whose sandbox wasn't confirmed deleted, by creation id, for Try again. */
|
||||
const unconfirmedCancels = new Map<string, CreateFromGitHubInput>()
|
||||
|
||||
/**
|
||||
* Try again for a cancelled create the first cleanup couldn't confirm: replays it by its key (or
|
||||
* finds it by its workspace id) and deletes it. True once deleted, or when there is nothing to do.
|
||||
*/
|
||||
export async function retryCancelledGitHubCreate(
|
||||
client: Pick<CreateFromGitHubClient, 'create' | 'destroy' | 'list'>,
|
||||
provisionId: string,
|
||||
options: Parameters<typeof reclaimCancelledCreate>[3] = {}
|
||||
): Promise<boolean> {
|
||||
const input = unconfirmedCancels.get(provisionId)
|
||||
if (!input?.idempotencyKey) {
|
||||
return true
|
||||
}
|
||||
const spec = createSpec(input)
|
||||
const ok = await reclaimCancelledCreate(
|
||||
client,
|
||||
hostedCreateBody(spec),
|
||||
input.idempotencyKey,
|
||||
options
|
||||
)
|
||||
if (ok) {
|
||||
unconfirmedCancels.delete(provisionId)
|
||||
}
|
||||
return ok
|
||||
}
|
||||
|
||||
export async function createCloudWorkspaceFromGitHub(
|
||||
client: CreateFromGitHubClient,
|
||||
input: CreateFromGitHubInput,
|
||||
io: { signal?: AbortSignal; onLog?: (chunk: string) => void } = {}
|
||||
): Promise<CloudWorkspaceResult<{ sandboxId: string }>> {
|
||||
try {
|
||||
const ready = await createSandboxWithProgress(
|
||||
client,
|
||||
{
|
||||
workspaceId: githubWorkspaceId(input.slug, input.provisionId),
|
||||
machineSize: 'standard',
|
||||
repo: { url: `https://github.com/${input.slug}.git` },
|
||||
...(input.title ? { workspaceName: input.title } : {}),
|
||||
...(input.idempotencyKey ? { idempotencyKey: input.idempotencyKey } : {})
|
||||
},
|
||||
{
|
||||
...(io.signal ? { signal: io.signal } : {}),
|
||||
onProgress: (progress) => io.onLog?.(hostedProgressLine(progress))
|
||||
}
|
||||
)
|
||||
const ready = await createSandboxWithProgress(client, createSpec(input), {
|
||||
...(io.signal ? { signal: io.signal } : {}),
|
||||
onProgress: (progress) => io.onLog?.(hostedProgressLine(progress))
|
||||
})
|
||||
return { ok: true, value: { sandboxId: ready.sandbox.id } }
|
||||
} catch (error) {
|
||||
if (error instanceof EnvironmentProviderError && error.reason === CANCEL_CLEANUP_UNCONFIRMED) {
|
||||
// The window must not say "Sandbox deleted" for this one; Try again retries the delete.
|
||||
if (input.provisionId) {
|
||||
unconfirmedCancels.set(input.provisionId, input)
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
code: 'failed',
|
||||
message: 'Cancelled.',
|
||||
reason: CANCEL_CLEANUP_UNCONFIRMED
|
||||
}
|
||||
}
|
||||
const result = hostedCreateFailure(error)
|
||||
if (result.stderr) {
|
||||
io.onLog?.(result.stderr)
|
||||
|
||||
@@ -15,7 +15,8 @@ import type { GitHubInstallRights } from '../../shared/hosted-sandbox/cloud-work
|
||||
import {
|
||||
CLOUD_CREATE_IDEMPOTENCY_KEY,
|
||||
CLOUD_CREATE_PROVISION_ID,
|
||||
createCloudWorkspaceFromGitHub
|
||||
createCloudWorkspaceFromGitHub,
|
||||
retryCancelledGitHubCreate
|
||||
} from './cloud-create-from-github'
|
||||
import { activeProvisionControllers } from '../ipc/provision-controllers'
|
||||
import {
|
||||
@@ -33,6 +34,7 @@ export const HOSTED_ACCOUNT_CHANNELS = [
|
||||
CH.gitHubInstallRights,
|
||||
CH.listGitHubRepositories,
|
||||
CH.createFromGitHub,
|
||||
CH.retryCancelledCreate,
|
||||
...HOSTED_GITHUB_USER_CHANNELS
|
||||
] as const
|
||||
|
||||
@@ -196,8 +198,14 @@ export function registerHostedAccountHandlers(
|
||||
}
|
||||
}
|
||||
})
|
||||
ipcMain.handle(CH.retryCancelledCreate, async (_event, args?: RetryCancelledCreateArgs) => {
|
||||
const provisionId = matching(args?.provisionId, CLOUD_CREATE_PROVISION_ID)
|
||||
return provisionId ? retryCancelledGitHubCreate(client, provisionId) : false
|
||||
})
|
||||
}
|
||||
|
||||
type RetryCancelledCreateArgs = { provisionId?: unknown } | undefined
|
||||
|
||||
type CreateFromGitHubArgs =
|
||||
| { repoUrl?: unknown; title?: unknown; provisionId?: unknown; idempotencyKey?: unknown }
|
||||
| undefined
|
||||
|
||||
@@ -4,7 +4,7 @@ import { describe, expect, it, vi } from 'vitest'
|
||||
import { EnvironmentProviderError } from '../environment-providers/environment-provider-driver'
|
||||
import { HostedEntitlementDeniedError } from './hosted-environment-provider'
|
||||
import { reclaimCancelledCreate } from './hosted-sandbox-create-reclaim'
|
||||
import { createSandboxWithProgress } from './hosted-sandbox-driver'
|
||||
import { CANCEL_CLEANUP_UNCONFIRMED, createSandboxWithProgress } from './hosted-sandbox-driver'
|
||||
|
||||
const READY = {
|
||||
sandbox: {
|
||||
@@ -26,7 +26,8 @@ function client() {
|
||||
create: vi.fn(async () => READY),
|
||||
bootPlan: vi.fn(async () => 'clone' as const),
|
||||
cloneStatus: vi.fn(async () => ({ state: 'done' as const })),
|
||||
destroy: vi.fn(async (id: string) => ({ id, status: 'destroyed' as const }))
|
||||
destroy: vi.fn(async (id: string) => ({ id, status: 'destroyed' as const })),
|
||||
list: vi.fn(async (): Promise<(typeof READY)['sandbox'][]> => [])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,6 +56,40 @@ describe('reclaimCancelledCreate', () => {
|
||||
expect(api.destroy).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('EP-5: while the first create still runs (409), deletes the sandbox it is making by workspace id', async () => {
|
||||
const api = client()
|
||||
api.create.mockRejectedValue(
|
||||
new EnvironmentProviderError('conflict', 'idempotency_in_progress')
|
||||
)
|
||||
api.list.mockResolvedValue([
|
||||
{ ...READY.sandbox, id: 'sbx_other', workspaceId: 'other-1' },
|
||||
{ ...READY.sandbox, status: 'creating' as never }
|
||||
])
|
||||
expect(await reclaimCancelledCreate(api, BODY, 'create:k1', quiet)).toBe(true)
|
||||
expect(api.destroy).toHaveBeenCalledTimes(1)
|
||||
expect(api.destroy).toHaveBeenCalledWith('sbx_1')
|
||||
expect(api.create).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('EP-5: a delete the control plane does not answer with destroyed is not confirmed', async () => {
|
||||
const api = client()
|
||||
api.destroy.mockImplementation(async (id: string) => ({ id, status: 'destroying' as never }))
|
||||
expect(await reclaimCancelledCreate(api, BODY, 'create:k1', { ...quiet, attempts: 2 })).toBe(
|
||||
false
|
||||
)
|
||||
})
|
||||
|
||||
it('EP-5: 409 with nothing listed yet keeps replaying until the create answers', async () => {
|
||||
const api = client()
|
||||
api.create.mockRejectedValueOnce(
|
||||
new EnvironmentProviderError('conflict', 'idempotency_in_progress')
|
||||
)
|
||||
expect(await reclaimCancelledCreate(api, BODY, 'create:k1', quiet)).toBe(true)
|
||||
expect(api.list).toHaveBeenCalledTimes(1)
|
||||
expect(api.create).toHaveBeenCalledTimes(2)
|
||||
expect(api.destroy).toHaveBeenCalledWith('sbx_1')
|
||||
})
|
||||
|
||||
it('gives up (false) when it can never tell', async () => {
|
||||
const api = client()
|
||||
api.create.mockRejectedValue(new EnvironmentProviderError('unavailable', 'down'))
|
||||
@@ -96,6 +131,42 @@ describe('createSandboxWithProgress cancel', () => {
|
||||
expect(api.create).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('EP-5: a cancel whose sandbox could not be confirmed deleted says so (reason), not a plain cancel', async () => {
|
||||
const api = client()
|
||||
const controller = new AbortController()
|
||||
api.create.mockImplementation(async (_body, signal) => {
|
||||
if (signal) {
|
||||
controller.abort()
|
||||
throw new EnvironmentProviderError('aborted', 'Cancelled.')
|
||||
}
|
||||
throw new EnvironmentProviderError('unavailable', 'down')
|
||||
})
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const outcome = expect(
|
||||
createSandboxWithProgress(api, spec, { signal: controller.signal })
|
||||
).rejects.toMatchObject({ code: 'aborted', reason: CANCEL_CLEANUP_UNCONFIRMED })
|
||||
// Every replay (2 min of them) fails: the retry waits run out.
|
||||
await vi.runAllTimersAsync()
|
||||
await outcome
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('EP-5: cancelled as it answered and the delete failed: not confirmed', async () => {
|
||||
const api = client()
|
||||
const controller = new AbortController()
|
||||
api.create.mockImplementationOnce(async () => {
|
||||
controller.abort()
|
||||
return READY
|
||||
})
|
||||
api.destroy.mockRejectedValueOnce(new EnvironmentProviderError('unavailable', 'down'))
|
||||
await expect(
|
||||
createSandboxWithProgress(api, spec, { signal: controller.signal })
|
||||
).rejects.toMatchObject({ code: 'aborted', reason: CANCEL_CLEANUP_UNCONFIRMED })
|
||||
})
|
||||
|
||||
it('not cancelled: no replay, no delete', async () => {
|
||||
const api = client()
|
||||
await expect(createSandboxWithProgress(api, spec, {})).resolves.toBe(READY)
|
||||
|
||||
@@ -3,12 +3,18 @@
|
||||
// and body: sending the same create again hands back the sandbox the first one made (or makes one, if
|
||||
// the first never arrived), and that sandbox is deleted. Shared by the owned create (the recipe) and
|
||||
// the GitHub-only create, which both go through createSandboxWithProgress.
|
||||
// EP-5: while the first create still runs, the replay answers 409; the sandbox it is making is found
|
||||
// by its workspace id and deleted directly (the control plane accepts a delete mid-create and the
|
||||
// create then frees its VM), so a cancel doesn't wait out the create. Only a confirmed delete (or
|
||||
// a refused create, which made nothing) counts as cleaned up.
|
||||
import { isEnvironmentProviderError } from '../environment-providers/environment-provider-driver'
|
||||
import type { HostedSandboxControlPlaneClient } from './hosted-sandbox-control-plane-client'
|
||||
|
||||
type CreateBody = Parameters<HostedSandboxControlPlaneClient['create']>[0]
|
||||
export type ReclaimClient = Pick<HostedSandboxControlPlaneClient, 'create' | 'destroy' | 'list'>
|
||||
|
||||
const RECLAIM_ATTEMPTS = 8
|
||||
/** 2 min at 5 s: past a create's longest run, under the cancel toast's 3 min limit. */
|
||||
const RECLAIM_ATTEMPTS = 24
|
||||
const RECLAIM_WAIT_MS = 5_000
|
||||
|
||||
/** The first create may still be running (409 in progress) or the service briefly unreachable. */
|
||||
@@ -26,7 +32,7 @@ function worthRetrying(error: unknown): boolean {
|
||||
* the create, so nothing was made); false when it still couldn't tell after every attempt.
|
||||
*/
|
||||
export async function reclaimCancelledCreate(
|
||||
client: Pick<HostedSandboxControlPlaneClient, 'create' | 'destroy'>,
|
||||
client: ReclaimClient,
|
||||
body: CreateBody,
|
||||
idempotencyKey: string,
|
||||
{
|
||||
@@ -48,17 +54,56 @@ export async function reclaimCancelledCreate(
|
||||
log(
|
||||
`[orca-cloud] cancelled create: replay ${attempt}/${attempts} failed (${isEnvironmentProviderError(error) ? error.code : 'error'})`
|
||||
)
|
||||
if (
|
||||
isEnvironmentProviderError(error, 'conflict') &&
|
||||
(await deleteByWorkspace(client, body.workspaceId, log))
|
||||
) {
|
||||
return true
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, waitMs))
|
||||
continue
|
||||
}
|
||||
try {
|
||||
await client.destroy(sandboxId)
|
||||
log(`[orca-cloud] cancelled create: deleted sandbox ${sandboxId}`)
|
||||
if (await deleted(client, sandboxId, log)) {
|
||||
return true
|
||||
} catch {
|
||||
await new Promise((resolve) => setTimeout(resolve, waitMs))
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, waitMs))
|
||||
}
|
||||
log('[orca-cloud] cancelled create: could not confirm the sandbox was deleted')
|
||||
return false
|
||||
}
|
||||
|
||||
/** True once the control plane answers the delete with the sandbox destroyed. */
|
||||
async function deleted(
|
||||
client: ReclaimClient,
|
||||
sandboxId: string,
|
||||
log: (message: string) => void
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const gone = await client.destroy(sandboxId)
|
||||
if (gone.status === 'destroyed') {
|
||||
log(`[orca-cloud] cancelled create: deleted sandbox ${sandboxId}`)
|
||||
return true
|
||||
}
|
||||
} catch {
|
||||
// Retried with the next attempt.
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/** The create still running: delete the sandbox it is making, found by the create's workspace id. */
|
||||
async function deleteByWorkspace(
|
||||
client: ReclaimClient,
|
||||
workspaceId: string,
|
||||
log: (message: string) => void
|
||||
): Promise<boolean> {
|
||||
const sandboxes = await client.list().catch(() => null)
|
||||
const live = sandboxes?.filter(
|
||||
(sandbox) => sandbox.workspaceId === workspaceId && sandbox.status !== 'destroyed'
|
||||
)
|
||||
if (!live || live.length === 0) {
|
||||
// Not listed yet (or the list failed): the next replay tries again.
|
||||
return false
|
||||
}
|
||||
const results = await Promise.all(live.map((sandbox) => deleted(client, sandbox.id, log)))
|
||||
return results.every(Boolean)
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// Orca Cloud workspaces: an EnvironmentProviderDriver that talks only to Orca's control plane.
|
||||
// Which provider runs the VM (Vercel today) is a control-plane detail the desktop never sees.
|
||||
import { CANCEL_CLEANUP_UNCONFIRMED } from '../../shared/hosted-sandbox/cloud-workspace-api'
|
||||
import {
|
||||
HOSTED_SANDBOX_MACHINE_SIZES,
|
||||
ORCA_CLOUD_IDLE_SLEEP_MINUTES,
|
||||
@@ -128,12 +129,31 @@ export function createHostedSandboxDriver(
|
||||
* create, then git's clone or update until it is done. Shared by the owned create (the recipe) and
|
||||
* the GitHub-only create (GH-CREATE-SHEET), so both sheets read the same phases.
|
||||
*/
|
||||
export { CANCEL_CLEANUP_UNCONFIRMED }
|
||||
|
||||
/** The create body for `spec`; a replay must send exactly this (the server keys replays by body). */
|
||||
export function hostedCreateBody(
|
||||
spec: EnvironmentCreateSpec
|
||||
): Parameters<HostedSandboxControlPlaneClient['create']>[0] {
|
||||
return {
|
||||
workspaceId: spec.workspaceId,
|
||||
machineSize: toMachineSize(spec.machineSize),
|
||||
// Named rows: Home shows the workspace's name, never an untitled sandbox (PC-8).
|
||||
...(spec.workspaceName?.trim() ? { title: spec.workspaceName.trim().slice(0, 120) } : {}),
|
||||
...(spec.region ? { region: spec.region } : {}),
|
||||
...(spec.repo ? { repo: spec.repo } : {})
|
||||
}
|
||||
}
|
||||
|
||||
export async function createSandboxWithProgress(
|
||||
client: Pick<HostedSandboxControlPlaneClient, 'create' | 'bootPlan' | 'cloneStatus' | 'destroy'>,
|
||||
client: Pick<
|
||||
HostedSandboxControlPlaneClient,
|
||||
'create' | 'bootPlan' | 'cloneStatus' | 'destroy' | 'list'
|
||||
>,
|
||||
spec: EnvironmentCreateSpec,
|
||||
ctx?: EnvironmentProviderCallContext
|
||||
): Promise<HostedSandboxReadyResponse> {
|
||||
const machineSize = toMachineSize(spec.machineSize)
|
||||
const body = hostedCreateBody(spec)
|
||||
ctx?.onProgress?.({ phase: 'provisioning' })
|
||||
// Names the repository step before the create answers (no relabel mid-run). Best effort: an
|
||||
// older control plane answers 404 and the step is named when the create answers.
|
||||
@@ -145,14 +165,6 @@ export async function createSandboxWithProgress(
|
||||
)
|
||||
.catch(() => undefined)
|
||||
}
|
||||
const body = {
|
||||
workspaceId: spec.workspaceId,
|
||||
machineSize,
|
||||
// Named rows: Home shows the workspace's name, never an untitled sandbox (PC-8).
|
||||
...(spec.workspaceName?.trim() ? { title: spec.workspaceName.trim().slice(0, 120) } : {}),
|
||||
...(spec.region ? { region: spec.region } : {}),
|
||||
...(spec.repo ? { repo: spec.repo } : {})
|
||||
}
|
||||
let ready: HostedSandboxReadyResponse
|
||||
try {
|
||||
ready = await client.create(body, ctx?.signal, spec.idempotencyKey)
|
||||
@@ -160,7 +172,10 @@ export async function createSandboxWithProgress(
|
||||
// GH-CREATE-CANCEL: cancelled in flight, the control plane may still have made it: find it by
|
||||
// its key and delete it before reporting the cancel.
|
||||
if (ctx?.signal?.aborted && spec.idempotencyKey) {
|
||||
await reclaimCancelledCreate(client, body, spec.idempotencyKey)
|
||||
// EP-5: the cancel may report "deleted" only when the control plane confirmed it.
|
||||
if (!(await reclaimCancelledCreate(client, body, spec.idempotencyKey))) {
|
||||
throw new EnvironmentProviderError('aborted', 'Cancelled.', CANCEL_CLEANUP_UNCONFIRMED)
|
||||
}
|
||||
}
|
||||
throw error
|
||||
}
|
||||
@@ -170,8 +185,15 @@ export async function createSandboxWithProgress(
|
||||
}
|
||||
if (ctx?.signal?.aborted) {
|
||||
// Cancelled just as it answered: nothing will use it, so it must not keep running (and billing).
|
||||
await client.destroy(ready.sandbox.id).catch(() => undefined)
|
||||
throw new EnvironmentProviderError('aborted', 'Cancelled.')
|
||||
const gone = await client
|
||||
.destroy(ready.sandbox.id)
|
||||
.then((sandbox) => sandbox.status === 'destroyed')
|
||||
.catch(() => false)
|
||||
throw new EnvironmentProviderError(
|
||||
'aborted',
|
||||
'Cancelled.',
|
||||
gone ? null : CANCEL_CLEANUP_UNCONFIRMED
|
||||
)
|
||||
}
|
||||
ctx?.onProgress?.({ phase: 'connecting' })
|
||||
return ready
|
||||
|
||||
@@ -28,6 +28,7 @@ export const orcaCloudWorkspaceApi = {
|
||||
gitHubInstallRights: (args) => ipcRenderer.invoke(CH.gitHubInstallRights, args),
|
||||
listGitHubRepositories: (args) => ipcRenderer.invoke(CH.listGitHubRepositories, args),
|
||||
createFromGitHub: (args) => ipcRenderer.invoke(CH.createFromGitHub, args),
|
||||
retryCancelledCreate: (args) => ipcRenderer.invoke(CH.retryCancelledCreate, args),
|
||||
onWakeProgress: (callback) => {
|
||||
const listener = (_event: Electron.IpcRendererEvent, event: CloudWakeProgressEvent): void =>
|
||||
callback(event)
|
||||
|
||||
@@ -7,7 +7,11 @@ const toasts = vi.hoisted(() => ({
|
||||
}))
|
||||
vi.mock('@/lib/orca-cloud-session-toasts', () => toasts)
|
||||
|
||||
import { beginOrcaCloudCancelCleanup, trackOrcaCloudCreateStep } from './orca-cloud-cancel-cleanup'
|
||||
import {
|
||||
beginOrcaCloudCancelCleanup,
|
||||
setOrcaCloudCancelRetry,
|
||||
trackOrcaCloudCreateStep
|
||||
} from './orca-cloud-cancel-cleanup'
|
||||
|
||||
function deferred<T>(): { promise: Promise<T>; resolve: (value: T) => void } {
|
||||
let resolve!: (value: T) => void
|
||||
@@ -131,4 +135,62 @@ describe('Orca Cloud cancel cleanup', () => {
|
||||
expect(cleanupCalls).toEqual(['mine'])
|
||||
expect(toasts.showOrcaCloudCleanupStartedToast).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('EP-5: a create step whose result says its sandbox was not confirmed deleted fails the cancel', async () => {
|
||||
const run = deferred<{ ok: boolean; reason?: string }>()
|
||||
void trackOrcaCloudCreateStep(
|
||||
'c-ep5',
|
||||
run.promise,
|
||||
(result) => result.ok || result.reason !== 'unconfirmed'
|
||||
)
|
||||
beginOrcaCloudCancelCleanup('c-ep5', [], { createStartedAt: 0, canceledAt: 1000 })
|
||||
run.resolve({ ok: false, reason: 'unconfirmed' })
|
||||
await flush()
|
||||
expect(toasts.showOrcaCloudCleanupDoneToast).not.toHaveBeenCalled()
|
||||
expect(toasts.showOrcaCloudCleanupFailedToast).toHaveBeenCalledWith(
|
||||
'c-ep5',
|
||||
expect.any(Function)
|
||||
)
|
||||
})
|
||||
|
||||
it('EP-5: a plain cancelled create (nothing left running) still reads as deleted', async () => {
|
||||
const run = deferred<{ ok: boolean; reason?: string }>()
|
||||
void trackOrcaCloudCreateStep(
|
||||
'c-ep5b',
|
||||
run.promise,
|
||||
(result) => result.ok || result.reason !== 'unconfirmed'
|
||||
)
|
||||
beginOrcaCloudCancelCleanup('c-ep5b', [], { createStartedAt: 0, canceledAt: 1000 })
|
||||
run.resolve({ ok: false, reason: null as never })
|
||||
await flush()
|
||||
expect(toasts.showOrcaCloudCleanupDoneToast).toHaveBeenCalledWith('c-ep5b')
|
||||
})
|
||||
|
||||
it("EP-5: Try again runs the create's own retry (no runtime records) and reads deleted only if it confirms", async () => {
|
||||
const retry = vi
|
||||
.fn<() => Promise<boolean>>()
|
||||
.mockResolvedValueOnce(false)
|
||||
.mockResolvedValueOnce(true)
|
||||
setOrcaCloudCancelRetry('c-ep5c', retry)
|
||||
beginOrcaCloudCancelCleanup('c-ep5c', [Promise.resolve(false)], {
|
||||
createStartedAt: 0,
|
||||
canceledAt: 1000
|
||||
})
|
||||
await flush()
|
||||
const tryAgain = () => toasts.showOrcaCloudCleanupFailedToast.mock.calls.at(-1)![1]()
|
||||
expect(toasts.showOrcaCloudCleanupFailedToast).toHaveBeenCalledTimes(1)
|
||||
|
||||
tryAgain()
|
||||
await flush()
|
||||
await flush()
|
||||
expect(retry).toHaveBeenCalledTimes(1)
|
||||
expect(toasts.showOrcaCloudCleanupFailedToast).toHaveBeenCalledTimes(2)
|
||||
expect(toasts.showOrcaCloudCleanupDoneToast).not.toHaveBeenCalled()
|
||||
|
||||
tryAgain()
|
||||
await flush()
|
||||
await flush()
|
||||
expect(retry).toHaveBeenCalledTimes(2)
|
||||
expect(toasts.showOrcaCloudCleanupDoneToast).toHaveBeenCalledWith('c-ep5c')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -30,37 +30,52 @@ type Cleanup = {
|
||||
// A provision that never settles must not leave "Deleting the sandbox…" spinning forever.
|
||||
const CLEANUP_TIMEOUT_MS = 3 * 60_000
|
||||
|
||||
// Orca Cloud creates still running, by creation id; after a cancel each ends once its own cleanup has.
|
||||
const inFlight = new Map<string, Set<Promise<unknown>>>()
|
||||
// Orca Cloud creates still running, by creation id, as "left nothing running" outcomes; after a
|
||||
// cancel each ends once its own cleanup has.
|
||||
const inFlight = new Map<string, Set<CleanupStep>>()
|
||||
const cleanups = new Map<string, Cleanup>()
|
||||
// EP-5: a step that couldn't confirm its sandbox deleted leaves its own Try again here.
|
||||
const retries = new Map<string, () => Promise<boolean>>()
|
||||
|
||||
export function isOrcaCloudCreateRequest(request: WorktreeCreationRequest): boolean {
|
||||
return request.ephemeralVmRecipe?.recipeId === ORCA_CLOUD_RECIPE_ID
|
||||
}
|
||||
|
||||
/** A create run a cancel must wait for: it may still bring the sandbox up or tear it down. */
|
||||
export function trackOrcaCloudCreateStep<T>(creationId: string, step: Promise<T>): Promise<T> {
|
||||
/**
|
||||
* A create run a cancel must wait for: it may still bring the sandbox up or tear it down.
|
||||
* `leftNothingRunning` reads its result (EP-5): false when it couldn't confirm its sandbox deleted,
|
||||
* so the cancel never says "Sandbox deleted" for it. A rejected step counts as not confirmed.
|
||||
*/
|
||||
export function trackOrcaCloudCreateStep<T>(
|
||||
creationId: string,
|
||||
step: Promise<T>,
|
||||
leftNothingRunning: (value: T) => boolean = () => true
|
||||
): Promise<T> {
|
||||
const steps = inFlight.get(creationId) ?? new Set()
|
||||
inFlight.set(creationId, steps)
|
||||
steps.add(step)
|
||||
const outcome = step.then(leftNothingRunning)
|
||||
// Read by a cancel's cleanup if there is one; never an unhandled rejection otherwise.
|
||||
outcome.catch(() => undefined)
|
||||
steps.add(outcome)
|
||||
const settled = (): void => {
|
||||
steps.delete(step)
|
||||
steps.delete(outcome)
|
||||
if (steps.size === 0 && inFlight.get(creationId) === steps) {
|
||||
inFlight.delete(creationId)
|
||||
}
|
||||
}
|
||||
step.then(settled, settled)
|
||||
outcome.then(settled, settled)
|
||||
const cleanup = cleanups.get(creationId)
|
||||
if (cleanup) {
|
||||
waitForStep(
|
||||
creationId,
|
||||
cleanup,
|
||||
step.then(() => true)
|
||||
)
|
||||
waitForStep(creationId, cleanup, outcome)
|
||||
}
|
||||
return step
|
||||
}
|
||||
|
||||
/** EP-5: how Try again deletes what a step of this create couldn't confirm deleted. */
|
||||
export function setOrcaCloudCancelRetry(creationId: string, retry: () => Promise<boolean>): void {
|
||||
retries.set(creationId, retry)
|
||||
}
|
||||
|
||||
/** The user canceled an Orca Cloud create; `steps` are the cleanups the cancel itself started. */
|
||||
export function beginOrcaCloudCancelCleanup(
|
||||
creationId: string,
|
||||
@@ -76,11 +91,7 @@ export function beginOrcaCloudCancelCleanup(
|
||||
cleanups.set(creationId, cleanup)
|
||||
showOrcaCloudCleanupStartedToast(creationId)
|
||||
for (const step of inFlight.get(creationId) ?? []) {
|
||||
waitForStep(
|
||||
creationId,
|
||||
cleanup,
|
||||
step.then(() => true)
|
||||
)
|
||||
waitForStep(creationId, cleanup, step)
|
||||
}
|
||||
for (const step of steps) {
|
||||
waitForStep(creationId, cleanup, step)
|
||||
@@ -120,6 +131,7 @@ function finish(creationId: string, cleanup: Cleanup, timedOut: boolean): void {
|
||||
if (failed) {
|
||||
showOrcaCloudCleanupFailedToast(creationId, () => retryCleanup(creationId, cleanup.scope))
|
||||
} else {
|
||||
retries.delete(creationId)
|
||||
showOrcaCloudCleanupDoneToast(creationId)
|
||||
}
|
||||
})
|
||||
@@ -166,7 +178,19 @@ function retryCleanup(creationId: string, scope: OrcaCloudCancelScope): void {
|
||||
if (cleanups.has(creationId)) {
|
||||
return
|
||||
}
|
||||
const step = leftoverRuntimeIds(scope, { timedOut: false, failedOnly: true }).then((ids) =>
|
||||
const retry = retries.get(creationId)
|
||||
// A create with no runtime records (GitHub-only) retries its own delete instead.
|
||||
const own = retry
|
||||
? retry()
|
||||
.catch(() => false)
|
||||
.then((ok) => {
|
||||
if (ok && retries.get(creationId) === retry) {
|
||||
retries.delete(creationId)
|
||||
}
|
||||
return ok
|
||||
})
|
||||
: Promise.resolve(true)
|
||||
const records = leftoverRuntimeIds(scope, { timedOut: false, failedOnly: true }).then((ids) =>
|
||||
ids === null
|
||||
? false
|
||||
: Promise.all(
|
||||
@@ -178,5 +202,5 @@ function retryCleanup(creationId: string, scope: OrcaCloudCancelScope): void {
|
||||
)
|
||||
).then((results) => results.every(Boolean))
|
||||
)
|
||||
beginOrcaCloudCancelCleanup(creationId, [step], scope, true)
|
||||
beginOrcaCloudCancelCleanup(creationId, [own, records], scope, true)
|
||||
}
|
||||
|
||||
@@ -10,7 +10,16 @@ type OpenOutcome = { ok: true; worktreeId: string | null; hostId?: string }
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
open: vi.fn<(workspace: unknown, options?: OpenOptions) => Promise<OpenOutcome>>(),
|
||||
launch: vi.fn()
|
||||
launch: vi.fn(),
|
||||
cleanupDone: vi.fn(),
|
||||
cleanupFailed: vi.fn<(creationId: string, tryAgain: () => void) => void>()
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/orca-cloud-session-toasts', async (importOriginal) => ({
|
||||
...(await importOriginal<object>()),
|
||||
showOrcaCloudCleanupStartedToast: vi.fn(),
|
||||
showOrcaCloudCleanupDoneToast: mocks.cleanupDone,
|
||||
showOrcaCloudCleanupFailedToast: mocks.cleanupFailed
|
||||
}))
|
||||
|
||||
vi.mock('@/components/cloud-workspaces/open-cloud-workspace', () => ({
|
||||
@@ -24,7 +33,10 @@ import { startCloudWorkspaceFromGitHub } from './start-cloud-workspace-from-gith
|
||||
import { resetGitHubCreateKeysForTest } from './github-cloud-create-key'
|
||||
|
||||
const createFromGitHub = vi.fn<(args: unknown) => Promise<unknown>>()
|
||||
const destroy = vi.fn(async () => ({ ok: true, value: undefined }))
|
||||
const destroy = vi.fn(
|
||||
async (_id: string) => ({ ok: true, value: undefined }) as { ok: boolean; value?: undefined }
|
||||
)
|
||||
const retryCancelledCreate = vi.fn(async (_args: { provisionId: string }) => true)
|
||||
|
||||
beforeEach(() => {
|
||||
resetGitHubCreateKeysForTest()
|
||||
@@ -34,7 +46,7 @@ beforeEach(() => {
|
||||
Object.assign(window, {
|
||||
api: {
|
||||
...window.api,
|
||||
orcaCloudWorkspace: { createFromGitHub },
|
||||
orcaCloudWorkspace: { createFromGitHub, retryCancelledCreate },
|
||||
cloudWorkspaces: {
|
||||
get: vi.fn(async (id: string) => ({ ok: true, value: { id } })),
|
||||
destroy,
|
||||
@@ -100,6 +112,72 @@ describe('Cancel leaves no workspace (GH-CREATE-CANCEL)', () => {
|
||||
})
|
||||
})
|
||||
|
||||
const settle = async (): Promise<void> => {
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 0))
|
||||
}
|
||||
}
|
||||
|
||||
describe('EP-5: Cancel says "Sandbox deleted" only once the control plane confirmed it', () => {
|
||||
// Earlier tests' cleanups can still finish late: every check is scoped to this test's creation.
|
||||
let id = ''
|
||||
// As in the app: the cancel lands while the create's steps are already tracked (a tick in).
|
||||
const cancelHere = async (): Promise<void> => {
|
||||
await Promise.resolve()
|
||||
id = creationId()
|
||||
cancel()
|
||||
}
|
||||
|
||||
it('cancelled in flight and main reclaimed it: Sandbox deleted', async () => {
|
||||
createFromGitHub.mockImplementationOnce(async () => {
|
||||
await cancelHere()
|
||||
return { ok: false, code: 'failed', message: 'Cancelled.' }
|
||||
})
|
||||
await startCloudWorkspaceFromGitHub(REQUEST)
|
||||
await settle()
|
||||
expect(mocks.cleanupDone).toHaveBeenCalledWith(id)
|
||||
expect(mocks.cleanupFailed).not.toHaveBeenCalledWith(id, expect.anything())
|
||||
})
|
||||
|
||||
it('cancelled in flight and main could not confirm the delete: not deleted; Try again retries it', async () => {
|
||||
createFromGitHub.mockImplementationOnce(async () => {
|
||||
await cancelHere()
|
||||
return {
|
||||
ok: false,
|
||||
code: 'failed',
|
||||
message: 'Cancelled.',
|
||||
reason: 'cancelCleanupUnconfirmed'
|
||||
}
|
||||
})
|
||||
await startCloudWorkspaceFromGitHub(REQUEST)
|
||||
await settle()
|
||||
expect(mocks.cleanupDone).not.toHaveBeenCalledWith(id)
|
||||
const failed = mocks.cleanupFailed.mock.calls.filter(([c]) => c === id)
|
||||
expect(failed).toHaveLength(1)
|
||||
failed[0]![1]()
|
||||
await settle()
|
||||
expect(retryCancelledCreate).toHaveBeenCalledWith({ provisionId: id })
|
||||
expect(mocks.cleanupDone).toHaveBeenCalledWith(id)
|
||||
})
|
||||
|
||||
it('cancelled after it started and the delete failed: not deleted; Try again deletes it again', async () => {
|
||||
createFromGitHub.mockImplementationOnce(async () => {
|
||||
await cancelHere()
|
||||
return { ok: true, value: { sandboxId: 'sbx_1' } }
|
||||
})
|
||||
destroy.mockResolvedValueOnce({ ok: false })
|
||||
await startCloudWorkspaceFromGitHub(REQUEST)
|
||||
await settle()
|
||||
expect(mocks.cleanupDone).not.toHaveBeenCalledWith(id)
|
||||
const failed = mocks.cleanupFailed.mock.calls.filter(([c]) => c === id)
|
||||
expect(failed).toHaveLength(1)
|
||||
failed[0]![1]()
|
||||
await settle()
|
||||
expect(destroy).toHaveBeenCalledTimes(2)
|
||||
expect(mocks.cleanupDone).toHaveBeenCalledWith(id)
|
||||
})
|
||||
})
|
||||
|
||||
describe('pending row hand-off (GH-CREATE-NO-ROW)', () => {
|
||||
it('records the paired server while opening, and the hand-off drops the pending row', async () => {
|
||||
let pairingHostIdDuringOpen: string | undefined
|
||||
|
||||
@@ -15,7 +15,8 @@ import { isTuiAgent } from '../../../../shared/agent-launch/catalog/tui-agent-co
|
||||
import { ORCA_CLOUD_RECIPE_ID } from '../../../../shared/hosted-sandbox/orca-cloud-recipe-id'
|
||||
import type { WorktreeCreationRequest } from '@/lib/workspace/creation/pending-worktree-creation'
|
||||
import { appendProvisioningLog } from '@/lib/workspace/creation/pending-creation-provisioning-log'
|
||||
import { trackOrcaCloudCreateStep } from '@/lib/orca-cloud-cancel-cleanup'
|
||||
import { setOrcaCloudCancelRetry, trackOrcaCloudCreateStep } from '@/lib/orca-cloud-cancel-cleanup'
|
||||
import { CANCEL_CLEANUP_UNCONFIRMED } from '../../../../shared/hosted-sandbox/cloud-workspace-api'
|
||||
import { launchFirstCloudAgent } from './github-cloud-first-agent'
|
||||
import {
|
||||
forgetGitHubCreateKey,
|
||||
@@ -158,8 +159,15 @@ async function startSandbox(creationId: string): Promise<string | null> {
|
||||
provisionId: creationId,
|
||||
idempotencyKey: githubCreateIdempotencyKey(creationId)
|
||||
})
|
||||
.catch(() => null)
|
||||
.catch(() => null),
|
||||
// EP-5: a cancel whose sandbox the control plane didn't confirm deleted is not "Sandbox deleted".
|
||||
(result) => result?.ok !== false || result.reason !== CANCEL_CLEANUP_UNCONFIRMED
|
||||
).finally(() => unsubscribe?.())
|
||||
if (created && !created.ok && created.reason === CANCEL_CLEANUP_UNCONFIRMED) {
|
||||
setOrcaCloudCancelRetry(creationId, () =>
|
||||
window.api.orcaCloudWorkspace.retryCancelledCreate({ provisionId: creationId })
|
||||
)
|
||||
}
|
||||
noteGitHubCreateOutcome(
|
||||
creationId,
|
||||
!created ||
|
||||
@@ -244,9 +252,16 @@ async function runGitHubCloudCreate(creationId: string): Promise<boolean> {
|
||||
async function discardSandbox(creationId: string, sandboxId: string): Promise<void> {
|
||||
startedSandboxes.delete(creationId)
|
||||
forgetGitHubCreateKey(creationId)
|
||||
const result = await window.api.cloudWorkspaces.destroy(sandboxId).catch(() => null)
|
||||
if (!result?.ok) {
|
||||
const destroy = (): Promise<boolean> =>
|
||||
window.api.cloudWorkspaces
|
||||
.destroy(sandboxId)
|
||||
.then((result) => result.ok)
|
||||
.catch(() => false)
|
||||
// EP-5: the cancel's toast waits for this delete and says "deleted" only if it went through.
|
||||
const ok = await trackOrcaCloudCreateStep(creationId, destroy(), (deleted) => deleted)
|
||||
if (!ok) {
|
||||
console.warn('[orca-cloud] cancelled GitHub create: could not delete sandbox', sandboxId)
|
||||
setOrcaCloudCancelRetry(creationId, destroy)
|
||||
}
|
||||
void refreshCloudWorktreeBindings().catch(() => undefined)
|
||||
}
|
||||
|
||||
@@ -22,6 +22,9 @@ export type CloudWorkspaceErrorCode =
|
||||
| 'not_found'
|
||||
| 'failed'
|
||||
|
||||
/** EP-5: the `reason` on a cancelled create whose sandbox the control plane didn't confirm deleted. */
|
||||
export const CANCEL_CLEANUP_UNCONFIRMED = 'cancelCleanupUnconfirmed'
|
||||
|
||||
export type CloudWorkspaceResult<T> =
|
||||
| { ok: true; value: T }
|
||||
| {
|
||||
@@ -143,6 +146,11 @@ export type OrcaCloudWorkspaceApi = {
|
||||
/** One per Create press; a retry after a lost answer reuses it and gets the same sandbox. */
|
||||
idempotencyKey?: string
|
||||
}) => Promise<CloudWorkspaceResult<{ sandboxId: string }>>
|
||||
/**
|
||||
* EP-5: Try again for a cancelled GitHub-only create whose sandbox wasn't confirmed deleted
|
||||
* (`reason: 'cancelCleanupUnconfirmed'`). True once the control plane confirms the delete.
|
||||
*/
|
||||
retryCancelledCreate: (args: { provisionId: string }) => Promise<boolean>
|
||||
onWakeProgress: (callback: (event: CloudWakeProgressEvent) => void) => () => void
|
||||
onOom: (callback: (event: CloudOomEvent) => void) => () => void
|
||||
}
|
||||
@@ -164,6 +172,7 @@ export const ORCA_CLOUD_WORKSPACE_CHANNELS = {
|
||||
gitHubInstallRights: 'orcaCloudWorkspace:gitHubInstallRights',
|
||||
listGitHubRepositories: 'orcaCloudWorkspace:listGitHubRepositories',
|
||||
createFromGitHub: 'orcaCloudWorkspace:createFromGitHub',
|
||||
retryCancelledCreate: 'orcaCloudWorkspace:retryCancelledCreate',
|
||||
wakeProgress: 'orcaCloudWorkspace:wakeProgress',
|
||||
oom: 'orcaCloudWorkspace:oom'
|
||||
} as const
|
||||
|
||||
Reference in New Issue
Block a user