diff --git a/cloud/apps/sandbox-control/src/lifecycle.ts b/cloud/apps/sandbox-control/src/lifecycle.ts index 7fa7463e73f..c35e0feb5ee 100644 --- a/cloud/apps/sandbox-control/src/lifecycle.ts +++ b/cloud/apps/sandbox-control/src/lifecycle.ts @@ -19,6 +19,7 @@ import type { Keyring } from './secrets-crypto' import { ORCAD_PORT, drainOrcad, runningOrcad, startOrcad, type OrcadReady } from './sandbox-vm-commands' import { claimPooledVm, deletePooledVm, maintainPool, type PoolConfig, type PooledVm } from './pool' import { + detachFromWorkspace, NO_SESSION, countActiveSandboxes, countActiveSandboxesForUser, @@ -400,6 +401,9 @@ export async function createSandbox( // A pooled VM that will not take the session is discarded; the caller still gets a workspace. console.log(JSON.stringify({ event: 'pool_adopt_failed', error: String(error).slice(0, 200) })) await failAndRelease(deps, (await getSandbox(deps.db, row.id)) ?? row) + // Destroy can stop short (e.g. the provider's delete snapshot is still being written); the cron + // finishes it. That row never held user data, so it lets go of the workspace now. + await detachFromWorkspace(deps.db, row.id) return createSandbox(deps, caller, request, { pool: false }) } } diff --git a/cloud/apps/sandbox-control/src/store.ts b/cloud/apps/sandbox-control/src/store.ts index a005736c857..2c41ccdcafa 100644 --- a/cloud/apps/sandbox-control/src/store.ts +++ b/cloud/apps/sandbox-control/src/store.ts @@ -98,6 +98,14 @@ export function getSandboxByProviderName(db: D1Database, name: string): Promise< return db.prepare('SELECT * FROM sandboxes WHERE provider_name = ?').bind(name).first() } +/** A failed attempt that is still being cleaned up stops blocking its workspace (unique live index). */ +export async function detachFromWorkspace(db: D1Database, id: string): Promise { + await db + .prepare(`UPDATE sandboxes SET workspace_id = workspace_id || '#abandoned:' || id WHERE id = ? AND status <> 'destroyed'`) + .bind(id) + .run() +} + export function liveSandboxForWorkspace( db: D1Database, accountId: string, diff --git a/cloud/apps/sandbox-control/test/pool.test.ts b/cloud/apps/sandbox-control/test/pool.test.ts index 871b33f2c90..2f0524ca2d6 100644 --- a/cloud/apps/sandbox-control/test/pool.test.ts +++ b/cloud/apps/sandbox-control/test/pool.test.ts @@ -75,6 +75,39 @@ describe('warm pool', () => { expect(h.vercel.boxes.get(`orca-test-${res.body.sandbox.id}`)!.params.tags.orcaPool).toBeUndefined() }) + it('a failed adopt whose cleanup stops short still gives the caller a fresh VM (no sandbox_destroying conflict)', async () => { + const { h, deps, poolRows } = await pooledHarness('1') + await maintainPool(deps()) + const pooledId = poolRows()[0]!.id as string + const pooledName = `orca-test-${pooledId}` + const realPolicy = h.vercel.updateNetworkPolicy.bind(h.vercel) + h.vercel.updateNetworkPolicy = async (sessionId, policy) => { + if (sessionId === h.vercel.boxes.get(pooledName)?.sessionId) { + throw new Error('policy update failed') + } + return realPolicy(sessionId, policy) + } + // The pooled VM's delete leaves a snapshot that will not delete yet: destroy stops at 'destroying'. + const realDeleteSnapshot = h.vercel.deleteSnapshot.bind(h.vercel) + h.vercel.deleteSnapshot = async (id) => { + if (h.vercel.snapshots.get(id)?.sandboxName === pooledName) { + throw new Error('snapshot still being written') + } + return realDeleteSnapshot(id) + } + const res = await h.call('POST', '/v1/sandboxes', { body: create }) + expect(res.status).toBe(200) + expect(res.body.sandbox.id).not.toBe(pooledId) + const abandoned = h.raw.prepare('SELECT status, workspace_id FROM sandboxes WHERE id = ?').get(pooledId) as { status: string; workspace_id: string } + expect(abandoned.status).not.toBe('destroyed') + expect(abandoned.workspace_id).toBe(`ws-1#abandoned:${pooledId}`) + // The cron finishes the abandoned row once the provider lets go. + h.vercel.deleteSnapshot = realDeleteSnapshot + h.advance(30 * MIN) + await h.reconcile() + expect((h.raw.prepare('SELECT status FROM sandboxes WHERE id = ?').get(pooledId) as { status: string }).status).toBe('destroyed') + }) + it('skips the pool for sizes and regions it does not hold', async () => { const { h, deps, poolRows } = await pooledHarness('1') await maintainPool(deps())