From 6eec750fe240fbc02de506fdef548b11238b3149 Mon Sep 17 00:00:00 2001
From: Neil <4138956+nwparker@users.noreply.github.com>
Date: Mon, 9 Nov 2026 05:28:11 -0800
Subject: [PATCH] fix(sandbox-control): a failed pool adopt no longer blocks
 its own fallback (sandbox_destroying conflict) (#44)

Reported by W5: when a pooled VM fails to adopt and its destroy stops short (the provider's delete
snapshot still being written), the row stays 'destroying' and keeps the workspace's live unique index,
so the fresh-VM retry hit conflict:sandbox_destroying. The abandoned row (never held user data) now
detaches from the workspace; the cron finishes destroying it. Regression test forces both failures.
---
 cloud/apps/sandbox-control/src/lifecycle.ts  |  4 +++
 cloud/apps/sandbox-control/src/store.ts      |  8 +++++
 cloud/apps/sandbox-control/test/pool.test.ts | 33 ++++++++++++++++++++
 3 files changed, 45 insertions(+)

diff --git a/cloud/apps/sandbox-control/src/lifecycle.ts b/cloud/apps/sandbox-control/src/lifecycle.ts
index 7fa7463e73f..c35e0feb5ee 100644
--- a/cloud/apps/sandbox-control/src/lifecycle.ts
+++ b/cloud/apps/sandbox-control/src/lifecycle.ts
@@ -19,6 +19,7 @@ import type { Keyring } from './secrets-crypto'
 import { ORCAD_PORT, drainOrcad, runningOrcad, startOrcad, type OrcadReady } from './sandbox-vm-commands'
 import { claimPooledVm, deletePooledVm, maintainPool, type PoolConfig, type PooledVm } from './pool'
 import {
+  detachFromWorkspace,
   NO_SESSION,
   countActiveSandboxes,
   countActiveSandboxesForUser,
@@ -400,6 +401,9 @@ export async function createSandbox(
       // A pooled VM that will not take the session is discarded; the caller still gets a workspace.
       console.log(JSON.stringify({ event: 'pool_adopt_failed', error: String(error).slice(0, 200) }))
       await failAndRelease(deps, (await getSandbox(deps.db, row.id)) ?? row)
+      // Destroy can stop short (e.g. the provider's delete snapshot is still being written); the cron
+      // finishes it. That row never held user data, so it lets go of the workspace now.
+      await detachFromWorkspace(deps.db, row.id)
       return createSandbox(deps, caller, request, { pool: false })
     }
   }
diff --git a/cloud/apps/sandbox-control/src/store.ts b/cloud/apps/sandbox-control/src/store.ts
index a005736c857..2c41ccdcafa 100644
--- a/cloud/apps/sandbox-control/src/store.ts
+++ b/cloud/apps/sandbox-control/src/store.ts
@@ -98,6 +98,14 @@ export function getSandboxByProviderName(db: D1Database, name: string): Promise<
   return db.prepare('SELECT * FROM sandboxes WHERE provider_name = ?').bind(name).first<SandboxRow>()
 }
 
+/** A failed attempt that is still being cleaned up stops blocking its workspace (unique live index). */
+export async function detachFromWorkspace(db: D1Database, id: string): Promise<void> {
+  await db
+    .prepare(`UPDATE sandboxes SET workspace_id = workspace_id || '#abandoned:' || id WHERE id = ? AND status <> 'destroyed'`)
+    .bind(id)
+    .run()
+}
+
 export function liveSandboxForWorkspace(
   db: D1Database,
   accountId: string,
diff --git a/cloud/apps/sandbox-control/test/pool.test.ts b/cloud/apps/sandbox-control/test/pool.test.ts
index 871b33f2c90..2f0524ca2d6 100644
--- a/cloud/apps/sandbox-control/test/pool.test.ts
+++ b/cloud/apps/sandbox-control/test/pool.test.ts
@@ -75,6 +75,39 @@ describe('warm pool', () => {
     expect(h.vercel.boxes.get(`orca-test-${res.body.sandbox.id}`)!.params.tags.orcaPool).toBeUndefined()
   })
 
+  it('a failed adopt whose cleanup stops short still gives the caller a fresh VM (no sandbox_destroying conflict)', async () => {
+    const { h, deps, poolRows } = await pooledHarness('1')
+    await maintainPool(deps())
+    const pooledId = poolRows()[0]!.id as string
+    const pooledName = `orca-test-${pooledId}`
+    const realPolicy = h.vercel.updateNetworkPolicy.bind(h.vercel)
+    h.vercel.updateNetworkPolicy = async (sessionId, policy) => {
+      if (sessionId === h.vercel.boxes.get(pooledName)?.sessionId) {
+        throw new Error('policy update failed')
+      }
+      return realPolicy(sessionId, policy)
+    }
+    // The pooled VM's delete leaves a snapshot that will not delete yet: destroy stops at 'destroying'.
+    const realDeleteSnapshot = h.vercel.deleteSnapshot.bind(h.vercel)
+    h.vercel.deleteSnapshot = async (id) => {
+      if (h.vercel.snapshots.get(id)?.sandboxName === pooledName) {
+        throw new Error('snapshot still being written')
+      }
+      return realDeleteSnapshot(id)
+    }
+    const res = await h.call('POST', '/v1/sandboxes', { body: create })
+    expect(res.status).toBe(200)
+    expect(res.body.sandbox.id).not.toBe(pooledId)
+    const abandoned = h.raw.prepare('SELECT status, workspace_id FROM sandboxes WHERE id = ?').get(pooledId) as { status: string; workspace_id: string }
+    expect(abandoned.status).not.toBe('destroyed')
+    expect(abandoned.workspace_id).toBe(`ws-1#abandoned:${pooledId}`)
+    // The cron finishes the abandoned row once the provider lets go.
+    h.vercel.deleteSnapshot = realDeleteSnapshot
+    h.advance(30 * MIN)
+    await h.reconcile()
+    expect((h.raw.prepare('SELECT status FROM sandboxes WHERE id = ?').get(pooledId) as { status: string }).status).toBe('destroyed')
+  })
+
   it('skips the pool for sizes and regions it does not hold', async () => {
     const { h, deps, poolRows } = await pooledHarness('1')
     await maintainPool(deps())