fix(sandbox-control): no snapshot leaks on any delete path, and nothing deletes a snapshot still in use

- snapshot-refs.ts: one check for what live things still boot from (goldens of any live status, pool
  images, active environment builds, live workspaces' own snapshots, and a live workspace's boot image
  until it has a snapshot of its own).
- Retired goldens beyond the rollback and deleted environments' builds are marked deleted only after the
  provider delete succeeded (a failed delete used to be marked and leaked for good), and are deferred
  while in use. A new hourly housekeeping step (sweepReleasedSnapshots) retries both.
- Team delete marks the team's environments deleted, so their build snapshots are freed (they leaked).
- Environment prune skips a build a live workspace still boots from.
- Orphan sweeps (per box and project-wide automatic) also respect pool and boot-image references.
- The golden smoke box is non-persistent: it no longer leaves a ~325 MB automatic snapshot behind.
This commit is contained in:
Neil
2026-11-14 20:57:17 -08:00
parent ace88d146c
commit bde0bed3ce
8 changed files with 421 additions and 19 deletions
@@ -10,6 +10,8 @@ import type { SandboxCaller } from './auth'
import { listBuilds, startBuild, TERMINAL, type BuildConfig, type BuildDeps } from './environment-builds'
import { canonicalRepoUrl, parseRepos, repoDir, type EnvironmentRow } from './environments'
import { normalizeRepo } from './secrets'
import type { SandboxProviderApi } from './sandbox-provider'
import { snapshotsInUse } from './snapshot-refs'
export const AUTO_ENVIRONMENT_USER = 'system:auto-environment'
export const AUTO_ENVIRONMENT_UNUSED_MS = 14 * 24 * 60 * 60_000
@@ -129,7 +131,7 @@ export async function touchEnvironment(db: D1Database, environmentId: string, no
* Shared by an admin's Delete and the automatic expiry.
*/
export async function deleteEnvironmentAndSnapshots(
deps: Pick<BuildDeps, 'db' | 'provider' | 'now'>,
deps: Pick<BuildDeps, 'db' | 'now'> & { provider: Pick<SandboxProviderApi, 'deleteSnapshot'> },
environmentId: string
): Promise<void> {
await deps.db
@@ -140,10 +142,38 @@ export async function deleteEnvironmentAndSnapshots(
.prepare(`SELECT id, snapshot_id FROM environment_builds WHERE environment_id = ? AND snapshot_id IS NOT NULL AND snapshot_deleted_at IS NULL`)
.bind(environmentId)
.all<{ id: string; snapshot_id: string }>()
for (const row of results) {
await deps.provider.deleteSnapshot(row.snapshot_id).catch(() => undefined)
await releaseBuildSnapshots(deps, results)
}
/**
* Deletes build snapshots nothing live boots from any more (snapshot-refs.ts) and marks them deleted.
* A snapshot still in use, or whose delete failed, keeps snapshot_deleted_at NULL: the hourly
* housekeeping sweep (retention.ts) retries it, so a provider hiccup never leaks it for good.
*/
export async function releaseBuildSnapshots(
deps: Pick<BuildDeps, 'db' | 'now'> & { provider: Pick<SandboxProviderApi, 'deleteSnapshot'> },
builds: readonly { id: string; snapshot_id: string }[]
): Promise<{ deleted: number; kept: number }> {
// No golden id applies: an environment build never boots from a golden snapshot id of its own.
const inUse = await snapshotsInUse(deps.db, { goldenSnapshotId: '' }, builds.map((b) => b.snapshot_id))
let deleted = 0
let kept = 0
for (const row of builds) {
if (inUse.has(row.snapshot_id)) {
kept++
continue
}
try {
await deps.provider.deleteSnapshot(row.snapshot_id)
} catch (error) {
log('environment_snapshot_delete_failed', { buildId: row.id, error: String(error).slice(0, 120) })
kept++
continue
}
await deps.db.prepare('UPDATE environment_builds SET snapshot_deleted_at = ? WHERE id = ?').bind(deps.now(), row.id).run()
deleted++
}
return { deleted, kept }
}
/** Cron: automatic environments no create used for 14 days go, with their snapshots. */
@@ -25,6 +25,7 @@ import { compileNetworkPolicy, gitBasicAuthorization, type CredentialRoute, type
import { SWAPPED_CREDENTIALS } from './provider-env'
import { ProviderError, type SandboxProviderApi } from './sandbox-provider'
import { ORCAD_PORT, startOrcad } from './sandbox-vm-commands'
import { snapshotsInUse } from './snapshot-refs'
import type { Keyring } from './secrets-crypto'
import { SANDBOX_VCPUS } from './machine-size'
@@ -681,8 +682,12 @@ export async function pruneSnapshots(deps: Pick<BuildDeps, 'db' | 'provider' | '
.bind(environmentId)
.all<BuildRow>()
let pruned = 0
for (const old of results.slice(KEPT_BUILDS)) {
if (old.id === env?.active_build_id) {
const candidates = results.slice(KEPT_BUILDS).filter((old) => old.id !== env?.active_build_id)
// A workspace still booting from an old build (no snapshot of its own yet) keeps it; a later prune
// or the housekeeping sweep takes it once nothing needs it.
const inUse = await snapshotsInUse(deps.db, { goldenSnapshotId: '' }, candidates.map((old) => old.snapshot_id!))
for (const old of candidates) {
if (inUse.has(old.snapshot_id!)) {
continue
}
try {
@@ -16,6 +16,7 @@ import { parseGitSource, resolveSourceHead, type AppRequest, type GitSource, typ
import type { GoldenImageProviderApi } from './golden-image-provider'
import { compileNetworkPolicy } from './network-policy'
import type { SandboxProviderApi } from './sandbox-provider'
import { snapshotsInUse, type SnapshotRefConfig } from './snapshot-refs'
import { ORCAD_PORT, startOrcad } from './sandbox-vm-commands'
export type GoldenImageDeps = {
@@ -461,7 +462,9 @@ async function stepSmoke(deps: GoldenImageDeps, row: GoldenRow): Promise<GoldenT
ports: [ORCAD_PORT],
networkPolicy: compileNetworkPolicy({ mode: 'allowlist', allowlist: deps.config.egressAllowlist }),
tags: { orcaGoldenSmoke: row.id, orcaEnv: deps.config.environment },
snapshotExpirationMs: 86_400_000
snapshotExpirationMs: 0,
// Throwaway: a persistent box would leave an automatic snapshot (~325 MB) behind its delete.
persistent: false
})
await startOrcad(provider, box, { sandboxId: `smoke-${row.id}` })
} catch (error) {
@@ -479,17 +482,50 @@ async function stepSmoke(deps: GoldenImageDeps, row: GoldenRow): Promise<GoldenT
db.prepare(`UPDATE golden_images SET status = 'retired', updated_at = ? WHERE status = 'active' AND region = ?`).bind(now, row.region),
db.prepare(`UPDATE golden_images SET status = 'active', activated_at = ?, updated_at = ? WHERE id = ?`).bind(now, now, row.id)
])
// Keep one retired image per region as the rollback; older ones are deleted.
for (const old of (await rows(db, row.region, ['retired'])).slice(1)) {
if (old.snapshot_id) {
await provider.deleteSnapshot(old.snapshot_id).catch(() => undefined)
}
await update(db, old.id, { status: 'deleted' }, now)
}
// The wrangler seed image may still be what the config hands out elsewhere: never deleted here.
await releaseRetiredGoldens({ ...deps, config: { goldenSnapshotId: deps.config.seedSnapshotId ?? '' } }, row.region)
console.log(JSON.stringify({ event: 'golden_image_activated', id: row.id, region: row.region, source: row.source, snapshotId: row.snapshot_id }))
return 'activated'
}
/**
* Keeps one retired image per region as the rollback; older ones are deleted once nothing live boots
* from them (snapshot-refs.ts). A row is marked deleted only after the provider delete succeeded, so a
* failed or deferred delete is retried by the next activation or the hourly housekeeping sweep.
*/
export async function releaseRetiredGoldens(
deps: { db: D1Database; provider: Pick<SandboxProviderApi, 'deleteSnapshot'>; now: () => number; config: SnapshotRefConfig },
region: string
): Promise<{ deleted: number; kept: number }> {
const stale = (await rows(deps.db, region, ['retired'])).slice(1)
const inUse = await snapshotsInUse(
deps.db,
deps.config,
stale.flatMap((old) => (old.snapshot_id ? [old.snapshot_id] : [])),
{ ignoreGoldenRows: stale.map((old) => old.id) }
)
let deleted = 0
let kept = 0
for (const old of stale) {
if (old.snapshot_id && inUse.has(old.snapshot_id)) {
kept++
continue
}
try {
if (old.snapshot_id) {
await deps.provider.deleteSnapshot(old.snapshot_id)
}
} catch (error) {
console.log(JSON.stringify({ event: 'golden_image_delete_failed', id: old.id, error: String(error).slice(0, 120) }))
kept++
continue
}
await update(deps.db, old.id, { status: 'deleted' }, deps.now())
deleted++
}
return { deleted, kept }
}
async function fail(deps: GoldenImageDeps, id: string, buildName: string | null, error: string): Promise<void> {
if (buildName) {
await deps.provider.deleteBuildSandbox(buildName).catch(() => undefined)
+1 -1
View File
@@ -1392,7 +1392,7 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise<Res
if (teamDelete.action === 'delete-check') {
return json({ workspaces: await teamWorkspaceCount(env.SANDBOX_DB, teamDelete.accountId) })
}
const purged = await purgeTeamSecrets(env.SANDBOX_DB, teamDelete.accountId)
const purged = await purgeTeamSecrets(env.SANDBOX_DB, teamDelete.accountId, (deps.now ?? Date.now)())
return purged.ok
? json({ purged: purged.purged })
: json({ error: 'workspaces_exist', workspaces: purged.workspaces }, 409)
+49 -2
View File
@@ -5,6 +5,9 @@ import { RETENTION_HOLD_REASONS } from './entitlements'
import { destroySandbox, goldenSnapshotIds, suspendSandbox, type LifecycleDeps } from './lifecycle'
import { getControlState, setControlState } from './guardrails'
import { ACTIVE_STATUSES, getSandboxByProviderName, listByStatus, type SandboxRow } from './store'
import { releaseBuildSnapshots } from './auto-environments'
import { releaseRetiredGoldens } from './golden-images'
import { snapshotsInUse } from './snapshot-refs'
const HOUR_MS = 60 * 60_000
const DAY_MS = 24 * HOUR_MS
@@ -55,6 +58,10 @@ export type HousekeepingSummary = {
orphanSandboxesDeleted: number
orphanSnapshotsSeen: number
orphanSnapshotsDeleted: number
/** Retired golden images and deleted environments' build snapshots freed this sweep. */
releasedSnapshotsDeleted: number
/** Ones still in use (or whose delete failed): retried next sweep. */
releasedSnapshotsKept: number
rowsMissingProvider: number
holdsOpened: number
holdsCleared: number
@@ -271,8 +278,10 @@ async function sweepOrphans(deps: LifecycleDeps, summary: HousekeepingSummary):
}
if (now - orphan.first_seen_at >= grace) {
await deps.provider.deleteSandbox(box.name)
for (const snapshot of await deps.provider.listSnapshots(box.name)) {
if (snapshot.status === 'created' && !goldenSnapshotIds(deps.config).includes(snapshot.id) && !(await isEnvironmentBuildSnapshot(deps.db, snapshot.id))) {
const left = (await deps.provider.listSnapshots(box.name)).filter((snapshot) => snapshot.status === 'created')
const inUse = await snapshotsInUse(deps.db, deps.config, left.map((snapshot) => snapshot.id))
for (const snapshot of left) {
if (!inUse.has(snapshot.id) && !(await isEnvironmentBuildSnapshot(deps.db, snapshot.id))) {
await deps.provider.deleteSnapshot(snapshot.id)
}
}
@@ -328,6 +337,11 @@ async function sweepSnapshots(deps: LifecycleDeps, sandboxNames: string[], summa
for (const r of results) {
owned.add(r.snapshot_id)
}
// Belt and braces for the D1-side references (pool images, live boot images, goldens of any status).
const automatic = listed.snapshots.filter((s) => s.status === 'created' && s.creationMethod === 'automatic' && !owned.has(s.id))
for (const id of await snapshotsInUse(deps.db, deps.config, automatic.map((s) => s.id))) {
owned.add(id)
}
for (const snapshot of listed.snapshots) {
if (snapshot.status !== 'created' || snapshot.creationMethod !== 'automatic' || owned.has(snapshot.id)) {
if (owned.has(snapshot.id)) {
@@ -346,6 +360,36 @@ async function sweepSnapshots(deps: LifecycleDeps, sandboxNames: string[], summa
}
}
// ---- released images ----
/**
* Snapshots Orca stopped needing but could not delete at the time: retired golden images beyond the
* one-per-region rollback, and the builds of deleted environments (an admin's delete, auto-environment
* expiry, a deleted team). Each is deleted only once nothing live boots from it (snapshot-refs.ts).
*/
async function sweepReleasedSnapshots(deps: LifecycleDeps, summary: HousekeepingSummary): Promise<void> {
const { results: builds } = await deps.db
.prepare(
`SELECT b.id, b.snapshot_id FROM environment_builds b JOIN environments e ON e.id = b.environment_id
WHERE e.deleted_at IS NOT NULL AND b.snapshot_id IS NOT NULL AND b.snapshot_deleted_at IS NULL LIMIT 50`
)
.all<{ id: string; snapshot_id: string }>()
const tallies = [await releaseBuildSnapshots(deps, builds)]
const { results: regions } = await deps.db
.prepare(`SELECT region FROM golden_images WHERE status = 'retired' GROUP BY region HAVING COUNT(*) > 1`)
.all<{ region: string }>()
for (const { region } of regions) {
tallies.push(await releaseRetiredGoldens(deps, region))
}
for (const tally of tallies) {
summary.releasedSnapshotsDeleted += tally.deleted
summary.releasedSnapshotsKept += tally.kept
}
if (summary.releasedSnapshotsDeleted + summary.releasedSnapshotsKept > 0) {
log('released_snapshots_swept', { deleted: summary.releasedSnapshotsDeleted, kept: summary.releasedSnapshotsKept })
}
}
// ---- entry points ----
/** Kill switch drain: snapshot-preserving suspend of every running sandbox (one batch per call). */
@@ -376,6 +420,8 @@ export async function housekeeping(
orphanSandboxesDeleted: 0,
orphanSnapshotsSeen: 0,
orphanSnapshotsDeleted: 0,
releasedSnapshotsDeleted: 0,
releasedSnapshotsKept: 0,
rowsMissingProvider: 0,
holdsOpened: 0,
holdsCleared: 0,
@@ -394,6 +440,7 @@ export async function housekeeping(
const notifier = options.notifier ?? LOG_NOTIFIER
for (const step of [
() => sweepOrphans(deps, summary),
() => sweepReleasedSnapshots(deps, summary),
() => probeHolds(deps, summary),
() => enforceHolds(deps, notifier, summary),
() => enforceIdle(deps, notifier, summary)
@@ -0,0 +1,75 @@
// Which provider snapshots something live may still boot from. Every path that deletes a snapshot it
// does not exclusively own (a retired golden image, an environment build, a sweep) asks here first,
// so a delete never pulls an image out from under a workspace, a warm-pool VM or a golden image.
//
// A snapshot is in use when it is:
// - a golden image that is not failed/deleted (active, smoking, building, or kept as a rollback),
// or one of the config's golden ids (GOLDEN_SNAPSHOT_ID, a preview's override);
// - the boot image of a warm-pool VM;
// - the active build of an environment that still exists;
// - one of a live workspace's own recorded snapshots;
// - the boot image (source_snapshot_id) of a live workspace that has no snapshot of its own yet.
// Once a persistent sandbox has stopped once it resumes from its own snapshot, so its source
// image no longer pins it (environment builds would otherwise be held for every workspace's life).
// "Live" = any workspace row not yet destroyed (creating, failed and destroying rows included).
export type SnapshotRefConfig = { goldenSnapshotId: string; goldenSnapshots?: Record<string, string> }
const CHUNK = 50
/** The subset of `ids` something live still references. */
export async function snapshotsInUse(
db: D1Database,
config: SnapshotRefConfig,
ids: readonly string[],
options: { ignoreGoldenRows?: readonly string[] } = {}
): Promise<Set<string>> {
const used = new Set<string>()
const configGoldens = new Set([config.goldenSnapshotId, ...Object.values(config.goldenSnapshots ?? {})].filter(Boolean))
const unique = [...new Set(ids)]
for (const id of unique) {
if (configGoldens.has(id)) {
used.add(id)
}
}
const ignored = options.ignoreGoldenRows ?? []
for (let i = 0; i < unique.length; i += CHUNK) {
const chunk = unique.slice(i, i + CHUNK)
const marks = chunk.map(() => '?').join(', ')
const ignoreMarks = ignored.map(() => '?').join(', ')
const queries = [
db
.prepare(
`SELECT snapshot_id AS id FROM golden_images WHERE snapshot_id IN (${marks})
AND status NOT IN ('failed', 'deleted')${ignored.length > 0 ? ` AND id NOT IN (${ignoreMarks})` : ''}`
)
.bind(...chunk, ...ignored),
db.prepare(`SELECT snapshot_id AS id FROM sandbox_pool WHERE snapshot_id IN (${marks})`).bind(...chunk),
db
.prepare(
`SELECT b.snapshot_id AS id FROM environments e JOIN environment_builds b ON b.id = e.active_build_id
WHERE e.deleted_at IS NULL AND b.snapshot_id IN (${marks})`
)
.bind(...chunk),
db
.prepare(
`SELECT ss.snapshot_id AS id FROM sandbox_snapshots ss JOIN sandboxes s ON s.id = ss.sandbox_id
WHERE ss.deleted_at IS NULL AND s.status <> 'destroyed' AND ss.snapshot_id IN (${marks})`
)
.bind(...chunk),
db
.prepare(
`SELECT s.source_snapshot_id AS id FROM sandboxes s
WHERE s.status <> 'destroyed' AND s.source_snapshot_id IN (${marks})
AND NOT EXISTS (SELECT 1 FROM sandbox_snapshots ss WHERE ss.sandbox_id = s.id AND ss.deleted_at IS NULL)`
)
.bind(...chunk)
]
for (const query of queries) {
for (const row of (await query.all<{ id: string }>()).results) {
used.add(row.id)
}
}
}
return used
}
+10 -3
View File
@@ -7,7 +7,8 @@
// A team can only be deleted once it has no cloud workspaces in any state (running, asleep or
// archived): the user deletes them first through the normal workspace delete, which destroys the
// sandbox and its snapshots. Then auth purges the team's cloud secrets (org-scoped and the team's
// repo-scoped ones; members' personal secrets are theirs) and deletes the team.
// repo-scoped ones; members' personal secrets are theirs) and deletes the team. The purge also marks
// the team's environments deleted, so housekeeping deletes their build snapshots.
const ACCOUNT_PATH = /^\/v1\/internal\/accounts\/((?:org|dev:org):[A-Za-z0-9_.:-]{1,150})\/(delete-check|secrets\/purge)$/
@@ -41,7 +42,8 @@ export async function teamWorkspaceCount(db: D1Database, accountId: string): Pro
*/
export async function purgeTeamSecrets(
db: D1Database,
accountId: string
accountId: string,
now: number = Date.now()
): Promise<{ ok: true; purged: number } | { ok: false; workspaces: number }> {
const workspaces = await teamWorkspaceCount(db, accountId)
if (workspaces > 0) {
@@ -58,7 +60,12 @@ export async function purgeTeamSecrets(
// The team's secret activity (names, who, when) goes with the team.
db.prepare('DELETE FROM secret_events WHERE account_id = ?').bind(accountId),
// So do its owner-change notices (workspace names, who): nobody can sign in to the team to read them.
db.prepare('DELETE FROM owner_transfer_notices WHERE account_id = ?').bind(accountId)
db.prepare('DELETE FROM owner_transfer_notices WHERE account_id = ?').bind(accountId),
// And its environments: their build snapshots (manual, no provider expiry) are freed by the hourly
// housekeeping sweep (retention.ts sweepReleasedSnapshots), which this route has no provider for.
db
.prepare('UPDATE environments SET deleted_at = ?, active_build_id = NULL WHERE account_id = ? AND deleted_at IS NULL')
.bind(now, accountId)
])
return { ok: true, purged: Number(result?.meta?.changes ?? 0) }
}
@@ -0,0 +1,202 @@
// Every delete path frees its snapshots, and nothing ever deletes a snapshot something live still
// boots from (snapshot-refs.ts): retired golden images, deleted environments' builds, team delete,
// the orphan sweeps, a provider delete that failed once.
import { describe, expect, it } from 'vitest'
import { DEFAULT_RETENTION, housekeeping } from '../src/retention'
import type { LifecycleDeps } from '../src/lifecycle'
import { snapshotsInUse } from '../src/snapshot-refs'
import { purgeTeamSecrets } from '../src/team-delete'
import { ProviderError } from '../src/sandbox-provider'
import { GOLDEN, createHarness } from './harness'
const HOUR = 60 * 60_000
const create = (workspaceId: string) => ({ body: { workspaceId, machineSize: 'standard' } })
async function setup() {
const h = await createHarness()
const deps: LifecycleDeps = {
db: h.env.SANDBOX_DB,
provider: h.vercel,
entitlements: {
checkStart: async () => h.billing.decision,
reportUsage: async () => h.billing.usageAnswer,
listSuspensions: async () => h.billing.suspensions
},
now: h.now,
retention: DEFAULT_RETENTION,
config: { environment: 'test', goldenSnapshotId: GOLDEN, defaultRegion: 'iad1' }
}
const sweep = () => housekeeping(deps, { force: true, notifier: { warn: async () => undefined } })
const snap = (id: string, creationMethod = 'manual', sandboxName = 'build') =>
h.vercel.snapshots.set(id, { id, status: 'created', sourceSessionId: 's', sandboxName, creationMethod, createdAt: h.now() })
const status = (id: string) => h.vercel.snapshots.get(id)?.status
return { h, deps, sweep, snap, status }
}
function golden(h: Awaited<ReturnType<typeof createHarness>>, id: string, status: string, snapshotId: string, createdAt: number) {
h.raw
.prepare(`INSERT INTO golden_images (id, source, status, snapshot_id, created_at, updated_at, region) VALUES (?, 'seed', ?, ?, ?, ?, 'iad1')`)
.run(id, status, snapshotId, createdAt, createdAt)
}
function environment(h: Awaited<ReturnType<typeof createHarness>>, id: string, accountId: string, builds: { id: string; snapshotId: string }[]) {
h.raw
.prepare(
`INSERT INTO environments (id, account_id, name, repos, install_script, env_var_names, machine_size, created_by, updated_by, created_at, updated_at)
VALUES (?, ?, ?, '[]', '', '[]', 'standard', 'u', 'u', 1, 1)`
)
.run(id, accountId, id)
builds.forEach((build, i) =>
h.raw
.prepare(
`INSERT INTO environment_builds (id, environment_id, account_id, number, status, config, base_snapshot_id, provider_name, snapshot_id, created_by, created_at, updated_at)
VALUES (?, ?, ?, ?, 'succeeded', '{}', ?, ?, ?, 'u', 1, 1)`
)
.run(build.id, id, accountId, i + 1, GOLDEN, `orca-test-envbuild-${build.id}`, build.snapshotId)
)
h.raw.prepare('UPDATE environments SET active_build_id = ? WHERE id = ?').run(builds.at(-1)?.id ?? null, id)
}
describe('workspace delete', () => {
it('deletes every automatic snapshot the workspace made, never the golden it booted from', async () => {
const { h, status } = await setup()
const id = (await h.call('POST', '/v1/sandboxes', create('ws-1'))).body.sandbox.id
await h.call('POST', `/v1/sandboxes/${id}/suspend`)
await h.call('POST', `/v1/sandboxes/${id}/resume`)
await h.call('POST', `/v1/sandboxes/${id}/suspend`)
const own = [...h.vercel.snapshots.values()].filter((s) => s.sandboxName === `orca-test-${id}`)
expect(own.length).toBe(2)
expect((await h.call('DELETE', `/v1/sandboxes/${id}`)).status).toBe(200)
expect(own.map((s) => status(s.id))).toEqual(['deleted', 'deleted'])
expect(h.vercel.boxes.size).toBe(0)
})
})
describe('snapshotsInUse', () => {
it('pins goldens, pool images, active environment builds, live workspaces’ own snapshots and un-stopped boot images', async () => {
const { h, deps } = await setup()
golden(h, 'g_active', 'active', 'snap_g_active', 3)
golden(h, 'g_failed', 'failed', 'snap_g_failed', 2)
environment(h, 'env_a', 'org:acme', [{ id: 'b1', snapshotId: 'snap_b1' }, { id: 'b2', snapshotId: 'snap_b2' }])
h.raw
.prepare(`INSERT INTO sandbox_pool (id, provider_name, machine_size, region, snapshot_id, status, created_at, updated_at) VALUES ('p1', 'orca-test-p1', 'standard', 'iad1', 'snap_pool', 'ready', 1, 1)`)
.run()
const id = (await h.call('POST', '/v1/sandboxes', create('ws-1'))).body.sandbox.id
h.raw.prepare('UPDATE sandboxes SET source_snapshot_id = ? WHERE id = ?').run('snap_boot', id)
const ids = [GOLDEN, 'snap_g_active', 'snap_g_failed', 'snap_b1', 'snap_b2', 'snap_pool', 'snap_boot', 'snap_free']
expect([...(await snapshotsInUse(deps.db, deps.config, ids))].sort()).toEqual(
[GOLDEN, 'snap_b2', 'snap_boot', 'snap_g_active', 'snap_pool'].sort()
)
// Once the workspace stopped once it resumes from its own snapshot: the boot image is free, its own one is pinned.
await h.call('POST', `/v1/sandboxes/${id}/suspend`)
const own = [...h.vercel.snapshots.values()].find((s) => s.sandboxName === `orca-test-${id}`)!.id
const after = await snapshotsInUse(deps.db, deps.config, ['snap_boot', own])
expect(after.has('snap_boot')).toBe(false)
expect(after.has(own)).toBe(true)
// A destroyed workspace pins nothing.
await h.call('DELETE', `/v1/sandboxes/${id}`)
expect((await snapshotsInUse(deps.db, deps.config, [own])).size).toBe(0)
})
})
describe('golden rotation', () => {
it('keeps one retired rollback, defers one a live workspace still boots from, retries a failed delete', async () => {
const { h, sweep, snap, status } = await setup()
golden(h, 'g4', 'active', 'snap_g4', 4)
golden(h, 'g3', 'retired', 'snap_g3', 3)
golden(h, 'g2', 'retired', 'snap_g2', 2)
golden(h, 'g1', 'retired', 'snap_g1', 1)
for (const id of ['snap_g4', 'snap_g3', 'snap_g2', 'snap_g1']) {
snap(id)
}
const ws = (await h.call('POST', '/v1/sandboxes', create('ws-1'))).body.sandbox.id
h.raw.prepare('UPDATE sandboxes SET source_snapshot_id = ? WHERE id = ?').run('snap_g2', ws)
const deleteSnapshot = h.vercel.deleteSnapshot
let failures = 1
h.vercel.deleteSnapshot = async (id) => {
if (id === 'snap_g1' && failures-- > 0) {
throw new ProviderError(503, null)
}
return deleteSnapshot(id)
}
const goldenStatus = (id: string) => (h.raw.prepare('SELECT status FROM golden_images WHERE id = ?').get(id) as { status: string }).status
expect((await sweep()).releasedSnapshotsKept).toBe(2)
expect([status('snap_g1'), goldenStatus('g1')]).toEqual(['created', 'retired'])
expect([status('snap_g2'), goldenStatus('g2')]).toEqual(['created', 'retired'])
const second = await sweep()
expect([second.releasedSnapshotsDeleted, second.releasedSnapshotsKept]).toEqual([1, 1])
expect([status('snap_g1'), goldenStatus('g1')]).toEqual(['deleted', 'deleted'])
expect(status('snap_g2')).toBe('created')
// The workspace stops once (now it has its own snapshot): its boot image is released.
await h.call('POST', `/v1/sandboxes/${ws}/suspend`)
expect((await sweep()).releasedSnapshotsDeleted).toBe(1)
expect([status('snap_g2'), goldenStatus('g2')]).toEqual(['deleted', 'deleted'])
// The newest retired (rollback) and the active image are never touched.
expect([status('snap_g3'), status('snap_g4')]).toEqual(['created', 'created'])
expect((await h.call('POST', `/v1/sandboxes/${ws}/resume`)).status).toBe(200)
})
})
describe('deleted environments and teams', () => {
it('a team delete marks its environments deleted; housekeeping frees their build snapshots unless a live workspace boots from one', async () => {
const { h, sweep, snap, status } = await setup()
environment(h, 'env_team', 'org:gone', [{ id: 'tb1', snapshotId: 'snap_tb1' }, { id: 'tb2', snapshotId: 'snap_tb2' }])
environment(h, 'env_other', 'org:stays', [{ id: 'ob1', snapshotId: 'snap_ob1' }])
for (const id of ['snap_tb1', 'snap_tb2', 'snap_ob1']) {
snap(id)
}
// Another account's live workspace was created from tb1 (it never stopped yet).
const ws = (await h.call('POST', '/v1/sandboxes', create('ws-1'))).body.sandbox.id
h.raw.prepare('UPDATE sandboxes SET source_snapshot_id = ? WHERE id = ?').run('snap_tb1', ws)
expect(await purgeTeamSecrets(h.env.SANDBOX_DB, 'org:gone', h.now())).toEqual({ ok: true, purged: 0 })
expect(h.raw.prepare('SELECT deleted_at IS NOT NULL AS d, active_build_id FROM environments WHERE id = ?').get('env_team')).toEqual({ d: 1, active_build_id: null })
const first = await sweep()
expect([first.releasedSnapshotsDeleted, first.releasedSnapshotsKept]).toEqual([1, 1])
expect([status('snap_tb1'), status('snap_tb2'), status('snap_ob1')]).toEqual(['created', 'deleted', 'created'])
await h.call('DELETE', `/v1/sandboxes/${ws}`)
expect((await sweep()).releasedSnapshotsDeleted).toBe(1)
expect(status('snap_tb1')).toBe('deleted')
const marked = h.raw.prepare(`SELECT id FROM environment_builds WHERE snapshot_deleted_at IS NOT NULL ORDER BY id`).all()
expect(marked).toEqual([{ id: 'tb1' }, { id: 'tb2' }])
})
it('an admin delete whose provider call fails keeps the build for the next sweep instead of marking it gone', async () => {
const { h, deps, sweep, snap, status } = await setup()
environment(h, 'env_a', 'org:acme', [{ id: 'b1', snapshotId: 'snap_b1' }])
snap('snap_b1')
const deleteSnapshot = h.vercel.deleteSnapshot
h.vercel.deleteSnapshot = async () => {
throw new ProviderError(500, null)
}
const { deleteEnvironmentAndSnapshots } = await import('../src/auto-environments')
await deleteEnvironmentAndSnapshots(deps, 'env_a')
expect(h.raw.prepare('SELECT snapshot_deleted_at FROM environment_builds WHERE id = ?').get('b1')).toEqual({ snapshot_deleted_at: null })
h.vercel.deleteSnapshot = deleteSnapshot
expect((await sweep()).releasedSnapshotsDeleted).toBe(1)
expect(status('snap_b1')).toBe('deleted')
})
})
describe('orphan snapshot sweep', () => {
it('never deletes an automatic snapshot a warm-pool VM or a live workspace boots from', async () => {
const { h, sweep, snap, status } = await setup()
snap('snap_pool_image', 'automatic', 'orca-test-long-gone-a')
snap('snap_boot_image', 'automatic', 'orca-test-long-gone-b')
snap('snap_free', 'automatic', 'orca-test-long-gone-c')
h.raw
.prepare(`INSERT INTO sandbox_pool (id, provider_name, machine_size, region, snapshot_id, status, created_at, updated_at) VALUES ('p1', 'orca-test-p1', 'standard', 'iad1', 'snap_pool_image', 'warming', 1, 1)`)
.run()
const ws = (await h.call('POST', '/v1/sandboxes', create('ws-1'))).body.sandbox.id
h.raw.prepare('UPDATE sandboxes SET source_snapshot_id = ? WHERE id = ?').run('snap_boot_image', ws)
await sweep()
h.advance(25 * HOUR)
expect((await sweep()).orphanSnapshotsDeleted).toBe(1)
expect([status('snap_pool_image'), status('snap_boot_image'), status('snap_free')]).toEqual(['created', 'created', 'deleted'])
})
})