diff --git a/cloud/apps/sandbox-control/src/auto-environments.ts b/cloud/apps/sandbox-control/src/auto-environments.ts index e13fa604ff4..d841c35a26c 100644 --- a/cloud/apps/sandbox-control/src/auto-environments.ts +++ b/cloud/apps/sandbox-control/src/auto-environments.ts @@ -10,6 +10,8 @@ import type { SandboxCaller } from './auth' import { listBuilds, startBuild, TERMINAL, type BuildConfig, type BuildDeps } from './environment-builds' import { canonicalRepoUrl, parseRepos, repoDir, type EnvironmentRow } from './environments' import { normalizeRepo } from './secrets' +import type { SandboxProviderApi } from './sandbox-provider' +import { snapshotsInUse } from './snapshot-refs' export const AUTO_ENVIRONMENT_USER = 'system:auto-environment' export const AUTO_ENVIRONMENT_UNUSED_MS = 14 * 24 * 60 * 60_000 @@ -129,7 +131,7 @@ export async function touchEnvironment(db: D1Database, environmentId: string, no * Shared by an admin's Delete and the automatic expiry. */ export async function deleteEnvironmentAndSnapshots( - deps: Pick, + deps: Pick & { provider: Pick }, environmentId: string ): Promise { await deps.db @@ -140,10 +142,38 @@ export async function deleteEnvironmentAndSnapshots( .prepare(`SELECT id, snapshot_id FROM environment_builds WHERE environment_id = ? AND snapshot_id IS NOT NULL AND snapshot_deleted_at IS NULL`) .bind(environmentId) .all<{ id: string; snapshot_id: string }>() - for (const row of results) { - await deps.provider.deleteSnapshot(row.snapshot_id).catch(() => undefined) + await releaseBuildSnapshots(deps, results) +} + +/** + * Deletes build snapshots nothing live boots from any more (snapshot-refs.ts) and marks them deleted. + * A snapshot still in use, or whose delete failed, keeps snapshot_deleted_at NULL: the hourly + * housekeeping sweep (retention.ts) retries it, so a provider hiccup never leaks it for good. + */ +export async function releaseBuildSnapshots( + deps: Pick & { provider: Pick }, + builds: readonly { id: string; snapshot_id: string }[] +): Promise<{ deleted: number; kept: number }> { + // No golden id applies: an environment build never boots from a golden snapshot id of its own. + const inUse = await snapshotsInUse(deps.db, { goldenSnapshotId: '' }, builds.map((b) => b.snapshot_id)) + let deleted = 0 + let kept = 0 + for (const row of builds) { + if (inUse.has(row.snapshot_id)) { + kept++ + continue + } + try { + await deps.provider.deleteSnapshot(row.snapshot_id) + } catch (error) { + log('environment_snapshot_delete_failed', { buildId: row.id, error: String(error).slice(0, 120) }) + kept++ + continue + } await deps.db.prepare('UPDATE environment_builds SET snapshot_deleted_at = ? WHERE id = ?').bind(deps.now(), row.id).run() + deleted++ } + return { deleted, kept } } /** Cron: automatic environments no create used for 14 days go, with their snapshots. */ diff --git a/cloud/apps/sandbox-control/src/environment-builds.ts b/cloud/apps/sandbox-control/src/environment-builds.ts index fd6102cbd6a..482a51f7e8f 100644 --- a/cloud/apps/sandbox-control/src/environment-builds.ts +++ b/cloud/apps/sandbox-control/src/environment-builds.ts @@ -25,6 +25,7 @@ import { compileNetworkPolicy, gitBasicAuthorization, type CredentialRoute, type import { SWAPPED_CREDENTIALS } from './provider-env' import { ProviderError, type SandboxProviderApi } from './sandbox-provider' import { ORCAD_PORT, startOrcad } from './sandbox-vm-commands' +import { snapshotsInUse } from './snapshot-refs' import type { Keyring } from './secrets-crypto' import { SANDBOX_VCPUS } from './machine-size' @@ -681,8 +682,12 @@ export async function pruneSnapshots(deps: Pick() let pruned = 0 - for (const old of results.slice(KEPT_BUILDS)) { - if (old.id === env?.active_build_id) { + const candidates = results.slice(KEPT_BUILDS).filter((old) => old.id !== env?.active_build_id) + // A workspace still booting from an old build (no snapshot of its own yet) keeps it; a later prune + // or the housekeeping sweep takes it once nothing needs it. + const inUse = await snapshotsInUse(deps.db, { goldenSnapshotId: '' }, candidates.map((old) => old.snapshot_id!)) + for (const old of candidates) { + if (inUse.has(old.snapshot_id!)) { continue } try { diff --git a/cloud/apps/sandbox-control/src/golden-images.ts b/cloud/apps/sandbox-control/src/golden-images.ts index ddf9fe48689..035caec020a 100644 --- a/cloud/apps/sandbox-control/src/golden-images.ts +++ b/cloud/apps/sandbox-control/src/golden-images.ts @@ -16,6 +16,7 @@ import { parseGitSource, resolveSourceHead, type AppRequest, type GitSource, typ import type { GoldenImageProviderApi } from './golden-image-provider' import { compileNetworkPolicy } from './network-policy' import type { SandboxProviderApi } from './sandbox-provider' +import { snapshotsInUse, type SnapshotRefConfig } from './snapshot-refs' import { ORCAD_PORT, startOrcad } from './sandbox-vm-commands' export type GoldenImageDeps = { @@ -461,7 +462,9 @@ async function stepSmoke(deps: GoldenImageDeps, row: GoldenRow): Promise undefined) - } - await update(db, old.id, { status: 'deleted' }, now) - } + // The wrangler seed image may still be what the config hands out elsewhere: never deleted here. + await releaseRetiredGoldens({ ...deps, config: { goldenSnapshotId: deps.config.seedSnapshotId ?? '' } }, row.region) console.log(JSON.stringify({ event: 'golden_image_activated', id: row.id, region: row.region, source: row.source, snapshotId: row.snapshot_id })) return 'activated' } +/** + * Keeps one retired image per region as the rollback; older ones are deleted once nothing live boots + * from them (snapshot-refs.ts). A row is marked deleted only after the provider delete succeeded, so a + * failed or deferred delete is retried by the next activation or the hourly housekeeping sweep. + */ +export async function releaseRetiredGoldens( + deps: { db: D1Database; provider: Pick; now: () => number; config: SnapshotRefConfig }, + region: string +): Promise<{ deleted: number; kept: number }> { + const stale = (await rows(deps.db, region, ['retired'])).slice(1) + const inUse = await snapshotsInUse( + deps.db, + deps.config, + stale.flatMap((old) => (old.snapshot_id ? [old.snapshot_id] : [])), + { ignoreGoldenRows: stale.map((old) => old.id) } + ) + let deleted = 0 + let kept = 0 + for (const old of stale) { + if (old.snapshot_id && inUse.has(old.snapshot_id)) { + kept++ + continue + } + try { + if (old.snapshot_id) { + await deps.provider.deleteSnapshot(old.snapshot_id) + } + } catch (error) { + console.log(JSON.stringify({ event: 'golden_image_delete_failed', id: old.id, error: String(error).slice(0, 120) })) + kept++ + continue + } + await update(deps.db, old.id, { status: 'deleted' }, deps.now()) + deleted++ + } + return { deleted, kept } +} + async function fail(deps: GoldenImageDeps, id: string, buildName: string | null, error: string): Promise { if (buildName) { await deps.provider.deleteBuildSandbox(buildName).catch(() => undefined) diff --git a/cloud/apps/sandbox-control/src/index.ts b/cloud/apps/sandbox-control/src/index.ts index 1e9c1e5ac8c..d59ad114e81 100644 --- a/cloud/apps/sandbox-control/src/index.ts +++ b/cloud/apps/sandbox-control/src/index.ts @@ -1392,7 +1392,7 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise= grace) { await deps.provider.deleteSandbox(box.name) - for (const snapshot of await deps.provider.listSnapshots(box.name)) { - if (snapshot.status === 'created' && !goldenSnapshotIds(deps.config).includes(snapshot.id) && !(await isEnvironmentBuildSnapshot(deps.db, snapshot.id))) { + const left = (await deps.provider.listSnapshots(box.name)).filter((snapshot) => snapshot.status === 'created') + const inUse = await snapshotsInUse(deps.db, deps.config, left.map((snapshot) => snapshot.id)) + for (const snapshot of left) { + if (!inUse.has(snapshot.id) && !(await isEnvironmentBuildSnapshot(deps.db, snapshot.id))) { await deps.provider.deleteSnapshot(snapshot.id) } } @@ -328,6 +337,11 @@ async function sweepSnapshots(deps: LifecycleDeps, sandboxNames: string[], summa for (const r of results) { owned.add(r.snapshot_id) } + // Belt and braces for the D1-side references (pool images, live boot images, goldens of any status). + const automatic = listed.snapshots.filter((s) => s.status === 'created' && s.creationMethod === 'automatic' && !owned.has(s.id)) + for (const id of await snapshotsInUse(deps.db, deps.config, automatic.map((s) => s.id))) { + owned.add(id) + } for (const snapshot of listed.snapshots) { if (snapshot.status !== 'created' || snapshot.creationMethod !== 'automatic' || owned.has(snapshot.id)) { if (owned.has(snapshot.id)) { @@ -346,6 +360,36 @@ async function sweepSnapshots(deps: LifecycleDeps, sandboxNames: string[], summa } } +// ---- released images ---- + +/** + * Snapshots Orca stopped needing but could not delete at the time: retired golden images beyond the + * one-per-region rollback, and the builds of deleted environments (an admin's delete, auto-environment + * expiry, a deleted team). Each is deleted only once nothing live boots from it (snapshot-refs.ts). + */ +async function sweepReleasedSnapshots(deps: LifecycleDeps, summary: HousekeepingSummary): Promise { + const { results: builds } = await deps.db + .prepare( + `SELECT b.id, b.snapshot_id FROM environment_builds b JOIN environments e ON e.id = b.environment_id + WHERE e.deleted_at IS NOT NULL AND b.snapshot_id IS NOT NULL AND b.snapshot_deleted_at IS NULL LIMIT 50` + ) + .all<{ id: string; snapshot_id: string }>() + const tallies = [await releaseBuildSnapshots(deps, builds)] + const { results: regions } = await deps.db + .prepare(`SELECT region FROM golden_images WHERE status = 'retired' GROUP BY region HAVING COUNT(*) > 1`) + .all<{ region: string }>() + for (const { region } of regions) { + tallies.push(await releaseRetiredGoldens(deps, region)) + } + for (const tally of tallies) { + summary.releasedSnapshotsDeleted += tally.deleted + summary.releasedSnapshotsKept += tally.kept + } + if (summary.releasedSnapshotsDeleted + summary.releasedSnapshotsKept > 0) { + log('released_snapshots_swept', { deleted: summary.releasedSnapshotsDeleted, kept: summary.releasedSnapshotsKept }) + } +} + // ---- entry points ---- /** Kill switch drain: snapshot-preserving suspend of every running sandbox (one batch per call). */ @@ -376,6 +420,8 @@ export async function housekeeping( orphanSandboxesDeleted: 0, orphanSnapshotsSeen: 0, orphanSnapshotsDeleted: 0, + releasedSnapshotsDeleted: 0, + releasedSnapshotsKept: 0, rowsMissingProvider: 0, holdsOpened: 0, holdsCleared: 0, @@ -394,6 +440,7 @@ export async function housekeeping( const notifier = options.notifier ?? LOG_NOTIFIER for (const step of [ () => sweepOrphans(deps, summary), + () => sweepReleasedSnapshots(deps, summary), () => probeHolds(deps, summary), () => enforceHolds(deps, notifier, summary), () => enforceIdle(deps, notifier, summary) diff --git a/cloud/apps/sandbox-control/src/snapshot-refs.ts b/cloud/apps/sandbox-control/src/snapshot-refs.ts new file mode 100644 index 00000000000..d5c49e1718d --- /dev/null +++ b/cloud/apps/sandbox-control/src/snapshot-refs.ts @@ -0,0 +1,75 @@ +// Which provider snapshots something live may still boot from. Every path that deletes a snapshot it +// does not exclusively own (a retired golden image, an environment build, a sweep) asks here first, +// so a delete never pulls an image out from under a workspace, a warm-pool VM or a golden image. +// +// A snapshot is in use when it is: +// - a golden image that is not failed/deleted (active, smoking, building, or kept as a rollback), +// or one of the config's golden ids (GOLDEN_SNAPSHOT_ID, a preview's override); +// - the boot image of a warm-pool VM; +// - the active build of an environment that still exists; +// - one of a live workspace's own recorded snapshots; +// - the boot image (source_snapshot_id) of a live workspace that has no snapshot of its own yet. +// Once a persistent sandbox has stopped once it resumes from its own snapshot, so its source +// image no longer pins it (environment builds would otherwise be held for every workspace's life). +// "Live" = any workspace row not yet destroyed (creating, failed and destroying rows included). + +export type SnapshotRefConfig = { goldenSnapshotId: string; goldenSnapshots?: Record } + +const CHUNK = 50 + +/** The subset of `ids` something live still references. */ +export async function snapshotsInUse( + db: D1Database, + config: SnapshotRefConfig, + ids: readonly string[], + options: { ignoreGoldenRows?: readonly string[] } = {} +): Promise> { + const used = new Set() + const configGoldens = new Set([config.goldenSnapshotId, ...Object.values(config.goldenSnapshots ?? {})].filter(Boolean)) + const unique = [...new Set(ids)] + for (const id of unique) { + if (configGoldens.has(id)) { + used.add(id) + } + } + const ignored = options.ignoreGoldenRows ?? [] + for (let i = 0; i < unique.length; i += CHUNK) { + const chunk = unique.slice(i, i + CHUNK) + const marks = chunk.map(() => '?').join(', ') + const ignoreMarks = ignored.map(() => '?').join(', ') + const queries = [ + db + .prepare( + `SELECT snapshot_id AS id FROM golden_images WHERE snapshot_id IN (${marks}) + AND status NOT IN ('failed', 'deleted')${ignored.length > 0 ? ` AND id NOT IN (${ignoreMarks})` : ''}` + ) + .bind(...chunk, ...ignored), + db.prepare(`SELECT snapshot_id AS id FROM sandbox_pool WHERE snapshot_id IN (${marks})`).bind(...chunk), + db + .prepare( + `SELECT b.snapshot_id AS id FROM environments e JOIN environment_builds b ON b.id = e.active_build_id + WHERE e.deleted_at IS NULL AND b.snapshot_id IN (${marks})` + ) + .bind(...chunk), + db + .prepare( + `SELECT ss.snapshot_id AS id FROM sandbox_snapshots ss JOIN sandboxes s ON s.id = ss.sandbox_id + WHERE ss.deleted_at IS NULL AND s.status <> 'destroyed' AND ss.snapshot_id IN (${marks})` + ) + .bind(...chunk), + db + .prepare( + `SELECT s.source_snapshot_id AS id FROM sandboxes s + WHERE s.status <> 'destroyed' AND s.source_snapshot_id IN (${marks}) + AND NOT EXISTS (SELECT 1 FROM sandbox_snapshots ss WHERE ss.sandbox_id = s.id AND ss.deleted_at IS NULL)` + ) + .bind(...chunk) + ] + for (const query of queries) { + for (const row of (await query.all<{ id: string }>()).results) { + used.add(row.id) + } + } + } + return used +} diff --git a/cloud/apps/sandbox-control/src/team-delete.ts b/cloud/apps/sandbox-control/src/team-delete.ts index 33c0f148283..2b614a7a44f 100644 --- a/cloud/apps/sandbox-control/src/team-delete.ts +++ b/cloud/apps/sandbox-control/src/team-delete.ts @@ -7,7 +7,8 @@ // A team can only be deleted once it has no cloud workspaces in any state (running, asleep or // archived): the user deletes them first through the normal workspace delete, which destroys the // sandbox and its snapshots. Then auth purges the team's cloud secrets (org-scoped and the team's -// repo-scoped ones; members' personal secrets are theirs) and deletes the team. +// repo-scoped ones; members' personal secrets are theirs) and deletes the team. The purge also marks +// the team's environments deleted, so housekeeping deletes their build snapshots. const ACCOUNT_PATH = /^\/v1\/internal\/accounts\/((?:org|dev:org):[A-Za-z0-9_.:-]{1,150})\/(delete-check|secrets\/purge)$/ @@ -41,7 +42,8 @@ export async function teamWorkspaceCount(db: D1Database, accountId: string): Pro */ export async function purgeTeamSecrets( db: D1Database, - accountId: string + accountId: string, + now: number = Date.now() ): Promise<{ ok: true; purged: number } | { ok: false; workspaces: number }> { const workspaces = await teamWorkspaceCount(db, accountId) if (workspaces > 0) { @@ -58,7 +60,12 @@ export async function purgeTeamSecrets( // The team's secret activity (names, who, when) goes with the team. db.prepare('DELETE FROM secret_events WHERE account_id = ?').bind(accountId), // So do its owner-change notices (workspace names, who): nobody can sign in to the team to read them. - db.prepare('DELETE FROM owner_transfer_notices WHERE account_id = ?').bind(accountId) + db.prepare('DELETE FROM owner_transfer_notices WHERE account_id = ?').bind(accountId), + // And its environments: their build snapshots (manual, no provider expiry) are freed by the hourly + // housekeeping sweep (retention.ts sweepReleasedSnapshots), which this route has no provider for. + db + .prepare('UPDATE environments SET deleted_at = ?, active_build_id = NULL WHERE account_id = ? AND deleted_at IS NULL') + .bind(now, accountId) ]) return { ok: true, purged: Number(result?.meta?.changes ?? 0) } } diff --git a/cloud/apps/sandbox-control/test/snapshot-leaks.test.ts b/cloud/apps/sandbox-control/test/snapshot-leaks.test.ts new file mode 100644 index 00000000000..d94b86068fc --- /dev/null +++ b/cloud/apps/sandbox-control/test/snapshot-leaks.test.ts @@ -0,0 +1,202 @@ +// Every delete path frees its snapshots, and nothing ever deletes a snapshot something live still +// boots from (snapshot-refs.ts): retired golden images, deleted environments' builds, team delete, +// the orphan sweeps, a provider delete that failed once. +import { describe, expect, it } from 'vitest' +import { DEFAULT_RETENTION, housekeeping } from '../src/retention' +import type { LifecycleDeps } from '../src/lifecycle' +import { snapshotsInUse } from '../src/snapshot-refs' +import { purgeTeamSecrets } from '../src/team-delete' +import { ProviderError } from '../src/sandbox-provider' +import { GOLDEN, createHarness } from './harness' + +const HOUR = 60 * 60_000 +const create = (workspaceId: string) => ({ body: { workspaceId, machineSize: 'standard' } }) + +async function setup() { + const h = await createHarness() + const deps: LifecycleDeps = { + db: h.env.SANDBOX_DB, + provider: h.vercel, + entitlements: { + checkStart: async () => h.billing.decision, + reportUsage: async () => h.billing.usageAnswer, + listSuspensions: async () => h.billing.suspensions + }, + now: h.now, + retention: DEFAULT_RETENTION, + config: { environment: 'test', goldenSnapshotId: GOLDEN, defaultRegion: 'iad1' } + } + const sweep = () => housekeeping(deps, { force: true, notifier: { warn: async () => undefined } }) + const snap = (id: string, creationMethod = 'manual', sandboxName = 'build') => + h.vercel.snapshots.set(id, { id, status: 'created', sourceSessionId: 's', sandboxName, creationMethod, createdAt: h.now() }) + const status = (id: string) => h.vercel.snapshots.get(id)?.status + return { h, deps, sweep, snap, status } +} + +function golden(h: Awaited>, id: string, status: string, snapshotId: string, createdAt: number) { + h.raw + .prepare(`INSERT INTO golden_images (id, source, status, snapshot_id, created_at, updated_at, region) VALUES (?, 'seed', ?, ?, ?, ?, 'iad1')`) + .run(id, status, snapshotId, createdAt, createdAt) +} + +function environment(h: Awaited>, id: string, accountId: string, builds: { id: string; snapshotId: string }[]) { + h.raw + .prepare( + `INSERT INTO environments (id, account_id, name, repos, install_script, env_var_names, machine_size, created_by, updated_by, created_at, updated_at) + VALUES (?, ?, ?, '[]', '', '[]', 'standard', 'u', 'u', 1, 1)` + ) + .run(id, accountId, id) + builds.forEach((build, i) => + h.raw + .prepare( + `INSERT INTO environment_builds (id, environment_id, account_id, number, status, config, base_snapshot_id, provider_name, snapshot_id, created_by, created_at, updated_at) + VALUES (?, ?, ?, ?, 'succeeded', '{}', ?, ?, ?, 'u', 1, 1)` + ) + .run(build.id, id, accountId, i + 1, GOLDEN, `orca-test-envbuild-${build.id}`, build.snapshotId) + ) + h.raw.prepare('UPDATE environments SET active_build_id = ? WHERE id = ?').run(builds.at(-1)?.id ?? null, id) +} + +describe('workspace delete', () => { + it('deletes every automatic snapshot the workspace made, never the golden it booted from', async () => { + const { h, status } = await setup() + const id = (await h.call('POST', '/v1/sandboxes', create('ws-1'))).body.sandbox.id + await h.call('POST', `/v1/sandboxes/${id}/suspend`) + await h.call('POST', `/v1/sandboxes/${id}/resume`) + await h.call('POST', `/v1/sandboxes/${id}/suspend`) + const own = [...h.vercel.snapshots.values()].filter((s) => s.sandboxName === `orca-test-${id}`) + expect(own.length).toBe(2) + expect((await h.call('DELETE', `/v1/sandboxes/${id}`)).status).toBe(200) + expect(own.map((s) => status(s.id))).toEqual(['deleted', 'deleted']) + expect(h.vercel.boxes.size).toBe(0) + }) +}) + +describe('snapshotsInUse', () => { + it('pins goldens, pool images, active environment builds, live workspaces’ own snapshots and un-stopped boot images', async () => { + const { h, deps } = await setup() + golden(h, 'g_active', 'active', 'snap_g_active', 3) + golden(h, 'g_failed', 'failed', 'snap_g_failed', 2) + environment(h, 'env_a', 'org:acme', [{ id: 'b1', snapshotId: 'snap_b1' }, { id: 'b2', snapshotId: 'snap_b2' }]) + h.raw + .prepare(`INSERT INTO sandbox_pool (id, provider_name, machine_size, region, snapshot_id, status, created_at, updated_at) VALUES ('p1', 'orca-test-p1', 'standard', 'iad1', 'snap_pool', 'ready', 1, 1)`) + .run() + const id = (await h.call('POST', '/v1/sandboxes', create('ws-1'))).body.sandbox.id + h.raw.prepare('UPDATE sandboxes SET source_snapshot_id = ? WHERE id = ?').run('snap_boot', id) + const ids = [GOLDEN, 'snap_g_active', 'snap_g_failed', 'snap_b1', 'snap_b2', 'snap_pool', 'snap_boot', 'snap_free'] + expect([...(await snapshotsInUse(deps.db, deps.config, ids))].sort()).toEqual( + [GOLDEN, 'snap_b2', 'snap_boot', 'snap_g_active', 'snap_pool'].sort() + ) + // Once the workspace stopped once it resumes from its own snapshot: the boot image is free, its own one is pinned. + await h.call('POST', `/v1/sandboxes/${id}/suspend`) + const own = [...h.vercel.snapshots.values()].find((s) => s.sandboxName === `orca-test-${id}`)!.id + const after = await snapshotsInUse(deps.db, deps.config, ['snap_boot', own]) + expect(after.has('snap_boot')).toBe(false) + expect(after.has(own)).toBe(true) + // A destroyed workspace pins nothing. + await h.call('DELETE', `/v1/sandboxes/${id}`) + expect((await snapshotsInUse(deps.db, deps.config, [own])).size).toBe(0) + }) +}) + +describe('golden rotation', () => { + it('keeps one retired rollback, defers one a live workspace still boots from, retries a failed delete', async () => { + const { h, sweep, snap, status } = await setup() + golden(h, 'g4', 'active', 'snap_g4', 4) + golden(h, 'g3', 'retired', 'snap_g3', 3) + golden(h, 'g2', 'retired', 'snap_g2', 2) + golden(h, 'g1', 'retired', 'snap_g1', 1) + for (const id of ['snap_g4', 'snap_g3', 'snap_g2', 'snap_g1']) { + snap(id) + } + const ws = (await h.call('POST', '/v1/sandboxes', create('ws-1'))).body.sandbox.id + h.raw.prepare('UPDATE sandboxes SET source_snapshot_id = ? WHERE id = ?').run('snap_g2', ws) + const deleteSnapshot = h.vercel.deleteSnapshot + let failures = 1 + h.vercel.deleteSnapshot = async (id) => { + if (id === 'snap_g1' && failures-- > 0) { + throw new ProviderError(503, null) + } + return deleteSnapshot(id) + } + const goldenStatus = (id: string) => (h.raw.prepare('SELECT status FROM golden_images WHERE id = ?').get(id) as { status: string }).status + + expect((await sweep()).releasedSnapshotsKept).toBe(2) + expect([status('snap_g1'), goldenStatus('g1')]).toEqual(['created', 'retired']) + expect([status('snap_g2'), goldenStatus('g2')]).toEqual(['created', 'retired']) + + const second = await sweep() + expect([second.releasedSnapshotsDeleted, second.releasedSnapshotsKept]).toEqual([1, 1]) + expect([status('snap_g1'), goldenStatus('g1')]).toEqual(['deleted', 'deleted']) + expect(status('snap_g2')).toBe('created') + + // The workspace stops once (now it has its own snapshot): its boot image is released. + await h.call('POST', `/v1/sandboxes/${ws}/suspend`) + expect((await sweep()).releasedSnapshotsDeleted).toBe(1) + expect([status('snap_g2'), goldenStatus('g2')]).toEqual(['deleted', 'deleted']) + // The newest retired (rollback) and the active image are never touched. + expect([status('snap_g3'), status('snap_g4')]).toEqual(['created', 'created']) + expect((await h.call('POST', `/v1/sandboxes/${ws}/resume`)).status).toBe(200) + }) +}) + +describe('deleted environments and teams', () => { + it('a team delete marks its environments deleted; housekeeping frees their build snapshots unless a live workspace boots from one', async () => { + const { h, sweep, snap, status } = await setup() + environment(h, 'env_team', 'org:gone', [{ id: 'tb1', snapshotId: 'snap_tb1' }, { id: 'tb2', snapshotId: 'snap_tb2' }]) + environment(h, 'env_other', 'org:stays', [{ id: 'ob1', snapshotId: 'snap_ob1' }]) + for (const id of ['snap_tb1', 'snap_tb2', 'snap_ob1']) { + snap(id) + } + // Another account's live workspace was created from tb1 (it never stopped yet). + const ws = (await h.call('POST', '/v1/sandboxes', create('ws-1'))).body.sandbox.id + h.raw.prepare('UPDATE sandboxes SET source_snapshot_id = ? WHERE id = ?').run('snap_tb1', ws) + + expect(await purgeTeamSecrets(h.env.SANDBOX_DB, 'org:gone', h.now())).toEqual({ ok: true, purged: 0 }) + expect(h.raw.prepare('SELECT deleted_at IS NOT NULL AS d, active_build_id FROM environments WHERE id = ?').get('env_team')).toEqual({ d: 1, active_build_id: null }) + + const first = await sweep() + expect([first.releasedSnapshotsDeleted, first.releasedSnapshotsKept]).toEqual([1, 1]) + expect([status('snap_tb1'), status('snap_tb2'), status('snap_ob1')]).toEqual(['created', 'deleted', 'created']) + + await h.call('DELETE', `/v1/sandboxes/${ws}`) + expect((await sweep()).releasedSnapshotsDeleted).toBe(1) + expect(status('snap_tb1')).toBe('deleted') + const marked = h.raw.prepare(`SELECT id FROM environment_builds WHERE snapshot_deleted_at IS NOT NULL ORDER BY id`).all() + expect(marked).toEqual([{ id: 'tb1' }, { id: 'tb2' }]) + }) + + it('an admin delete whose provider call fails keeps the build for the next sweep instead of marking it gone', async () => { + const { h, deps, sweep, snap, status } = await setup() + environment(h, 'env_a', 'org:acme', [{ id: 'b1', snapshotId: 'snap_b1' }]) + snap('snap_b1') + const deleteSnapshot = h.vercel.deleteSnapshot + h.vercel.deleteSnapshot = async () => { + throw new ProviderError(500, null) + } + const { deleteEnvironmentAndSnapshots } = await import('../src/auto-environments') + await deleteEnvironmentAndSnapshots(deps, 'env_a') + expect(h.raw.prepare('SELECT snapshot_deleted_at FROM environment_builds WHERE id = ?').get('b1')).toEqual({ snapshot_deleted_at: null }) + h.vercel.deleteSnapshot = deleteSnapshot + expect((await sweep()).releasedSnapshotsDeleted).toBe(1) + expect(status('snap_b1')).toBe('deleted') + }) +}) + +describe('orphan snapshot sweep', () => { + it('never deletes an automatic snapshot a warm-pool VM or a live workspace boots from', async () => { + const { h, sweep, snap, status } = await setup() + snap('snap_pool_image', 'automatic', 'orca-test-long-gone-a') + snap('snap_boot_image', 'automatic', 'orca-test-long-gone-b') + snap('snap_free', 'automatic', 'orca-test-long-gone-c') + h.raw + .prepare(`INSERT INTO sandbox_pool (id, provider_name, machine_size, region, snapshot_id, status, created_at, updated_at) VALUES ('p1', 'orca-test-p1', 'standard', 'iad1', 'snap_pool_image', 'warming', 1, 1)`) + .run() + const ws = (await h.call('POST', '/v1/sandboxes', create('ws-1'))).body.sandbox.id + h.raw.prepare('UPDATE sandboxes SET source_snapshot_id = ? WHERE id = ?').run('snap_boot_image', ws) + await sweep() + h.advance(25 * HOUR) + expect((await sweep()).orphanSnapshotsDeleted).toBe(1) + expect([status('snap_pool_image'), status('snap_boot_image'), status('snap_free')]).toEqual(['created', 'created', 'deleted']) + }) +})