diff --git a/cloud/apps/sandbox-control/src/lifecycle.ts b/cloud/apps/sandbox-control/src/lifecycle.ts index 5079dd9d3bf..6b0708619e4 100644 --- a/cloud/apps/sandbox-control/src/lifecycle.ts +++ b/cloud/apps/sandbox-control/src/lifecycle.ts @@ -873,7 +873,21 @@ export async function scrubStoppedRecord(deps: LifecycleDeps, row: SandboxRow): if (!recordMayHoldValues(deps, row)) { return } - const outcome = await scrubRecordPolicy(deps, row) + // Review R2: the row may have resumed since it was selected (or since its stop). Only a still-stopped row + // with no session is scrubbed; a scrub that raced a resume is undone by a fresh apply right after. + const stopped = (fresh: SandboxRow | null): boolean => + fresh !== null && (fresh.status === 'suspended' || fresh.status === 'archived') && !fresh.session_id + const before = await getSandbox(deps.db, row.id) + if (!before || !stopped(before)) { + return + } + const outcome = await scrubRecordPolicy(deps, before) + const after = await getSandbox(deps.db, row.id) + if (after && !stopped(after) && after.session_id && (after.status === 'running' || after.status === 'resuming')) { + console.log(JSON.stringify({ event: 'policy_scrub_raced_resume', sandboxId: row.id })) + await applySandboxPolicy(deps, after, 'scrub_race').catch(() => undefined) + return + } const ok = outcome === 'scrubbed' await deps.db .prepare('UPDATE sandboxes SET policy_scrub_pending = ?, policy_scrubbed_at = COALESCE(?, policy_scrubbed_at) WHERE id = ?') @@ -882,7 +896,11 @@ export async function scrubStoppedRecord(deps: LifecycleDeps, row: SandboxRow): console.log(JSON.stringify({ event: 'policy_record_scrub', sandboxId: row.id, outcome: typeof outcome === 'string' ? outcome : outcome.failed })) } -/** Stopped rows whose record may still hold values: a failed scrub, or an apply since the last scrub. */ +/** + * Stopped rows whose record may still hold values: a failed scrub, or an apply since the last scrub. + * 'failed' rows are left out on purpose (review R4): failAndRelease destroys their provider sandbox, and the + * cron retries a destroy that didn't finish, so no record is left to scrub. + */ export async function rowsToScrub(db: D1Database, limit = 25): Promise { const { results } = await db .prepare( diff --git a/cloud/apps/sandbox-control/test/egress-record-scrub.test.ts b/cloud/apps/sandbox-control/test/egress-record-scrub.test.ts index 44c35b0762a..9160398d9df 100644 --- a/cloud/apps/sandbox-control/test/egress-record-scrub.test.ts +++ b/cloud/apps/sandbox-control/test/egress-record-scrub.test.ts @@ -113,3 +113,33 @@ describe('A4: the post-resume write retries, and a lasting failure is visible, n expect(record).not.toContain(KEY) }) }) + +describe('A4 review R2: a scrub never strips a workspace that resumed meanwhile', () => { + it('a row that resumed before the scrub is skipped; one that resumed during it gets a fresh apply', async () => { + const t = await setup() + const life = (await lifecycleDeps(t.h.env, t.h.deps))! + const { scrubStoppedRecord } = await import('../src/lifecycle') + // Selected while stopped, resumed before the scrub runs: nothing is written. + expect((await t.h.call('POST', `/v1/sandboxes/${t.id}/suspend`, { bearer: t.mia })).status).toBe(200) + const stale = await t.row() + expect((await t.h.call('POST', `/v1/sandboxes/${t.id}/resume`, { bearer: t.mia })).status).toBe(200) + t.box().recordPolicy = undefined + await scrubStoppedRecord(life, stale) + expect(t.box().recordPolicy).toBeUndefined() + // Resumes while the PATCH is in flight: the live policy is re-applied with its values. + expect((await t.h.call('POST', `/v1/sandboxes/${t.id}/suspend`, { bearer: t.mia })).status).toBe(200) + const deps: LifecycleDeps = { + ...life, + provider: { + ...life.provider, + updateRecordNetworkPolicy: async (name, policy) => { + await life.provider.updateRecordNetworkPolicy!(name, policy) + expect((await t.h.call('POST', `/v1/sandboxes/${t.id}/resume`, { bearer: t.mia })).status).toBe(200) + t.box().policy = { allow: {} } + } + } + } + await scrubStoppedRecord(deps, await t.row()) + expect(JSON.stringify(t.box().policy)).toContain(KEY) + }) +})