diff --git a/cloud/apps/sandbox-control/src/env.ts b/cloud/apps/sandbox-control/src/env.ts index 690bddcc30e..4d985b4def6 100644 --- a/cloud/apps/sandbox-control/src/env.ts +++ b/cloud/apps/sandbox-control/src/env.ts @@ -42,6 +42,8 @@ export type Env = { BILLING_URL?: string /** Service binding to the billing Worker; Workers on one account cannot reach each other over workers.dev. */ BILLING?: Fetcher + /** Durable Object that advances environment builds on alarms (EnvironmentBuildRunner in index.ts). */ + ENV_BUILD_RUNNER?: DurableObjectNamespace CONTROL_PLANE_TOKEN?: string /** `stub` allows every start and records nothing: local dev only, refused when SANDBOX_ENV=prod. */ ENTITLEMENTS_MODE?: string diff --git a/cloud/apps/sandbox-control/src/environment-builds.ts b/cloud/apps/sandbox-control/src/environment-builds.ts index 9e993a9016f..00c5e9019a0 100644 --- a/cloud/apps/sandbox-control/src/environment-builds.ts +++ b/cloud/apps/sandbox-control/src/environment-builds.ts @@ -107,6 +107,11 @@ export type BuildDeps = { githubFetch?: typeof fetch /** Keeps finalization running after a watcher disconnects (Worker ctx.waitUntil). */ waitUntil?: (promise: Promise) => void + /** + * Hands a started build to its server-side runner (a Durable Object alarm in the Worker), so it + * advances whether or not anyone is watching. Absent in tests that tick builds by hand. + */ + schedule?: (build: { id: string; environmentId: string }) => Promise config: { environment: string; goldenSnapshotId: string; region: string; guardrails?: GuardrailConfig } } @@ -250,7 +255,12 @@ export async function startBuild(deps: BuildDeps, caller: SandboxCaller, environ const row = (await getBuild(deps.db, environment.id, id))! log('environment_build_started', { buildId: id, environmentId: environment.id, number: row.number, repos: repos.length }) try { - return await bootBuild(deps, row, config) + const booted = await bootBuild(deps, row, config) + // Progress is server-side from here: a client that never opens the log stream still gets a build. + await deps.schedule?.({ id: booted.id, environmentId: booted.environment_id }).catch((error: unknown) => + log('environment_build_schedule_failed', { buildId: booted.id, error: String(error).slice(0, 120) }) + ) + return booted } catch (error) { const current = (await getBuild(deps.db, environment.id, id)) ?? row const code = @@ -740,6 +750,43 @@ export async function reportBuildUsage(deps: Pick): number | null { + switch (row.status) { + case 'starting': + case 'running': + return 3_000 + case 'snapshotting': + return 10_000 + default: + return null + } +} + +/** + * One server-side step for a build (the runner's alarm): advance it and say when to look again. + * Independent of readers: the log stream and GET also advance, all through the same CAS steps. + */ +export async function tickBuild(deps: BuildDeps, environmentId: string, buildId: string): Promise { + const row = await getBuild(deps.db, environmentId, buildId) + if (!row) { + return null + } + const next = await advanceBuild(deps, row) + if (TERMINAL.has(next.status) && next.usage_reported_at === null) { + await reportBuildUsage(deps, (await getBuild(deps.db, environmentId, buildId)) ?? next) + } + return nextBuildTickMs(next) +} + +/** Unfinished builds (the cron re-arms their runners in case one was lost). */ +export async function unfinishedBuilds(db: D1Database): Promise<{ id: string; environment_id: string }[]> { + const { results } = await db + .prepare(`SELECT id, environment_id FROM environment_builds WHERE status IN ('starting', 'running', 'snapshotting') LIMIT 50`) + .all<{ id: string; environment_id: string }>() + return results +} + /** Cron: advance every unfinished build (finalize the ones nobody is watching). */ export async function advanceAllBuilds(deps: BuildDeps): Promise { const { results } = await deps.db diff --git a/cloud/apps/sandbox-control/src/index.ts b/cloud/apps/sandbox-control/src/index.ts index 3dde83b32f9..643718cdf6c 100644 --- a/cloud/apps/sandbox-control/src/index.ts +++ b/cloud/apps/sandbox-control/src/index.ts @@ -98,7 +98,7 @@ import { SshKeyRequestSchema, updateSshKey } from './ssh-access' import { audit } from './api/audit' import { getSandbox, getSandboxByProviderName, listAccountSandboxes, type SandboxRow } from './store' import { createVercelSandboxApi } from './vercel-sandbox-api' -import { advanceAllBuilds, type BuildDeps } from './environment-builds' +import { advanceAllBuilds, tickBuild, unfinishedBuilds, type BuildDeps } from './environment-builds' import { environmentErrorResponse, handleEnvironmentRoute, isEnvironmentRoute } from './environment-routes' import { EnvironmentError } from './environments' import type { GoldenImageProviderApi } from './golden-image-provider' @@ -178,6 +178,8 @@ export type Deps = { retention?: RetentionConfig /** Anonymous github.com reachability (tests stub it). */ publicRepoCheck?: (repoUrl: string) => Promise + /** Build runner override (tests tick builds by hand). */ + scheduleBuild?: BuildDeps['schedule'] /** ctx.waitUntil from the Worker entry; tests collect the promises. */ waitUntil?: (work: Promise) => void /** Route groups mounted on the public API (routines, see src/routines). */ @@ -523,8 +525,22 @@ async function lifecycleDeps(env: Env, deps: Deps): Promise { + const stub = runners.get(runners.idFromName(build.id)) + const response = await stub.fetch('https://runner/start', { method: 'POST', body: JSON.stringify(build) }) + await response.body?.cancel() + } +} + +function buildDeps(life: LifecycleDeps, deps: Deps, env?: Env): BuildDeps { return { + schedule: deps.scheduleBuild ?? (env ? buildScheduler(env) : undefined), db: life.db, // SAFETY: the Vercel client implements both; test fakes for builds do too. provider: life.provider as BuildDeps['provider'], @@ -978,7 +994,7 @@ export async function handleRequest(request: Request, env: Env, deps: Deps = {}) if (!life) { return fail('not_configured', 503) } - return await handleEnvironmentRoute(request, url, caller, buildDeps(life, deps), () => readJson(request)) + return await handleEnvironmentRoute(request, url, caller, buildDeps(life, deps, env), () => readJson(request)) } if (/^\/v1\/sandboxes\/[A-Za-z0-9_]+\/shares(?:\/|$)|^\/v1\/shares\//.test(url.pathname)) { const caller = await authenticateDesktop(env, request, (deps.now ?? Date.now)()) @@ -1393,6 +1409,23 @@ export default { } }) ) + // Environment builds: every minute and on their own, so a failing reconcile never stalls one. + // Advances what nobody watches and re-arms each build's runner in case one was lost. + ctx.waitUntil( + lifecycleDeps(env, {}).then(async (life) => { + if (!life) { + return + } + const deps = buildDeps(life, {}, env) + const builds = await advanceAllBuilds(deps).catch(() => 0) + if (builds > 0) { + console.log(JSON.stringify({ event: 'environment_builds_advanced', builds })) + } + for (const build of await unfinishedBuilds(env.SANDBOX_DB).catch(() => [])) { + await deps.schedule?.({ id: build.id, environmentId: build.environment_id }).catch(() => undefined) + } + }) + ) if (new Date(controller.scheduledTime).getUTCMinutes() % 5 !== 0) { return } @@ -1402,10 +1435,6 @@ export default { // Counts only; no ids, tokens or pairing material. const summary = await reconcile(life) console.log(JSON.stringify({ event: 'reconcile', ...summary })) - const builds = await advanceAllBuilds(buildDeps(life, {})).catch(() => 0) - if (builds > 0) { - console.log(JSON.stringify({ event: 'environment_builds_advanced', builds })) - } const retried = await retryPendingMessages(life).catch(() => -1) if (retried !== 0) { console.log(JSON.stringify({ event: 'message_retry', retried })) @@ -1430,3 +1459,48 @@ export default { ) } } satisfies ExportedHandler + +/** + * One per environment build: wakes on an alarm, advances the build one step (finalize, snapshot, + * smoke boot, billing all run here) and re-arms until the build is final. Builds therefore finish + * with no client watching, at most a few seconds after their VM command ends. + */ +export class EnvironmentBuildRunner implements DurableObject { + constructor( + private readonly state: DurableObjectState, + private readonly env: Env + ) {} + + async fetch(request: Request): Promise { + const body = (await request.json().catch(() => null)) as { id?: string; environmentId?: string } | null + if (!body?.id || !body.environmentId) { + return new Response('bad request', { status: 400 }) + } + await this.state.storage.put('build', { id: body.id, environmentId: body.environmentId }) + if ((await this.state.storage.getAlarm()) === null) { + await this.state.storage.setAlarm(Date.now() + 2_000) + } + return new Response(null, { status: 204 }) + } + + async alarm(): Promise { + const build = await this.state.storage.get<{ id: string; environmentId: string }>('build') + const life = build ? await lifecycleDeps(this.env, {}) : null + if (!build || !life) { + await this.state.storage.deleteAll() + return + } + let next: number | null + try { + next = await tickBuild(buildDeps(life, {}, this.env), build.environmentId, build.id) + } catch (error) { + console.log(JSON.stringify({ event: 'environment_build_tick_failed', buildId: build.id, error: String(error).slice(0, 200) })) + next = 15_000 + } + if (next === null) { + await this.state.storage.deleteAll() + return + } + await this.state.storage.setAlarm(Date.now() + next) + } +} diff --git a/cloud/apps/sandbox-control/test/environments.test.ts b/cloud/apps/sandbox-control/test/environments.test.ts index a1b6c5beac7..2fa3826444d 100644 --- a/cloud/apps/sandbox-control/test/environments.test.ts +++ b/cloud/apps/sandbox-control/test/environments.test.ts @@ -124,6 +124,46 @@ describe('org environments', () => { expect(refused.body.reason).toBe('hours_exhausted') }) + it('finishes a build server-side with nobody streaming or polling it (regression: W9 poll-only stall)', async () => { + const scheduled: { id: string; environmentId: string }[] = [] + const h = await createHarness({}, { scheduleBuild: async (build) => void scheduled.push(build) }) + h.vercel.buildResult = RESULT + const admin = await h.token('alice', 'acme', 'sandbox', { orgRole: 'admin' }) + const env = await h.call('POST', '/v1/environments', { bearer: admin, body: { name: 'Default', repos: [{ url: 'acme/web' }], installScript: 'echo hi' } }) + const started = await h.call('POST', `/v1/environments/${env.body.environment.id}/builds`, { bearer: admin }) + expect(started.body.build.status).toBe('running') + // The build is handed to its runner at start; no reader ever looks at it again. + expect(scheduled).toEqual([{ id: started.body.build.id, environmentId: env.body.environment.id }]) + const { tickBuild, unfinishedBuilds } = await import('../src/environment-builds') + const deps = { + db: h.env.SANDBOX_DB, + provider: h.vercel, + entitlements: { checkStart: async () => h.billing.decision, reportUsage: async (s: never) => { h.billing.usage.push(s); return h.billing.usageAnswer }, listSuspensions: async () => [] }, + now: h.now, + config: { environment: 'test', goldenSnapshotId: 'snap_golden', region: 'iad1' } + } + // While the VM command runs, the runner keeps a short cadence. + expect(await tickBuild(deps, env.body.environment.id, started.body.build.id)).toBe(3_000) + expect(await unfinishedBuilds(h.env.SANDBOX_DB)).toHaveLength(1) + const command = [...h.vercel.buildCommands.values()].at(-1)! + command.logs = log('::orca-step:: install\nhi\n') + command.exitCode = 0 + h.advance(3_000) + // The next alarm finalizes it (snapshot, smoke boot, activation, billing) and stops. + expect(await tickBuild(deps, env.body.environment.id, started.body.build.id)).toBeNull() + const row = h.raw.prepare('SELECT status, usage_reported_at, log_bytes FROM environment_builds WHERE id = ?').get(started.body.build.id) as { + status: string + usage_reported_at: number | null + log_bytes: number + } + expect(row.status).toBe('succeeded') + expect(row.usage_reported_at).not.toBeNull() + expect(row.log_bytes).toBeGreaterThan(0) + expect(await unfinishedBuilds(h.env.SANDBOX_DB)).toEqual([]) + const environment = (await h.call('GET', `/v1/environments/${env.body.environment.id}`, { bearer: admin })).body.environment + expect(environment.activeBuildId).toBe(started.body.build.id) + }) + it('refuses a second concurrent build and can cancel one', async () => { const { adminCall, envId } = await setup() const started = await adminCall('POST', `/v1/environments/${envId}/builds`) diff --git a/cloud/apps/sandbox-control/wrangler.dev.jsonc b/cloud/apps/sandbox-control/wrangler.dev.jsonc index 1a373b1f18d..650b04dd9df 100644 --- a/cloud/apps/sandbox-control/wrangler.dev.jsonc +++ b/cloud/apps/sandbox-control/wrangler.dev.jsonc @@ -22,6 +22,9 @@ // Same-account Workers cannot call each other over workers.dev (error 1042); BILLING_URL names the origin. "services": [{ "binding": "BILLING", "service": "orca-billing-dev" }], // Every minute for routines (src/routines); the rest of the cron work runs on minute % 5 == 0. + // Environment builds advance on Durable Object alarms (EnvironmentBuildRunner), not on readers. + "durable_objects": { "bindings": [{ "name": "ENV_BUILD_RUNNER", "class_name": "EnvironmentBuildRunner" }] }, + "migrations": [{ "tag": "env-build-runner-v1", "new_sqlite_classes": ["EnvironmentBuildRunner"] }], "triggers": { "crons": ["* * * * *"] }, "vars": { "SANDBOX_ENV": "dev", diff --git a/cloud/apps/sandbox-control/wrangler.jsonc b/cloud/apps/sandbox-control/wrangler.jsonc index 677c810b07c..1958c91974d 100644 --- a/cloud/apps/sandbox-control/wrangler.jsonc +++ b/cloud/apps/sandbox-control/wrangler.jsonc @@ -17,6 +17,9 @@ } ], // Usage settlement, lease/deadline enforcement and stuck-row reconciliation. + // Environment builds advance on Durable Object alarms (EnvironmentBuildRunner), not on readers. + "durable_objects": { "bindings": [{ "name": "ENV_BUILD_RUNNER", "class_name": "EnvironmentBuildRunner" }] }, + "migrations": [{ "tag": "env-build-runner-v1", "new_sqlite_classes": ["EnvironmentBuildRunner"] }], "triggers": { "crons": ["*/5 * * * *"] }, "vars": { "SANDBOX_ENV": "local",