mirror of
https://github.com/stablyai/orca.git
synced 2026-11-18 03:45:08 +00:00
fix(cloud): env builds progress server-side on a Durable Object alarm, not only while streamed (#118)
* fix(sandbox-control): read the orgRole claim that Orca Cloud auth mints (#29) Auth mints the live org role as orgRole on sandbox tokens (same claim as billing tokens, stablyai/orca-cloud#695); the verifier read org_role, so admins would never have been recognized. * wip(cloud): environment builds control plane * wip(cloud): environment build boot path, staleness, routes * wip(cloud): env build tests, anonymous staleness fallback * wip(cloud): migration 0008 (claimed), quieter recovered clone errors * fix(cloud): env build review: interrupted-snapshot recovery, bounded log reads, secret redaction, activation guard, hook/prelude fixes * fix(cloud): wait briefly for the build exit code after its log ends; print the base image and package manager * wip(desktop): org environments client, IPC, CLI, settings UI, orcad disk usage + low-disk banner * perf(cloud): smoke-boot each environment build before activating it; skip the redundant prelude fetch * fix(cloud): a GitHub token error fails the build as github_unavailable instead of cloning anonymously * perf(cloud): warm pool VMs per hot environment build, so a create from an org environment skips the provider boot * wip(desktop): cloud environment UI round-3 fixes * fix(cloud): pool-per-build review: smoke race safety, provider hiccups skip smoke, usage-based hot builds with caps, bounded boots, index * fix(desktop): Configure with an agent hands the prompt over at launch (trust dialog swallowed the pasted Enter); CLI builds table widths * polish(desktop): W10 P2s — InlineError action slot, shared focus-ring, copy * feat(cloud): build event stream heartbeat every 20 s * fix(desktop): FE review — stable virtualizer, shadow-buffer replays, heartbeat-aware build watch, CLI follower restarts, repo import path checks, poll races * feat(cloud): meter environment build sandbox time to the member who started it; no setup-script nudge for workspaces booted from an env build * fix(cloud): environment builds progress server-side on a Durable Object alarm, not only while someone streams their log
This commit is contained in:
@@ -42,6 +42,8 @@ export type Env = {
|
||||
BILLING_URL?: string
|
||||
/** Service binding to the billing Worker; Workers on one account cannot reach each other over workers.dev. */
|
||||
BILLING?: Fetcher
|
||||
/** Durable Object that advances environment builds on alarms (EnvironmentBuildRunner in index.ts). */
|
||||
ENV_BUILD_RUNNER?: DurableObjectNamespace
|
||||
CONTROL_PLANE_TOKEN?: string
|
||||
/** `stub` allows every start and records nothing: local dev only, refused when SANDBOX_ENV=prod. */
|
||||
ENTITLEMENTS_MODE?: string
|
||||
|
||||
@@ -107,6 +107,11 @@ export type BuildDeps = {
|
||||
githubFetch?: typeof fetch
|
||||
/** Keeps finalization running after a watcher disconnects (Worker ctx.waitUntil). */
|
||||
waitUntil?: (promise: Promise<unknown>) => void
|
||||
/**
|
||||
* Hands a started build to its server-side runner (a Durable Object alarm in the Worker), so it
|
||||
* advances whether or not anyone is watching. Absent in tests that tick builds by hand.
|
||||
*/
|
||||
schedule?: (build: { id: string; environmentId: string }) => Promise<void>
|
||||
config: { environment: string; goldenSnapshotId: string; region: string; guardrails?: GuardrailConfig }
|
||||
}
|
||||
|
||||
@@ -250,7 +255,12 @@ export async function startBuild(deps: BuildDeps, caller: SandboxCaller, environ
|
||||
const row = (await getBuild(deps.db, environment.id, id))!
|
||||
log('environment_build_started', { buildId: id, environmentId: environment.id, number: row.number, repos: repos.length })
|
||||
try {
|
||||
return await bootBuild(deps, row, config)
|
||||
const booted = await bootBuild(deps, row, config)
|
||||
// Progress is server-side from here: a client that never opens the log stream still gets a build.
|
||||
await deps.schedule?.({ id: booted.id, environmentId: booted.environment_id }).catch((error: unknown) =>
|
||||
log('environment_build_schedule_failed', { buildId: booted.id, error: String(error).slice(0, 120) })
|
||||
)
|
||||
return booted
|
||||
} catch (error) {
|
||||
const current = (await getBuild(deps.db, environment.id, id)) ?? row
|
||||
const code =
|
||||
@@ -740,6 +750,43 @@ export async function reportBuildUsage(deps: Pick<BuildDeps, 'db' | 'entitlement
|
||||
return true
|
||||
}
|
||||
|
||||
/** Runner cadence: how soon to look again at a build in this state; null once it is final. */
|
||||
export function nextBuildTickMs(row: Pick<BuildRow, 'status'>): number | null {
|
||||
switch (row.status) {
|
||||
case 'starting':
|
||||
case 'running':
|
||||
return 3_000
|
||||
case 'snapshotting':
|
||||
return 10_000
|
||||
default:
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One server-side step for a build (the runner's alarm): advance it and say when to look again.
|
||||
* Independent of readers: the log stream and GET also advance, all through the same CAS steps.
|
||||
*/
|
||||
export async function tickBuild(deps: BuildDeps, environmentId: string, buildId: string): Promise<number | null> {
|
||||
const row = await getBuild(deps.db, environmentId, buildId)
|
||||
if (!row) {
|
||||
return null
|
||||
}
|
||||
const next = await advanceBuild(deps, row)
|
||||
if (TERMINAL.has(next.status) && next.usage_reported_at === null) {
|
||||
await reportBuildUsage(deps, (await getBuild(deps.db, environmentId, buildId)) ?? next)
|
||||
}
|
||||
return nextBuildTickMs(next)
|
||||
}
|
||||
|
||||
/** Unfinished builds (the cron re-arms their runners in case one was lost). */
|
||||
export async function unfinishedBuilds(db: D1Database): Promise<{ id: string; environment_id: string }[]> {
|
||||
const { results } = await db
|
||||
.prepare(`SELECT id, environment_id FROM environment_builds WHERE status IN ('starting', 'running', 'snapshotting') LIMIT 50`)
|
||||
.all<{ id: string; environment_id: string }>()
|
||||
return results
|
||||
}
|
||||
|
||||
/** Cron: advance every unfinished build (finalize the ones nobody is watching). */
|
||||
export async function advanceAllBuilds(deps: BuildDeps): Promise<number> {
|
||||
const { results } = await deps.db
|
||||
|
||||
@@ -98,7 +98,7 @@ import { SshKeyRequestSchema, updateSshKey } from './ssh-access'
|
||||
import { audit } from './api/audit'
|
||||
import { getSandbox, getSandboxByProviderName, listAccountSandboxes, type SandboxRow } from './store'
|
||||
import { createVercelSandboxApi } from './vercel-sandbox-api'
|
||||
import { advanceAllBuilds, type BuildDeps } from './environment-builds'
|
||||
import { advanceAllBuilds, tickBuild, unfinishedBuilds, type BuildDeps } from './environment-builds'
|
||||
import { environmentErrorResponse, handleEnvironmentRoute, isEnvironmentRoute } from './environment-routes'
|
||||
import { EnvironmentError } from './environments'
|
||||
import type { GoldenImageProviderApi } from './golden-image-provider'
|
||||
@@ -178,6 +178,8 @@ export type Deps = {
|
||||
retention?: RetentionConfig
|
||||
/** Anonymous github.com reachability (tests stub it). */
|
||||
publicRepoCheck?: (repoUrl: string) => Promise<boolean>
|
||||
/** Build runner override (tests tick builds by hand). */
|
||||
scheduleBuild?: BuildDeps['schedule']
|
||||
/** ctx.waitUntil from the Worker entry; tests collect the promises. */
|
||||
waitUntil?: (work: Promise<unknown>) => void
|
||||
/** Route groups mounted on the public API (routines, see src/routines). */
|
||||
@@ -523,8 +525,22 @@ async function lifecycleDeps(env: Env, deps: Deps): Promise<LifecycleDeps | null
|
||||
}
|
||||
}
|
||||
|
||||
function buildDeps(life: LifecycleDeps, deps: Deps): BuildDeps {
|
||||
/** Arms the build's runner (one Durable Object per build); a no-op where the binding is absent. */
|
||||
function buildScheduler(env: Env): BuildDeps['schedule'] {
|
||||
const runners = env.ENV_BUILD_RUNNER
|
||||
if (!runners) {
|
||||
return undefined
|
||||
}
|
||||
return async (build) => {
|
||||
const stub = runners.get(runners.idFromName(build.id))
|
||||
const response = await stub.fetch('https://runner/start', { method: 'POST', body: JSON.stringify(build) })
|
||||
await response.body?.cancel()
|
||||
}
|
||||
}
|
||||
|
||||
function buildDeps(life: LifecycleDeps, deps: Deps, env?: Env): BuildDeps {
|
||||
return {
|
||||
schedule: deps.scheduleBuild ?? (env ? buildScheduler(env) : undefined),
|
||||
db: life.db,
|
||||
// SAFETY: the Vercel client implements both; test fakes for builds do too.
|
||||
provider: life.provider as BuildDeps['provider'],
|
||||
@@ -978,7 +994,7 @@ export async function handleRequest(request: Request, env: Env, deps: Deps = {})
|
||||
if (!life) {
|
||||
return fail('not_configured', 503)
|
||||
}
|
||||
return await handleEnvironmentRoute(request, url, caller, buildDeps(life, deps), () => readJson(request))
|
||||
return await handleEnvironmentRoute(request, url, caller, buildDeps(life, deps, env), () => readJson(request))
|
||||
}
|
||||
if (/^\/v1\/sandboxes\/[A-Za-z0-9_]+\/shares(?:\/|$)|^\/v1\/shares\//.test(url.pathname)) {
|
||||
const caller = await authenticateDesktop(env, request, (deps.now ?? Date.now)())
|
||||
@@ -1393,6 +1409,23 @@ export default {
|
||||
}
|
||||
})
|
||||
)
|
||||
// Environment builds: every minute and on their own, so a failing reconcile never stalls one.
|
||||
// Advances what nobody watches and re-arms each build's runner in case one was lost.
|
||||
ctx.waitUntil(
|
||||
lifecycleDeps(env, {}).then(async (life) => {
|
||||
if (!life) {
|
||||
return
|
||||
}
|
||||
const deps = buildDeps(life, {}, env)
|
||||
const builds = await advanceAllBuilds(deps).catch(() => 0)
|
||||
if (builds > 0) {
|
||||
console.log(JSON.stringify({ event: 'environment_builds_advanced', builds }))
|
||||
}
|
||||
for (const build of await unfinishedBuilds(env.SANDBOX_DB).catch(() => [])) {
|
||||
await deps.schedule?.({ id: build.id, environmentId: build.environment_id }).catch(() => undefined)
|
||||
}
|
||||
})
|
||||
)
|
||||
if (new Date(controller.scheduledTime).getUTCMinutes() % 5 !== 0) {
|
||||
return
|
||||
}
|
||||
@@ -1402,10 +1435,6 @@ export default {
|
||||
// Counts only; no ids, tokens or pairing material.
|
||||
const summary = await reconcile(life)
|
||||
console.log(JSON.stringify({ event: 'reconcile', ...summary }))
|
||||
const builds = await advanceAllBuilds(buildDeps(life, {})).catch(() => 0)
|
||||
if (builds > 0) {
|
||||
console.log(JSON.stringify({ event: 'environment_builds_advanced', builds }))
|
||||
}
|
||||
const retried = await retryPendingMessages(life).catch(() => -1)
|
||||
if (retried !== 0) {
|
||||
console.log(JSON.stringify({ event: 'message_retry', retried }))
|
||||
@@ -1430,3 +1459,48 @@ export default {
|
||||
)
|
||||
}
|
||||
} satisfies ExportedHandler<Env>
|
||||
|
||||
/**
|
||||
* One per environment build: wakes on an alarm, advances the build one step (finalize, snapshot,
|
||||
* smoke boot, billing all run here) and re-arms until the build is final. Builds therefore finish
|
||||
* with no client watching, at most a few seconds after their VM command ends.
|
||||
*/
|
||||
export class EnvironmentBuildRunner implements DurableObject {
|
||||
constructor(
|
||||
private readonly state: DurableObjectState,
|
||||
private readonly env: Env
|
||||
) {}
|
||||
|
||||
async fetch(request: Request): Promise<Response> {
|
||||
const body = (await request.json().catch(() => null)) as { id?: string; environmentId?: string } | null
|
||||
if (!body?.id || !body.environmentId) {
|
||||
return new Response('bad request', { status: 400 })
|
||||
}
|
||||
await this.state.storage.put('build', { id: body.id, environmentId: body.environmentId })
|
||||
if ((await this.state.storage.getAlarm()) === null) {
|
||||
await this.state.storage.setAlarm(Date.now() + 2_000)
|
||||
}
|
||||
return new Response(null, { status: 204 })
|
||||
}
|
||||
|
||||
async alarm(): Promise<void> {
|
||||
const build = await this.state.storage.get<{ id: string; environmentId: string }>('build')
|
||||
const life = build ? await lifecycleDeps(this.env, {}) : null
|
||||
if (!build || !life) {
|
||||
await this.state.storage.deleteAll()
|
||||
return
|
||||
}
|
||||
let next: number | null
|
||||
try {
|
||||
next = await tickBuild(buildDeps(life, {}, this.env), build.environmentId, build.id)
|
||||
} catch (error) {
|
||||
console.log(JSON.stringify({ event: 'environment_build_tick_failed', buildId: build.id, error: String(error).slice(0, 200) }))
|
||||
next = 15_000
|
||||
}
|
||||
if (next === null) {
|
||||
await this.state.storage.deleteAll()
|
||||
return
|
||||
}
|
||||
await this.state.storage.setAlarm(Date.now() + next)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -124,6 +124,46 @@ describe('org environments', () => {
|
||||
expect(refused.body.reason).toBe('hours_exhausted')
|
||||
})
|
||||
|
||||
it('finishes a build server-side with nobody streaming or polling it (regression: W9 poll-only stall)', async () => {
|
||||
const scheduled: { id: string; environmentId: string }[] = []
|
||||
const h = await createHarness({}, { scheduleBuild: async (build) => void scheduled.push(build) })
|
||||
h.vercel.buildResult = RESULT
|
||||
const admin = await h.token('alice', 'acme', 'sandbox', { orgRole: 'admin' })
|
||||
const env = await h.call('POST', '/v1/environments', { bearer: admin, body: { name: 'Default', repos: [{ url: 'acme/web' }], installScript: 'echo hi' } })
|
||||
const started = await h.call('POST', `/v1/environments/${env.body.environment.id}/builds`, { bearer: admin })
|
||||
expect(started.body.build.status).toBe('running')
|
||||
// The build is handed to its runner at start; no reader ever looks at it again.
|
||||
expect(scheduled).toEqual([{ id: started.body.build.id, environmentId: env.body.environment.id }])
|
||||
const { tickBuild, unfinishedBuilds } = await import('../src/environment-builds')
|
||||
const deps = {
|
||||
db: h.env.SANDBOX_DB,
|
||||
provider: h.vercel,
|
||||
entitlements: { checkStart: async () => h.billing.decision, reportUsage: async (s: never) => { h.billing.usage.push(s); return h.billing.usageAnswer }, listSuspensions: async () => [] },
|
||||
now: h.now,
|
||||
config: { environment: 'test', goldenSnapshotId: 'snap_golden', region: 'iad1' }
|
||||
}
|
||||
// While the VM command runs, the runner keeps a short cadence.
|
||||
expect(await tickBuild(deps, env.body.environment.id, started.body.build.id)).toBe(3_000)
|
||||
expect(await unfinishedBuilds(h.env.SANDBOX_DB)).toHaveLength(1)
|
||||
const command = [...h.vercel.buildCommands.values()].at(-1)!
|
||||
command.logs = log('::orca-step:: install\nhi\n')
|
||||
command.exitCode = 0
|
||||
h.advance(3_000)
|
||||
// The next alarm finalizes it (snapshot, smoke boot, activation, billing) and stops.
|
||||
expect(await tickBuild(deps, env.body.environment.id, started.body.build.id)).toBeNull()
|
||||
const row = h.raw.prepare('SELECT status, usage_reported_at, log_bytes FROM environment_builds WHERE id = ?').get(started.body.build.id) as {
|
||||
status: string
|
||||
usage_reported_at: number | null
|
||||
log_bytes: number
|
||||
}
|
||||
expect(row.status).toBe('succeeded')
|
||||
expect(row.usage_reported_at).not.toBeNull()
|
||||
expect(row.log_bytes).toBeGreaterThan(0)
|
||||
expect(await unfinishedBuilds(h.env.SANDBOX_DB)).toEqual([])
|
||||
const environment = (await h.call('GET', `/v1/environments/${env.body.environment.id}`, { bearer: admin })).body.environment
|
||||
expect(environment.activeBuildId).toBe(started.body.build.id)
|
||||
})
|
||||
|
||||
it('refuses a second concurrent build and can cancel one', async () => {
|
||||
const { adminCall, envId } = await setup()
|
||||
const started = await adminCall('POST', `/v1/environments/${envId}/builds`)
|
||||
|
||||
@@ -22,6 +22,9 @@
|
||||
// Same-account Workers cannot call each other over workers.dev (error 1042); BILLING_URL names the origin.
|
||||
"services": [{ "binding": "BILLING", "service": "orca-billing-dev" }],
|
||||
// Every minute for routines (src/routines); the rest of the cron work runs on minute % 5 == 0.
|
||||
// Environment builds advance on Durable Object alarms (EnvironmentBuildRunner), not on readers.
|
||||
"durable_objects": { "bindings": [{ "name": "ENV_BUILD_RUNNER", "class_name": "EnvironmentBuildRunner" }] },
|
||||
"migrations": [{ "tag": "env-build-runner-v1", "new_sqlite_classes": ["EnvironmentBuildRunner"] }],
|
||||
"triggers": { "crons": ["* * * * *"] },
|
||||
"vars": {
|
||||
"SANDBOX_ENV": "dev",
|
||||
|
||||
@@ -17,6 +17,9 @@
|
||||
}
|
||||
],
|
||||
// Usage settlement, lease/deadline enforcement and stuck-row reconciliation.
|
||||
// Environment builds advance on Durable Object alarms (EnvironmentBuildRunner), not on readers.
|
||||
"durable_objects": { "bindings": [{ "name": "ENV_BUILD_RUNNER", "class_name": "EnvironmentBuildRunner" }] },
|
||||
"migrations": [{ "tag": "env-build-runner-v1", "new_sqlite_classes": ["EnvironmentBuildRunner"] }],
|
||||
"triggers": { "crons": ["*/5 * * * *"] },
|
||||
"vars": {
|
||||
"SANDBOX_ENV": "local",
|
||||
|
||||
Reference in New Issue
Block a user