From d88c87f17db3fa663975045044b0e2bc0d64e28a Mon Sep 17 00:00:00 2001 From: Neil Date: Sun, 15 Nov 2026 01:06:08 -0800 Subject: [PATCH] fix(sandbox-control): a boot from an older environment snapshot starts with this Worker's entry points (probe folded into the runtime refresh command; install only when behind) --- .../apps/sandbox-control/src/golden-images.ts | 14 ++++- cloud/apps/sandbox-control/src/lifecycle.ts | 37 +++++++++---- .../test/entry-points-before-start.test.ts | 52 +++++++++++++++++++ 3 files changed, 90 insertions(+), 13 deletions(-) create mode 100644 cloud/apps/sandbox-control/test/entry-points-before-start.test.ts diff --git a/cloud/apps/sandbox-control/src/golden-images.ts b/cloud/apps/sandbox-control/src/golden-images.ts index 035caec020a..915646cc5e8 100644 --- a/cloud/apps/sandbox-control/src/golden-images.ts +++ b/cloud/apps/sandbox-control/src/golden-images.ts @@ -362,13 +362,13 @@ export function entryPointsVersion(): string { // One round trip: the image's scripts stamp (missing = 0: images from before stamps) and whether // this Worker's entry points are already installed. -const ENTRY_POINTS_PROBE = [ +export const ENTRY_POINTS_PROBE = [ 'stamp="$(cat /opt/orca/bin/.scripts-stamp 2>/dev/null || true)"', 'echo "stamp=${stamp:-0}"', `if [ -e /opt/orca/bin/.entry-points-${VM_SCRIPTS_HASH} ]; then echo marker=1; else echo marker=0; fi` ].join('\n') -function parseEntryPointsProbe(stdout: string): { stamp: number; marker: boolean } | null { +export function parseEntryPointsProbe(stdout: string): { stamp: number; marker: boolean } | null { const marker = /^marker=([01])$/m.exec(stdout)?.[1] if (marker === undefined) { return null @@ -399,6 +399,16 @@ export async function ensureEntryPoints(deps: { provider: Pick }, sessionId: string): Promise { const env = provisionEnv({ manifest: MANIFEST, provisionScript: PROVISION_SCRIPT, vmScripts: VM_SCRIPT_SOURCES, vmScriptsStamp: VM_SCRIPTS_STAMP, buildId: 'sync' }) const scriptEnv = Object.fromEntries(Object.entries(env).filter(([key]) => key.startsWith('ORCA_VM_SCRIPT_'))) diff --git a/cloud/apps/sandbox-control/src/lifecycle.ts b/cloud/apps/sandbox-control/src/lifecycle.ts index 855ad508a65..4c01ce7fa3f 100644 --- a/cloud/apps/sandbox-control/src/lifecycle.ts +++ b/cloud/apps/sandbox-control/src/lifecycle.ts @@ -15,7 +15,7 @@ import { } from './guardrails' import { DEFAULT_EGRESS_ALLOWLIST, DEFAULT_EGRESS_PRESET, KEYS_WITHHELD_BODY, compileNetworkPolicy, egressProxyMissing, gitBasicAuthorization, type CredentialRoute, type EgressPreset, type PlatformHostSuffixesConfigError } from './network-policy' import { applyQueuedRevokes } from './grant-revokes' -import { ensureEntryPoints } from './golden-images' +import { ENTRY_POINTS_PROBE, ensureEntryPoints, entryPointsBehind, installEntryPoints, parseEntryPointsProbe } from './golden-images' import { RUNTIME_REFRESH_SCRIPT, type RuntimeBundle } from './runtime-bundles' import { memberPlaceholder, type MemberGitIdentity } from './github-user-tokens' import { connectedMembers, memberGitConfigs } from './member-git' @@ -638,7 +638,9 @@ export const MEMBERS_COMMAND = '/opt/orca/bin/orca-sandbox-members' /** * ENV-STALE boot step: a sandbox whose orcad is behind its region's active golden (booted from an environment * build snapshot, or resumed from an older image) installs the golden's runtime bundle before orcad starts. - * Best effort and quiet: any failure keeps the old orcad, and every outcome is logged with its timing. + * The same command probes the VM's entry points (ENTRY-POINTS-BEFORE-START): scripts older than this + * Worker's are installed before the start script runs, so the common case adds no round trip. + * Best effort and quiet: any failure keeps the old orcad and scripts, and every outcome is logged with its timing. */ export async function refreshRuntime(deps: LifecycleDeps, row: SandboxRow, sessionId: string): Promise { if (!deps.runtimeRefresh) { @@ -647,28 +649,41 @@ export async function refreshRuntime(deps: LifecycleDeps, row: SandboxRow, sessi const started = deps.now() let result = 'error' let target: string | null = null + let entryPoints = 'unknown' try { const plan = await deps.runtimeRefresh(row.region, started) - if (!plan) { - result = 'no_bundle' - return result - } - target = plan.bundle.commit + target = plan?.bundle.commit ?? null const run = await deps.provider.runCommand(sessionId, { command: 'bash', - args: ['-c', RUNTIME_REFRESH_SCRIPT], - env: { WANT: plan.bundle.commit, SHA: plan.bundle.sha256, RT: plan.bundle.runtime, URL: plan.url }, + args: ['-c', plan ? `${ENTRY_POINTS_PROBE}\n${RUNTIME_REFRESH_SCRIPT}` : ENTRY_POINTS_PROBE], + env: plan ? { WANT: plan.bundle.commit, SHA: plan.bundle.sha256, RT: plan.bundle.runtime, URL: plan.url } : {}, sudo: true, timeoutMs: 40_000 }) - result = run.stdout.trim().split('\n').pop() ?? 'error' + result = plan ? (run.stdout.trim().split('\n').pop() ?? 'error') : 'no_bundle' + const image = parseEntryPointsProbe(run.stdout) + entryPoints = entryPointsBehind(image) ? 'behind' : image ? 'current' : 'unknown' + if (entryPoints === 'behind') { + entryPoints = await installEntryPoints(deps, sessionId).then( + () => 'installed', + () => 'install_failed' + ) + } return result } catch { return result } finally { const [outcome, from] = result.split(' ') console.log( - JSON.stringify({ event: 'runtime_refresh', sandboxId: row.id, outcome, from: from ?? null, to: target, ms: deps.now() - started }) + JSON.stringify({ + event: 'runtime_refresh', + sandboxId: row.id, + outcome, + from: from ?? null, + to: target, + entryPoints, + ms: deps.now() - started + }) ) } } diff --git a/cloud/apps/sandbox-control/test/entry-points-before-start.test.ts b/cloud/apps/sandbox-control/test/entry-points-before-start.test.ts new file mode 100644 index 00000000000..cc6afc9f219 --- /dev/null +++ b/cloud/apps/sandbox-control/test/entry-points-before-start.test.ts @@ -0,0 +1,52 @@ +// ENTRY-POINTS-BEFORE-START: a boot from an environment build made on an older golden ran that golden's +// orca-sandbox-start (no agent prefetch, no OpenCode first run) until the environment was rebuilt. +// refreshRuntime's one command also probes the entry points; older ones are installed before the start. +import { describe, expect, it, vi } from 'vitest' +import { VM_SCRIPTS_STAMP } from '../image/vm-scripts.generated' +import { entryPointsBehind } from '../src/golden-images' +import { refreshRuntime, type LifecycleDeps } from '../src/lifecycle' +import type { SandboxRow } from '../src/store' + +type Call = { args?: string[]; env?: Record } + +function setup(probe: string, plan: boolean) { + const calls: Call[] = [] + const runCommand = vi.fn(async (_session: string, call: Call) => { + calls.push(call) + return calls.length === 1 ? { exitCode: 0, stdout: `${probe}\n${plan ? 'current abc' : ''}`, stderr: '' } : { exitCode: 0, stdout: 'synced\n', stderr: '' } + }) + const deps = { + provider: { runCommand }, + now: () => 0, + runtimeRefresh: async () => (plan ? { bundle: { commit: 'abc', sha256: 'f', runtime: '22' }, url: 'https://x' } : null) + } as unknown as LifecycleDeps + const row = { id: 'sbx_1', region: 'iad1' } as unknown as SandboxRow + return { calls, deps, row } +} + +describe('ENTRY-POINTS-BEFORE-START', () => { + it('installs older entry points before the start, in the same round trip as the runtime check', async () => { + const t = setup(`stamp=${VM_SCRIPTS_STAMP - 1}\nmarker=0`, true) + await expect(refreshRuntime(t.deps, t.row, 'sess')).resolves.toBe('current abc') + expect(t.calls).toHaveLength(2) + expect(t.calls[0]!.args!.at(-1)).toContain('stamp=') + expect(t.calls[0]!.args!.at(-1)).toContain('SOURCE_COMMIT') + expect(t.calls[1]!.args!.at(-1)).toContain('install -o root') + }) + + it('adds nothing when the image carries this Worker scripts (same stamp or marker)', async () => { + for (const probe of [`stamp=${VM_SCRIPTS_STAMP}\nmarker=0`, 'stamp=0\nmarker=1']) { + const t = setup(probe, true) + await refreshRuntime(t.deps, t.row, 'sess') + expect(t.calls).toHaveLength(1) + } + }) + + it('still probes without a runtime bundle, and never downgrades a newer image', async () => { + const t = setup(`stamp=${VM_SCRIPTS_STAMP - 1}\nmarker=0`, false) + await expect(refreshRuntime(t.deps, t.row, 'sess')).resolves.toBe('no_bundle') + expect(t.calls).toHaveLength(2) + expect(entryPointsBehind({ stamp: VM_SCRIPTS_STAMP + 1, marker: false })).toBe(false) + expect(entryPointsBehind(null)).toBe(false) + }) +})