fix(sandbox-control): a boot from an older environment snapshot starts with this Worker's entry points (probe folded into the runtime refresh command; install only when behind)

This commit is contained in:
Neil
2026-11-15 01:06:08 -08:00
parent 8f38bcf3ce
commit d88c87f17d
3 changed files with 90 additions and 13 deletions
@@ -362,13 +362,13 @@ export function entryPointsVersion(): string {
// One round trip: the image's scripts stamp (missing = 0: images from before stamps) and whether
// this Worker's entry points are already installed.
const ENTRY_POINTS_PROBE = [
export const ENTRY_POINTS_PROBE = [
'stamp="$(cat /opt/orca/bin/.scripts-stamp 2>/dev/null || true)"',
'echo "stamp=${stamp:-0}"',
`if [ -e /opt/orca/bin/.entry-points-${VM_SCRIPTS_HASH} ]; then echo marker=1; else echo marker=0; fi`
].join('\n')
function parseEntryPointsProbe(stdout: string): { stamp: number; marker: boolean } | null {
export function parseEntryPointsProbe(stdout: string): { stamp: number; marker: boolean } | null {
const marker = /^marker=([01])$/m.exec(stdout)?.[1]
if (marker === undefined) {
return null
@@ -399,6 +399,16 @@ export async function ensureEntryPoints(deps: { provider: Pick<SandboxProviderAp
await installEntryPoints(deps, sessionId)
}
/**
* ENTRY-POINTS-BEFORE-START: a boot from an environment build made on an older golden would run that
* golden's orca-sandbox-start (no agent prefetch, no OpenCode first run) until the environment is
* rebuilt, hours later. True when the image's scripts are older than this Worker's: no marker for this
* set and a lower stamp (an equal stamp is the same scripts: the stamp only moves when they change).
*/
export function entryPointsBehind(image: { stamp: number; marker: boolean } | null): boolean {
return image !== null && !image.marker && image.stamp < VM_SCRIPTS_STAMP
}
export async function installEntryPoints(deps: { provider: Pick<SandboxProviderApi, 'runCommand'> }, sessionId: string): Promise<void> {
const env = provisionEnv({ manifest: MANIFEST, provisionScript: PROVISION_SCRIPT, vmScripts: VM_SCRIPT_SOURCES, vmScriptsStamp: VM_SCRIPTS_STAMP, buildId: 'sync' })
const scriptEnv = Object.fromEntries(Object.entries(env).filter(([key]) => key.startsWith('ORCA_VM_SCRIPT_')))
+26 -11
View File
@@ -15,7 +15,7 @@ import {
} from './guardrails'
import { DEFAULT_EGRESS_ALLOWLIST, DEFAULT_EGRESS_PRESET, KEYS_WITHHELD_BODY, compileNetworkPolicy, egressProxyMissing, gitBasicAuthorization, type CredentialRoute, type EgressPreset, type PlatformHostSuffixesConfigError } from './network-policy'
import { applyQueuedRevokes } from './grant-revokes'
import { ensureEntryPoints } from './golden-images'
import { ENTRY_POINTS_PROBE, ensureEntryPoints, entryPointsBehind, installEntryPoints, parseEntryPointsProbe } from './golden-images'
import { RUNTIME_REFRESH_SCRIPT, type RuntimeBundle } from './runtime-bundles'
import { memberPlaceholder, type MemberGitIdentity } from './github-user-tokens'
import { connectedMembers, memberGitConfigs } from './member-git'
@@ -638,7 +638,9 @@ export const MEMBERS_COMMAND = '/opt/orca/bin/orca-sandbox-members'
/**
* ENV-STALE boot step: a sandbox whose orcad is behind its region's active golden (booted from an environment
* build snapshot, or resumed from an older image) installs the golden's runtime bundle before orcad starts.
* Best effort and quiet: any failure keeps the old orcad, and every outcome is logged with its timing.
* The same command probes the VM's entry points (ENTRY-POINTS-BEFORE-START): scripts older than this
* Worker's are installed before the start script runs, so the common case adds no round trip.
* Best effort and quiet: any failure keeps the old orcad and scripts, and every outcome is logged with its timing.
*/
export async function refreshRuntime(deps: LifecycleDeps, row: SandboxRow, sessionId: string): Promise<string | null> {
if (!deps.runtimeRefresh) {
@@ -647,28 +649,41 @@ export async function refreshRuntime(deps: LifecycleDeps, row: SandboxRow, sessi
const started = deps.now()
let result = 'error'
let target: string | null = null
let entryPoints = 'unknown'
try {
const plan = await deps.runtimeRefresh(row.region, started)
if (!plan) {
result = 'no_bundle'
return result
}
target = plan.bundle.commit
target = plan?.bundle.commit ?? null
const run = await deps.provider.runCommand(sessionId, {
command: 'bash',
args: ['-c', RUNTIME_REFRESH_SCRIPT],
env: { WANT: plan.bundle.commit, SHA: plan.bundle.sha256, RT: plan.bundle.runtime, URL: plan.url },
args: ['-c', plan ? `${ENTRY_POINTS_PROBE}\n${RUNTIME_REFRESH_SCRIPT}` : ENTRY_POINTS_PROBE],
env: plan ? { WANT: plan.bundle.commit, SHA: plan.bundle.sha256, RT: plan.bundle.runtime, URL: plan.url } : {},
sudo: true,
timeoutMs: 40_000
})
result = run.stdout.trim().split('\n').pop() ?? 'error'
result = plan ? (run.stdout.trim().split('\n').pop() ?? 'error') : 'no_bundle'
const image = parseEntryPointsProbe(run.stdout)
entryPoints = entryPointsBehind(image) ? 'behind' : image ? 'current' : 'unknown'
if (entryPoints === 'behind') {
entryPoints = await installEntryPoints(deps, sessionId).then(
() => 'installed',
() => 'install_failed'
)
}
return result
} catch {
return result
} finally {
const [outcome, from] = result.split(' ')
console.log(
JSON.stringify({ event: 'runtime_refresh', sandboxId: row.id, outcome, from: from ?? null, to: target, ms: deps.now() - started })
JSON.stringify({
event: 'runtime_refresh',
sandboxId: row.id,
outcome,
from: from ?? null,
to: target,
entryPoints,
ms: deps.now() - started
})
)
}
}
@@ -0,0 +1,52 @@
// ENTRY-POINTS-BEFORE-START: a boot from an environment build made on an older golden ran that golden's
// orca-sandbox-start (no agent prefetch, no OpenCode first run) until the environment was rebuilt.
// refreshRuntime's one command also probes the entry points; older ones are installed before the start.
import { describe, expect, it, vi } from 'vitest'
import { VM_SCRIPTS_STAMP } from '../image/vm-scripts.generated'
import { entryPointsBehind } from '../src/golden-images'
import { refreshRuntime, type LifecycleDeps } from '../src/lifecycle'
import type { SandboxRow } from '../src/store'
type Call = { args?: string[]; env?: Record<string, string> }
function setup(probe: string, plan: boolean) {
const calls: Call[] = []
const runCommand = vi.fn(async (_session: string, call: Call) => {
calls.push(call)
return calls.length === 1 ? { exitCode: 0, stdout: `${probe}\n${plan ? 'current abc' : ''}`, stderr: '' } : { exitCode: 0, stdout: 'synced\n', stderr: '' }
})
const deps = {
provider: { runCommand },
now: () => 0,
runtimeRefresh: async () => (plan ? { bundle: { commit: 'abc', sha256: 'f', runtime: '22' }, url: 'https://x' } : null)
} as unknown as LifecycleDeps
const row = { id: 'sbx_1', region: 'iad1' } as unknown as SandboxRow
return { calls, deps, row }
}
describe('ENTRY-POINTS-BEFORE-START', () => {
it('installs older entry points before the start, in the same round trip as the runtime check', async () => {
const t = setup(`stamp=${VM_SCRIPTS_STAMP - 1}\nmarker=0`, true)
await expect(refreshRuntime(t.deps, t.row, 'sess')).resolves.toBe('current abc')
expect(t.calls).toHaveLength(2)
expect(t.calls[0]!.args!.at(-1)).toContain('stamp=')
expect(t.calls[0]!.args!.at(-1)).toContain('SOURCE_COMMIT')
expect(t.calls[1]!.args!.at(-1)).toContain('install -o root')
})
it('adds nothing when the image carries this Worker scripts (same stamp or marker)', async () => {
for (const probe of [`stamp=${VM_SCRIPTS_STAMP}\nmarker=0`, 'stamp=0\nmarker=1']) {
const t = setup(probe, true)
await refreshRuntime(t.deps, t.row, 'sess')
expect(t.calls).toHaveLength(1)
}
})
it('still probes without a runtime bundle, and never downgrades a newer image', async () => {
const t = setup(`stamp=${VM_SCRIPTS_STAMP - 1}\nmarker=0`, false)
await expect(refreshRuntime(t.deps, t.row, 'sess')).resolves.toBe('no_bundle')
expect(t.calls).toHaveLength(2)
expect(entryPointsBehind({ stamp: VM_SCRIPTS_STAMP + 1, marker: false })).toBe(false)
expect(entryPointsBehind(null)).toBe(false)
})
})