mirror of
https://github.com/stablyai/orca.git
synced 2026-11-17 03:15:01 +00:00
fix(sandbox-control): a boot from an older environment snapshot starts with this Worker's entry points (probe folded into the runtime refresh command; install only when behind)
This commit is contained in:
@@ -362,13 +362,13 @@ export function entryPointsVersion(): string {
|
||||
|
||||
// One round trip: the image's scripts stamp (missing = 0: images from before stamps) and whether
|
||||
// this Worker's entry points are already installed.
|
||||
const ENTRY_POINTS_PROBE = [
|
||||
export const ENTRY_POINTS_PROBE = [
|
||||
'stamp="$(cat /opt/orca/bin/.scripts-stamp 2>/dev/null || true)"',
|
||||
'echo "stamp=${stamp:-0}"',
|
||||
`if [ -e /opt/orca/bin/.entry-points-${VM_SCRIPTS_HASH} ]; then echo marker=1; else echo marker=0; fi`
|
||||
].join('\n')
|
||||
|
||||
function parseEntryPointsProbe(stdout: string): { stamp: number; marker: boolean } | null {
|
||||
export function parseEntryPointsProbe(stdout: string): { stamp: number; marker: boolean } | null {
|
||||
const marker = /^marker=([01])$/m.exec(stdout)?.[1]
|
||||
if (marker === undefined) {
|
||||
return null
|
||||
@@ -399,6 +399,16 @@ export async function ensureEntryPoints(deps: { provider: Pick<SandboxProviderAp
|
||||
await installEntryPoints(deps, sessionId)
|
||||
}
|
||||
|
||||
/**
|
||||
* ENTRY-POINTS-BEFORE-START: a boot from an environment build made on an older golden would run that
|
||||
* golden's orca-sandbox-start (no agent prefetch, no OpenCode first run) until the environment is
|
||||
* rebuilt, hours later. True when the image's scripts are older than this Worker's: no marker for this
|
||||
* set and a lower stamp (an equal stamp is the same scripts: the stamp only moves when they change).
|
||||
*/
|
||||
export function entryPointsBehind(image: { stamp: number; marker: boolean } | null): boolean {
|
||||
return image !== null && !image.marker && image.stamp < VM_SCRIPTS_STAMP
|
||||
}
|
||||
|
||||
export async function installEntryPoints(deps: { provider: Pick<SandboxProviderApi, 'runCommand'> }, sessionId: string): Promise<void> {
|
||||
const env = provisionEnv({ manifest: MANIFEST, provisionScript: PROVISION_SCRIPT, vmScripts: VM_SCRIPT_SOURCES, vmScriptsStamp: VM_SCRIPTS_STAMP, buildId: 'sync' })
|
||||
const scriptEnv = Object.fromEntries(Object.entries(env).filter(([key]) => key.startsWith('ORCA_VM_SCRIPT_')))
|
||||
|
||||
@@ -15,7 +15,7 @@ import {
|
||||
} from './guardrails'
|
||||
import { DEFAULT_EGRESS_ALLOWLIST, DEFAULT_EGRESS_PRESET, KEYS_WITHHELD_BODY, compileNetworkPolicy, egressProxyMissing, gitBasicAuthorization, type CredentialRoute, type EgressPreset, type PlatformHostSuffixesConfigError } from './network-policy'
|
||||
import { applyQueuedRevokes } from './grant-revokes'
|
||||
import { ensureEntryPoints } from './golden-images'
|
||||
import { ENTRY_POINTS_PROBE, ensureEntryPoints, entryPointsBehind, installEntryPoints, parseEntryPointsProbe } from './golden-images'
|
||||
import { RUNTIME_REFRESH_SCRIPT, type RuntimeBundle } from './runtime-bundles'
|
||||
import { memberPlaceholder, type MemberGitIdentity } from './github-user-tokens'
|
||||
import { connectedMembers, memberGitConfigs } from './member-git'
|
||||
@@ -638,7 +638,9 @@ export const MEMBERS_COMMAND = '/opt/orca/bin/orca-sandbox-members'
|
||||
/**
|
||||
* ENV-STALE boot step: a sandbox whose orcad is behind its region's active golden (booted from an environment
|
||||
* build snapshot, or resumed from an older image) installs the golden's runtime bundle before orcad starts.
|
||||
* Best effort and quiet: any failure keeps the old orcad, and every outcome is logged with its timing.
|
||||
* The same command probes the VM's entry points (ENTRY-POINTS-BEFORE-START): scripts older than this
|
||||
* Worker's are installed before the start script runs, so the common case adds no round trip.
|
||||
* Best effort and quiet: any failure keeps the old orcad and scripts, and every outcome is logged with its timing.
|
||||
*/
|
||||
export async function refreshRuntime(deps: LifecycleDeps, row: SandboxRow, sessionId: string): Promise<string | null> {
|
||||
if (!deps.runtimeRefresh) {
|
||||
@@ -647,28 +649,41 @@ export async function refreshRuntime(deps: LifecycleDeps, row: SandboxRow, sessi
|
||||
const started = deps.now()
|
||||
let result = 'error'
|
||||
let target: string | null = null
|
||||
let entryPoints = 'unknown'
|
||||
try {
|
||||
const plan = await deps.runtimeRefresh(row.region, started)
|
||||
if (!plan) {
|
||||
result = 'no_bundle'
|
||||
return result
|
||||
}
|
||||
target = plan.bundle.commit
|
||||
target = plan?.bundle.commit ?? null
|
||||
const run = await deps.provider.runCommand(sessionId, {
|
||||
command: 'bash',
|
||||
args: ['-c', RUNTIME_REFRESH_SCRIPT],
|
||||
env: { WANT: plan.bundle.commit, SHA: plan.bundle.sha256, RT: plan.bundle.runtime, URL: plan.url },
|
||||
args: ['-c', plan ? `${ENTRY_POINTS_PROBE}\n${RUNTIME_REFRESH_SCRIPT}` : ENTRY_POINTS_PROBE],
|
||||
env: plan ? { WANT: plan.bundle.commit, SHA: plan.bundle.sha256, RT: plan.bundle.runtime, URL: plan.url } : {},
|
||||
sudo: true,
|
||||
timeoutMs: 40_000
|
||||
})
|
||||
result = run.stdout.trim().split('\n').pop() ?? 'error'
|
||||
result = plan ? (run.stdout.trim().split('\n').pop() ?? 'error') : 'no_bundle'
|
||||
const image = parseEntryPointsProbe(run.stdout)
|
||||
entryPoints = entryPointsBehind(image) ? 'behind' : image ? 'current' : 'unknown'
|
||||
if (entryPoints === 'behind') {
|
||||
entryPoints = await installEntryPoints(deps, sessionId).then(
|
||||
() => 'installed',
|
||||
() => 'install_failed'
|
||||
)
|
||||
}
|
||||
return result
|
||||
} catch {
|
||||
return result
|
||||
} finally {
|
||||
const [outcome, from] = result.split(' ')
|
||||
console.log(
|
||||
JSON.stringify({ event: 'runtime_refresh', sandboxId: row.id, outcome, from: from ?? null, to: target, ms: deps.now() - started })
|
||||
JSON.stringify({
|
||||
event: 'runtime_refresh',
|
||||
sandboxId: row.id,
|
||||
outcome,
|
||||
from: from ?? null,
|
||||
to: target,
|
||||
entryPoints,
|
||||
ms: deps.now() - started
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
// ENTRY-POINTS-BEFORE-START: a boot from an environment build made on an older golden ran that golden's
|
||||
// orca-sandbox-start (no agent prefetch, no OpenCode first run) until the environment was rebuilt.
|
||||
// refreshRuntime's one command also probes the entry points; older ones are installed before the start.
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { VM_SCRIPTS_STAMP } from '../image/vm-scripts.generated'
|
||||
import { entryPointsBehind } from '../src/golden-images'
|
||||
import { refreshRuntime, type LifecycleDeps } from '../src/lifecycle'
|
||||
import type { SandboxRow } from '../src/store'
|
||||
|
||||
type Call = { args?: string[]; env?: Record<string, string> }
|
||||
|
||||
function setup(probe: string, plan: boolean) {
|
||||
const calls: Call[] = []
|
||||
const runCommand = vi.fn(async (_session: string, call: Call) => {
|
||||
calls.push(call)
|
||||
return calls.length === 1 ? { exitCode: 0, stdout: `${probe}\n${plan ? 'current abc' : ''}`, stderr: '' } : { exitCode: 0, stdout: 'synced\n', stderr: '' }
|
||||
})
|
||||
const deps = {
|
||||
provider: { runCommand },
|
||||
now: () => 0,
|
||||
runtimeRefresh: async () => (plan ? { bundle: { commit: 'abc', sha256: 'f', runtime: '22' }, url: 'https://x' } : null)
|
||||
} as unknown as LifecycleDeps
|
||||
const row = { id: 'sbx_1', region: 'iad1' } as unknown as SandboxRow
|
||||
return { calls, deps, row }
|
||||
}
|
||||
|
||||
describe('ENTRY-POINTS-BEFORE-START', () => {
|
||||
it('installs older entry points before the start, in the same round trip as the runtime check', async () => {
|
||||
const t = setup(`stamp=${VM_SCRIPTS_STAMP - 1}\nmarker=0`, true)
|
||||
await expect(refreshRuntime(t.deps, t.row, 'sess')).resolves.toBe('current abc')
|
||||
expect(t.calls).toHaveLength(2)
|
||||
expect(t.calls[0]!.args!.at(-1)).toContain('stamp=')
|
||||
expect(t.calls[0]!.args!.at(-1)).toContain('SOURCE_COMMIT')
|
||||
expect(t.calls[1]!.args!.at(-1)).toContain('install -o root')
|
||||
})
|
||||
|
||||
it('adds nothing when the image carries this Worker scripts (same stamp or marker)', async () => {
|
||||
for (const probe of [`stamp=${VM_SCRIPTS_STAMP}\nmarker=0`, 'stamp=0\nmarker=1']) {
|
||||
const t = setup(probe, true)
|
||||
await refreshRuntime(t.deps, t.row, 'sess')
|
||||
expect(t.calls).toHaveLength(1)
|
||||
}
|
||||
})
|
||||
|
||||
it('still probes without a runtime bundle, and never downgrades a newer image', async () => {
|
||||
const t = setup(`stamp=${VM_SCRIPTS_STAMP - 1}\nmarker=0`, false)
|
||||
await expect(refreshRuntime(t.deps, t.row, 'sess')).resolves.toBe('no_bundle')
|
||||
expect(t.calls).toHaveLength(2)
|
||||
expect(entryPointsBehind({ stamp: VM_SCRIPTS_STAMP + 1, marker: false })).toBe(false)
|
||||
expect(entryPointsBehind(null)).toBe(false)
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user